Changeset 415

Timestamp:

02/03/2010 06:45:12 PM (7 months ago)

Author:

matt

Message:

Work on NUMERIC, DATETIME, and BOOLEAN field types.
Changes instances of test_index to testindex.

Location:

projects/whoosh/trunk

Files:

• src/whoosh/fields.py (modified) (8 diffs)
• src/whoosh/qparser/default.py (modified) (3 diffs)
• src/whoosh/qparser/simple.py (modified) (2 diffs)
• src/whoosh/util.py (modified) (2 diffs)
• tests/test_highlighting.py (modified) (1 diff)
• tests/test_misc.py (modified) (4 diffs)
• tests/test_writing.py (modified) (2 diffs)

projects/whoosh/trunk/src/whoosh/fields.py

@@ -19 +19 @@
 """
 
-import datetime
-import re
+import datetime, re, struct
 
 from whoosh.analysis import IDAnalyzer, RegexAnalyzer, KeywordAnalyzer
@@ -32 +31 @@
 class UnknownFieldError(Exception):
     pass
-
 
 
@@ -68 +66 @@
     
     format = vector = scorable = stored = unique = None
+    parse_query = None
     indexed = True
     __inittypes__ = dict(format=Format, vector=Format,
@@ -108 +107 @@
         if not self.format:
             raise Exception("%s field cannot index without a format" % self.__class__)
-        
         if not isinstance(value, unicode):
             raise ValueError("%r is not unicode" % value)
-        
         return self.format.word_values(value, mode="index", **kwargs)
+    
+    def process_text(self, qstring, mode='', **kwargs):
+        if not self.format:
+            raise Exception("%s field has no format" % self)
+        return (t.text for t in self.format.analyze(qstring, mode=mode, **kwargs))
     
 
@@ -123 +125 @@
     __inittypes__ = dict(stored=bool, unique=bool, field_boost=float)
     
-    def __init__(self, stored = False, unique = False, field_boost = 1.0):
+    def __init__(self, stored=False, unique=False, field_boost=1.0):
         """
         :param stored: Whether the value of this field is stored with the document.
         """
-        self.format = Existence(analyzer = IDAnalyzer(), field_boost = field_boost)
+        self.format = Existence(analyzer=IDAnalyzer(), field_boost= field_boost)
         self.stored = stored
         self.unique = unique
@@ -139 +141 @@
     __inittypes__ = dict(stored=bool, unique=bool, expression=bool, field_boost=float)
     
-    def __init__(self, stored = False, unique = False, expression = None, field_boost = 1.0):
+    def __init__(self, stored=False, unique=False, expression=None, field_boost=1.0):
         """
         :param stored: Whether the value of this field is stored with the document.
@@ -155 +157 @@
 
 
+class NUMERIC(FieldType):
+    def __init__(self, type=int, stored=False, unique=False, field_boost=1.0):
+        self.type = type
+        self.stored = stored
+        self.unique = unique
+        self.format = Existence(analyzer=IDAnalyzer(), field_boost= field_boost)
+    
+    def index(self, num):
+        method = getattr(self, self.type.__name__ + "_to_text")
+        return [(method(num), 1, '')]
+    
+    def to_text(self, x):
+        ntype = self.type
+        method = getattr(self, ntype.__name__ + "_to_text")
+        return method(ntype(x))
+    
+    def process_text(self, text, **kwargs):
+        return (self.to_text(text), )
+    
+    def parse_query(self, fieldname, qstring, boost=1.0):
+        from whoosh import query
+        return query.Term(fieldname, self.to_text(qstring), boost=boost)
+    
+    @staticmethod
+    def int_to_text(x):
+        x += (1<<(4<<2))-1 # 4 means 32-bits
+        return u"%08x" % x
+    
+    @staticmethod
+    def text_to_int(text):
+        x = int(text, 16)
+        x -= (1<<(4<<2))-1
+        return x
+    
+    @staticmethod
+    def long_to_text(x):
+        x += (1<<(8<<2))-1
+        return u"%016x" % x
+    
+    @staticmethod
+    def text_to_long(text):
+        x = long(text, 16)
+        x -= (1<<(8<<2))-1
+        return x
+    
+    @staticmethod
+    def float_to_text(x):
+        x = struct.unpack("<q", struct.pack("<d", x))[0]
+        x += (1<<(8<<2))-1
+        return u"%016x" % x
+    
+    @staticmethod
+    def text_to_float(text):
+        x = long(text, 16)
+        x -= (1<<(8<<2))-1
+        x = struct.unpack("<d", struct.pack("<q", x))[0]
+        return x
+    
+
 class DATETIME(FieldType):
     __inittypes__ = dict(stored=bool, unique=bool)
     
-    def __init__(self, stored = True, unique = False):
+    def __init__(self, stored=False, unique=False):
         """
         :param stored: Whether the value of this field is stored with the document.
@@ -172 +233 @@
             raise ValueError("Value of DATETIME field must be a datetime object: %r" % dt)
         
-        text = dt.isoformat()
+        text = dt.isoformat() # 2010-02-02T17:06:19.109000
         text = text.replace(" ", "").replace(":", "").replace("-", "").replace(".", "")
         return [(text, 1, '')]
+    
+    def process_text(self, text, **kwargs):
+        text = text.replace(" ", "").replace(":", "").replace("-", "").replace(".", "")
+        return (text, )
+    
+    def parse_query(self, fieldname, qstring, boost=1.0):
+        text = self.process_text(qstring)
+        from whoosh import query
+        return query.Prefix(fieldname, text, boost=boost)
+    
+
+class BOOLEAN(FieldType):
+    strings = (u"t", u"f")
+    trues = frozenset((u"t", u"true", u"yes", u"1"))
+    falses = frozenset((u"f", u"false", u"no", u"0"))
+    
+    __inittypes__ = dict(stored=bool)
+    
+    def __init__(self, stored=False):
+        self.stored = stored
+        self.format = Existence()
+    
+    def index(self, bit):
+        if not isinstance(bit, bool):
+            raise ValueError("Value of BOOL field must be a bool object: %r" % bit)
+        return [(self.strings[int(bit)], 1, '')]
+    
+    def parse_query(self, fieldname, qstring, boost=1.0):
+        from whoosh import query
+        text = None
+        if qstring in self.falses:
+            text = self.strings[0]
+        elif qstring in self.trues:
+            text = self.strings[1]
+        
+        if text is None:
+            return query.NullQuery
+        return query.Term(fieldname, text, boost=boost)
     
 

projects/whoosh/trunk/src/whoosh/qparser/default.py

@@ -182 +182 @@
     
     def get_term_text(self, field, text, **kwargs):
-        if not field.format:
-            raise Exception("%s field has no format" % field)
-        
         # Just take the first token
-        for token in field.format.analyze(text, mode="query", **kwargs):
-            return token.text
+        for t in field.process_text(text, mode="query", **kwargs):
+            return t
     
     def make_term(self, fieldname, text):
         field = self._field(fieldname)
         if field:
-            text = self.get_term_text(field, text)
-        if not text:
+            if field.parse_query:
+                return field.parse_query(fieldname, text)
+            else:
+                text = self.get_term_text(field, text)
+        
+        if text is None:
             return NullQuery
         return self.termclass(fieldname, text)
@@ -200 +201 @@
         field = self._field(fieldname)
         if field:
-            texts = [t.text for t in field.format.analyze(text, mode="query")]
+            if field.parse_query:
+                return field.parse_query(fieldname, text)
+            
+            texts = list(field.process_text(text, mode="query"))
             if not texts:
                 return self.termclass(fieldname, u'')
@@ -213 +217 @@
         field = self._field(fieldname)
         if field:
-            ptext = self.get_term_text(field, text, tokenize=False, removestops=False)
-            if ptext: text = ptext
+            text = self.get_term_text(field, text, tokenize=False, removestops=False)
         return Wildcard(fieldname, text)
     

projects/whoosh/trunk/src/whoosh/qparser/simple.py

@@ -88 +88 @@
         if self.schema:
             field = self.schema[fieldname]
-            if not field.format:
-                raise Exception("%s field has no format" % field)
-            return [token.text for token in field.format.analyze(text, mode="query", **kwargs)]
+            return list(field.process_text(text, mode="query", **kwargs))
         else:
             return [text]
     
     def make_basic_clause(self, fieldname, text, boost=1.0):
+        if self.schema:
+            field = self.schema[fieldname]
+            if field.parse_query:
+                return field.parse_query(fieldname, text, boost=boost)
+        
         parts = self.get_term_text(fieldname, text)
         if len(parts) > 1:
@@ -100 +103 @@
         else:
             return self.termclass(fieldname, parts[0], boost=boost)
-        
+    
     def make_clause(self, text, boost=1.0):
         return self.make_basic_clause(self.fieldname, text, boost=boost)

projects/whoosh/trunk/src/whoosh/util.py

@@ -19 +19 @@
 """
 
+from array import array
 import codecs, re
 
@@ -164 +165 @@
         yield chr(i) + w[i:].encode("utf8")
         last = w
-        
+
+
+def to_7bit(x, islong):
+    if not islong:
+        shift = 31
+        nchars = 5
+    else:
+        shift = 62
+        nchars = 10
+    
+    buffer = array("c", "\x00" * nchars)
+    x += (1<<shift)-1
+    while x:
+        buffer[nchars-1] = chr(x & 0x7f)
+        x >>= 7
+        nchars -= 1
+        return buffer.tostring()
+
+def from_7bit(text):
+    if len(text) == 5:
+        shift = 31
+    elif len(text) == 10:
+        shift = 62
+    else:
+        raise ValueError("text is not 5 or 10 bytes")
+    
+    x = 0
+    for char in text:
+        x <<= 7
+        char = ord(char)
+        if char > 0x7f:
+            raise Exception
+        x |= char
+    x -= (1<<shift)-1
+    return x
 
 def prefix_decode_all(ls):

projects/whoosh/trunk/tests/test_highlighting.py

@@ -58 +58 @@
 
 
-
 if __name__ == '__main__':
     unittest.main()

projects/whoosh/trunk/tests/test_misc.py

@@ -4 +4 @@
 
 from whoosh.filedb.filestore import FileStorage
+from whoosh.support.filelock import try_for
 
 
@@ -13 +14 @@
     def destroy_dir(self, name):
         try:
-            os.rmdir("test_index")
+            os.rmdir("testindex")
         except:
             pass
@@ -19 +20 @@
     def clean_file(self, path):
         if os.path.exists(path):
-            os.remove(path)
+            try:
+                os.remove(path)
+            except:
+                pass
     
     def test_filelock_simple(self):
-        self.make_dir("test_index")
-        st = FileStorage("test_index")
+        self.make_dir("testindex")
+        st = FileStorage("testindex")
         lock1 = st.lock("testlock")
         lock2 = st.lock("testlock")
@@ -34 +38 @@
         lock2.release()
         
-        self.clean_file("test_index/testlock")
-        self.destroy_dir("test_index")
+        self.clean_file("testindex/testlock")
+        self.destroy_dir("testindex")
     
     def test_threaded_filelock(self):
-        self.make_dir("test_index")
-        st = FileStorage("test_index")
+        self.make_dir("testindex")
+        st = FileStorage("testindex")
         lock1 = st.lock("testlock")
         result = []
         
+        # The thread function tries to acquire the lock and
+        # then quits
         def fn():
             lock2 = st.lock("testlock")
-            lock2.acquire(blocking=True)
-            result.append(True)
-            lock2.release()
-            
+            gotit = try_for(lock2.acquire, 1.0, 0.1)
+            if gotit:
+                result.append(True)
+                lock2.release()
         t = threading.Thread(target=fn)
+        
+        # Acquire the lock in this thread
         lock1.acquire()
+        # Start the other thread trying to acquire the lock
         t.start()
-        time.sleep(0.1)
+        # Wait for a bit
+        time.sleep(0.15)
+        # Release the lock
         lock1.release()
-        del lock1
-        time.sleep(0.1)
+        # Wait for the other thread to finish
+        t.join()
+        # If the other thread got the lock, it should have
+        # appended something to the "results" list.
         self.assertEqual(len(result), 1)
         
-        self.clean_file("test_index/testlock")
-        self.destroy_dir("test_index")
+        self.clean_file("testindex/testlock")
+        self.destroy_dir("testindex")
 
 

projects/whoosh/trunk/tests/test_writing.py

@@ -13 +13 @@
     def destroy_dir(self, name):
         try:
-            os.rmdir("test_index")
+            os.rmdir("testindex")
         except:
             pass
@@ -22 +22 @@
     
     def test_asyncwriter(self):
-        self.make_dir("test_index")
+        self.make_dir("testindex")
         schema = fields.Schema(id=fields.ID, text=fields.TEXT)
-        ix = index.create_in("test_index", schema)
+        ix = index.create_in("testindex", schema)
         
         domain = (u"alfa", u"bravo", u"charlie", u"delta", u"echo", u"foxtrot", u"golf", u"hotel", u"india")