Not indexed value support (MissingValue, EmptyValue)

src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py

@@ -24,13 +24,23 @@
 from BTrees.OOBTree import difference
 from BTrees.OOBTree import OOSet
 from Persistence import PersistentMapping
-from zope.interface import implementer
-
-from Products.PluginIndexes.interfaces import ITransposeQuery
+from zope.interface import implementer_only
+
+from Products.PluginIndexes.interfaces import (
+    ILimitedResultIndex,
+    IQueryIndex,
+    ISortIndex,
+    IUniqueValueIndex,
+    IRequestCacheIndex,
+    ITransposeQuery,
+    MissingValue,
+    EmptyValue,
+)
 from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
 from Products.PluginIndexes.unindex import _marker
 from Products.ZCatalog.query import IndexQuery
 
+
 LOG = logging.getLogger('CompositeIndex')
 
 QUERY_OPTIONS = {
@@ -172,7 +182,8 @@ def __repr__(self):
                 'attributes: {0.attributes}>').format(self)
 
 
-@implementer(ITransposeQuery)
+@implementer_only(ILimitedResultIndex, IQueryIndex, IUniqueValueIndex,
+                  ISortIndex, IRequestCacheIndex, ITransposeQuery)
 class CompositeIndex(KeywordIndex):
 
     """Index for composition of simple fields.
@@ -380,6 +391,14 @@ def make_query(self, query):
             if c.meta_type == 'BooleanIndex':
                 rec.keys = [int(bool(v)) for v in rec.keys[:]]
 
+            # cannot currently support KeywordIndex's
+            # MissigValue/EmptyValue feature
+            if c.meta_type == 'KeywordIndex':
+                if MissingValue in rec.keys:
+                    continue
+                if EmptyValue in rec.keys:
+                    continue
+
             # rec with 'not' parameter
             not_parm = rec.get('not', None)
             if not_parm:

src/Products/PluginIndexes/KeywordIndex/KeywordIndex.py

@@ -16,10 +16,19 @@
 from BTrees.OOBTree import difference
 from BTrees.OOBTree import OOSet
 from App.special_dtml import DTMLFile
+from zope.interface import implementer
 
 from Products.PluginIndexes.unindex import UnIndex
 from Products.PluginIndexes.util import safe_callable
-
+from Products.PluginIndexes.interfaces import (
+    NotIndexedValue,
+    IIndexingMissingValue,
+    MissingValue,
+    IIndexingEmptyValue,
+    EmptyValue,
+)
+
+_marker = []
 LOG = getLogger('Zope.KeywordIndex')
 
 try:
@@ -29,6 +38,7 @@
     basestring = (bytes, str)
 
 
+@implementer(IIndexingMissingValue, IIndexingEmptyValue)
 class KeywordIndex(UnIndex):
     """Like an UnIndex only it indexes sequences of items.
 
@@ -58,24 +68,28 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         # we'll do so.
 
         newKeywords = self._get_object_keywords(obj, attr)
-
         oldKeywords = self._unindex.get(documentId, None)
 
         if oldKeywords is None:
             # we've got a new document, let's not futz around.
+            if isinstance(newKeywords, NotIndexedValue):
+                return self.insertNotIndexed(newKeywords, documentId)
+
             try:
                 for kw in newKeywords:
                     self.insertForwardIndexEntry(kw, documentId)
-                if newKeywords:
-                    self._unindex[documentId] = list(newKeywords)
+                self._unindex[documentId] = list(newKeywords)
             except TypeError:
                 return 0
         else:
             # we have an existing entry for this document, and we need
             # to figure out if any of the keywords have actually changed
             if type(oldKeywords) is not OOSet:
                 oldKeywords = OOSet(oldKeywords)
-            newKeywords = OOSet(newKeywords)
+            if isinstance(newKeywords, NotIndexedValue):
+                newKeywords = OOSet()
+            else:
+                newKeywords = OOSet(newKeywords)
             fdiff = difference(oldKeywords, newKeywords)
             rdiff = difference(newKeywords, oldKeywords)
             if fdiff or rdiff:
@@ -84,22 +98,32 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
                     self._unindex[documentId] = list(newKeywords)
                 else:
                     del self._unindex[documentId]
+
                 if fdiff:
                     self.unindex_objectKeywords(documentId, fdiff)
                 if rdiff:
                     for kw in rdiff:
                         self.insertForwardIndexEntry(kw, documentId)
+
+        # maybe a previous attribute was a not indexable value
+        if newKeywords:
+            self.removeNotIndexed(EmptyValue, documentId)
+
         return 1
 
     def _get_object_keywords(self, obj, attr):
-        newKeywords = getattr(obj, attr, ())
+        newKeywords = getattr(obj, attr, None)
         if safe_callable(newKeywords):
             try:
                 newKeywords = newKeywords()
             except (AttributeError, TypeError):
-                return ()
+                return MissingValue
+
+        if newKeywords is None:
+            return MissingValue
+
         if not newKeywords:
-            return ()
+            return EmptyValue
         elif isinstance(newKeywords, basestring):
             return (newKeywords,)
         else:
@@ -122,13 +146,25 @@ def unindex_objectKeywords(self, documentId, keywords):
     def unindex_object(self, documentId):
         """ carefully unindex the object with integer id 'documentId'"""
 
-        keywords = self._unindex.get(documentId, None)
+        keywords = self._unindex.get(documentId, _marker)
 
-        # Couldn't we return 'None' immediately
-        # if keywords is 'None' (or _marker)???
+        if keywords is _marker:
+            res = 0
+            res += self.removeNotIndexed(MissingValue, documentId)
+            res += self.removeNotIndexed(EmptyValue, documentId)
 
-        if keywords is not None:
-            self._increment_counter()
+            if res:
+                self._increment_counter()
+            else:
+                LOG.debug('%(context)s: Attempt to unindex nonexistent '
+                          'document with id %(doc_id)s', dict(
+                              context=self.__class__.__name__,
+                              doc_id=documentId),
+                          exc_info=True)
+
+            return None
+
+        self._increment_counter()
 
         self.unindex_objectKeywords(documentId, keywords)
         try:

src/Products/PluginIndexes/KeywordIndex/tests.py

@@ -16,6 +16,11 @@
 from OFS.SimpleItem import SimpleItem
 from Testing.makerequest import makerequest
 
+from Products.PluginIndexes.interfaces import (
+    MissingValue,
+    EmptyValue,
+)
+
 
 class Dummy(object):
 
@@ -67,20 +72,30 @@ def setUp(self):
                         (5, Dummy(['a', 'b', 'c', 'e'])),
                         (6, Dummy(['a', 'b', 'c', 'e', 'f'])),
                         (7, Dummy(['0'])),
+                        (8, Dummy([])),
+                        (9, Dummy(None)),
                         ]
         self._noop_req = {'bar': 123}
         self._all_req = {'foo': ['a']}
         self._some_req = {'foo': ['e']}
         self._overlap_req = {'foo': ['c', 'e']}
         self._string_req = {'foo': 'a'}
         self._zero_req = {'foo': ['0']}
+        self._miss_req_1 = {'foo': [MissingValue]}
+        self._miss_req_2 = {'foo': ['a', MissingValue]}
+        self._empty_req_1 = {'foo': [EmptyValue]}
+        self._empty_req_2 = {'foo': [EmptyValue, 'f']}
 
         self._not_1 = {'foo': {'query': 'f', 'not': 'f'}}
         self._not_2 = {'foo': {'query': ['e', 'f'], 'not': 'f'}}
         self._not_3 = {'foo': {'not': '0'}}
         self._not_4 = {'foo': {'not': ['0', 'e']}}
         self._not_5 = {'foo': {'not': ['0', 'no-value']}}
         self._not_6 = {'foo': 'c', 'bar': {'query': 123, 'not': 1}}
+        self._not_7 = {'foo': {'not': [MissingValue]}}
+        self._not_8 = {'foo': {'not': []}}
+        self._not_9 = {'foo': {'not': [EmptyValue, ]}}
+        self._not_10 = {'foo': {'not': [EmptyValue, 'f']}}
 
     def _populateIndex(self):
         for k, v in self._values:
@@ -152,38 +167,53 @@ def testEmpty(self):
         self._checkApply(self._some_req, [])
         self._checkApply(self._overlap_req, [])
         self._checkApply(self._string_req, [])
+        self._checkApply(self._miss_req_1, [])
+        self._checkApply(self._miss_req_2, [])
 
     def testPopulated(self):
         self._populateIndex()
         values = self._values
 
-        assert len(self._index.referencedObjects()) == len(values)
+        self.assertEqual(len(self._index.referencedObjects()), len(values) - 2)
         assert self._index.getEntryForObject(1234) is None
         assert (self._index.getEntryForObject(1234, self._marker)
                 is self._marker)
         self._index.unindex_object(1234)  # nothrow
-        self.assertEqual(self._index.indexSize(), len(values) - 1)
+        self.assertEqual(self._index.indexSize(), len(values) - 3)
 
-        for k, v in values:
-            entry = self._index.getEntryForObject(k)
+        for k, v in values[:8]:
+            entry = self._index.getEntryForObject(k, None)
             entry.sort()
             kw = sorted(set(v.foo()))
             self.assertEqual(entry, kw)
 
-        assert len(list(self._index.uniqueValues('foo'))) == len(values) - 1
+        empty_indexed = self._index.getNotIndexed(EmptyValue)
+        self.assertEqual(8 in empty_indexed, True)
+
+        miss_indexed = self._index.getNotIndexed(MissingValue)
+        self.assertEqual(9 in miss_indexed, True)
+
+        assert len(list(self._index.uniqueValues('foo'))) == len(values) - 3
         assert self._index._apply_index(self._noop_req) is None
 
-        self._checkApply(self._all_req, values[:-1])
+        self._checkApply(self._all_req, values[:-3])
         self._checkApply(self._some_req, values[5:7])
         self._checkApply(self._overlap_req, values[2:7])
-        self._checkApply(self._string_req, values[:-1])
+        self._checkApply(self._string_req, values[:-3])
+        self._checkApply(self._miss_req_1, values[9:10])
+        self._checkApply(self._miss_req_2, values[:7] + values[9:10])
+        self._checkApply(self._empty_req_1, values[8:9])
+        self._checkApply(self._empty_req_2, values[6:7] + values[8:9])
 
         self._checkApply(self._not_1, [])
         self._checkApply(self._not_2, values[5:6])
-        self._checkApply(self._not_3, values[:7])
-        self._checkApply(self._not_4, values[:5])
-        self._checkApply(self._not_5, values[:7])
+        self._checkApply(self._not_3, values[:7] + values[9:10])
+        self._checkApply(self._not_4, values[:5] + values[9:10])
+        self._checkApply(self._not_5, values[:7] + values[9:10])
         self._checkApply(self._not_6, values[2:7])
+        self._checkApply(self._not_7, values[:8])
+        self._checkApply(self._not_8, values[:8] + values[9:10])
+        self._checkApply(self._not_9, values[:8] + values[9:10])
 
     def testReindexChange(self):
         self._populateIndex()

src/Products/PluginIndexes/interfaces.py

@@ -283,3 +283,37 @@ def make_query(query):
 
     def getIndexNames():
         """ returns index names that are optimized by index """
+
+
+class NotIndexedValue(object):
+    """generic marker class for not indexed values"""
+
+    def __nonzero__(self):
+        return 0
+
+
+class IIndexingMissingValue(Interface):
+    """Marker interface to mark indexes with support the
+    `MissingValue` query term."""
+
+
+class MissingValue(NotIndexedValue):
+    """MissingValue can be used as query "term" to query
+    for objects the index does not have a value for
+    (like a not defined value)"""
+
+
+MissingValue = MissingValue()
+
+
+class IIndexingEmptyValue(Interface):
+    """Marker interface to mark indexes with support the
+    `EmptyValue` query term."""
+
+
+class EmptyValue(NotIndexedValue):
+    """EmptyValue can be used as query "term" to query
+    for objects with an empty value (like an empty set)"""
+
+
+EmptyValue = EmptyValue()