Not indexed value support (MissingValue, EmptyValue)

src/Products/PluginIndexes/CompositeIndex/CompositeIndex.py

@@ -24,13 +24,23 @@
 from BTrees.OOBTree import difference
 from BTrees.OOBTree import OOSet
 from Persistence import PersistentMapping
-from zope.interface import implementer
-
-from Products.PluginIndexes.interfaces import ITransposeQuery
+from zope.interface import implementer_only
+
+from Products.PluginIndexes.interfaces import (
+    ILimitedResultIndex,
+    IQueryIndex,
+    ISortIndex,
+    IUniqueValueIndex,
+    IRequestCacheIndex,
+    ITransposeQuery,
+    missing,
+    empty,
+)
 from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
 from Products.PluginIndexes.unindex import _marker
 from Products.ZCatalog.query import IndexQuery
 
+
 LOG = logging.getLogger('CompositeIndex')
 
 QUERY_OPTIONS = {
@@ -172,7 +182,8 @@ def __repr__(self):
                 'attributes: {0.attributes}>').format(self)
 
 
-@implementer(ITransposeQuery)
+@implementer_only(ILimitedResultIndex, IQueryIndex, IUniqueValueIndex,
+                  ISortIndex, IRequestCacheIndex, ITransposeQuery)
 class CompositeIndex(KeywordIndex):
 
     """Index for composition of simple fields.
@@ -380,6 +391,14 @@ def make_query(self, query):
             if c.meta_type == 'BooleanIndex':
                 rec.keys = [int(bool(v)) for v in rec.keys[:]]
 
+            # cannot currently support KeywordIndex's
+            # missing/empty feature
+            if c.meta_type == 'KeywordIndex':
+                if missing in rec.keys:
+                    continue
+                if empty in rec.keys:
+                    continue
+
             # rec with 'not' parameter
             not_parm = rec.get('not', None)
             if not_parm:

src/Products/PluginIndexes/CompositeIndex/tests/testCompositeIndex.py

@@ -21,32 +21,29 @@
 types = ['Document', 'News', 'File', 'Image']
 default_pages = [True, False, False, False, False, False]
 subjects = list(map(lambda x: 'subject_{0}'.format(x), range(6)))
-keywords = list(map(lambda x: 'keyword_{0}'.format(x), range(6)))
 
 
 class TestObject(object):
 
     def __init__(self, id, portal_type, review_state,
-                 is_default_page=False, subject=(), keyword=()):
+                 is_default_page=False, subject=()):
         self.id = id
         self.portal_type = portal_type
         self.review_state = review_state
         self.is_default_page = is_default_page
         self.subject = subject
-        self.keyword = keyword
 
     def getPhysicalPath(self):
         return ['', self.id, ]
 
     def __repr__(self):
         return ('< {id}, {portal_type}, {review_state},\
-        {is_default_page}, {subject} , {keyword}>'.format(
+        {is_default_page}, {subject}>'.format(
             id=self.id,
             portal_type=self.portal_type,
             review_state=self.review_state,
             is_default_page=self.is_default_page,
-            subject=self.subject,
-            keyword=self.keyword))
+            subject=self.subject))
 
 
 class RandomTestObject(TestObject):
@@ -63,11 +60,10 @@ def __init__(self, id):
         is_default_page = default_pages[i]
 
         subject = random.sample(subjects, random.randint(1, len(subjects)))
-        keyword = random.sample(keywords, random.randint(1, len(keywords)))
 
         super(RandomTestObject, self).__init__(id, portal_type,
                                                review_state, is_default_page,
-                                               subject, keyword)
+                                               subject)
 
 
 # Pseudo ContentLayer class to support quick
@@ -92,7 +88,7 @@ def setUp(self):
                          KeywordIndex('subject',
                                       extra={
                                           'indexed_attrs':
-                                          'keyword,subject'}
+                                          'subject'}
                                       ),
                          CompositeIndex('comp01',
                                         extra=[{'id': 'portal_type',
@@ -107,7 +103,7 @@ def setUp(self):
                                                {'id': 'subject',
                                                 'meta_type': 'KeywordIndex',
                                                 'attributes':
-                                                'keyword,subject'}
+                                                'subject'}
                                                ])
                          ]
 
@@ -206,9 +202,6 @@ def testPerformance(self):
                    ('query02_default_two_indexes',
                     {'portal_type': {'query': 'Document'},
                      'subject': {'query': 'subject_2'}}),
-                   ('query02_default_two_indexes_zero_hits',
-                    {'portal_type': {'query': 'Document'},
-                     'subject': {'query': ['keyword_1', 'keyword_2']}}),
                    ('query03_default_two_indexes',
                     {'portal_type': {'query': 'Document'},
                      'subject': {'query': ['subject_1', 'subject_3']}}),
@@ -340,8 +333,7 @@ def testSearch(self):
                          subject=('subject_1', 'subject_2'))
         self.populateIndexes(3, obj)
         obj = TestObject('obj_4', 'Event', 'private',
-                         subject=('subject_1', 'subject_2'),
-                         keyword=('keyword_1', ))
+                         subject=('subject_1', 'subject_2'))
         self.populateIndexes(4, obj)
 
         queries = [
@@ -379,12 +371,6 @@ def testSearch(self):
              'is_default_page': {'query': False},
              'subject': {'query': ('subject_1', 'subject_2'),
                          'operator': 'and'}},
-            # query on five attributes with
-            {'review_state': {'not': ('pending', 'visible')},
-             'portal_type': {'query': ('News', 'Document')},
-             'is_default_page': {'query': False},
-             'subject': {'query': ('subject_1', )},
-             'keyword': {'query': ('keyword_1',)}},
         ]
 
         for query in queries:

src/Products/PluginIndexes/KeywordIndex/KeywordIndex.py

@@ -11,15 +11,24 @@
 #
 ##############################################################################
 
+import sys
 from logging import getLogger
 
 from BTrees.OOBTree import difference
 from BTrees.OOBTree import OOSet
 from App.special_dtml import DTMLFile
+from zope.interface import implementer
 
 from Products.PluginIndexes.unindex import UnIndex
 from Products.PluginIndexes.util import safe_callable
-
+from Products.PluginIndexes.interfaces import (
+    IIndexingMissingValue,
+    missing,
+    IIndexingEmptyValue,
+    empty,
+)
+
+_marker = []
 LOG = getLogger('Zope.KeywordIndex')
 
 try:
@@ -29,6 +38,7 @@
     basestring = (bytes, str)
 
 
+@implementer(IIndexingMissingValue, IIndexingEmptyValue)
 class KeywordIndex(UnIndex):
     """Like an UnIndex only it indexes sequences of items.
 
@@ -38,6 +48,10 @@ class KeywordIndex(UnIndex):
     """
     meta_type = 'KeywordIndex'
     query_options = ('query', 'range', 'not', 'operator')
+    special_values = {TypeError: missing,
+                      AttributeError: missing,
+                      None: missing,
+                      (): empty}
 
     manage_options = (
         {'label': 'Settings', 'action': 'manage_main'},
@@ -58,59 +72,120 @@ def _index_object(self, documentId, obj, threshold=None, attr=''):
         # we'll do so.
 
         newKeywords = self._get_object_keywords(obj, attr)
+        oldKeywords = self._unindex.get(documentId, _marker)
 
-        oldKeywords = self._unindex.get(documentId, None)
-
-        if oldKeywords is None:
+        if oldKeywords is _marker:
             # we've got a new document, let's not futz around.
-            try:
+            if newKeywords in (missing, empty):
+                self.insertSpecialIndexEntry(newKeywords, documentId)
+            else:
+                keys = list()
                 for kw in newKeywords:
-                    self.insertForwardIndexEntry(kw, documentId)
-                if newKeywords:
-                    self._unindex[documentId] = list(newKeywords)
-            except TypeError:
-                return 0
+                    try:
+                        self.insertForwardIndexEntry(kw, documentId)
+                        keys.append(kw)
+                    except TypeError:
+                        # key is not valid for this Btree so we have to
+                        # roll back insertForwardIndexEntry
+                        LOG.error('%(context)s: Unable to insert forward '
+                                  'index entry for document with id '
+                                  '%(doc_id)s and keyword %(kw)r '
+                                  'for index %{index}r.', dict(
+                                      context=self.__class__.__name__,
+                                      kw=kw,
+                                      doc_id=documentId,
+                                      index=self.id))
+
+                        self.unindex_objectKeywords(documentId, keys)
+                        return 0
+
+                newKeywords = OOSet(newKeywords)
+
+            self._unindex[documentId] = newKeywords
+
         else:
             # we have an existing entry for this document, and we need
             # to figure out if any of the keywords have actually changed
-            if type(oldKeywords) is not OOSet:
-                oldKeywords = OOSet(oldKeywords)
-            newKeywords = OOSet(newKeywords)
-            fdiff = difference(oldKeywords, newKeywords)
-            rdiff = difference(newKeywords, oldKeywords)
+            if oldKeywords in (missing, empty):
+                self.removeSpecialIndexEntry(oldKeywords, documentId)
+                oldSet = OOSet()
+            else:
+                if not isinstance(oldKeywords, OOSet):
+                    oldKeywords = OOSet(oldKeywords)
+                oldSet = oldKeywords
+
+            if newKeywords in (missing, empty):
+                self.insertSpecialIndexEntry(newKeywords, documentId)
+                newSet = OOSet()
+            else:
+                newSet = newKeywords = OOSet(newKeywords)
+
+            fdiff = difference(oldSet, newSet)
+            rdiff = difference(newSet, oldSet)
             if fdiff or rdiff:
                 # if we've got forward or reverse changes
-                if newKeywords:
-                    self._unindex[documentId] = list(newKeywords)
-                else:
-                    del self._unindex[documentId]
                 if fdiff:
                     self.unindex_objectKeywords(documentId, fdiff)
                 if rdiff:
                     for kw in rdiff:
                         self.insertForwardIndexEntry(kw, documentId)
+
+            self._unindex[documentId] = newKeywords
+
         return 1
 
     def _get_object_keywords(self, obj, attr):
-        newKeywords = getattr(obj, attr, ())
+        newKeywords = getattr(obj, attr, None)
+
+        def _getSpecialValueFor(datum):
+            try:
+                special_value = self.special_values[datum]
+            except TypeError:
+                raise KeyError(datum)
+
+            if self.providesSpecialIndex(special_value):
+                return special_value
+            raise KeyError(datum)
+
         if safe_callable(newKeywords):
             try:
                 newKeywords = newKeywords()
             except (AttributeError, TypeError):
-                return ()
-        if not newKeywords:
-            return ()
-        elif isinstance(newKeywords, basestring):
-            return (newKeywords,)
+                LOG.debug('%(context)s: Cannot determine datum for attribute '
+                          '%(attr)s of object %(obj)r', dict(
+                              context=self.__class__.__name__,
+                              attr=attr,
+                              obj=obj),
+                          exc_info=True)
+
+                newKeywords = sys.exc_info()[0]
+                try:
+                    return _getSpecialValueFor(newKeywords)
+                except KeyError:
+                    return _marker
+
+        try:
+            return _getSpecialValueFor(newKeywords)
+        except KeyError:
+            pass
+
+        # normalize datum
+        if isinstance(newKeywords, basestring):
+            newKeywords = (newKeywords,)
         else:
             try:
                 # unique
                 newKeywords = set(newKeywords)
             except TypeError:
                 # Not a sequence
-                return (newKeywords,)
+                newKeywords = (newKeywords,)
             else:
-                return tuple(newKeywords)
+                newKeywords = tuple(newKeywords)
+
+        try:
+            return _getSpecialValueFor(newKeywords)
+        except KeyError:
+            return newKeywords
 
     def unindex_objectKeywords(self, documentId, keywords):
         """ carefully unindex the object with integer id 'documentId'"""
@@ -122,13 +197,27 @@ def unindex_objectKeywords(self, documentId, keywords):
     def unindex_object(self, documentId):
         """ carefully unindex the object with integer id 'documentId'"""
 
-        keywords = self._unindex.get(documentId, None)
+        keywords = self._unindex.get(documentId, _marker)
 
-        # Couldn't we return 'None' immediately
-        # if keywords is 'None' (or _marker)???
+        if keywords is _marker:
+            return
 
-        if keywords is not None:
-            self._increment_counter()
+        self._increment_counter()
+
+        if keywords in (missing, empty):
+            try:
+                if not self.removeSpecialIndexEntry(keywords, documentId):
+                    raise KeyError
+                del self._unindex[documentId]
+
+            except KeyError:
+                LOG.debug('%(context)s: Attempt to unindex nonexistent '
+                          'document with id %(doc_id)s', dict(
+                              context=self.__class__.__name__,
+                              doc_id=documentId),
+                          exc_info=True)
+
+            return None
 
         self.unindex_objectKeywords(documentId, keywords)
         try: