Reuse deleted docs set

This should reduce memory requirements in case there is many of them. Issue whoosh-community#492 Signed-off-by: Michal Čihař <[email protected]>
cclauss · Jan 9, 2019 · 60a9ff9 · 60a9ff9
1 parent f4e4ea9
commit 60a9ff9
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 2 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,7 @@ python:
   - "3.7"
 
 install:
-  - pip install pytest nose codecov coverage
+  - pip install pytest nose codecov coverage cached-property
 
 script:
   - nosetests  --with-coverage

diff --git a/setup.py b/setup.py
@@ -43,6 +43,7 @@ def run_tests(self):
         url="http://bitbucket.org/mchaput/whoosh",
 
         zip_safe=True,
+        require=['cached-property'],
         tests_require=['pytest'],
         cmdclass={'test': PyTest},
 

diff --git a/src/whoosh/reading.py b/src/whoosh/reading.py
@@ -32,6 +32,8 @@
 from bisect import bisect_right
 from heapq import heapify, heapreplace, heappop, nlargest
 
+from cached_property import cached_property
+
 from whoosh import columns
 from whoosh.compat import abstractmethod
 from whoosh.compat import xrange, zip_, next, iteritems
@@ -815,6 +817,10 @@ def doc_frequency(self, fieldname, text):
         except KeyError:
             return 0
 
+    @cached_property
+    def deleted_docs_set(self):
+        return frozenset(self._perdoc.deleted_docs())
+
     def postings(self, fieldname, text, scorer=None):
         from whoosh.matching.wrappers import FilterMatcher
 
@@ -825,7 +831,7 @@ def postings(self, fieldname, text, scorer=None):
         text = self._text_to_bytes(fieldname, text)
         format_ = self.schema[fieldname].format
         matcher = self._terms.matcher(fieldname, text, format_, scorer=scorer)
-        deleted = frozenset(self._perdoc.deleted_docs())
+        deleted = self.deleted_docs_set
         if deleted:
             matcher = FilterMatcher(matcher, deleted, exclude=True)
         return matcher