Protect from MemoryError occuring when parsing large documents

and4r1lh0 · Aug 1, 2018 · 752a22e · 752a22e
1 parent fa3ac17
commit 752a22e
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/noisy.py b/noisy.py
@@ -238,6 +238,9 @@ def crawl(self):
 
                 except requests.exceptions.RequestException:
                     logging.warn("Error connecting to root url: {}".format(url))
+
+                except MemoryError:
+                    logging.warn("Error: content at url: {} is exhausting the memory".format(url))
 
                 except self.CrawlerTimedOut:
                     logging.info("Timeout has exceeded, exiting")