Add a filter by content option (dadoonet#585)

You can filter out documents you would like to index by adding one or more
regular expression that match the extracted content.
Documents which are not matching will be simply ignored and not indexed.

If you define the following `fs.filters` property in your
`~/.fscrawler/test/_settings.json` file:

```json
{
 "name" : "test",
 "fs": {
   "filters": [
     ".*foo.*",
     "^4\\d{3}([\\ \\-]?)\\d{4}\\1\\d{4}\\1\\d{4}$"
   ]
 }
}
```

With this example, only documents which contains the word `foo` and a VISA credit card number
with the form like `4012888888881881`, `4012 8888 8888 1881` or `4012-8888-8888-1881`
will be indexed.

Closes dadoonet#463.

core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParser.java

@@ -63,7 +63,6 @@
 import java.util.stream.Collectors;
 
 import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName;
-import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isExcluded;
 import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit;
 import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isIndexable;
 import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate;
@@ -485,10 +484,15 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
                 }
 
                 // We index the data structure
-                esIndex(esClientManager.bulkProcessorDoc(), fsSettings.getElasticsearch().getIndex(),
-                        generateIdFromFilename(filename, dirname),
-                        DocParser.toJson(doc),
-                        fsSettings.getElasticsearch().getPipeline());
+                if (isIndexable(doc.getContent(), fsSettings.getFs().getFilters())) {
+                    esIndex(esClientManager.bulkProcessorDoc(), fsSettings.getElasticsearch().getIndex(),
+                            generateIdFromFilename(filename, dirname),
+                            DocParser.toJson(doc),
+                            fsSettings.getElasticsearch().getPipeline());
+                } else {
+                    logger.debug("We ignore file [{}] because it does not match all the patterns {}", filename,
+                            fsSettings.getFs().getFilters());
+                }
             } else {
                 if (fsSettings.getFs().isJsonSupport()) {
                     // We index the json content directly

docs/source/admin/fs/local-fs.rst

@@ -16,6 +16,8 @@ Here is a list of Local FS settings (under ``fs.`` prefix)`:
 +----------------------------+-----------------------+---------------------------------+
 | ``fs.excludes``            | ``["~*"]``            | `Includes and excludes`_        |
 +----------------------------+-----------------------+---------------------------------+
+| ``fs.filters``             | ``null``              | `Filter content`_               |
++----------------------------+-----------------------+---------------------------------+
 | ``fs.json_support``        | ``false``             | `Indexing JSon docs`_           |
 +----------------------------+-----------------------+---------------------------------+
 | ``fs.xml_support``         | ``false``             | `Indexing XML docs`_            |
@@ -174,6 +176,35 @@ If you define the following ``fs.excludes`` property in your
 Then all files but the ones in ``/folderB/subfolderA``, ``/folderB/subfolderB`` and
 ``/folderB/subfolderC`` will be indexed.
 
+Filter content
+^^^^^^^^^^^^^^
+
+.. versionadded:: 2.5
+
+You can filter out documents you would like to index by adding one or more
+regular expression that match the extracted content.
+Documents which are not matching will be simply ignored and not indexed.
+
+If you define the following ``fs.filters`` property in your
+``~/.fscrawler/test/_settings.json`` file:
+
+.. code:: json
+
+   {
+     "name" : "test",
+     "fs": {
+       "filters": [
+         ".*foo.*",
+         "^4\\d{3}([\\ \\-]?)\\d{4}\\1\\d{4}\\1\\d{4}$"
+       ]
+     }
+   }
+
+With this example, only documents which contains the word ``foo`` and a VISA credit card number
+with the form like ``4012888888881881``, ``4012 8888 8888 1881`` or ``4012-8888-8888-1881``
+will be indexed.
+
+
 Indexing JSon docs
 ^^^^^^^^^^^^^^^^^^
 

framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java

@@ -56,6 +56,7 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.TimeZone;
+import java.util.regex.Pattern;
 
 public class FsCrawlerUtil {
     public static final String INDEX_SUFFIX_FOLDER = "_folder";
@@ -210,6 +211,40 @@ public static boolean isIncluded(String filename, List<String> includes) {
         return false;
     }
 
+    /**
+     * We check if we can index the content or skip it
+     *
+     * @param content Content to parse
+     * @param filters regular expressions that all needs to match if we want to index. If empty
+     *                we consider it always matches.
+     */
+    public static boolean isIndexable(String content, List<String> filters) {
+        logger.debug("content = [{}], filters = {}", content, filters);
+
+        if (isNullOrEmpty(content)) {
+            logger.trace("Null or empty content always matches.");
+            return true;
+        }
+
+        if (filters == null || filters.isEmpty()) {
+            logger.trace("No pattern always matches.");
+            return true;
+        }
+
+        for (String filter : filters) {
+            Pattern pattern = Pattern.compile(filter, Pattern.MULTILINE | Pattern.UNIX_LINES);
+            logger.trace("Testing filter [{}]", filter);
+            if (!pattern.matcher(content).find()) {
+                logger.trace("Filter [{}] is not matching.", filter);
+                return false;
+            } else {
+                logger.trace("Filter [{}] is matching.", filter);
+            }
+        }
+
+        return true;
+    }
+
     public static String computeVirtualPathName(String rootPath, String realPath) {
         String result = "/";
         if (realPath != null && realPath.length() > rootPath.length()) {

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/FsCrawlerTestFiltersIT.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs.test.integration;
+
+import fr.pilato.elasticsearch.crawler.fs.settings.Fs;
+import org.elasticsearch.action.search.SearchRequest;
+import org.junit.Test;
+
+/**
+ * Test filters crawler settings
+ */
+public class FsCrawlerTestFiltersIT extends AbstractFsCrawlerITCase {
+    @Test
+    public void test_filter_one_term() throws Exception {
+        Fs fs = startCrawlerDefinition()
+                .addFilter(".*foo.*")
+                .build();
+        startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), null);
+        countTestHelper(new SearchRequest(getCrawlerName()), 2L, null);
+    }
+
+    @Test
+    public void test_filter_visa_pattern() throws Exception {
+        Fs fs = startCrawlerDefinition()
+                .addFilter("^4\\d{3}([\\ \\-]?)\\d{4}\\1\\d{4}\\1\\d{4}$")
+                .build();
+        startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), null);
+        countTestHelper(new SearchRequest(getCrawlerName()), 2L, null);
+    }
+
+    @Test
+    public void test_filter_visa_pattern_plus_foo() throws Exception {
+        Fs fs = startCrawlerDefinition()
+                .addFilter("^4\\d{3}([\\ \\-]?)\\d{4}\\1\\d{4}\\1\\d{4}$")
+                .addFilter(".*foo.*")
+                .build();
+        startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), null);
+        countTestHelper(new SearchRequest(getCrawlerName()), 1L, null);
+    }
+}

integration-tests/src/test/resources-binary/samples/test_filter_one_term/foo.txt

@@ -0,0 +1 @@
+This is containing foo as one of the words.

integration-tests/src/test/resources-binary/samples/test_filter_one_term/sample.txt

@@ -0,0 +1 @@
+This file contains some words.

integration-tests/src/test/resources-binary/samples/test_filter_one_term/visa-number-with-foo.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "foo".

integration-tests/src/test/resources-binary/samples/test_filter_one_term/visa-number.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "f o o" without the spaces.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern/foo.txt

@@ -0,0 +1 @@
+This is containing foo as one of the words.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern/sample.txt

@@ -0,0 +1 @@
+This file contains some words.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern/visa-number-with-foo.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "foo".

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern/visa-number.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "f o o" without the spaces.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern_plus_foo/foo.txt

@@ -0,0 +1 @@
+This is containing foo as one of the words.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern_plus_foo/sample.txt

@@ -0,0 +1 @@
+This file contains some words.

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern_plus_foo/visa-number-with-foo.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "foo".

integration-tests/src/test/resources-binary/samples/test_filter_visa_pattern_plus_foo/visa-number.txt

@@ -0,0 +1,8 @@
+This file contains
+
+4012-8888-8888-1881
+
+^^^ This is a fake VISA number.
+
+This will be indexed unless we check for a word which
+is not inside this text, like "f o o" without the spaces.