CoreStopWordDictionary支持自定义过滤逻辑

elfgzp · Nov 13, 2015 · 1f29a35 · 1f29a35
1 parent 504a9eb
commit 1f29a35
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 34 deletions.
diff --git a/src/main/java/com/hankcs/hanlp/corpus/tag/Nature.java b/src/main/java/com/hankcs/hanlp/corpus/tag/Nature.java
@@ -783,4 +783,14 @@ public boolean startsWith(char prefix)
     {
         return toString().charAt(0) == prefix;
     }
+
+    /**
+     * 词性的首字母<br>
+     *     词性根据开头的几个字母可以判断大的类别
+     * @return
+     */
+    public char firstChar()
+    {
+        return toString().charAt(0);
+    }
 }
diff --git a/src/main/java/com/hankcs/hanlp/dictionary/stopword/CoreStopWordDictionary.java b/src/main/java/com/hankcs/hanlp/dictionary/stopword/CoreStopWordDictionary.java
@@ -61,56 +61,56 @@ public static boolean contains(String key)
     }
 
     /**
-     * 核心停用词典的核心过滤器
+     * 核心停用词典的核心过滤器，词性属于名词、动词、副词、形容词，并且不在停用词表中才不会被过滤
      */
     public static Filter FILTER = new Filter()
     {
         @Override
         public boolean shouldInclude(Term term)
         {
-            return CoreStopWordDictionary.shouldInclude(term);
+            // 除掉停用词
+            String nature = term.nature != null ? term.nature.toString() : "空";
+            char firstChar = nature.charAt(0);
+            switch (firstChar)
+            {
+                case 'm':
+                case 'b':
+                case 'c':
+                case 'e':
+                case 'o':
+                case 'p':
+                case 'q':
+                case 'u':
+                case 'y':
+                case 'z':
+                case 'r':
+                case 'w':
+                {
+                    return false;
+                }
+                default:
+                {
+                    if (term.word.length() > 1 && !CoreStopWordDictionary.contains(term.word))
+                    {
+                        return true;
+                    }
+                }
+                break;
+            }
+
+            return false;
         }
     };
 
     /**
-     * 是否应当将这个term纳入计算，词性属于名词、动词、副词、形容词，并且不在停用词表中
+     * 是否应当将这个term纳入计算
      *
      * @param term
      * @return 是否应当
      */
     public static boolean shouldInclude(Term term)
     {
-        // 除掉停用词
-        String nature = term.nature != null ? term.nature.toString() : "空";
-        char firstChar = nature.charAt(0);
-        switch (firstChar)
-        {
-            case 'm':
-            case 'b':
-            case 'c':
-            case 'e':
-            case 'o':
-            case 'p':
-            case 'q':
-            case 'u':
-            case 'y':
-            case 'z':
-            case 'r':
-            case 'w':
-            {
-                return false;
-            }
-            default:
-            {
-                if (term.word.length() > 1 && !CoreStopWordDictionary.contains(term.word))
-                {
-                    return true;
-                }
-            }
-            break;
-        }
-
-        return false;
+        return FILTER.shouldInclude(term);
     }
 
     /**

diff --git a/src/test/java/com/hankcs/demo/DemoStopWord.java b/src/test/java/com/hankcs/demo/DemoStopWord.java
@@ -12,6 +12,7 @@
 
 
 import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
+import com.hankcs.hanlp.dictionary.stopword.Filter;
 import com.hankcs.hanlp.seg.common.Term;
 import com.hankcs.hanlp.tokenizer.BasicTokenizer;
 import com.hankcs.hanlp.tokenizer.NotionalTokenizer;
@@ -38,5 +39,20 @@ public static void main(String[] args)
         System.out.println(termList);
         CoreStopWordDictionary.apply(termList);
         System.out.println(termList);
+        // 还可以自定义过滤逻辑
+        CoreStopWordDictionary.FILTER = new Filter()
+        {
+            @Override
+            public boolean shouldInclude(Term term)
+            {
+                switch (term.nature)
+                {
+                    case nz:
+                    return !CoreStopWordDictionary.contains(term.word);
+                }
+                return false;
+            }
+        };
+        System.out.println(NotionalTokenizer.segment(text));
     }
 }