Skip to content

Commit

Permalink
移除对日志库的依赖,返璞归真
Browse files Browse the repository at this point in the history
  • Loading branch information
hankcs committed Oct 28, 2014
1 parent b69c507 commit 85163de
Show file tree
Hide file tree
Showing 51 changed files with 219 additions and 523 deletions.
13 changes: 0 additions & 13 deletions .idea/libraries/Maven__ch_qos_logback_logback_classic_0_9_28.xml

This file was deleted.

13 changes: 0 additions & 13 deletions .idea/libraries/Maven__ch_qos_logback_logback_core_0_9_28.xml

This file was deleted.

13 changes: 0 additions & 13 deletions .idea/libraries/Maven__org_slf4j_jcl_over_slf4j_1_6_4.xml

This file was deleted.

13 changes: 0 additions & 13 deletions .idea/libraries/Maven__org_slf4j_slf4j_api_1_6_4.xml

This file was deleted.

4 changes: 0 additions & 4 deletions HanLP.iml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,6 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: junit:junit:4.7" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.6.4" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:0.9.28" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:0.9.28" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jcl-over-slf4j:1.6.4" level="project" />
<orderEntry type="library" name="Maven: org.ahocorasick:ahocorasick:0.2.4" level="project" />
</component>
</module>
Expand Down
7 changes: 7 additions & 0 deletions main/java/com/hankcs/hanlp/HanLP.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@
*/
package com.hankcs.hanlp;

import com.hankcs.hanlp.utility.Predefine;

import java.util.Properties;
import java.util.logging.Level;

/**
* 常用接口静态化
Expand Down Expand Up @@ -76,6 +79,10 @@ public static class Config
System.out.println("没有找到HanLP.properties,将采用默认配置");
}
}
if (!DEBUG)
{
Predefine.logger.setLevel(Level.OFF);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,13 @@
package com.hankcs.hanlp.corpus.dictionary;

import com.hankcs.hanlp.corpus.document.sentence.word.Word;
import com.hankcs.hanlp.corpus.tag.NR;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.utility.Predefine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;

import static com.hankcs.hanlp.utility.Predefine.logger;
/**
* @author hankcs
*/
public abstract class CommonDictionaryMaker implements ISaveAble
{
static Logger logger = LoggerFactory.getLogger(CommonDictionaryMaker.class);
static boolean verbose = false;
/**
* 语料库中的单词
Expand Down
36 changes: 23 additions & 13 deletions main/java/com/hankcs/hanlp/corpus/dictionary/DictionaryMaker.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,18 @@

import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
import com.hankcs.hanlp.corpus.dictionary.item.Item;
import com.hankcs.hanlp.corpus.dictionary.item.SimpleItem;
import com.hankcs.hanlp.corpus.document.sentence.word.IWord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.*;

import static com.hankcs.hanlp.utility.Predefine.logger;

/**
* @author hankcs
*/
public class DictionaryMaker implements ISaveAble
{
static Logger logger = LoggerFactory.getLogger(DictionaryMaker.class);
BinTrie<Item> trie;

public DictionaryMaker()
Expand All @@ -36,6 +34,7 @@ public DictionaryMaker()

/**
* 向词典中加入一个词语
*
* @param word 词语
*/
public void add(IWord word)
Expand All @@ -54,6 +53,7 @@ public void add(IWord word)

/**
* 读取所有条目
*
* @param path
* @return
*/
Expand All @@ -69,15 +69,15 @@ public static List<Item> loadAsItemList(String path)
Item item = Item.create(line);
if (item == null)
{
logger.warn("使用【{}】创建Item失败", line);
logger.warning("使用【" + line + "】创建Item失败");
return null;
}
itemList.add(item);
}
}
catch (Exception e)
{
logger.warn("读取词典{}发生异常", path, e);
logger.warning("读取词典" + path + "发生异常" + e);
return null;
}

Expand All @@ -86,6 +86,7 @@ public static List<Item> loadAsItemList(String path)

/**
* 插入全部条目
*
* @param itemList
*/
public void addAll(List<Item> itemList)
Expand All @@ -98,6 +99,7 @@ public void addAll(List<Item> itemList)

/**
* 插入新条目,不执行合并
*
* @param itemList
*/
public void addAllNotCombine(List<Item> itemList)
Expand All @@ -110,6 +112,7 @@ public void addAllNotCombine(List<Item> itemList)

/**
* 插入条目
*
* @param item
*/
public void add(Item item)
Expand All @@ -128,6 +131,7 @@ public void add(Item item)

/**
* 插入条目,但是不合并,如果已有则忽略
*
* @param item
*/
public void addNotCombine(Item item)
Expand All @@ -142,6 +146,7 @@ public void addNotCombine(Item item)

/**
* 合并两部词典
*
* @param pathA
* @param pathB
* @return
Expand All @@ -157,6 +162,7 @@ public static DictionaryMaker combine(String pathA, String pathB)

/**
* 合并多部词典
*
* @param pathArray
* @return
*/
Expand All @@ -165,43 +171,45 @@ public static DictionaryMaker combine(String[] pathArray)
DictionaryMaker dictionaryMaker = new DictionaryMaker();
for (String path : pathArray)
{
logger.trace("正在处理{}", path);
logger.warning("正在处理" + path);
dictionaryMaker.addAll(DictionaryMaker.loadAsItemList(path));
}
return dictionaryMaker;
}

/**
* 对除第一个之外的词典执行标准化,并且合并
*
* @param pathArray
* @return
*/
public static DictionaryMaker combineWithNormalization(String[] pathArray)
{
DictionaryMaker dictionaryMaker = new DictionaryMaker();
logger.trace("正在处理主词典{}", pathArray[0]);
logger.info("正在处理主词典" + pathArray[0]);
dictionaryMaker.addAll(DictionaryMaker.loadAsItemList(pathArray[0]));
for (int i = 1; i < pathArray.length; ++i)
{
logger.trace("正在处理副词典{},将执行新词合并模式", pathArray[i]);
logger.info("正在处理副词典" + pathArray[i] + ",将执行新词合并模式");
dictionaryMaker.addAllNotCombine(DictionaryMaker.loadAsItemList(pathArray[i]));
}
return dictionaryMaker;
}

/**
* 合并,只补充除第一个词典外其他词典的新词
*
* @param pathArray
* @return
*/
public static DictionaryMaker combineWhenNotInclude(String[] pathArray)
{
DictionaryMaker dictionaryMaker = new DictionaryMaker();
logger.trace("正在处理主词典{}", pathArray[0]);
logger.info("正在处理主词典" + pathArray[0]);
dictionaryMaker.addAll(DictionaryMaker.loadAsItemList(pathArray[0]));
for (int i = 1; i < pathArray.length; ++i)
{
logger.trace("正在处理副词典{},并且过滤已有词典", pathArray[i]);
logger.info("正在处理副词典" + pathArray[i] + ",并且过滤已有词典");
dictionaryMaker.addAllNotCombine(DictionaryMaker.normalizeFrequency(DictionaryMaker.loadAsItemList(pathArray[i])));
}
return dictionaryMaker;
Expand Down Expand Up @@ -232,7 +240,7 @@ public boolean saveTxtTo(String path)
}
catch (Exception e)
{
logger.warn("保存到{}失败", path, e);
logger.warning("保存到" + path + "失败" + e);
return false;
}

Expand All @@ -246,6 +254,7 @@ public static interface Filter

/**
* 允许保存之前对其做一些调整
*
* @param path
* @param filter
* @return
Expand All @@ -268,7 +277,7 @@ public boolean saveTxtTo(String path, Filter filter)
}
catch (Exception e)
{
logger.warn("保存到{}失败", path, e);
logger.warning("保存到" + path + "失败" + e);
return false;
}

Expand All @@ -277,6 +286,7 @@ public boolean saveTxtTo(String path, Filter filter)

/**
* 调整频次,按排序后的次序给定频次
*
* @param itemList
* @return 处理后的列表
*/
Expand Down
27 changes: 13 additions & 14 deletions main/java/com/hankcs/hanlp/corpus/dictionary/EasyDictionary.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,39 +14,39 @@
import com.hankcs.hanlp.collection.trie.DoubleArrayTrie;
import com.hankcs.hanlp.dictionary.BaseSearcher;
import com.hankcs.hanlp.corpus.tag.Nature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.*;

import static com.hankcs.hanlp.utility.Predefine.logger;

/**
* 一个通用的、满足特定格式的双数组词典
*
* @author hankcs
*/
public class EasyDictionary
{
static Logger logger = LoggerFactory.getLogger(EasyDictionary.class);
DoubleArrayTrie<Attribute> trie = new DoubleArrayTrie<Attribute>();

public static EasyDictionary create(String path)
{
EasyDictionary dictionary = new EasyDictionary();
if (dictionary.load(path))
{
return dictionary;
}
else
else
{
logger.warn("从{}读取失败", path);
logger.warning("从" + path + "读取失败");
}

return null;
}

private boolean load(String path)
{
logger.info("通用词典开始加载:{}", path);
logger.info("通用词典开始加载:" + path);
List<String> wordList = new ArrayList<String>();
List<Attribute> attributeList = new ArrayList<Attribute>();
BufferedReader br = null;
Expand All @@ -68,23 +68,22 @@ private boolean load(String path)
}
attributeList.add(attribute);
}
logger.info("通用词典读入词条{} 属性{}", wordList.size(), attributeList.size());
logger.info("通用词典读入词条" + wordList.size() + " 属性" + attributeList.size());
br.close();
}
catch (FileNotFoundException e)
{
logger.error("通用词典" + path + "不存在!", e);
logger.severe("通用词典" + path + "不存在!" + e);
return false;
}
catch (IOException e)
{
logger.error("通用词典" + path + "读取错误!", e);
logger.severe("通用词典" + path + "读取错误!" + e);
return false;
}

logger.trace("通用词典DAT构建结果:{}", trie.build(wordList, attributeList));
logger.info("通用词典加载成功:{}个词条", trie.size());
logger.trace("“一”对应的属性:{}", trie.getValueAt(trie.exactMatchSearch("一")).toString());
logger.info("通用词典DAT构建结果:" + trie.build(wordList, attributeList));
logger.info("通用词典加载成功:" + trie.size() +"个词条" );
return true;
}

Expand Down
Loading

0 comments on commit 85163de

Please sign in to comment.