From 8a60cb9d83095ef7d58ad0f98f259d0e5908a297 Mon Sep 17 00:00:00 2001
From: Oneplus <oneplus.lau@gmail.com>
Date: Tue, 14 Jan 2014 20:49:59 +0800
Subject: [PATCH] revision segmentor

---
 src/segmentor/decoder.cpp      |  136 +--
 src/segmentor/decoder.h        |  103 +--
 src/segmentor/extractor.cpp    |  124 +--
 src/segmentor/extractor.h      |   18 +-
 src/segmentor/featurespace.cpp |  142 ++--
 src/segmentor/featurespace.h   |  184 ++--
 src/segmentor/model.cpp        |  126 +--
 src/segmentor/model.h          |   84 +-
 src/segmentor/options.h        |   31 +-
 src/segmentor/parameter.h      |  256 +++---
 src/segmentor/rulebase.h       |  340 ++++----
 src/segmentor/segmentor.cpp    | 1456 +++++++++++++++-----------------
 src/segmentor/segmentor.h      |  206 ++---
 src/segmentor/segmentreader.h  |  184 ++--
 src/segmentor/segmentwriter.h  |  116 +--
 15 files changed, 1741 insertions(+), 1765 deletions(-)

diff --git a/src/segmentor/decoder.cpp b/src/segmentor/decoder.cpp
index 07b3c5d1b..369e8218a 100644
--- a/src/segmentor/decoder.cpp
+++ b/src/segmentor/decoder.cpp
@@ -5,104 +5,104 @@ namespace segmentor {
 
 
 void Decoder::decode(Instance * inst) {
-    init_lattice(inst);
-    viterbi_decode(inst);
-    get_result(inst);
-    free_lattice();
+  init_lattice(inst);
+  viterbi_decode(inst);
+  get_result(inst);
+  free_lattice();
 }
 
 void Decoder::init_lattice(const Instance * inst) {
-    int len = inst->size();
-    lattice.resize(len, L);
-    lattice = NULL;
+  int len = inst->size();
+  lattice.resize(len, L);
+  lattice = NULL;
 }
 
 void Decoder::viterbi_decode(const Instance * inst) {
-    int len = inst->size();
-    for (int i = 0; i < len; ++ i) {
-        for (int l = 0; l < L; ++ l) {
-            if (false == base.legal_emit(inst->chartypes[i], l)) {
-                continue;
-            }
-
-            if (i == 0) {
-                LatticeItem * item = new LatticeItem(i, l, inst->uni_scores[i][l], NULL);
-                lattice_insert(lattice[i][l], item);
-            } else {
-                for (int pl = 0; pl < L; ++ pl) {
-                    if (false == base.legal_trans(pl, l)) {
-                        continue;
-                    }
-
-                    double score = 0.;
-                    const LatticeItem * prev = lattice[i-1][pl];
-
-                    if (!prev) {
-                        continue;
-                    }
-
-                    // std::cout << i << " " << pl << " " << l << std::endl;
-                    score = inst->uni_scores[i][l] + inst->bi_scores[pl][l] + prev->score;
-                    const LatticeItem * item = new LatticeItem(i, l, score, prev);
-                    lattice_insert(lattice[i][l], item);
-                }
-            }   //  end for if i == 0
+  int len = inst->size();
+  for (int i = 0; i < len; ++ i) {
+    for (int l = 0; l < L; ++ l) {
+      if (false == base.legal_emit(inst->chartypes[i], l)) {
+        continue;
+      }
+
+      if (i == 0) {
+        LatticeItem * item = new LatticeItem(i, l, inst->uni_scores[i][l], NULL);
+        lattice_insert(lattice[i][l], item);
+      } else {
+        for (int pl = 0; pl < L; ++ pl) {
+          if (false == base.legal_trans(pl, l)) {
+            continue;
+          }
+
+          double score = 0.;
+          const LatticeItem * prev = lattice[i-1][pl];
+
+          if (!prev) {
+            continue;
+          }
+
+          // std::cout << i << " " << pl << " " << l << std::endl;
+          score = inst->uni_scores[i][l] + inst->bi_scores[pl][l] + prev->score;
+          const LatticeItem * item = new LatticeItem(i, l, score, prev);
+          lattice_insert(lattice[i][l], item);
         }
+      }   //  end for if i == 0
     }
+  }
 }
 
 void Decoder::get_result(Instance * inst) {
-    int len = inst->size();
-    const LatticeItem * best_item = NULL;
-    for (int l = 0; l < L; ++ l) {
-        if (!lattice[len-1][l]) {
-            continue;
-        }
-        if (best_item == NULL || (lattice[len-1][l]->score > best_item->score)) {
-            best_item = lattice[len - 1][l];
-        }
+  int len = inst->size();
+  const LatticeItem * best_item = NULL;
+  for (int l = 0; l < L; ++ l) {
+    if (!lattice[len-1][l]) {
+      continue;
     }
+    if (best_item == NULL || (lattice[len-1][l]->score > best_item->score)) {
+      best_item = lattice[len - 1][l];
+    }
+  }
 
-    const LatticeItem * item = best_item;
-    inst->predicted_tagsidx.resize(len);
+  const LatticeItem * item = best_item;
+  inst->predicted_tagsidx.resize(len);
 
-    while (item) {
-        inst->predicted_tagsidx[item->i] = item->l;
-        // std::cout << item->i << " " << item->l << std::endl;
-        item = item->prev;
-    }
+  while (item) {
+    inst->predicted_tagsidx[item->i] = item->l;
+    // std::cout << item->i << " " << item->l << std::endl;
+    item = item->prev;
+  }
 }
 
 void Decoder::free_lattice() {
-    for (int i = 0; i < lattice.nrows(); ++ i) {
-        for (int j = 0; j < lattice.ncols(); ++ j) {
-            if (lattice[i][j]) delete lattice[i][j];
-        }
+  for (int i = 0; i < lattice.nrows(); ++ i) {
+    for (int j = 0; j < lattice.ncols(); ++ j) {
+      if (lattice[i][j]) delete lattice[i][j];
     }
+  }
 }
 
 /*void KBestDecoder::decode(Instance * inst, KBestDecodeResult & result) {
-    init_lattice(inst);
-    kbest_viterbi_decode(inst);
-    get_result(result);
-    free_lattice();
+  init_lattice(inst);
+  kbest_viterbi_decode(inst);
+  get_result(result);
+  free_lattice();
 }
 
 void KBestDecoder::init_lattice(const Instance * inst) {
-    int len = inst->len();
-    lattice.resize(len, L);
+  int len = inst->len();
+  lattice.resize(len, L);
 
-    for (int i = 0; i < len; ++ i) {
-        for (int l = 0; l < L; ++ l) {
-            lattice[i][l] = new KHeap<LatticeItem>(k);
-        }
+  for (int i = 0; i < len; ++ i) {
+    for (int l = 0; l < L; ++ l) {
+      lattice[i][l] = new KHeap<LatticeItem>(k);
     }
+  }
 }
 
 void KBestDecoder::kbest_viterbi_decode(const Instance * inst) {
 }*/
 
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
 
diff --git a/src/segmentor/decoder.h b/src/segmentor/decoder.h
index 747f6bd59..c5a920322 100644
--- a/src/segmentor/decoder.h
+++ b/src/segmentor/decoder.h
@@ -13,75 +13,78 @@ namespace segmentor {
 // data structure for lattice item
 class LatticeItem {
 public:
-    LatticeItem (int _i, int _l, double _score, const LatticeItem * _prev) : 
-        i(_i),
-        l(_l),
-        score(_score),
-        prev(_prev) {}
-
-    LatticeItem (int _l, double _score) : 
-        i(0),
-        l(_l),
-        score(_score),
-        prev(0) {}
+  LatticeItem (int _i, int _l, double _score, const LatticeItem * _prev) :
+    i(_i),
+    l(_l),
+    score(_score),
+    prev(_prev) {}
+
+  LatticeItem (int _l, double _score) :
+    i(0),
+    l(_l),
+    score(_score),
+    prev(0) {}
 
 public:
-    int                 i;
-    int                 l;
-    double              score;
-    const LatticeItem * prev;
+  int         i;
+  int         l;
+  double        score;
+  const LatticeItem * prev;
 };
 
 class Decoder {
 public:
-    Decoder (int _L, rulebase::RuleBase & _base) : L(_L), base(_base) {}
-    void decode(Instance * inst);
+  Decoder (int _L, rulebase::RuleBase & _base) : L(_L), base(_base) {}
+  void decode(Instance * inst);
 
 private:
-    void init_lattice(const Instance * inst);
-    void viterbi_decode(const Instance * inst);
-    void get_result(Instance * inst);
-    void free_lattice();
+  void init_lattice(const Instance * inst);
+  void viterbi_decode(const Instance * inst);
+  void get_result(Instance * inst);
+  void free_lattice();
 
 private:
-    int L;
-
-    math::Mat< const LatticeItem * > lattice;
-    rulebase::RuleBase base;
-
-    void lattice_insert(const LatticeItem * &position, const LatticeItem * const item) {
-        if (position == NULL) {
-            position = item;
-        } else if (position->score < item->score) {
-            delete position;
-            position = item;
-        } else {
-            delete item;
-        }
+  int L;
+
+  math::Mat< const LatticeItem * > lattice;
+  rulebase::RuleBase base;
+
+  void lattice_insert(const LatticeItem * &position,
+                      const LatticeItem * const item) {
+    if (position == NULL) {
+      position = item;
+    } else if (position->score < item->score) {
+      delete position;
+      position = item;
+    } else {
+      delete item;
     }
+  }
 };
 
-// maintain kest best list of 
-/*class KBestDecoder {
+// Source code for k-best decoding, it's not implemented.
+/*
+class KBestDecoder {
 public:
-    typedef std::vector< std::vector<int> > KBestDecodeResult;
+  typedef std::vector< std::vector<int> > KBestDecodeResult;
 
 public:
-    KBestDecoder (int _L) : L(_L) {}
+  KBestDecoder (int _L) : L(_L) {}
 
-    void decode(Instance * inst, KBestDecodeResult & result);
+  void decode(Instance * inst, KBestDecodeResult & result);
 private:
-    void init_lattice(const Instance * inst);
-    void kbest_viterbi_decode(const Instance * inst);
-    void get_result(KBestDecodeResult & result);
-    void free_lattice();
+  void init_lattice(const Instance * inst);
+  void kbest_viterbi_decode(const Instance * inst);
+  void get_result(KBestDecodeResult & result);
+  void free_lattice();
 
 private:
-    int L;
+  int L;
 
-    Mat< KHeap< const LatticeItem * > > lattice;
-};*/
+  Mat< KHeap< const LatticeItem * > > lattice;
+};
+*/
 
-}           //  end for namespace segmentor
-}           //  end for namespace ltp
-#endif      //  end for __LTP_SEGMENTOR_DECODER_H__
+}       //  end for namespace segmentor
+}       //  end for namespace ltp
+#endif    //  end for __LTP_SEGMENTOR_DECODER_H__
diff --git a/src/segmentor/extractor.cpp b/src/segmentor/extractor.cpp
index eef00e893..6fc7ee054 100644
--- a/src/segmentor/extractor.cpp
+++ b/src/segmentor/extractor.cpp
@@ -11,90 +11,90 @@ Extractor * Extractor::instance_ = 0;
 std::vector<Template *> Extractor::templates;
 
 Extractor * Extractor::extractor() {
-    if (0 == instance_) {
-        instance_ = new Extractor;
-    }
+  if (0 == instance_) {
+    instance_ = new Extractor;
+  }
 
-    return instance_;
+  return instance_;
 }
 
 int Extractor::num_templates() {
-    if (0 == instance_) {
-        instance_ = new Extractor;
-    }
+  if (0 == instance_) {
+    instance_ = new Extractor;
+  }
 
-    return templates.size();
+  return templates.size();
 }
 
 Extractor::Extractor() {
-    templates.push_back(new Template("1={c-2}"));
-    templates.push_back(new Template("2={c-1}"));
-    templates.push_back(new Template("3={c-0}"));
-    templates.push_back(new Template("4={c+1}"));
-    templates.push_back(new Template("5={c+2}"));
-    templates.push_back(new Template("6={c-2}-{c-1}"));
-    templates.push_back(new Template("7={c-1}-{c-0}"));
-    templates.push_back(new Template("8={c-0}-{c+1}"));
-    templates.push_back(new Template("9={c+1}-{c+2}"));
-    //templates.push_back(new Template("10={c-2}-{c-0}"));
-    //templates.push_back(new Template("11={c-1}-{c+1}"));
-    //templates.push_back(new Template("12={c-0}-{c+2}"));
-    //templates.push_back(new Template("13={c-1}-{c-0}-{c+1}"));
-    templates.push_back(new Template("14={ct-1}"));
-    templates.push_back(new Template("15={ct-0}"));
-    templates.push_back(new Template("16={ct+1}"));
-    templates.push_back(new Template("17={lex1}"));
-    templates.push_back(new Template("18={lex2}"));
-    templates.push_back(new Template("19={lex3}"));
-    //templates.push_back(new Template("17={dup-1}"));
-    //templates.push_back(new Template("18={dup-0}"));
-    //templates.push_back(new Template("19={dup2-2}"));
-    //templates.push_back(new Template("20={dup2-1}"));
-    //templates.push_back(new Template("21={dup2-0}"));
+  templates.push_back(new Template("1={c-2}"));
+  templates.push_back(new Template("2={c-1}"));
+  templates.push_back(new Template("3={c-0}"));
+  templates.push_back(new Template("4={c+1}"));
+  templates.push_back(new Template("5={c+2}"));
+  templates.push_back(new Template("6={c-2}-{c-1}"));
+  templates.push_back(new Template("7={c-1}-{c-0}"));
+  templates.push_back(new Template("8={c-0}-{c+1}"));
+  templates.push_back(new Template("9={c+1}-{c+2}"));
+  //templates.push_back(new Template("10={c-2}-{c-0}"));
+  //templates.push_back(new Template("11={c-1}-{c+1}"));
+  //templates.push_back(new Template("12={c-0}-{c+2}"));
+  //templates.push_back(new Template("13={c-1}-{c-0}-{c+1}"));
+  templates.push_back(new Template("14={ct-1}"));
+  templates.push_back(new Template("15={ct-0}"));
+  templates.push_back(new Template("16={ct+1}"));
+  templates.push_back(new Template("17={lex1}"));
+  templates.push_back(new Template("18={lex2}"));
+  templates.push_back(new Template("19={lex3}"));
+  //templates.push_back(new Template("17={dup-1}"));
+  //templates.push_back(new Template("18={dup-0}"));
+  //templates.push_back(new Template("19={dup2-2}"));
+  //templates.push_back(new Template("20={dup2-1}"));
+  //templates.push_back(new Template("21={dup2-0}"));
 }
 
 Extractor::~Extractor() {
-    for (int i = 0; i < templates.size(); ++ i) {
-        delete templates[i];
-    }
+  for (int i = 0; i < templates.size(); ++ i) {
+    delete templates[i];
+  }
 }
 
 int Extractor::extract1o(Instance * inst, int idx, std::vector< StringVec > & cache) {
-    int len = inst->size();
+  int len = inst->size();
 
-    Template::Data data;
+  Template::Data data;
 
 #define EQU(x, y) (inst->forms[(x)] == inst->forms[(y)])
 #define TYPE(x) (strutils::to_str(inst->chartypes[(x)]&0x07))
 
-    data.set( "c-2",    (idx-2 < 0 ? BOS : inst->forms[idx-2]) );
-    data.set( "c-1",    (idx-1 < 0 ? BOS : inst->forms[idx-1]) );
-    data.set( "c-0",    inst->forms[idx] );
-    data.set( "c+1",    (idx+1 >= len ? EOS : inst->forms[idx+1]) );
-    data.set( "c+2",    (idx+2 >= len ? EOS : inst->forms[idx+2]) );
-    data.set( "ct-1",   (idx-1 < 0 ? BOT : TYPE(idx-1)) );
-    data.set( "ct-0",   TYPE(idx) );
-    data.set( "ct+1",   (idx+1 >= len ? EOT : TYPE(idx+1)) );
-    // data.set( "dup-1",  (idx-1 > 0 && EQU(idx-1, idx) ? "1" : "0") );
-    // data.set( "dup-0",  (idx+1 < len && EQU(idx, idx+1) ? "1" : "0") );
-    // data.set( "dup2-2", (idx-2 > 0 && EQU(idx-2, idx) ? "1" : "0") );
-    // data.set( "dup2-1", (idx-1 > 0 && idx+1 < len && EQU(idx-1, idx+1) ? "1" : "0") );
-    // data.set( "dup2-0", (idx+2 < len && EQU(idx, idx+2) ? "1" : "0") );
-    data.set( "lex1",   strutils::to_str(inst->lexicon_match_state[idx] & 0x0f));
-    data.set( "lex2",   strutils::to_str((inst->lexicon_match_state[idx]>>4) & 0x0f));
-    data.set( "lex3",   strutils::to_str((inst->lexicon_match_state[idx]>>8) & 0x0f));
+  data.set( "c-2",  (idx-2 < 0 ? BOS : inst->forms[idx-2]) );
+  data.set( "c-1",  (idx-1 < 0 ? BOS : inst->forms[idx-1]) );
+  data.set( "c-0",  inst->forms[idx] );
+  data.set( "c+1",  (idx+1 >= len ? EOS : inst->forms[idx+1]) );
+  data.set( "c+2",  (idx+2 >= len ? EOS : inst->forms[idx+2]) );
+  data.set( "ct-1",   (idx-1 < 0 ? BOT : TYPE(idx-1)) );
+  data.set( "ct-0",   TYPE(idx) );
+  data.set( "ct+1",   (idx+1 >= len ? EOT : TYPE(idx+1)) );
+  // data.set( "dup-1",  (idx-1 > 0 && EQU(idx-1, idx) ? "1" : "0") );
+  // data.set( "dup-0",  (idx+1 < len && EQU(idx, idx+1) ? "1" : "0") );
+  // data.set( "dup2-2", (idx-2 > 0 && EQU(idx-2, idx) ? "1" : "0") );
+  // data.set( "dup2-1", (idx-1 > 0 && idx+1 < len && EQU(idx-1, idx+1) ? "1" : "0") );
+  // data.set( "dup2-0", (idx+2 < len && EQU(idx, idx+2) ? "1" : "0") );
+  data.set( "lex1",   strutils::to_str(inst->lexicon_match_state[idx] & 0x0f));
+  data.set( "lex2",   strutils::to_str((inst->lexicon_match_state[idx]>>4) & 0x0f));
+  data.set( "lex3",   strutils::to_str((inst->lexicon_match_state[idx]>>8) & 0x0f));
 
 #undef TYPE
 #undef EQU
 
-    string feat;
-    feat.reserve(1024);
-    for (int i = 0; i < templates.size(); ++ i) {
-        templates[i]->render(data, feat);
-        cache[i].push_back(feat);
-    }
-    return 0;
+  string feat;
+  feat.reserve(1024);
+  for (int i = 0; i < templates.size(); ++ i) {
+    templates[i]->render(data, feat);
+    cache[i].push_back(feat);
+  }
+  return 0;
 }
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
diff --git a/src/segmentor/extractor.h b/src/segmentor/extractor.h
index e3b49a7d0..f6f85f31f 100644
--- a/src/segmentor/extractor.h
+++ b/src/segmentor/extractor.h
@@ -16,18 +16,18 @@ using namespace ltp::utility;
 
 class Extractor {
 public:
-    static Extractor * extractor();
-    static int num_templates();
-    static int extract1o(Instance * inst, int idx, vector< StringVec > & cache);
+  static Extractor * extractor();
+  static int num_templates();
+  static int extract1o(Instance * inst, int idx, vector< StringVec > & cache);
 protected:
-    Extractor();
-    ~Extractor();
+  Extractor();
+  ~Extractor();
 private:
-    static Extractor * instance_;
-    static vector< Template * > templates;
+  static Extractor * instance_;
+  static vector< Template * > templates;
 };
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp 
+}     //  end for namespace segmentor
+}     //  end for namespace ltp 
 
 #endif  //  end for __LTP_SEGMENTOR_EXTRACTOR_H__
diff --git a/src/segmentor/featurespace.cpp b/src/segmentor/featurespace.cpp
index bc7dcc517..ade57c708 100644
--- a/src/segmentor/featurespace.cpp
+++ b/src/segmentor/featurespace.cpp
@@ -5,103 +5,113 @@
 namespace ltp {
 namespace segmentor {
 
-FeatureSpace::FeatureSpace(int num_labels) : 
-    _num_labels(num_labels), 
+FeatureSpace::FeatureSpace(int num_labels)
+  : _num_labels(num_labels),
     _offset(0) {
 
-    // allocate dictionary according to number of templates
-    _num_dicts = Extractor::num_templates();
+  // allocate dictionary according to number of templates
+  _num_dicts = Extractor::num_templates();
 
-    dicts = new utility::SmartMap<int>[ _num_dicts ];
+  dicts = new utility::SmartMap<int>[ _num_dicts ];
 }
 
 FeatureSpace::~FeatureSpace(void) {
-    delete [](dicts);
+  delete [](dicts);
 }
 
-int FeatureSpace::retrieve(int tid, const char * key, bool create) {
-    int val;
+int
+FeatureSpace::retrieve(int tid, const char * key, bool create) {
+  int val;
 
-    if (dicts[tid].get(key, val)) {
-        return val;
-    } else {
-        if (create) {
-            val = _offset;
-            dicts[tid].set(key, val);
-            ++ _offset;
+  if (dicts[tid].get(key, val)) {
+    return val;
+  } else {
+    if (create) {
+      val = _offset;
+      dicts[tid].set(key, val);
+      ++ _offset;
 
-            return val;
-        }
-    }
- 
-    return -1;
-}
-
-int FeatureSpace::index(int tid, const char * key, int lid) {
-    int idx = retrieve(tid, key, false);
-    if (idx < 0) {
-        return -1;
+      return val;
     }
+  }
 
-    return idx * _num_labels + lid;
+  return -1;
 }
 
-int FeatureSpace::index(int prev_lid, int lid) {
-    return _offset * _num_labels + prev_lid * _num_labels + lid;
-}
+int
+FeatureSpace::index(int tid, const char * key, int lid) {
+  int idx = retrieve(tid, key, false);
+  if (idx < 0) {
+    return -1;
+  }
 
-int FeatureSpace::num_features() {
-    return _offset;
+  return idx * _num_labels + lid;
 }
 
-int FeatureSpace::dim() {
-    return _offset * _num_labels + _num_labels * _num_labels;
+int
+FeatureSpace::index(int prev_lid, int lid) {
+  return _offset * _num_labels + prev_lid * _num_labels + lid;
 }
 
-int FeatureSpace::get_offset() {
-    return _offset;
+int
+FeatureSpace::num_features() {
+  return _offset;
 }
 
-void FeatureSpace::set_num_labels(int num_labels) {
-    _num_labels = num_labels;
+int
+FeatureSpace::dim() {
+  return _offset * _num_labels + _num_labels * _num_labels;
 }
-void FeatureSpace::dump(std::ostream & ofs) {
-    char chunk[16];
-    unsigned sz = _num_dicts;
-    strncpy(chunk, "featurespace", 16);
 
-    ofs.write(chunk, 16);
-    ofs.write(reinterpret_cast<const char *>(&_offset), sizeof(int));
-    ofs.write(reinterpret_cast<const char *>(&sz), sizeof(unsigned int));
+int
+FeatureSpace::get_offset() {
+  return _offset;
+}
 
-    for (int i = 0; i < _num_dicts; ++ i) {
-        dicts[i].dump(ofs);
-    }
+void
+FeatureSpace::set_num_labels(int num_labels) {
+  _num_labels = num_labels;
 }
 
-bool FeatureSpace::load(int num_labels, std::istream & ifs) {
-    _num_labels = num_labels;
-    char chunk[16];
-    unsigned int sz;
-    ifs.read(chunk, 16);
-    if (strcmp(chunk, "featurespace")) {
-        return false;
-    }
+void
+FeatureSpace::dump(std::ostream & ofs) {
+  char chunk[16];
+  unsigned sz = _num_dicts;
+  strncpy(chunk, "featurespace", 16);
 
-    ifs.read(reinterpret_cast<char *>(&_offset), sizeof(int));
-    ifs.read(reinterpret_cast<char *>(&sz), sizeof(unsigned int));
+  ofs.write(chunk, 16);
+  ofs.write(reinterpret_cast<const char *>(&_offset), sizeof(int));
+  ofs.write(reinterpret_cast<const char *>(&sz), sizeof(unsigned int));
 
-    if (sz != _num_dicts) {
-        return false;
-    }
+  for (int i = 0; i < _num_dicts; ++ i) {
+    dicts[i].dump(ofs);
+  }
+}
 
-    for (unsigned i = 0; i < sz; ++ i) {
-        if (!dicts[i].load(ifs)) {
-            return false;
-        }
+bool
+FeatureSpace::load(int num_labels, std::istream & ifs) {
+  _num_labels = num_labels;
+  char chunk[16];
+  unsigned int sz;
+  ifs.read(chunk, 16);
+  if (strcmp(chunk, "featurespace")) {
+    return false;
+  }
+
+  ifs.read(reinterpret_cast<char *>(&_offset), sizeof(int));
+  ifs.read(reinterpret_cast<char *>(&sz), sizeof(unsigned int));
+
+  if (sz != _num_dicts) {
+    return false;
+  }
+
+  for (unsigned i = 0; i < sz; ++ i) {
+    if (!dicts[i].load(ifs)) {
+      return false;
     }
+  }
 
-    return true;
+  return true;
 }
 
 }   //  end for namespace segmentor
diff --git a/src/segmentor/featurespace.h b/src/segmentor/featurespace.h
index 9e4bcbecc..88eddf9ff 100644
--- a/src/segmentor/featurespace.h
+++ b/src/segmentor/featurespace.h
@@ -11,108 +11,108 @@ namespace segmentor {
 
 class FeatureSpaceIterator {
 public:
-    FeatureSpaceIterator() : 
-        _dicts(NULL),
-        _i(0),
-        _state(0) {
-        // should be careful about the empty dicts
+  FeatureSpaceIterator()
+    : _dicts(NULL),
+      _i(0),
+      _state(0) {
+    // should be careful about the empty dicts
+  }
+
+  FeatureSpaceIterator(utility::SmartMap<int> * dicts, int num_dicts)
+    : _dicts(dicts),
+      _num_dicts(num_dicts),
+      _i(0),
+      _state(0) {
+    ++ (*this);
+  }
+
+  ~FeatureSpaceIterator() {
+  }
+
+  const char * key() { return _j.key(); }
+  int id() { return (*_j.value()); }
+  int tid() { return _i; }
+
+  bool operator ==(const FeatureSpaceIterator & other) const {
+    return ((_dicts + _i) == other._dicts);
+  }
+
+  bool operator !=(const FeatureSpaceIterator & other) const {
+    return ((_dicts + _i) != other._dicts);
+  }
+
+  FeatureSpaceIterator & operator = (const FeatureSpaceIterator & other) {
+    if (this != &other) {
+      _dicts  = other._dicts;
+      _i    = other._i;
+      _state  = other._state;
     }
 
-    FeatureSpaceIterator(utility::SmartMap<int> * dicts,int num_dicts) : 
-        _dicts(dicts), 
-	_num_dicts(num_dicts),
-        _i(0), 
-        _state(0) {
-        ++ (*this);
-    }
-
-    ~FeatureSpaceIterator() {
-    }
-
-    const char * key() { return _j.key(); }
-    int id() { return (*_j.value()); }
-    int tid() { return _i; }
-
-    bool operator ==(const FeatureSpaceIterator & other) const { return ((_dicts + _i) == other._dicts); }
-    bool operator !=(const FeatureSpaceIterator & other) const { return ((_dicts + _i) != other._dicts); }
-
-    FeatureSpaceIterator & operator = (const FeatureSpaceIterator & other) {
-        if (this != &other) {
-            _dicts  = other._dicts;
-            _i      = other._i;
-            _state  = other._state;
+    return *this;
+  }
+
+  void operator ++() {
+    switch (_state) {
+      case 0:
+        for (_i = 0; _i < _num_dicts; ++ _i) {
+          for (_j = _dicts[_i].begin(); _j != _dicts[_i].end(); ++ _j) {
+            _state = 1;
+            return;
+      case 1:;
+          }
         }
-
-        return *this;
     }
+  }
 
-    void operator ++() {
-        switch (_state) {
-            case 0:
-                for (_i = 0;_i<_num_dicts ; ++ _i) {
-                   /* if (_dicts[_i].begin() == _dicts[_i].end()) {
-                        _state = 1;
-                        return;
-                    }*/
-                    for (_j = _dicts[_i].begin(); _j != _dicts[_i].end(); ++ _j) {
-                        _state = 1;
-                        return;
-            case 1:;
-                    }
-                }
-        }
-    }
-
-    int _i;
-    int _state;
-    int _num_dicts;
-    utility::SmartMap<int>::const_iterator  _j;
-    utility::SmartMap<int> * _dicts;
+  int _i;
+  int _state;
+  int _num_dicts;
+  utility::SmartMap<int>::const_iterator  _j;
+  utility::SmartMap<int> * _dicts;
 };
 
 class FeatureSpace {
 public:
-    FeatureSpace(int num_labels = 1);
-    ~FeatureSpace();
-
-    int retrieve(int tid, const char * key, bool create);
-    int index(int tid, const char * key, int lid = 0);
-    int index(int prev_lid, int lid);
-    int num_features();
-    int id_label_to_id(int idx);
-    int dim();
-    int get_offset();
-    void set_num_labels(int num_labeles);
-
-    /*
-     * dump the feature space to a output stream
-     *
-     *  @param[in]  ofs     the output stream
-     */
-    void dump(std::ostream & ofs);
-
-    /*
-     * load the feature space from a input stream
-     *
-     *  @param[in]  num_labels  the number of labels
-     *  @param[in]  ifs         the input stream
-     */
-    bool load(int num_labeles, std::istream & ifs);
-
-    FeatureSpaceIterator begin() {
-        return FeatureSpaceIterator(dicts,_num_dicts);
-    }
-
-    FeatureSpaceIterator end() {
-        return FeatureSpaceIterator(dicts + _num_dicts,_num_dicts);
-    }
+  FeatureSpace(int num_labels = 1);
+  ~FeatureSpace();
+
+  int retrieve(int tid, const char * key, bool create);
+  int index(int tid, const char * key, int lid = 0);
+  int index(int prev_lid, int lid);
+  int num_features();
+  int dim();
+  int get_offset();
+  void set_num_labels(int num_labeles);
+
+  /*
+   * dump the feature space to a output stream
+   *
+   *  @param[in]  ofs   the output stream
+   */
+  void dump(std::ostream & ofs);
+
+  /*
+   * load the feature space from a input stream
+   *
+   *  @param[in]  num_labels  the number of labels
+   *  @param[in]  ifs     the input stream
+   */
+  bool load(int num_labeles, std::istream & ifs);
+
+  FeatureSpaceIterator begin() {
+    return FeatureSpaceIterator(dicts, _num_dicts);
+  }
+
+  FeatureSpaceIterator end() {
+    return FeatureSpaceIterator(dicts + _num_dicts, _num_dicts);
+  }
 private:
-    int _offset;
-    int _num_labels;
-    int _num_dicts;
-    utility::SmartMap<int> * dicts;
+  int _offset;
+  int _num_labels;
+  int _num_dicts;
+  utility::SmartMap<int> * dicts;
 };
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
 #endif  //  end for __LTP_SEGMENTOR_FEATURE_SPACE_H__
diff --git a/src/segmentor/model.cpp b/src/segmentor/model.cpp
index 902272aa7..054eae5f0 100644
--- a/src/segmentor/model.cpp
+++ b/src/segmentor/model.cpp
@@ -9,77 +9,79 @@ Model::Model() {
 Model::~Model() {
 }
 
-void Model::save(std::ostream & ofs) {
-    // write a signature into the file
-    char chunk[16] = {'o','t','c','w','s', '\0'};
-    ofs.write(chunk, 16);
+void
+Model::save(std::ostream & ofs) {
+  // write a signature into the file
+  char chunk[16] = {'o','t','c','w','s', '\0'};
+  ofs.write(chunk, 16);
 
-    int off = ofs.tellp();
+  int off = ofs.tellp();
 
-    unsigned labels_offset      = 0;
-    unsigned lexicon_offset     = 0;
-    unsigned feature_offset     = 0;
-    unsigned parameter_offset   = 0;
+  unsigned labels_offset    = 0;
+  unsigned lexicon_offset   = 0;
+  unsigned feature_offset   = 0;
+  unsigned parameter_offset   = 0;
 
-    write_uint(ofs, 0); //  the label offset
-    write_uint(ofs, 0); //  the internal lexicon offset
-    write_uint(ofs, 0); //  the features offset
-    write_uint(ofs, 0); //  the parameter offset
+  write_uint(ofs, 0); //  the label offset
+  write_uint(ofs, 0); //  the internal lexicon offset
+  write_uint(ofs, 0); //  the features offset
+  write_uint(ofs, 0); //  the parameter offset
 
-    labels_offset = ofs.tellp();
-    labels.dump(ofs);
+  labels_offset = ofs.tellp();
+  labels.dump(ofs);
 
-    lexicon_offset = ofs.tellp();
-    internal_lexicon.dump(ofs);
+  lexicon_offset = ofs.tellp();
+  internal_lexicon.dump(ofs);
 
-    feature_offset = ofs.tellp();
-    space.dump(ofs);
+  feature_offset = ofs.tellp();
+  space.dump(ofs);
 
-    parameter_offset = ofs.tellp();
-    param.dump(ofs);
+  parameter_offset = ofs.tellp();
+  param.dump(ofs);
 
-    ofs.seekp(off);
-    write_uint(ofs, labels_offset);
-    write_uint(ofs, lexicon_offset);
-    write_uint(ofs, feature_offset);
-    write_uint(ofs, parameter_offset);
+  ofs.seekp(off);
+  write_uint(ofs, labels_offset);
+  write_uint(ofs, lexicon_offset);
+  write_uint(ofs, feature_offset);
+  write_uint(ofs, parameter_offset);
 }
 
-bool Model::load(std::istream & ifs) {
-    char chunk[16];
-    ifs.read(chunk, 16);
-
-    if (strcmp(chunk, "otcws")) {
-        return false;
-    }
-
-    unsigned labels_offset    = read_uint(ifs);
-    unsigned lexicon_offset   = read_uint(ifs);
-    unsigned feature_offset   = read_uint(ifs);
-    unsigned parameter_offset = read_uint(ifs);
-
-    ifs.seekg(labels_offset);
-    if (!labels.load(ifs)) {
-        return false;
-    }
-
-    ifs.seekg(lexicon_offset);
-    if (!internal_lexicon.load(ifs)) {
-        return false;
-    }
-
-    ifs.seekg(feature_offset);
-    if (!space.load(labels.size(), ifs)) {
-        return false;
-    }
-
-    ifs.seekg(parameter_offset);
-    if (!param.load(ifs)) {
-        return false;
-    }
-
-    return true;
+bool
+Model::load(std::istream & ifs) {
+  char chunk[16];
+  ifs.read(chunk, 16);
+
+  if (strcmp(chunk, "otcws")) {
+    return false;
+  }
+
+  unsigned labels_offset  = read_uint(ifs);
+  unsigned lexicon_offset   = read_uint(ifs);
+  unsigned feature_offset   = read_uint(ifs);
+  unsigned parameter_offset = read_uint(ifs);
+
+  ifs.seekg(labels_offset);
+  if (!labels.load(ifs)) {
+    return false;
+  }
+
+  ifs.seekg(lexicon_offset);
+  if (!internal_lexicon.load(ifs)) {
+    return false;
+  }
+
+  ifs.seekg(feature_offset);
+  if (!space.load(labels.size(), ifs)) {
+    return false;
+  }
+
+  ifs.seekg(parameter_offset);
+  if (!param.load(ifs)) {
+    return false;
+  }
+
+  return true;
 }
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
diff --git a/src/segmentor/model.h b/src/segmentor/model.h
index 29b0e3fe1..3824f1f07 100644
--- a/src/segmentor/model.h
+++ b/src/segmentor/model.h
@@ -13,51 +13,53 @@ using namespace ltp::utility;
 
 class Model {
 public:
-    Model();
-    ~Model();
-
-    /*
-     * get number of labels;
-     *
-     *  @return     int     the number of labels
-     */
-    inline int num_labels(void) {
-        return labels.size();
-    }
-
-    /*
-     * save the model to a output stream
-     *
-     *  @param[out] ofs     the output stream
-     */
-    void save(std::ostream & ofs);
-
-    /*
-     * load the model from an input stream
-     *
-     *  @param[in]  ifs     the input stream
-     */
-    bool load(std::istream & ifs);
+  Model();
+  ~Model();
+
+  /*
+   * get number of labels;
+   *
+   *  @return   int   the number of labels
+   */
+  inline int num_labels(void) {
+    return labels.size();
+  }
+
+  /*
+   * save the model to a output stream
+   *
+   *  @param[out] ofs   the output stream
+   */
+  void save(std::ostream & ofs);
+
+  /*
+   * load the model from an input stream
+   *
+   *  @param[in]  ifs   the input stream
+   */
+  bool load(std::istream & ifs);
+
 public:
-    IndexableSmartMap   labels;
-    FeatureSpace        space;
-    Parameters          param;
+  IndexableSmartMap labels;
+  FeatureSpace      space;
+  Parameters        param;
+
+  SmartMap<bool>    internal_lexicon;
+  SmartMap<bool>    external_lexicon;
 
-    SmartMap<bool>      internal_lexicon;
-    SmartMap<bool>      external_lexicon;
 private:
-    void write_uint(std::ostream & out, unsigned int val) {
-        out.write(reinterpret_cast<const char *>(&val), sizeof(unsigned int));
-    }
-
-    unsigned int read_uint(std::istream & in) {
-        char p[4];
-        in.read(reinterpret_cast<char*>(p), sizeof(unsigned int));
-        return *reinterpret_cast<const unsigned int*>(p);
-    }
+  void write_uint(std::ostream & out, unsigned int val) {
+    out.write(reinterpret_cast<const char *>(&val), sizeof(unsigned int));
+  }
+
+  unsigned int read_uint(std::istream & in) {
+    char p[4];
+    in.read(reinterpret_cast<char*>(p), sizeof(unsigned int));
+    return *reinterpret_cast<const unsigned int*>(p);
+  }
 };
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
 
 #endif  //  end for __LTP_SEGMENTOR_MODEL_H__
diff --git a/src/segmentor/options.h b/src/segmentor/options.h
index db3d489ad..298f4200b 100644
--- a/src/segmentor/options.h
+++ b/src/segmentor/options.h
@@ -7,28 +7,27 @@ namespace ltp {
 namespace segmentor {
 
 struct ModelOptions {
-    std::string     model_file;
+  std::string   model_file;
 };
 
 struct TrainOptions {
-    std::string     train_file;
-    std::string     holdout_file;
-    std::string     model_name;
-    std::string     algorithm;
-    std::string     use_update;
-    int             max_iter;
-    int             display_interval;
-    int		    min_update;
+  std::string       train_file;
+  std::string       holdout_file;
+  std::string       model_name;
+  std::string       algorithm;
+  int               max_iter;
+  int               display_interval;
+  int               rare_feature_threshold;
 };
 
 struct TestOptions {
-    std::string     test_file;
-    std::string     model_file;
-    std::string     lexicon_file;
+  std::string       test_file;
+  std::string       model_file;
+  std::string       lexicon_file;
 };
 
 struct DumpOptions {
-    std::string     model_file;
+  std::string       model_file;
 };
 
 extern ModelOptions model_opt;
@@ -36,7 +35,7 @@ extern TrainOptions train_opt;
 extern TestOptions  test_opt;
 extern DumpOptions  dump_opt;
 
-}           //  end for namespace segmentor
-}           //  end for namespace ltp
+}       //  end for namespace segmentor
+}       //  end for namespace ltp
 
-#endif      //  end for __LTP_SEGMENTOR_OPTIONS_H__
+#endif    //  end for __LTP_SEGMENTOR_OPTIONS_H__
diff --git a/src/segmentor/parameter.h b/src/segmentor/parameter.h
index a3fbb5a44..119662021 100644
--- a/src/segmentor/parameter.h
+++ b/src/segmentor/parameter.h
@@ -12,151 +12,151 @@ using namespace ltp::math;
 
 class Parameters {
 public:
-    int _dim;
-    double * _W;
-    double * _W_sum;
-    int *    _W_time;
-
-    Parameters() :
-        _dim(0),
-        _W(0),
-        _W_sum(0),
-        _W_time(0) {}
-
-    ~Parameters() {
-        dealloc();
+  int _dim;
+  double * _W;
+  double * _W_sum;
+  int *    _W_time;
+
+  Parameters() :
+    _dim(0),
+    _W(0),
+    _W_sum(0),
+    _W_time(0) {}
+
+  ~Parameters() {
+    dealloc();
+  }
+
+  void realloc(int dim) {
+    dealloc();
+    _dim = dim;
+
+    if (dim > 0) {
+      _W = new double[dim];
+      _W_sum = new double[dim];
+      _W_time = new int[dim];
     }
 
-    void realloc(int dim) {
-        dealloc();
-        _dim = dim;
-
-        if (dim > 0) {
-            _W = new double[dim];
-            _W_sum = new double[dim];
-            _W_time = new int[dim];
-        }
-
-        for (int i = 0; i < dim; ++ i) {
-            _W[i] = 0;
-            _W_sum[i] = 0;
-            _W_time[i] = 0;
-        }
+    for (int i = 0; i < dim; ++ i) {
+      _W[i] = 0;
+      _W_sum[i] = 0;
+      _W_time[i] = 0;
     }
-
-    void dealloc() {
-        if (_W && _W == _W_sum) {
-            delete [](_W);
-            _W = 0;
-            _W_sum = 0;
-        } else {
-            if (_W) {
-                delete [](_W);
-                _W = 0;
-            }
-            if (_W_sum) {
-                delete [](_W_sum);
-                _W_sum = 0;
-            }
-        }
-
-        if (_W_time) {
-            delete [](_W_time);
-            _W_time = 0;
-        }
+  }
+
+  void dealloc() {
+    if (_W && _W == _W_sum) {
+      delete [](_W);
+      _W = 0;
+      _W_sum = 0;
+    } else {
+      if (_W) {
+        delete [](_W);
+        _W = 0;
+      }
+      if (_W_sum) {
+        delete [](_W_sum);
+        _W_sum = 0;
+      }
     }
 
-    void add(int idx, int now, double scale = 1.) {
-        int elapsed = now - _W_time[idx];
-        double upd = scale;
-        double cur_val = _W[idx];
-
-        _W[idx]         = cur_val + upd;
-        _W_sum[idx]    += elapsed * cur_val + upd;
-        _W_time[idx]    = now;
+    if (_W_time) {
+      delete [](_W_time);
+      _W_time = 0;
     }
-
-    void add(const SparseVec & vec, int now, double scale = 1.) {
-        for (SparseVec::const_iterator itx = vec.begin();
-                itx != vec.end();
-                itx ++) {
-            int idx = itx->first;
-            int elapsed = now - _W_time[idx];
-            double upd = scale * itx->second;
-            double cur_val = _W[idx];
-
-            _W[idx]         = cur_val + upd;
-            _W_sum[idx]    += elapsed * cur_val + upd;
-            _W_time[idx]    = now;
-        }
+  }
+
+  void add(int idx, int now, double scale = 1.) {
+    int elapsed = now - _W_time[idx];
+    double upd = scale;
+    double cur_val = _W[idx];
+
+    _W[idx]       = cur_val + upd;
+    _W_sum[idx]   += elapsed * cur_val + upd;
+    _W_time[idx]  = now;
+  }
+
+  void add(const SparseVec & vec, int now, double scale = 1.) {
+    for (SparseVec::const_iterator itx = vec.begin();
+        itx != vec.end();
+        itx ++) {
+      int idx = itx->first;
+      int elapsed = now - _W_time[idx];
+      double upd = scale * itx->second;
+      double cur_val = _W[idx];
+
+      _W[idx]       = cur_val + upd;
+      _W_sum[idx]   += elapsed * cur_val + upd;
+      _W_time[idx]  = now;
     }
-
-    double dot(const SparseVec & vec, bool use_avg = false) const {
-        const double * const p = (use_avg ? _W_sum : _W);
-        double ret = 0.;
-        for (SparseVec::const_iterator itx = vec.begin();
-                itx != vec.end();
-                ++ itx) {
-            ret += p[itx->first] * itx->second;
-        }
-        return ret;
+  }
+
+  double dot(const SparseVec & vec, bool use_avg = false) const {
+    const double * const p = (use_avg ? _W_sum : _W);
+    double ret = 0.;
+    for (SparseVec::const_iterator itx = vec.begin();
+        itx != vec.end();
+        ++ itx) {
+      ret += p[itx->first] * itx->second;
     }
-
-    double dot(const FeatureVector * vec, bool use_avg = false) const {
-        const double * const p = (use_avg ? _W_sum : _W);
-        double ret = 0.;
-        for (int i = 0; i < vec->n; ++ i) {
-            if (vec->val) {
-                ret += p[vec->idx[i] + vec->loff] * vec->val[i];
-            } else {
-                ret += p[vec->idx[i] + vec->loff];
-            }
-        }
-        return ret;
+    return ret;
+  }
+
+  double dot(const FeatureVector * vec, bool use_avg = false) const {
+    const double * const p = (use_avg ? _W_sum : _W);
+    double ret = 0.;
+    for (int i = 0; i < vec->n; ++ i) {
+      if (vec->val) {
+        ret += p[vec->idx[i] + vec->loff] * vec->val[i];
+      } else {
+        ret += p[vec->idx[i] + vec->loff];
+      }
     }
-
-    double dot(const int idx, bool use_avg = false) const {
-        const double * const p = (use_avg ? _W_sum : _W);
-        return p[idx];
+    return ret;
+  }
+
+  double dot(const int idx, bool use_avg = false) const {
+    const double * const p = (use_avg ? _W_sum : _W);
+    return p[idx];
+  }
+
+  void flush(int now) {
+    for(int i = 0; i < _dim; ++i) {
+      _W_sum[i]  += (now - _W_time[i]) * _W[i];
+      _W_time[i] = now;
     }
-
-    void flush(int now) {
-        for(int i = 0; i < _dim; ++i) {
-            _W_sum[i] += (now - _W_time[i]) * _W[i];
-            _W_time[i] = now;
-        }
+  }
+
+  void dump(std::ostream & out, bool use_avg = true) {
+    const double * p = (use_avg ? _W_sum : _W);
+    char chunk[16] = {'p', 'a', 'r', 'a', 'm', 0};
+    out.write(chunk, 16);
+    out.write(reinterpret_cast<const char *>(&_dim), sizeof(int));
+    if (_dim > 0) {
+      out.write(reinterpret_cast<const char *>(p), sizeof(double) * _dim);
     }
+  }
 
-    void dump(std::ostream & out, bool use_avg = true) {
-        const double * p = (use_avg ? _W_sum : _W);
-        char chunk[16] = {'p', 'a', 'r', 'a', 'm', 0};
-        out.write(chunk, 16);
-        out.write(reinterpret_cast<const char *>(&_dim), sizeof(int));
-        if (_dim > 0) {
-            out.write(reinterpret_cast<const char *>(p), sizeof(double) * _dim);
-        }
+  bool load(std::istream & in) {
+    char chunk[16];
+    in.read(chunk, 16);
+    if (strcmp(chunk, "param")) {
+      return false;
     }
 
-    bool load(std::istream & in) {
-        char chunk[16];
-        in.read(chunk, 16);
-        if (strcmp(chunk, "param")) {
-            return false;
-        }
-
-        in.read(reinterpret_cast<char *>(&_dim), sizeof(int));
-        if (_dim > 0) {
-            _W = new double[_dim];
-            in.read(reinterpret_cast<char *>(_W), sizeof(double) * _dim);
-            _W_sum = _W;
-        }
-
-        return true;
+    in.read(reinterpret_cast<char *>(&_dim), sizeof(int));
+    if (_dim > 0) {
+      _W = new double[_dim];
+      in.read(reinterpret_cast<char *>(_W), sizeof(double) * _dim);
+      _W_sum = _W;
     }
+
+    return true;
+  }
 };
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
 
 #endif  //  end for __LTP_SEGMENTOR_PARAMETER_H__
 
diff --git a/src/segmentor/rulebase.h b/src/segmentor/rulebase.h
index a635f9d50..3786748f2 100644
--- a/src/segmentor/rulebase.h
+++ b/src/segmentor/rulebase.h
@@ -28,224 +28,224 @@ const int CHAR_URI = strutils::chartypes::CHAR_PUNC+2;
 
 const unsigned HAVE_SPACE_ON_LEFT  = (1<<3);
 const unsigned HAVE_SPACE_ON_RIGHT = (1<<4);
-const unsigned HAVE_ENG_ON_LEFT    = (1<<5);
+const unsigned HAVE_ENG_ON_LEFT  = (1<<5);
 const unsigned HAVE_ENG_ON_RIGHT   = (1<<6);
-const unsigned HAVE_URI_ON_LEFT    = (1<<7);
+const unsigned HAVE_URI_ON_LEFT  = (1<<7);
 const unsigned HAVE_URI_ON_RIGHT   = (1<<8);
 
 static boost::regex engpattern("(([A-Za-z]+)([\\-'\\.][A-Za-z]+)*)");
 static boost::regex uripattern("((https?|ftp|file)"
-        "://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|])");
+    "://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|])");
 
 static bool flags_clear_check(int * flags, int left, int right) {
-    for (int i = left; i < right; ++ i) {
-        if (flags[i]) return false;
-    }
-    return true;
+  for (int i = left; i < right; ++ i) {
+    if (flags[i]) return false;
+  }
+  return true;
 }
 
 static void flags_color(int * flags, int left, int right, int color) {
-    for (int i = left; i < right; ++ i) {
-        flags[i] = color;
-    }
+  for (int i = left; i < right; ++ i) {
+    flags[i] = color;
+  }
 }
 
 inline int preprocess(const std::string & sentence,
-        std::vector<std::string> & raw_forms,
-        std::vector<std::string> & forms,
-        std::vector<int> & chartypes) {
+    std::vector<std::string> & raw_forms,
+    std::vector<std::string> & forms,
+    std::vector<int> & chartypes) {
 
-    std::string sent = ltp::strutils::chomp(sentence);
-    // std::cerr << sent << std::endl;
+  std::string sent = ltp::strutils::chomp(sentence);
+  // std::cerr << sent << std::endl;
 
-    int len = sent.size();
-    if (0 == len) {
-        return 0;
-    }
+  int len = sent.size();
+  if (0 == len) {
+    return 0;
+  }
+
+  std::string::const_iterator start, end;
+  boost::match_results<std::string::const_iterator> what;
+
+  int ret = 0;
+  int * flags = new int[len];
 
-    std::string::const_iterator start, end;
-    boost::match_results<std::string::const_iterator> what;
+  for (int i = 0; i < len; ++ i) {
+    flags[i] = 0;
+  }
 
-    int ret = 0;
-    int * flags = new int[len];
+  start = sent.begin();
+  end = sent.end();
 
-    for (int i = 0; i < len; ++ i) {
-        flags[i] = 0;
+  while (boost::regex_search(start, end, what, uripattern, boost::match_default)) {
+    int left = what[0].first - sent.begin();
+    int right = what[0].second - sent.begin();
+
+    if (flags_clear_check(flags, left, right)) {
+      flags[left] = URI_BEG;
+      flags_color(flags, left+1, right, URI_MID);
     }
 
-    start = sent.begin();
-    end = sent.end();
+    start = what[0].second;
+  }
 
-    while (boost::regex_search(start, end, what, uripattern, boost::match_default)) {
-        int left = what[0].first - sent.begin();
-        int right = what[0].second - sent.begin();
+  start = sent.begin();
+  end   = sent.end();
 
-        if (flags_clear_check(flags, left, right)) {
-            flags[left] = URI_BEG;
-            flags_color(flags, left+1, right, URI_MID);
-        }
+  while (boost::regex_search(start, end, what, engpattern, boost::match_default)) {
+    int left = what[0].first - sent.begin();
+    int right = what[0].second - sent.begin();
 
-        start = what[0].second;
+    if (flags_clear_check(flags, left, right)) {
+      flags[left] = ENG_BEG;
+      flags_color(flags, left+1, right, ENG_MID);
     }
 
-    start = sent.begin();
-    end   = sent.end();
+    start = what[0].second;
+  }
+
+  std::string form = "";
+  unsigned left  = 0;
 
-    while (boost::regex_search(start, end, what, engpattern, boost::match_default)) {
-        int left = what[0].first - sent.begin();
-        int right = what[0].second - sent.begin();
+  for (int i = 0; i < len; ) {
+    int flag = 0;
+    if ((flag = flags[i])) {
+      form = "";
 
-        if (flags_clear_check(flags, left, right)) {
-            flags[left] = ENG_BEG;
-            flags_color(flags, left+1, right, ENG_MID);
+      for (; i<len && flags[i]; ++ i) {
+        form += sent[i];
+      }
+      raw_forms.push_back(form);
+
+      if (flag == ENG_BEG) {
+        forms.push_back( __eng__ );
+        if (chartypes.size() > 0) {
+          chartypes.back() |= HAVE_ENG_ON_RIGHT;
         }
 
-        start = what[0].second;
-    }
+        chartypes.push_back(CHAR_ENG);
+        chartypes.back() |= left;
+        left = HAVE_ENG_ON_LEFT;
+      } else if (flag == URI_BEG) {
+        forms.push_back( __uri__ );
+        if (chartypes.size() > 0) {
+          chartypes.back() |= HAVE_URI_ON_RIGHT;
+        }
 
-    std::string form = "";
-    unsigned left  = 0;
-
-    for (int i = 0; i < len; ) {
-        int flag = 0;
-        if ((flag = flags[i])) {
-            form = "";
-
-            for (; i<len && flags[i]; ++ i) {
-                form += sent[i];
-            }
-            raw_forms.push_back(form);
-
-            if (flag == ENG_BEG) {
-                forms.push_back( __eng__ );
-                if (chartypes.size() > 0) {
-                    chartypes.back() |= HAVE_ENG_ON_RIGHT;
-                }
-
-                chartypes.push_back(CHAR_ENG);
-                chartypes.back() |= left;
-                left = HAVE_ENG_ON_LEFT;
-            } else if (flag == URI_BEG) {
-                forms.push_back( __uri__ );
-                if (chartypes.size() > 0) {
-                    chartypes.back() |= HAVE_URI_ON_RIGHT;
-                }
-
-                chartypes.push_back(CHAR_URI);
-                chartypes.back() |= left;
-                left = HAVE_URI_ON_LEFT;
-            }
-            ++ ret;
+        chartypes.push_back(CHAR_URI);
+        chartypes.back() |= left;
+        left = HAVE_URI_ON_LEFT;
+      }
+      ++ ret;
+    } else {
+      if ((sent[i]&0x80)==0) {
+        if ((sent[i] != ' ') && (sent[i] != '\t')) {
+          raw_forms.push_back(sent.substr(i, 1));
+          chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
+          forms.push_back("");
+          strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
+          chartypes.back() |= left;
+          left = 0;
         } else {
-            if ((sent[i]&0x80)==0) {
-                if ((sent[i] != ' ') && (sent[i] != '\t')) {
-                    raw_forms.push_back(sent.substr(i, 1));
-                    chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
-                    forms.push_back("");
-                    strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
-                    chartypes.back() |= left;
-                    left = 0;
-                } else {
-                    left = HAVE_SPACE_ON_LEFT;
-                    if (chartypes.size()>0) {
-                        chartypes.back() |= HAVE_SPACE_ON_RIGHT;
-                    }
-                }
-                ++ i;
-            } else if ((sent[i]&0xE0)==0xC0) {
-                raw_forms.push_back(sent.substr(i, 2));
-                chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
-                forms.push_back("");
-                strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
-                chartypes.back() |= left;
-                left = 0;
-                i += 2;
-            } else if ((sent[i]&0xF0)==0xE0) {
-                raw_forms.push_back(sent.substr(i, 3));
-                chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
-                forms.push_back("");
-                strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
-                chartypes.back() |= left;
-                i += 3;
-            } else if ((sent[i]&0xF8)==0xF0) {
-                raw_forms.push_back(sent.substr(i, 4));
-                chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
-                forms.push_back("");
-                strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
-                chartypes.back() |= left;
-                i += 4;
-            } else {
-                delete [](flags);
-                return -1;
-            }
-
-            ++ ret;
+          left = HAVE_SPACE_ON_LEFT;
+          if (chartypes.size()>0) {
+            chartypes.back() |= HAVE_SPACE_ON_RIGHT;
+          }
         }
+        ++ i;
+      } else if ((sent[i]&0xE0)==0xC0) {
+        raw_forms.push_back(sent.substr(i, 2));
+        chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
+        forms.push_back("");
+        strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
+        chartypes.back() |= left;
+        left = 0;
+        i += 2;
+      } else if ((sent[i]&0xF0)==0xE0) {
+        raw_forms.push_back(sent.substr(i, 3));
+        chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
+        forms.push_back("");
+        strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
+        chartypes.back() |= left;
+        i += 3;
+      } else if ((sent[i]&0xF8)==0xF0) {
+        raw_forms.push_back(sent.substr(i, 4));
+        chartypes.push_back(strutils::chartypes::chartype(raw_forms.back()));
+        forms.push_back("");
+        strutils::chartypes::sbc2dbc(raw_forms.back(), forms.back());
+        chartypes.back() |= left;
+        i += 4;
+      } else {
+        delete [](flags);
+        return -1;
+      }
+
+      ++ ret;
     }
+  }
 
-    delete [](flags);
-    return ret;
+  delete [](flags);
+  return ret;
 }
 
 class RuleBase {
 public:
-    RuleBase(utility::IndexableSmartMap & labels, int style = 4) {
-        // only 4 tag style is supported
-        if (style == 4) {
-            __trans__ = 0;
-            __s_idx__ = labels.index( __s__ );
-            __b_idx__ = labels.index( __b__ );
-            __i_idx__ = labels.index( __i__ );
-            __e_idx__ = labels.index( __e__ );
-
-            if (__s_idx__>=0 && __b_idx__>=0 && __i_idx__>=0 && __e_idx__>=0) {
-                __trans__ |= (1<<((__s_idx__<<2) + __s_idx__));
-                __trans__ |= (1<<((__s_idx__<<2) + __b_idx__));
-                __trans__ |= (1<<((__b_idx__<<2) + __i_idx__));
-                __trans__ |= (1<<((__b_idx__<<2) + __e_idx__));
-                __trans__ |= (1<<((__i_idx__<<2) + __i_idx__));
-                __trans__ |= (1<<((__i_idx__<<2) + __e_idx__));
-                __trans__ |= (1<<((__e_idx__<<2) + __s_idx__));
-                __trans__ |= (1<<((__e_idx__<<2) + __b_idx__));
-            } else {
-                __trans__ = 0xffff;
-            }
-        }
+  RuleBase(utility::IndexableSmartMap & labels, int style = 4) {
+    // only 4 tag style is supported
+    if (style == 4) {
+      __trans__ = 0;
+      __s_idx__ = labels.index( __s__ );
+      __b_idx__ = labels.index( __b__ );
+      __i_idx__ = labels.index( __i__ );
+      __e_idx__ = labels.index( __e__ );
+
+      if (__s_idx__>=0 && __b_idx__>=0 && __i_idx__>=0 && __e_idx__>=0) {
+        __trans__ |= (1<<((__s_idx__<<2) + __s_idx__));
+        __trans__ |= (1<<((__s_idx__<<2) + __b_idx__));
+        __trans__ |= (1<<((__b_idx__<<2) + __i_idx__));
+        __trans__ |= (1<<((__b_idx__<<2) + __e_idx__));
+        __trans__ |= (1<<((__i_idx__<<2) + __i_idx__));
+        __trans__ |= (1<<((__i_idx__<<2) + __e_idx__));
+        __trans__ |= (1<<((__e_idx__<<2) + __s_idx__));
+        __trans__ |= (1<<((__e_idx__<<2) + __b_idx__));
+      } else {
+        __trans__ = 0xffff;
+      }
     }
+  }
 
-    ~RuleBase() {
-    }
+  ~RuleBase() {
+  }
 
-    inline bool legal_trans(int prev, int curr) {
-        return (__trans__ & (1<<((prev<<2) + curr))) > 0;
-    }
+  inline bool legal_trans(int prev, int curr) {
+    return (__trans__ & (1<<((prev<<2) + curr))) > 0;
+  }
 
-    inline bool legal_emit(int type, int curr) {
-        if (((type & 0x07) == CHAR_ENG) || ((type & 0x07) == CHAR_URI)) {
-            return (curr == __s_idx__);
-        }
+  inline bool legal_emit(int type, int curr) {
+    if (((type & 0x07) == CHAR_ENG) || ((type & 0x07) == CHAR_URI)) {
+      return (curr == __s_idx__);
+    }
 
-        /*if ((type & HAVE_SPACE_ON_LEFT)) {
-            return (curr == __s_idx__ || curr == __b_idx__);
-        }
+    /*if ((type & HAVE_SPACE_ON_LEFT)) {
+      return (curr == __s_idx__ || curr == __b_idx__);
+    }
 
-        if ((type & HAVE_SPACE_ON_RIGHT)) {
-            return (curr == __s_idx__ || curr == __e_idx__);
-        }*/
+    if ((type & HAVE_SPACE_ON_RIGHT)) {
+      return (curr == __s_idx__ || curr == __e_idx__);
+    }*/
 
-        return true;
-    }
+    return true;
+  }
 private:
-    unsigned __trans__;
+  unsigned __trans__;
 
-    int __s_idx__;
-    int __b_idx__;
-    int __i_idx__;
-    int __e_idx__;
+  int __s_idx__;
+  int __b_idx__;
+  int __i_idx__;
+  int __e_idx__;
 };
 
-}       //  end for rulebase
-}       //  end for namespace segmentor
-}       //  end for namespace ltp 
+}     //  end for rulebase
+}     //  end for namespace segmentor
+}     //  end for namespace ltp 
 
 #endif  //  end for __LTP_SEGMENTOR_RULE_BASE_H__
diff --git a/src/segmentor/segmentor.cpp b/src/segmentor/segmentor.cpp
index 58f55466c..92446c2a3 100644
--- a/src/segmentor/segmentor.cpp
+++ b/src/segmentor/segmentor.cpp
@@ -11,14 +11,14 @@
 
 #include <iostream>
 #include <fstream>
-#include <strstream>
+#include <sstream>
 #include <algorithm>    //  std::sort
 #include <functional>   //  std::greater
 
 #if _WIN32
 #include <Windows.h>
 #define sleep Sleep
-#endif	//	end for _WIN32
+#endif  //  end for _WIN32
 
 namespace ltp {
 namespace segmentor {
@@ -30,914 +30,864 @@ Segmentor::Segmentor() :
 }
 
 Segmentor::Segmentor(ltp::utility::ConfigParser & cfg) :
-    model(0),
-    decoder(0),
-    baseAll(0) {
-    parse_cfg(cfg);
+  model(0),
+  decoder(0),
+  baseAll(0) {
+  parse_cfg(cfg);
 }
 
 Segmentor::~Segmentor() {
-    if (model) {
-        delete model;
-    }
+  if (model) {
+    delete model;
+  }
 
-    if (decoder) {
-        delete decoder;
-    }
+  if (decoder) {
+    delete decoder;
+  }
 
-    if(baseAll) {
-        delete baseAll;
-    }
+  if(baseAll) {
+    delete baseAll;
+  }
 }
 
-void Segmentor::run(void) {
-    if (__TRAIN__) {
-        train();
-    }
+void
+Segmentor::run(void) {
+  if (__TRAIN__) {
+    train();
+  }
 
-    if (__TEST__) {
-        test();
-    }
+  if (__TEST__) {
+    test();
+  }
 
-    if (__DUMP__) {
-        dump();
-    }
+  if (__DUMP__) {
+    dump();
+  }
 
-    for (int i = 0; i < train_dat.size(); ++ i) {
-        if (train_dat[i]) {
-            delete train_dat[i];
-        }
+  for (int i = 0; i < train_dat.size(); ++ i) {
+    if (train_dat[i]) {
+      delete train_dat[i];
     }
+  }
 }
 
-bool Segmentor::parse_cfg(ltp::utility::ConfigParser & cfg) {
-    std::string strbuf;
-    int         intbuf;
-
-    __TRAIN__ = false;
-
-    train_opt.train_file    = "";
-    train_opt.holdout_file  = "";
-    train_opt.algorithm     = "pa";
-    train_opt.model_name    = "";
-    train_opt.max_iter      = 10;
-    train_opt.display_interval = 5000;
-    train_opt.min_update = 10;
-    train_opt.use_update = "false";
-
-    if (cfg.has_section("train")) {
-        TRACE_LOG("Training mode specified.");
-        __TRAIN__ = true;
-
-        if (cfg.get("train", "train-file", strbuf)) {
-            train_opt.train_file = strbuf;
-        } else {
-            ERROR_LOG("train-file config item is not found.");
-            return false;
-        }
+bool
+Segmentor::parse_cfg(ltp::utility::ConfigParser & cfg) {
+  std::string strbuf;
+  int         intbuf;
 
-        if (cfg.get("train", "holdout-file", strbuf)) {
-            train_opt.holdout_file = strbuf;
-        } else {
-            ERROR_LOG("holdout-file config item is not found.");
-            return false;
-        }
+  __TRAIN__ = false;
 
-        if (cfg.get("train", "algorithm", strbuf)) {
-            train_opt.algorithm = strbuf;
-        } else {
-            WARNING_LOG("algorithm is not configed, [PA] is set as default");
-        }
+  train_opt.train_file              = "";
+  train_opt.holdout_file            = "";
+  train_opt.algorithm               = "pa";
+  train_opt.model_name              = "";
+  train_opt.max_iter                = 10;
+  train_opt.display_interval        = 5000;
+  train_opt.rare_feature_threshold  = 0;
 
-        if (cfg.get("train", "min_update", strbuf)) {
-            train_opt.min_update = atoi(strbuf.c_str());
-	    std::cout<<"min_update:"<<train_opt.min_update<<std::endl;
-        } else {
-            WARNING_LOG("min_update is not configed, 10 is set as default");
-        }
-        if (cfg.get("train", "use_update", strbuf)) {
-            train_opt.use_update = strbuf;
-	    std::cout<<"use_update:"<<train_opt.use_update<<std::endl;
-        } else {
-            WARNING_LOG("use_update is not configed, false is set as default");
-        }
+  if (cfg.has_section("train")) {
+    TRACE_LOG("Training mode specified.");
+    __TRAIN__ = true;
 
-        train_opt.model_name = train_opt.train_file + "." + train_opt.algorithm;
-        if (cfg.get("train", "model-name", strbuf)) {
-            train_opt.model_name = strbuf;
-        } else {
-            WARNING_LOG("model name is not configed, [%s] is set as default",
-                    train_opt.model_name.c_str());
-        }
+    if (cfg.get("train", "train-file", strbuf)) {
+      train_opt.train_file = strbuf;
+    } else {
+      ERROR_LOG("train-file config item is not found.");
+      return false;
+    }
 
-        if (cfg.get_integer("train", "max-iter", intbuf)) {
-            train_opt.max_iter = intbuf;
-        } else {
-            WARNING_LOG("max-iter is not configed, [10] is set as default.");
-        }
+    if (cfg.get("train", "holdout-file", strbuf)) {
+      train_opt.holdout_file = strbuf;
+    } else {
+      ERROR_LOG("holdout-file config item is not found.");
+      return false;
+    }
+
+    if (cfg.get("train", "algorithm", strbuf)) {
+      train_opt.algorithm = strbuf;
+    } else {
+      WARNING_LOG("algorithm is not configed, [PA] is set as default");
     }
 
-    __TEST__ = false;
+    if (cfg.get("train", "rare-feature-threshold", strbuf)) {
+      train_opt.rare_feature_threshold = atoi(strbuf.c_str());
+    } else {
+      WARNING_LOG("rare feature threshold is not configed, 10 is set as default");
+    }
 
-    test_opt.test_file = "";
-    test_opt.model_file = "";
-    test_opt.lexicon_file = "";
+    train_opt.model_name = train_opt.train_file + "." + train_opt.algorithm;
+    if (cfg.get("train", "model-name", strbuf)) {
+      train_opt.model_name = strbuf;
+    } else {
+      WARNING_LOG("model name is not configed, [%s] is set as default",
+                  train_opt.model_name.c_str());
+    }
 
-    if (cfg.has_section("test")) {
-        __TEST__ = true;
+    if (cfg.get_integer("train", "max-iter", intbuf)) {
+      train_opt.max_iter = intbuf;
+    } else {
+      WARNING_LOG("max-iter is not configed, [10] is set as default.");
+    }
+  }
 
-        if (cfg.get("test", "test-file", strbuf)) {
-            test_opt.test_file = strbuf;
-        } else {
-            ERROR_LOG("test-file config item is not set.");
-            return false;
-        }
+  __TEST__ = false;
 
-        if (cfg.get("test", "model-file", strbuf)) {
-            test_opt.model_file = strbuf;
-        } else {
-            ERROR_LOG("model-file is not configed. ");
-            return false;
-        }
+  test_opt.test_file    = "";
+  test_opt.model_file   = "";
+  test_opt.lexicon_file = "";
 
-        if (cfg.get("test", "lexicon-file", strbuf)) {
-            test_opt.lexicon_file = strbuf;
-        }
+  if (cfg.has_section("test")) {
+    __TEST__ = true;
+
+    if (cfg.get("test", "test-file", strbuf)) {
+      test_opt.test_file = strbuf;
+    } else {
+      ERROR_LOG("test-file config item is not set.");
+      return false;
     }
 
-    __DUMP__ = false;
+    if (cfg.get("test", "model-file", strbuf)) {
+      test_opt.model_file = strbuf;
+    } else {
+      ERROR_LOG("model-file is not configed.");
+      return false;
+    }
+
+    if (cfg.get("test", "lexicon-file", strbuf)) {
+      test_opt.lexicon_file = strbuf;
+    }
+  }
 
-    dump_opt.model_file = "";
-    if (cfg.has_section("dump")) {
-        __DUMP__ = true;
+  __DUMP__ = false;
 
-        if (cfg.get("dump", "model-file", strbuf)) {
-            dump_opt.model_file = strbuf;
-        } else {
-            ERROR_LOG("model-file is not configed.");
-            return false;
-        }
+  dump_opt.model_file = "";
+  if (cfg.has_section("dump")) {
+    __DUMP__ = true;
+
+    if (cfg.get("dump", "model-file", strbuf)) {
+      dump_opt.model_file = strbuf;
+    } else {
+      ERROR_LOG("model-file is not configed.");
+      return false;
     }
+  }
 
-    return true;
+  return true;
 }
 
-bool Segmentor::read_instance(const char * train_file) {
-    std::ifstream ifs(train_file);
+bool
+Segmentor::read_instance(const char * train_file) {
+  std::ifstream ifs(train_file);
 
-    if (!ifs) {
-        return false;
-    }
+  if (!ifs) {
+    return false;
+  }
 
-    SegmentReader reader(ifs, true);
-    train_dat.clear();
+  SegmentReader reader(ifs, true);
+  train_dat.clear();
 
-    Instance * inst = NULL;
+  Instance * inst = NULL;
 
-    while ((inst = reader.next())) {
-        train_dat.push_back(inst);
+  while ((inst = reader.next())) {
+    train_dat.push_back(inst);
+  }
+
+  return true;
+}
+
+void
+Segmentor::build_configuration(void) {
+  // model->labels.push( __dummy__ );
+  for (int i = 0; i < train_dat.size(); ++ i) {
+    Instance * inst = train_dat[i];
+    int len = inst->size();
+
+    inst->tagsidx.resize(len);
+    for (int j = 0; j < len; ++ j) {
+      // build labels dictionary
+      inst->tagsidx[j] = model->labels.push( inst->tags[j] );
     }
 
-    return true;
+  }
+  TRACE_LOG("Label sets is built");
+
+  SmartMap<bool> wordfreq;
+  long long total_freq = 0;
+  for (int i = 0; i < train_dat.size(); ++ i) {
+    //
+    Instance * inst = train_dat[i];
+    int len = inst->size();
+    int buff = 0;
+
+    for (int j = 0; j < inst->words.size(); ++ j) {
+      wordfreq.set(inst->words[j].c_str(), true);
+    }
+    total_freq += inst->words.size();
+  }
+
+  std::vector<int> freqs;
+  for (SmartMap<bool>::const_iterator itx = wordfreq.begin();
+      itx != wordfreq.end();
+      ++ itx) {
+    freqs.push_back(itx.frequency());
+  }
+
+  long long accumulate_freq = 0;
+  std::sort(freqs.begin(), freqs.end(), std::greater<int>());
+  int target = freqs[int(freqs.size() * 0.2)];
+  for (int i = 0; i < freqs.size(); ++ i) {
+    accumulate_freq += freqs[i];
+    if (accumulate_freq > total_freq * 0.9) {
+      target = freqs[i];
+      break;
+    }
+  }
+
+  for (SmartMap<bool>::const_iterator itx = wordfreq.begin();
+      itx != wordfreq.end();
+      ++ itx) {
+    if (itx.frequency() >= target && strutils::codecs::length(itx.key()) > 1) {
+    // if (itx.frequency() >= target) {
+      model->internal_lexicon.set(itx.key(), true);
+    }
+  }
+
+  TRACE_LOG("Collecting interanl lexicon is done.");
+  TRACE_LOG("Total word frequency : %ld", total_freq);
+  TRACE_LOG("Vocabulary size: %d", wordfreq.size());
+  TRACE_LOG("Trancation word frequency : %d", target);
+  TRACE_LOG("Internal lexicon size : %d", model->internal_lexicon.size());
 }
 
+void
+Segmentor::extract_features(Instance * inst, bool create) {
+  const int N = Extractor::num_templates();
+  const int L = model->num_labels();
 
-void Segmentor::build_configuration(void) {
-    // model->labels.push( __dummy__ );
-    for (int i = 0; i < train_dat.size(); ++ i) {
-        Instance * inst = train_dat[i];
-        int len = inst->size();
+  vector< StringVec > cache;
+  vector< int > cache_again;
 
-        inst->tagsidx.resize(len);
-        for (int j = 0; j < len; ++ j) {
-            // build labels dictionary
-            inst->tagsidx[j] = model->labels.push( inst->tags[j] );
-        }
+  cache.resize(N);
+  int len = inst->size();
 
-    }
-    TRACE_LOG("Label sets is built");
+  // allocate the uni_features
+  inst->uni_features.resize(len, L);  inst->uni_features = 0;
+  inst->uni_scores.resize(len, L);  inst->uni_scores = NEG_INF;
+  inst->bi_scores.resize(L, L);     inst->bi_scores = NEG_INF;
 
-    SmartMap<bool> wordfreq;
-    long long total_freq = 0;
-    for (int i = 0; i < train_dat.size(); ++ i) {
-        //
-        Instance * inst = train_dat[i];
-        int len = inst->size();
-        int buff = 0;
+  // cache lexicon features.
+  if (0 == inst->lexicon_match_state.size()) {
+    inst->lexicon_match_state.resize(len, 0);
 
-        for (int j = 0; j < inst->words.size(); ++ j) {
-            wordfreq.set(inst->words[j].c_str(), true);
+    for (int i = 0; i < len; ++ i) {
+      std::string word; word.reserve(32);
+      for (int j = i; j<i+5 && j < len; ++ j) {
+        word = word + inst->forms[j];
+
+        // it's not a lexicon word
+        if (!model->internal_lexicon.get(word.c_str())
+            && !model->external_lexicon.get(word.c_str())) {
+          continue;
         }
-        total_freq += inst->words.size();
-    }
 
-    std::vector<int> freqs;
-    for (SmartMap<bool>::const_iterator itx = wordfreq.begin(); 
-            itx != wordfreq.end(); 
-            ++ itx) {
-        freqs.push_back(itx.frequency());
-    }
+        int l = j+1-i;
 
-    long long accumulate_freq = 0;
-    std::sort(freqs.begin(), freqs.end(), std::greater<int>());
-    int target = freqs[int(freqs.size() * 0.2)];
-    for (int i = 0; i < freqs.size(); ++ i) {
-        accumulate_freq += freqs[i];
-        if (accumulate_freq > total_freq * 0.9) {
-            target = freqs[i];
-            break;
+        if (l > (inst->lexicon_match_state[i] & 0x0F)) {
+          inst->lexicon_match_state[i] &= 0xfff0;
+          inst->lexicon_match_state[i] |= l;
+        }
+
+        if (l > ((inst->lexicon_match_state[j]>>4) & 0x0F)) {
+          inst->lexicon_match_state[j] &= 0xff0f;
+          inst->lexicon_match_state[j] |= (l<<4);
         }
-    }
 
-    for (SmartMap<bool>::const_iterator itx = wordfreq.begin();
-            itx != wordfreq.end();
-            ++ itx) {
-        if (itx.frequency() >= target && strutils::codecs::length(itx.key()) > 1) {
-        // if (itx.frequency() >= target) {
-            model->internal_lexicon.set(itx.key(), true);
+        for (int k = i+1; k < j; ++k) {
+          if (l>((inst->lexicon_match_state[k]>>8) & 0x0F)) {
+            inst->lexicon_match_state[k] &= 0xf0ff;
+            inst->lexicon_match_state[k] |= (l<<8);
+          }
         }
+      }
     }
+  }
 
-    TRACE_LOG("Collecting interanl lexicon is done.");
-    TRACE_LOG("Total word frequency : %ld", total_freq);
-    TRACE_LOG("Vocabulary size: %d", wordfreq.size());
-    TRACE_LOG("Trancation word frequency : %d", target);
-    TRACE_LOG("Internal lexicon size : %d", model->internal_lexicon.size());
-}
+  for (int pos = 0; pos < len; ++ pos) {
+    for (int n = 0; n < N; ++ n) {
+      cache[n].clear();
+    }
+    cache_again.clear();
 
-void Segmentor::extract_features(Instance * inst, bool create) {
-    const int N = Extractor::num_templates();
-    const int L = model->num_labels();
+    Extractor::extract1o(inst, pos, cache);
 
-    vector< StringVec > cache;
-    vector< int > cache_again;
+    for (int tid = 0; tid < cache.size(); ++ tid) {
+      for (int itx = 0; itx < cache[tid].size(); ++ itx) {
+        if (create) {
+          model->space.retrieve(tid, cache[tid][itx], true);
+        }
 
-    cache.resize(N);
-    int len = inst->size();
+        int idx = model->space.index(tid, cache[tid][itx]);
 
-    // allocate the uni_features
-    inst->uni_features.resize(len, L);  inst->uni_features = 0;
-    inst->uni_scores.resize(len, L);    inst->uni_scores = NEG_INF;
-    inst->bi_scores.resize(L, L);       inst->bi_scores = NEG_INF;
-
-    // cache lexicon features.
-    if (0 == inst->lexicon_match_state.size()) {
-        inst->lexicon_match_state.resize(len, 0);
-
-        for (int i = 0; i < len; ++ i) {
-            std::string word; word.reserve(32);
-            for (int j = i; j<i+5 && j < len; ++ j) {
-                word = word + inst->forms[j];
-
-                // it's not a lexicon word
-                if (!model->internal_lexicon.get(word.c_str()) 
-                        && !model->external_lexicon.get(word.c_str())) {
-                    continue;
-                }
-
-                int l = j+1-i;
-
-                if (l > (inst->lexicon_match_state[i] & 0x0F)) {
-                    inst->lexicon_match_state[i] &= 0xfff0;
-                    inst->lexicon_match_state[i] |= l;
-                }
-
-                if (l > ((inst->lexicon_match_state[j]>>4) & 0x0F)) {
-                    inst->lexicon_match_state[j] &= 0xff0f;
-                    inst->lexicon_match_state[j] |= (l<<4);
-                }
-
-                for (int k = i+1; k < j; ++k) {
-                    if (l>((inst->lexicon_match_state[k]>>8) & 0x0F)) {
-                        inst->lexicon_match_state[k] &= 0xf0ff;
-                        inst->lexicon_match_state[k] |= (l<<8);
-                    }
-                }
-            }
+        if (idx >= 0) {
+          cache_again.push_back(idx);
         }
+      }
     }
 
-    for (int pos = 0; pos < len; ++ pos) {
-        for (int n = 0; n < N; ++ n) {
-            cache[n].clear();
-        }
-        cache_again.clear();
-
-        Extractor::extract1o(inst, pos, cache);
+    int num_feat = cache_again.size();
 
-        for (int tid = 0; tid < cache.size(); ++ tid) {
-            for (int itx = 0; itx < cache[tid].size(); ++ itx) {
-                if (create) {
-                    model->space.retrieve(tid, cache[tid][itx], true);
-                }
+    if (num_feat > 0) {
+      int l = 0;
+      int * idx = new int[num_feat];
+      for (int j = 0; j < num_feat; ++ j) {
+        idx[j] = cache_again[j];
+      }
 
-                int idx = model->space.index(tid, cache[tid][itx]);
-		//cout<<"id should be"<<model->space.retrieve(tid,cache[tid][itx],false)<<endl;
-		//cout<<"id_label_to_id is "<<model->space.id_label_to_id(idx)<<endl;
+      inst->uni_features[pos][l] = new FeatureVector;
+      inst->uni_features[pos][l]->n = num_feat;
+      inst->uni_features[pos][l]->val = 0;
+      inst->uni_features[pos][l]->loff = 0;
+      inst->uni_features[pos][l]->idx = idx;
 
-                if (idx >= 0) {
-                    cache_again.push_back(idx);
-                }
-            }
-        }
-
-        int num_feat = cache_again.size();
-
-        if (num_feat > 0) {
-            int l = 0;
-            int * idx = new int[num_feat];
-            for (int j = 0; j < num_feat; ++ j) {
-                idx[j] = cache_again[j];
-            }
-
-            inst->uni_features[pos][l] = new FeatureVector;
-            inst->uni_features[pos][l]->n = num_feat;
-            inst->uni_features[pos][l]->val = 0;
-            inst->uni_features[pos][l]->loff = 0;
-            inst->uni_features[pos][l]->idx = idx;
-
-            for (l = 1; l < L; ++ l) {
-                inst->uni_features[pos][l] = new FeatureVector;
-                inst->uni_features[pos][l]->n = num_feat;
-                inst->uni_features[pos][l]->idx = idx;
-                inst->uni_features[pos][l]->val = 0;
-                inst->uni_features[pos][l]->loff = l;
-            }
-        }
+      for (l = 1; l < L; ++ l) {
+        inst->uni_features[pos][l] = new FeatureVector;
+        inst->uni_features[pos][l]->n = num_feat;
+        inst->uni_features[pos][l]->idx = idx;
+        inst->uni_features[pos][l]->val = 0;
+        inst->uni_features[pos][l]->loff = l;
+      }
     }
+  }
 }
 
-void Segmentor::build_words(Instance * inst,
-        const std::vector<int> & tagsidx,
-        std::vector<std::string> & words,
-        int beg_tag0,
-        int beg_tag1) {
-    std::string word = "";
-    int len = inst->size();
-
-    // should check the tagsidx size
-    word = inst->raw_forms[0];
-    for (int i = 1; i < len; ++ i) {
-        int tag = tagsidx[i];
-        if (tag == beg_tag0 || tag == beg_tag1) {
-            words.push_back(word);
-            word = inst->raw_forms[i];
-        } else {
-            word += inst->raw_forms[i];
-        }
+void
+Segmentor::build_words(Instance * inst,
+                       const std::vector<int> & tagsidx,
+                       std::vector<std::string> & words,
+                       int beg_tag0,
+                       int beg_tag1) {
+
+  std::string word = "";
+  int len = inst->size();
+
+  // should check the tagsidx size
+  word = inst->raw_forms[0];
+  for (int i = 1; i < len; ++ i) {
+    int tag = tagsidx[i];
+    if (tag == beg_tag0 || tag == beg_tag1) {
+      words.push_back(word);
+      word = inst->raw_forms[i];
+    } else {
+      word += inst->raw_forms[i];
     }
+  }
 
-    words.push_back(word);
+  words.push_back(word);
 }
 
-void Segmentor::build_feature_space(void) {
-    // build feature space, it a wrapper for
-    // featurespace.build_feature_space
-    int N = Extractor::num_templates();
-    int L = model->num_labels();
-    model->space.set_num_labels(L);
-
-    for (int i = 0; i < train_dat.size(); ++ i) {
-        extract_features(train_dat[i], true);
-        if ((i + 1) % train_opt.display_interval == 0) {
-            TRACE_LOG("[%d] instances is extracted.", (i+1));
-        }
+void
+Segmentor::build_feature_space(void) {
+  // build feature space, it a wrapper for
+  // featurespace.build_feature_space
+  int N = Extractor::num_templates();
+  int L = model->num_labels();
+  model->space.set_num_labels(L);
+
+  for (int i = 0; i < train_dat.size(); ++ i) {
+    extract_features(train_dat[i], true);
+    if ((i + 1) % train_opt.display_interval == 0) {
+      TRACE_LOG("[%d] instances is extracted.", (i+1));
     }
+  }
 }
 
-void Segmentor::calculate_scores(Instance * inst, bool use_avg) {
-    int len = inst->size();
-    int L = model->num_labels();
-    for (int i = 0; i < len; ++ i) {
-        for (int l = 0; l < L; ++ l) {
-            FeatureVector * fv = inst->uni_features[i][l];
-            if (!fv) {
-                continue;
-            }
+void
+Segmentor::calculate_scores(Instance * inst, bool use_avg) {
+  int len = inst->size();
+  int L = model->num_labels();
+  for (int i = 0; i < len; ++ i) {
+    for (int l = 0; l < L; ++ l) {
+      FeatureVector * fv = inst->uni_features[i][l];
+      if (!fv) {
+        continue;
+      }
 
-            inst->uni_scores[i][l] = model->param.dot(inst->uni_features[i][l], use_avg);
-        }
+      inst->uni_scores[i][l] = model->param.dot(inst->uni_features[i][l], use_avg);
     }
+  }
 
-    for (int pl = 0; pl < L; ++ pl) {
-        for (int l = 0; l < L; ++ l) {
-            int idx = model->space.index(pl, l);
-            inst->bi_scores[pl][l] = model->param.dot(idx, use_avg);
-        }
+  for (int pl = 0; pl < L; ++ pl) {
+    for (int l = 0; l < L; ++ l) {
+      int idx = model->space.index(pl, l);
+      inst->bi_scores[pl][l] = model->param.dot(idx, use_avg);
     }
+  }
 }
 
-void Segmentor::collect_features(Instance * inst, const std::vector<int> & tagsidx, math::SparseVec & vec) {
-    int len = inst->size();
+void
+Segmentor::collect_features(Instance * inst,
+                            const std::vector<int> & tagsidx,
+                            math::SparseVec & vec) {
+  int len = inst->size();
 
-    vec.zero();
-    for (int i = 0; i < len; ++ i) {
-        int l = tagsidx[i];
-        const FeatureVector * fv = inst->uni_features[i][l];
+  vec.zero();
+  for (int i = 0; i < len; ++ i) {
+    int l = tagsidx[i];
+    const FeatureVector * fv = inst->uni_features[i][l];
 
-        if (!fv) {
-            continue;
-        }
+    if (!fv) {
+      continue;
+    }
 
-        vec.add(fv->idx, fv->val, fv->n, fv->loff, 1.);
+    vec.add(fv->idx, fv->val, fv->n, fv->loff, 1.);
 
-        if (i > 0) {
-            int prev_lid = tagsidx[i-1];
-            int idx = model->space.index(prev_lid, l);
-            vec.add(idx, 1.);
-        }
+    if (i > 0) {
+      int prev_lid = tagsidx[i-1];
+      int idx = model->space.index(prev_lid, l);
+      vec.add(idx, 1.);
     }
+  }
 }
 
-Model * Segmentor::truncate(void) {
-    Model * new_model = new Model;
-    // copy the label indexable map to the new model
-    for (int i = 0; i < model->labels.size(); ++ i) {
-        const char * key = model->labels.at(i);
-        new_model->labels.push(key);
-    }
-
-    TRACE_LOG("building labels map is done");
-
-    int L = new_model->num_labels();
-    new_model->space.set_num_labels(L);
-
-    // iterate over the feature space and see if the parameter value equals to zero
-    for (FeatureSpaceIterator itx = model->space.begin(); 
-            itx != model->space.end(); 
-            ++ itx) {
-        const char * key = itx.key();
-        int tid = itx.tid();
-        int id = model->space.index(tid, key);
-
-        bool flag = false;
-        for (int l = 0; l < L; ++ l) {
-            double p = model->param.dot(id + l);
-            if (p != 0.) {
-                flag = true;
-            }
-        }
+// Perform model truncation on the model, according to these two conditions
+//  (1) Erase the group of parameters that it all the parameter in this group
+//      is equals to zero.
+//  (2) (optional) Erase the group of parameters the total updated time is
+//      lower than the pre-defined threshold.
+Model *
+Segmentor::erase_rare_features(const int * feature_updated_times) {
+  Model * new_model = new Model;
+  // copy the label indexable map to the new model
+  for (int i = 0; i < model->labels.size(); ++ i) {
+    const char * key = model->labels.at(i);
+    new_model->labels.push(key);
+  }
+
+  TRACE_LOG("building labels map is done");
+
+  int L = new_model->num_labels();
+  new_model->space.set_num_labels(L);
+
+  // Iterate over the feature space
+  for (FeatureSpaceIterator itx = model->space.begin();
+      itx != model->space.end();
+      ++ itx) {
+    const char * key = itx.key();
+    int tid = itx.tid();
+    int id  = model->space.index(tid, key);
+
+    // Enumerate each feature with the same feature prefix, check if it's zero.
+    bool flag = false;
+    for (int l = 0; l < L; ++ l) {
+      double p = model->param.dot(id + l);
+      if (p != 0.) {
+        flag = true;
+      }
+    }
+
+    if (!flag) {
+      continue;
+    }
+
+    // Check if this feature's updated time.
+    int idx = model->space.retrieve(tid, key, false);
+    if (feature_updated_times
+        && (feature_updated_times[idx] < train_opt.rare_feature_threshold)) {
+      continue;
+    }
+
+    new_model->space.retrieve(tid, key, true);
+  }
+
+  TRACE_LOG("Scanning old features space, building new feature space is done");
+  new_model->param.realloc(new_model->space.dim());
+  TRACE_LOG("Parameter dimension of new model is [%d]", new_model->space.dim());
+
+  for (FeatureSpaceIterator itx = new_model->space.begin();
+      itx != new_model->space.end();
+      ++ itx) {
+    const char * key = itx.key();
+    int tid = itx.tid();
+
+    int old_id = model->space.index(tid, key);
+    int new_id = new_model->space.index(tid, key);
+
+    for (int l = 0; l < L; ++ l) {
+      // pay attention to this place, use average should be set true
+      // some dirty code
+      new_model->param._W[new_id + l]      = model->param._W[old_id + l];
+      new_model->param._W_sum[new_id + l]  = model->param._W_sum[old_id + l];
+      new_model->param._W_time[new_id + l] = model->param._W_time[old_id + l];
+    }
+  }
+
+  for (int pl = 0; pl < L; ++ pl) {
+    for (int l = 0; l < L; ++ l) {
+      int old_id = model->space.index(pl, l);
+      int new_id = new_model->space.index(pl, l);
+
+      new_model->param._W[new_id]      = model->param._W[old_id];
+      new_model->param._W_sum[new_id]  = model->param._W_sum[old_id];
+      new_model->param._W_time[new_id] = model->param._W_time[old_id];
+    }
+  }
+  TRACE_LOG("Building new model is done");
+
+  for (SmartMap<bool>::const_iterator itx = model->internal_lexicon.begin();
+      itx != model->internal_lexicon.end();
+      ++ itx) {
+    new_model->internal_lexicon.set(itx.key(), true);
+  }
+
+  return new_model;
+}
 
-        if (!flag) {
-            continue;
-        }
+void
+Segmentor::train(void) {
+  const char * train_file = train_opt.train_file.c_str();
 
-        new_model->space.retrieve(tid, key, true);
-    }
-    TRACE_LOG("Scanning old features space, building new feature space is done");
+  // read in training instance
+  if (!read_instance(train_file)) {
+    ERROR_LOG("Training file not exist.");
+    return;
+  }
+  TRACE_LOG("Read in [%d] instances.", train_dat.size());
+
+  model = new Model;
+  // build tag dictionary, map string tag to index
+  TRACE_LOG("Start build configuration");
+  build_configuration();
+  TRACE_LOG("Build configuration is done.");
+  TRACE_LOG("Number of labels: [%d]", model->labels.size());
+
+  // build feature space from the training instance
+  TRACE_LOG("Start building feature space.");
+  build_feature_space();
+  TRACE_LOG("Building feature space is done.");
+  TRACE_LOG("Number of features: [%d]", model->space.num_features());
+
+  model->param.realloc(model->space.dim());
+  TRACE_LOG("Allocate [%d] dimensition parameter.", model->space.dim());
+
+  int nr_feature_groups = model->space.get_offset();
+  int * feature_group_updated_time = NULL;
+
+  // If the rare feature threshold is used, allocate memory for the
+  // feature group updated time.
+  if (train_opt.rare_feature_threshold > 0) {
+    feature_group_updated_time = new int[nr_feature_groups];
+    for (int i = 0; i < nr_feature_groups; ++ i) {
+      feature_group_updated_time[i] = 0;
+    }
+  }
+
+  TRACE_LOG("Allocate [%d] update counters.", nr_feature_groups);
+
+  SegmentWriter writer(cout);
+
+  if (train_opt.algorithm == "mira") {
+    // use mira to train model
+    // it's still not implemented.
+  } else {
+    // use pa or average perceptron algorithm
+    rulebase::RuleBase base(model->labels);
+    decoder = new Decoder(model->num_labels(), base);
+    TRACE_LOG("Allocated plain decoder");
 
-    new_model->param.realloc(new_model->space.dim());
-    TRACE_LOG("Parameter dimension of new model is [%d]", new_model->space.dim());
+    int best_iteration = -1;
+    double best_p = -1.;
+    double best_r = -1.;
+    double best_f = -1.;
 
-    for (FeatureSpaceIterator itx = new_model->space.begin();
-            itx != new_model->space.end();
-            ++ itx) {
-        const char * key = itx.key();
-        int tid = itx.tid();
+    for (int iter = 0; iter < train_opt.max_iter; ++ iter) {
+      TRACE_LOG("Training iteraition [%d]", (iter + 1));
 
-        int old_id = model->space.index(tid, key);
-        int new_id = new_model->space.index(tid, key);
+      for (int i = 0; i < train_dat.size(); ++ i) {
+        // extract_features(train_dat[i]);
+
+        Instance * inst = train_dat[i];
+        calculate_scores(inst, false);
+        decoder->decode(inst);
 
-        for (int l = 0; l < L; ++ l) {
-            // pay attention to this place, use average should be set true
-            // some dirty code
-            new_model->param._W[new_id + l]         = model->param._W[old_id + l];
-            new_model->param._W_sum[new_id + l]     = model->param._W_sum[old_id + l];
-            new_model->param._W_time[new_id + l]    = model->param._W_time[old_id + l];
+        if (inst->features.dim() == 0) {
+          collect_features(inst, inst->tagsidx, inst->features);
         }
-    }
+        collect_features(inst, inst->predicted_tagsidx, inst->predicted_features);
 
-    for (int pl = 0; pl < L; ++ pl) {
-        for (int l = 0; l < L; ++ l) {
-            int old_id = model->space.index(pl, l);
-            int new_id = new_model->space.index(pl, l);
+        if (train_opt.algorithm == "pa") {
+          SparseVec update_features;
+          update_features.zero();
+          update_features.add(train_dat[i]->features, 1.);
+          update_features.add(train_dat[i]->predicted_features, -1.);
+          update_features.update_counter(feature_group_updated_time,
+                                         nr_feature_groups,
+                                         model->num_labels());
 
-            new_model->param._W[new_id]         = model->param._W[old_id];
-            new_model->param._W_sum[new_id]     = model->param._W_sum[old_id];
-            new_model->param._W_time[new_id]    = model->param._W_time[old_id];
-        }
-    }
-    TRACE_LOG("Building new model is done");
+          double error = train_dat[i]->num_errors();
+          double score = model->param.dot(update_features, false);
+          double norm  = update_features.L2();
 
-    for (SmartMap<bool>::const_iterator itx = model->internal_lexicon.begin();
-            itx != model->internal_lexicon.end();
-            ++ itx) {
-        new_model->internal_lexicon.set(itx.key(), true);
-    }
+          double step = 0.;
+          if (norm < EPS) {
+            step = 0;
+          } else {
+            step = (error - score) / norm;
+          }
 
-    return new_model;
-}
+          model->param.add(update_features,
+                           iter * train_dat.size() + i + 1,
+                           step);
 
-Model * Segmentor::truncate_prune(int * updates) {
-    Model * new_model = new Model;
-    // copy the label indexable map to the new model
-    for (int i = 0; i < model->labels.size(); ++ i) {
-        const char * key = model->labels.at(i);
-        new_model->labels.push(key);
-    }
-
-    TRACE_LOG("building labels map is done");
-
-    int L = new_model->num_labels();
-    new_model->space.set_num_labels(L);
-
-    // iterate over the feature space and see if the parameter value equals to zero
-    for (FeatureSpaceIterator itx = model->space.begin(); 
-            itx != model->space.end(); 
-            ++ itx) {
-        const char * key = itx.key();
-        int tid = itx.tid();
-        int id = model->space.index(tid, key);
-
-        bool flag = false;
-        for (int l = 0; l < L; ++ l) {
-            double p = model->param.dot(id + l);
-            if (p != 0.) {
-                flag = true;
-            }
-        }
+        } else if (train_opt.algorithm == "ap") {
+          SparseVec update_features;
+          update_features.zero();
+          update_features.add(train_dat[i]->features, 1.);
+          update_features.add(train_dat[i]->predicted_features, -1.);
 
-        if (!flag) {
-            continue;
+
+          model->param.add(update_features,
+                           iter * train_dat.size() + i + 1,
+                           1.);
         }
-	int idx=model->space.retrieve(tid,key,false);
-	if(updates[idx]<train_opt.min_update) {
-	    continue;
-	}
-        new_model->space.retrieve(tid, key, true);
-    }
-    TRACE_LOG("Scanning old features space, building new feature space is done");
-
-    new_model->param.realloc(new_model->space.dim());
-    TRACE_LOG("Parameter dimension of new model is [%d]", new_model->space.dim());
-
-    for (FeatureSpaceIterator itx = new_model->space.begin();
-            itx != new_model->space.end();
-            ++ itx) {
-        const char * key = itx.key();
-        int tid = itx.tid();
-
-        int old_id = model->space.index(tid, key);
-        int new_id = new_model->space.index(tid, key);
-
-        for (int l = 0; l < L; ++ l) {
-            // pay attention to this place, use average should be set true
-            // some dirty code
-            new_model->param._W[new_id + l]         = model->param._W[old_id + l];
-            new_model->param._W_sum[new_id + l]     = model->param._W_sum[old_id + l];
-            new_model->param._W_time[new_id + l]    = model->param._W_time[old_id + l];
+
+        if ((i+1) % train_opt.display_interval == 0) {
+          TRACE_LOG("[%d] instances is trained.", i+1);
         }
-    }
+      }
+      model->param.flush( train_dat.size() * (iter + 1) );
 
-    for (int pl = 0; pl < L; ++ pl) {
-        for (int l = 0; l < L; ++ l) {
-            int old_id = model->space.index(pl, l);
-            int new_id = new_model->space.index(pl, l);
+      Model * new_model = NULL;
+      new_model = erase_rare_features(feature_group_updated_time);
 
-            new_model->param._W[new_id]         = model->param._W[old_id];
-            new_model->param._W_sum[new_id]     = model->param._W_sum[old_id];
-            new_model->param._W_time[new_id]    = model->param._W_time[old_id];
-        }
-    }
-    TRACE_LOG("Building new model is done");
+      swap(model, new_model);
+
+      double p, r, f;
+      evaluate(p,r,f);
+
+      if (f > best_f) {
+        best_p = p;
+        best_r = r;
+        best_f = f;
+        best_iteration = iter;
+      }
 
-    for (SmartMap<bool>::const_iterator itx = model->internal_lexicon.begin();
-            itx != model->internal_lexicon.end();
-            ++ itx) {
-        new_model->internal_lexicon.set(itx.key(), true);
+      std::string saved_model_file = (train_opt.model_name
+                                      + "."
+                                      + strutils::to_str(iter)
+                                      + ".model");
+      std::ofstream ofs(saved_model_file.c_str(), std::ofstream::binary);
+
+      swap(model, new_model);
+      new_model->save(ofs);
+      delete new_model;
+      TRACE_LOG("Model for iteration [%d] is saved to [%s]",
+                iter + 1,
+                saved_model_file.c_str());
     }
 
-    return new_model;
-}
-void Segmentor::train(void) {
-    const char * train_file = train_opt.train_file.c_str();
-
-    // read in training instance
-    if (!read_instance(train_file)) {
-        ERROR_LOG("Training file not exist.");
-        return;
-    }
-    TRACE_LOG("Read in [%d] instances.", train_dat.size());
-
-    model = new Model;
-    // build tag dictionary, map string tag to index
-    TRACE_LOG("Start build configuration");
-    build_configuration();
-    TRACE_LOG("Build configuration is done.");
-    TRACE_LOG("Number of labels: [%d]", model->labels.size());
-
-    // build feature space from the training instance
-    TRACE_LOG("Start building feature space.");
-    build_feature_space();
-    TRACE_LOG("Building feature space is done.");
-    TRACE_LOG("Number of features: [%d]", model->space.num_features());
-
-    model->param.realloc(model->space.dim());
-    TRACE_LOG("Allocate [%d] dimensition parameter.", model->space.dim());
-
-    int offset_model = model->space.get_offset();
-    int *updates=new int [offset_model];
-    for(int i=0;i<offset_model;i++) {
-	updates[i]=0;
-    }
-    TRACE_LOG("Allocate [%d] update counters.", offset_model);
-
-    SegmentWriter writer(cout);
-
-    if (train_opt.algorithm == "mira") {
-        // use mira algorithm
-        /*kbest_decoder = new KBestDecoder(L);
-
-        for (int iter = 0; iter < train_opt.max_iter; ++ iter) {
-            for (int i = 0; i < train_dat.size(); ++ i) {
-                extract_features(train_dat[i]);
-                calculate_scores(train_dat[i]);
-
-                KBestDecoder::KBestDecodeResult result;
-                kbest_decoder->decode(train_dat[i], result);
-            }
-        }*/
-    } else {
-        // use pa or average perceptron algorithm
-        rulebase::RuleBase base(model->labels);
-        decoder = new Decoder(model->num_labels(), base);
-        TRACE_LOG("Allocated plain decoder");
-
-	int best_result=-1;
-	double best_p=-1;
-	double best_r=-1;
-	double best_f=-1;
-        for (int iter = 0; iter < train_opt.max_iter; ++ iter) {
-            TRACE_LOG("Training iteraition [%d]", (iter + 1));
-            for (int i = 0; i < train_dat.size(); ++ i) {
-                // extract_features(train_dat[i]);
-
-                Instance * inst = train_dat[i];
-                calculate_scores(inst, false);
-                decoder->decode(inst);
-
-                if (inst->features.dim() == 0) {
-                    collect_features(inst, inst->tagsidx, inst->features);
-                }
-                collect_features(inst, inst->predicted_tagsidx, inst->predicted_features);
-
-                // writer.debug(inst, true);
-
-                if (train_opt.algorithm == "pa") {
-                    SparseVec update_features;
-                    update_features.zero();
-                    update_features.add(train_dat[i]->features, 1.);
-                    update_features.add(train_dat[i]->predicted_features, -1.);
-
-		    
-		    update_features.update_counter(updates,offset_model,model->num_labels());
-
-                    double error = train_dat[i]->num_errors();
-                    double score = model->param.dot(update_features, false);
-                    double norm = update_features.L2();
-
-                    double step = 0.;
-                    if (norm < EPS) {
-                       step = 0;
-                    } else {
-                        step = (error - score) / norm;
-                    }
-
-                    model->param.add(update_features,
-                            iter * train_dat.size() + i + 1,
-                            step);
-                } else if (train_opt.algorithm == "ap") {
-                    SparseVec update_features;
-                    update_features.zero();
-                    update_features.add(train_dat[i]->features, 1.);
-                    update_features.add(train_dat[i]->predicted_features, -1.);
-
-
-                    model->param.add(update_features,
-                            iter * train_dat.size() + i + 1,
-                            1.);
-                }
-
-                if ((i+1) % train_opt.display_interval == 0) {
-                    TRACE_LOG("[%d] instances is trained.", i+1);
-                }
-            }
-            model->param.flush( train_dat.size() * (iter + 1) );
-
-	    Model * new_model; 
-	    if(train_opt.use_update=="true")
-            new_model = truncate_prune(updates);
-	    else
-            new_model = truncate();
-	    swap(model, new_model);
-	    double p,r,f;
-            evaluate(p,r,f);
-
-	    if(f>best_f){
-		best_p=p;
-		best_r=r;
-		best_f=f;
-		best_result=iter;
-	    }
-            std::string saved_model_file = (train_opt.model_name + "." + strutils::to_str(iter) + ".model");
-            std::ofstream ofs(saved_model_file.c_str(), std::ofstream::binary);
-
-            swap(model, new_model);
-            new_model->save(ofs);
-            delete new_model;
-            TRACE_LOG("Model for iteration [%d] is saved to [%s]",
-                    iter + 1,
-                    saved_model_file.c_str());
-        }
-	delete updates;
-        TRACE_LOG("Best result is :");
-        TRACE_LOG("P: %lf ;R: %lf;F: %lf;iter: %d",best_p,best_r,best_f,best_result);
+    if (feature_group_updated_time) {
+      delete [] feature_group_updated_time;
     }
+
+    TRACE_LOG("Best result (iteratin = %d) P = %lf | R = %lf | F = %lf",
+              best_iteration,
+              best_p,
+              best_r,
+              best_f);
+  }
 }
 
+void
+Segmentor::evaluate(double &p, double &r, double &f) {
+  const char * holdout_file = train_opt.holdout_file.c_str();
 
-void Segmentor::evaluate(double &p,double &r,double &f) {
-    const char * holdout_file = train_opt.holdout_file.c_str();
+  ifstream ifs(holdout_file);
 
-    ifstream ifs(holdout_file);
+  if (!ifs) {
+    ERROR_LOG("Failed to open holdout file.");
+    return;
+  }
 
-    if (!ifs) {
-        ERROR_LOG("Failed to open holdout file.");
-        return;
-    }
+  SegmentReader reader(ifs, true);
+  Instance * inst = NULL;
 
-    SegmentReader reader(ifs, true);
-    Instance * inst = NULL;
+  int num_recalled_words = 0;
+  int num_predicted_words = 0;
+  int num_gold_words = 0;
 
-    int num_recalled_words = 0;
-    int num_predicted_words = 0;
-    int num_gold_words = 0;
+  int beg_tag0 = model->labels.index( __b__ );
+  int beg_tag1 = model->labels.index( __s__ );
 
-    int beg_tag0 = model->labels.index( __b__ );
-    int beg_tag1 = model->labels.index( __s__ );
+  int L = model->num_labels();
 
-    int L = model->num_labels();
+  while ((inst = reader.next())) {
+    int len = inst->size();
+    inst->tagsidx.resize(len);
+    for (int i = 0; i < len; ++ i) {
+      inst->tagsidx[i] = model->labels.index(inst->tags[i]);
+    }
 
-    while ((inst = reader.next())) {
-        int len = inst->size();
-        inst->tagsidx.resize(len);
-        for (int i = 0; i < len; ++ i) {
-            inst->tagsidx[i] = model->labels.index(inst->tags[i]);
-        }
+    extract_features(inst);
+    calculate_scores(inst, true);
 
-        extract_features(inst);
-        calculate_scores(inst, true);
+    decoder->decode(inst);
 
-        decoder->decode(inst);
+    if (inst->words.size() == 0) {
+      build_words(inst, inst->tagsidx, inst->words, beg_tag0, beg_tag1);
+    }
+    build_words(inst,
+                inst->predicted_tagsidx,
+                inst->predicted_words,
+                beg_tag0,
+                beg_tag1);
 
-        if (inst->words.size() == 0) {
-            build_words(inst, inst->tagsidx, inst->words, beg_tag0, beg_tag1);
-        }
-        build_words(inst, inst->predicted_tagsidx, inst->predicted_words, beg_tag0, beg_tag1);
+    num_recalled_words += inst->num_recalled_words();
+    num_predicted_words += inst->num_predicted_words();
+    num_gold_words += inst->num_gold_words();
 
-        num_recalled_words += inst->num_recalled_words();
-        num_predicted_words += inst->num_predicted_words();
-        num_gold_words += inst->num_gold_words();
+    delete inst;
+  }
 
-        delete inst;
-    }
+  p = (double)num_recalled_words / num_predicted_words;
+  r = (double)num_recalled_words / num_gold_words;
+  f = 2 * p * r / (p + r);
 
-    double p_tmp = (double)num_recalled_words / num_predicted_words;
-    double r_tmp = (double)num_recalled_words / num_gold_words;
-    double f_tmp = 2 * p_tmp * r_tmp / (p_tmp + r_tmp);
+  TRACE_LOG("P: %lf ( %d / %d )", p, num_recalled_words, num_predicted_words);
+  TRACE_LOG("R: %lf ( %d / %d )", r, num_recalled_words, num_gold_words);
+  TRACE_LOG("F: %lf" , f);
+  return;
+}
 
-    p=p_tmp;
-    r=r_tmp;
-    f=f_tmp;
+void
+Segmentor::test(void) {
+  // load model
+  const char * model_file = test_opt.model_file.c_str();
+  ifstream mfs(model_file, std::ifstream::binary);
 
-    TRACE_LOG("P: %lf ( %d / %d )", p_tmp, num_recalled_words, num_predicted_words);
-    TRACE_LOG("R: %lf ( %d / %d )", r_tmp, num_recalled_words, num_gold_words);
-    TRACE_LOG("F: %lf" , f_tmp); 
+  if (!mfs) {
+    ERROR_LOG("Failed to load model");
     return;
-}
-void Segmentor::test(void) {
-    // load model
-    const char * model_file = test_opt.model_file.c_str();
-    ifstream mfs(model_file, std::ifstream::binary);
+  }
 
-    if (!mfs) {
-        ERROR_LOG("Failed to load model");
-        return;
-    }
-
-    model = new Model;
-    if (!model->load(mfs)) {
-        ERROR_LOG("Failed to load model");
-        return;
-    }
+  model = new Model;
+  if (!model->load(mfs)) {
+    ERROR_LOG("Failed to load model");
+    return;
+  }
 
-    TRACE_LOG("Number of labels                 [%d]", model->num_labels());
-    TRACE_LOG("Number of features               [%d]", model->space.num_features());
-    TRACE_LOG("Number of dimension              [%d]", model->space.dim());
+  TRACE_LOG("Number of labels     [%d]", model->num_labels());
+  TRACE_LOG("Number of features   [%d]", model->space.num_features());
+  TRACE_LOG("Number of dimension  [%d]", model->space.dim());
 
-    // load exteranl lexicon
-    const char * lexicon_file =test_opt.lexicon_file.c_str();
+  // load exteranl lexicon
+  const char * lexicon_file =test_opt.lexicon_file.c_str();
 
-    if (NULL != lexicon_file) {
-        ifstream lfs(lexicon_file);
+  if (NULL != lexicon_file) {
+    ifstream lfs(lexicon_file);
 
-        if (lfs) {
-            std::string buffer;
-            while (std::getline(lfs, buffer)) {
-                buffer = strutils::chomp(buffer);
-                if (buffer.size() == 0) {
-                    continue;
-                }
-                model->external_lexicon.set(buffer.c_str(), true);
-            }
+    if (lfs) {
+      std::string buffer;
+      while (std::getline(lfs, buffer)) {
+        buffer = strutils::chomp(buffer);
+        if (buffer.size() == 0) {
+          continue;
         }
+        model->external_lexicon.set(buffer.c_str(), true);
+      }
     }
+  }
 
-    const char * test_file = test_opt.test_file.c_str();
+  const char * test_file = test_opt.test_file.c_str();
 
-    ifstream ifs(test_file);
+  ifstream ifs(test_file);
 
-    if (!ifs) {
-        ERROR_LOG("Failed to open holdout file.");
-        return;
-    }
+  if (!ifs) {
+    ERROR_LOG("Failed to open holdout file.");
+    return;
+  }
 
-    rulebase::RuleBase base(model->labels);
-    Decoder * decoder = new Decoder(model->num_labels(), base);
-    SegmentReader reader(ifs);
-    SegmentWriter writer(cout);
-    Instance * inst = NULL;
+  rulebase::RuleBase base(model->labels);
+  Decoder * decoder = new Decoder(model->num_labels(), base);
+  SegmentReader reader(ifs);
+  SegmentWriter writer(cout);
+  Instance * inst = NULL;
 
-    int beg_tag0 = model->labels.index( __b__ );
-    int beg_tag1 = model->labels.index( __s__ );
+  int beg_tag0 = model->labels.index( __b__ );
+  int beg_tag1 = model->labels.index( __s__ );
 
-    double before = get_time();
+  double before = get_time();
 
-    while ((inst = reader.next())) {
-        int len = inst->size();
-        inst->tagsidx.resize(len);
+  while ((inst = reader.next())) {
+    int len = inst->size();
+    inst->tagsidx.resize(len);
 
-        extract_features(inst);
-        calculate_scores(inst, true);
-        decoder->decode(inst);
+    extract_features(inst);
+    calculate_scores(inst, true);
+    decoder->decode(inst);
 
-        build_words(inst, 
-                inst->predicted_tagsidx, 
-                inst->predicted_words, 
-                beg_tag0, 
+    build_words(inst,
+                inst->predicted_tagsidx,
+                inst->predicted_words,
+                beg_tag0,
                 beg_tag1);
 
-        writer.write(inst);
-        delete inst;
-    }
+    writer.write(inst);
+    delete inst;
+  }
 
-    double after = get_time();
-    TRACE_LOG("Eclipse time %lf", after - before);
-    return;
+  double after = get_time();
+  TRACE_LOG("Eclipse time %lf", after - before);
+  return;
 }
 
 
 void Segmentor::dump() {
-    // load model
-    const char * model_file = dump_opt.model_file.c_str();
-    ifstream mfs(model_file, std::ifstream::binary);
-
-    if (!mfs) {
-        ERROR_LOG("Failed to load model");
-        return;
-    }
+  // load model
+  const char * model_file = dump_opt.model_file.c_str();
+  ifstream mfs(model_file, std::ifstream::binary);
 
-    model = new Model;
-    if (!model->load(mfs)) {
-        ERROR_LOG("Failed to load model");
-        return;
-    }
-
-    int L = model->num_labels();
-    TRACE_LOG("Number of labels                 [%d]", model->num_labels());
-    TRACE_LOG("Number of features               [%d]", model->space.num_features());
-    TRACE_LOG("Number of dimension              [%d]", model->space.dim());
-
-    for (FeatureSpaceIterator itx = model->space.begin(); itx != model->space.end(); ++ itx) {
-        const char * key = itx.key();
-        int tid = itx.tid();
-        int id = model->space.index(tid, key);
+  if (!mfs) {
+    ERROR_LOG("Failed to load model");
+    return;
+  }
 
-        for (int l = 0; l < L; ++ l) {
-            std::cout << key << " ( " << id + l << " ) "
+  model = new Model;
+  if (!model->load(mfs)) {
+    ERROR_LOG("Failed to load model");
+    return;
+  }
+
+  int L = model->num_labels();
+  TRACE_LOG("Number of labels     [%d]", model->num_labels());
+  TRACE_LOG("Number of features   [%d]", model->space.num_features());
+  TRACE_LOG("Number of dimension  [%d]", model->space.dim());
+
+  for (FeatureSpaceIterator itx = model->space.begin();
+       itx != model->space.end();
+       ++ itx) {
+    const char * key = itx.key();
+    int tid = itx.tid();
+    int id = model->space.index(tid, key);
+
+    for (int l = 0; l < L; ++ l) {
+      std::cout << key
+                << " ( " << id + l << " ) "
                 << " --> "
                 << model->param.dot(id + l)
                 << std::endl;
-        }
     }
+  }
 
-    for (int pl = 0; pl < L; ++ pl) {
-        for (int l = 0; l < L; ++ l) {
-            int id = model->space.index(pl, l);
-            std::cout << pl << " --> " << l << " " << model->param.dot(id) << std::endl;
-        }
+  for (int pl = 0; pl < L; ++ pl) {
+    for (int l = 0; l < L; ++ l) {
+      int id = model->space.index(pl, l);
+      std::cout << pl << " --> " << l
+                << " " << model->param.dot(id)
+                << std::endl;
     }
+  }
 }
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
diff --git a/src/segmentor/segmentor.h b/src/segmentor/segmentor.h
index ce46990b5..f865f2501 100644
--- a/src/segmentor/segmentor.h
+++ b/src/segmentor/segmentor.h
@@ -11,115 +11,125 @@ namespace segmentor {
 
 class Segmentor {
 public:
-    Segmentor();
-    Segmentor(ltp::utility::ConfigParser & cfg);
-    ~Segmentor();
+  Segmentor();
+  Segmentor(ltp::utility::ConfigParser & cfg);
+  ~Segmentor();
 
-    void run();
+  void run();
 
 private:
-    /*
-     * parse the configuration, return true on success, otherwise false
-     *
-     *  @param[in]  cfg     the config class
-     *  @return     bool    return true on success, otherwise false
-     */
-    bool parse_cfg(ltp::utility::ConfigParser & cfg);
-
-    /*
-     * read instances from file and store them in train_dat
-     *
-     *  @param[in]  file_name   the filename
-     *  @return     bool        true on success, otherwise false
-     */
-    bool read_instance( const char * file_name );
-    void build_configuration(void);
-    void build_feature_space(void);
-
-    /*
-     * the training process
-     */
-    void train(void);
-
-    /*
-     * the evaluating process
-     */
-    void evaluate(double &p,double &r,double &f);
-
-    /*
-     * the testing process
-     */
-    void test(void);
-
-    /*
-     * the dumping model process
-     */
-    void dump(void);
-
-    /*
-     * do feature trauncation on the model. create a model duplation
-     * on the model and return their
-     *
-     *  @return Model   the duplication of the model
-     */
-    Model * truncate(void);
-    Model * truncate_prune(int *);
+  /*
+   * parse the configuration, return true on success, otherwise false
+   *
+   *  @param[in]  cfg         the config class
+   *  @return     bool        return true on success, otherwise false
+   */
+  bool parse_cfg(ltp::utility::ConfigParser & cfg);
+
+  /*
+   * read instances from file and store them in train_dat
+   *
+   *  @param[in]  file_name   the filename
+   *  @return     bool        true on success, otherwise false
+   */
+  bool read_instance( const char * file_name );
+
+  /*
+   * build tag sets, collect internal word map, record word frequency.
+   */
+  void build_configuration(void);
+
+  /*
+   *
+   *
+   */
+  void build_feature_space(void);
+
+  /*
+   * the training process
+   */
+  void train(void);
+
+  /*
+   * the evaluating process
+   */
+  void evaluate(double &p, double &r, double &f);
+
+  /*
+   * the testing process
+   */
+  void test(void);
+
+  /*
+   * the dumping model process
+   */
+  void dump(void);
+
+  /*
+   * do feature selection by erasing the rare feature. create a new model
+   * without rare feature (only witness a few times) according the original
+   * model.
+   *
+   *  @param[in]  nr_updates  the number of update times
+   *  @return     Model       the model without rare feature
+   */
+  Model * erase_rare_features(const int * nr_updates = NULL);
 protected:
-    /*
-     * extract features from one instance,
-     *
-     */
-    void extract_features(Instance * inst, bool create = false);
-
-    /*
-     * build words from tags for certain instance
-     *
-     *  @param[in/out]  inst    the instance
-     *  @param[out]     words   the output words
-     *  @param[in]      tagsidx the index of tags
-     *  @param[in]      begtag0 first of the word begin tag
-     *  @param[in]      begtag1 second of the word begin tag
-     */
-    void build_words(Instance * inst, 
-            const std::vector<int> & tagsidx,
-            std::vector<std::string> & words,
-            int beg_tag0,
-            int beg_tag1 = -1);
-
-    /*
-     * cache all the score for the certain instance.
-     *
-     *  @param[in/out]  inst    the instance
-     *  @param[in]      use_avg use to specify use average parameter
-     */
-    void calculate_scores(Instance * inst, bool use_avg);
-
-    /*
-     * collect feature when given the tags index
-     *
-     *  @param[in]      inst    the instance
-     *  @param[in]      tagsidx the tags index
-     *  @param[out]     vec     the output sparse vector
-     */
-    void collect_features(Instance * inst, 
-            const std::vector<int> & tagsidx, 
-            ltp::math::SparseVec & vec);
+  /*
+   * extract features from one instance,
+   *
+   */
+  void extract_features(Instance * inst, bool create = false);
+
+  /*
+   * build words from tags for certain instance
+   *
+   *  @param[in/out]  inst      the instance
+   *  @param[in]      tagsidx   the index of tags
+   *  @param[out]     words     the output words
+   *  @param[in]      begtag0   first of the word begin tag
+   *  @param[in]      begtag1   second of the word begin tag
+   */
+  void build_words(Instance * inst,
+                   const std::vector<int> & tagsidx,
+                   std::vector<std::string> & words,
+                   int beg_tag0,
+                   int beg_tag1 = -1);
+
+  /*
+   * cache all the score for the certain instance.
+   *
+   *  @param[in/out]  inst      the instance
+   *  @param[in]      use_avg   use to specify use average parameter
+   */
+  void calculate_scores(Instance * inst, bool use_avg);
+
+  /*
+   * collect feature when given the tags index
+   *
+   *  @param[in]    inst    the instance
+   *  @param[in]    tagsidx the tags index
+   *  @param[out]   vec     the output sparse vector
+   */
+  void collect_features(Instance * inst,
+                        const std::vector<int> & tagsidx,
+                        ltp::math::SparseVec & vec);
 
 private:
-    bool    __TRAIN__;
-    bool    __TEST__;
-    bool    __DUMP__;
+  bool  __TRAIN__;
+  bool  __TEST__;
+  bool  __DUMP__;
 
 private:
-    std::vector< Instance * > train_dat;
+  std::vector< Instance * > train_dat;
 
 protected:
-    Model * model;
-    Decoder * decoder;
-    rulebase::RuleBase* baseAll;
+  Model *              model;
+  Decoder *            decoder;
+  rulebase::RuleBase * baseAll;
 };
 
-}       //  end for namespace segmentor
-}       //  end for namespace ltp
+}     //  end for namespace segmentor
+}     //  end for namespace ltp
 
 #endif  //  end for __LTP_SEGMENTOR_SEGMENTOR_H__
diff --git a/src/segmentor/segmentreader.h b/src/segmentor/segmentreader.h
index 530c64de1..6e4f6ba51 100644
--- a/src/segmentor/segmentreader.h
+++ b/src/segmentor/segmentreader.h
@@ -15,107 +15,107 @@ using namespace ltp::strutils;
 
 class SegmentReader {
 public:
-    SegmentReader(istream & _ifs, bool _segmented = false, int _style = 4) : 
-        ifs(_ifs),
-        segmented(_segmented),
-        style(_style) {}
-
-    Instance * next() {
-        if (ifs.eof()) {
-            return 0;
-        }
+  SegmentReader(istream & _ifs, bool _segmented = false, int _style = 4) : 
+    ifs(_ifs),
+    segmented(_segmented),
+    style(_style) {}
+
+  Instance * next() {
+    if (ifs.eof()) {
+      return 0;
+    }
 
-        Instance * inst = new Instance;
-        std::string  line;
+    Instance * inst = new Instance;
+    std::string  line;
 
-        std::getline(ifs, line);
+    std::getline(ifs, line);
 
-        line = chomp(line);
-        if (line.size() == 0) {
-            delete inst;
-            return 0;
-        }
+    line = chomp(line);
+    if (line.size() == 0) {
+      delete inst;
+      return 0;
+    }
 
-        if (segmented) {
-            std::vector<std::string> words = split(line);
-            inst->words = words;
-
-            for (int i = 0; i < words.size(); ++ i) {
-                // std::vector<std::string> chars;
-                // int num_chars = codecs::decode(words[i], chars);
-                int num_chars = rulebase::preprocess(words[i],
-                        inst->raw_forms,
-                        inst->forms,
-                        inst->chartypes);
-
-                // support different style
-                if (style == 2) {
-                    for (int j = 0; j < num_chars; ++ j) {
-                        // inst->forms.push_back(chars[j]);
-                        if (j == 0) {
-                            inst->tags.push_back( __b__ );
-                        } else {
-                            inst->tags.push_back( __i__ );
-                        }
-                    }
-                } else if (style == 4) {
-                    for(int j = 0; j < num_chars; ++ j) {
-                        // inst->forms.push_back(chars[j]);
-                        if (1 == num_chars) {
-                            inst->tags.push_back( __s__ );
-                        } else {
-                            if (0 == j) {
-                                inst->tags.push_back( __b__ );
-                            } else if (num_chars - 1 == j) {
-                                inst->tags.push_back( __e__ );
-                            } else {
-                                inst->tags.push_back( __i__ );
-                            }
-                        }
-                    }
-                } else if (style == 6) {
-                    for (int j = 0; j < num_chars; ++ j) {
-                        // inst->forms.push_back(chars[j]);
-
-                        if (1 == num_chars) {
-                            inst->tags.push_back( __s__ );
-                        } else {
-                            if (0 == j) {
-                                inst->tags.push_back( __b__ );
-                            } else if (1 == j) {
-                                inst->tags.push_back( __b2__ );
-                            } else if (2 == j) {
-                                inst->tags.push_back( __b3__ );
-                            } else if (num_chars - 1 == j) {
-                                inst->tags.push_back( __e__ );
-                            } else {
-                                inst->tags.push_back( __i__ );
-                            }
-                        }
-                    }
-                }
+    if (segmented) {
+      std::vector<std::string> words = split(line);
+      inst->words = words;
+
+      for (int i = 0; i < words.size(); ++ i) {
+        // std::vector<std::string> chars;
+        // int num_chars = codecs::decode(words[i], chars);
+        int num_chars = rulebase::preprocess(words[i],
+            inst->raw_forms,
+            inst->forms,
+            inst->chartypes);
+
+        // support different style
+        if (style == 2) {
+          for (int j = 0; j < num_chars; ++ j) {
+            // inst->forms.push_back(chars[j]);
+            if (j == 0) {
+              inst->tags.push_back( __b__ );
+            } else {
+              inst->tags.push_back( __i__ );
             }
-        } else {
-            int ret = rulebase::preprocess(line,
-                    inst->raw_forms,
-                    inst->forms,
-                    inst->chartypes);
-
-            if (ret < 0) {
-                delete inst;
-                return 0;
+          }
+        } else if (style == 4) {
+          for(int j = 0; j < num_chars; ++ j) {
+            // inst->forms.push_back(chars[j]);
+            if (1 == num_chars) {
+              inst->tags.push_back( __s__ );
+            } else {
+              if (0 == j) {
+                inst->tags.push_back( __b__ );
+              } else if (num_chars - 1 == j) {
+                inst->tags.push_back( __e__ );
+              } else {
+                inst->tags.push_back( __i__ );
+              }
             }
+          }
+        } else if (style == 6) {
+          for (int j = 0; j < num_chars; ++ j) {
+            // inst->forms.push_back(chars[j]);
+
+            if (1 == num_chars) {
+              inst->tags.push_back( __s__ );
+            } else {
+              if (0 == j) {
+                inst->tags.push_back( __b__ );
+              } else if (1 == j) {
+                inst->tags.push_back( __b2__ );
+              } else if (2 == j) {
+                inst->tags.push_back( __b3__ );
+              } else if (num_chars - 1 == j) {
+                inst->tags.push_back( __e__ );
+              } else {
+                inst->tags.push_back( __i__ );
+              }
+            }
+          }
         }
-
-        return inst;
+      }
+    } else {
+      int ret = rulebase::preprocess(line,
+          inst->raw_forms,
+          inst->forms,
+          inst->chartypes);
+
+      if (ret < 0) {
+        delete inst;
+        return 0;
+      }
     }
+
+    return inst;
+  }
 private:
-    istream &   ifs;
-    int         style;
-    bool        segmented;
+  istream &   ifs;
+  int     style;
+  bool    segmented;
 };
 
-}           //  end for namespace segmentor
-}           //  end for namespace ltp
+}       //  end for namespace segmentor
+}       //  end for namespace ltp
 
-#endif      //  end for __LTP_SEGMENTOR_READER_H__
+#endif    //  end for __LTP_SEGMENTOR_READER_H__
diff --git a/src/segmentor/segmentwriter.h b/src/segmentor/segmentwriter.h
index 59a580bc0..4a5fd21c0 100644
--- a/src/segmentor/segmentwriter.h
+++ b/src/segmentor/segmentwriter.h
@@ -9,76 +9,76 @@ namespace segmentor {
 
 class SegmentWriter {
 public:
-    SegmentWriter(std::ostream & _ofs) : ofs(_ofs) {}
+  SegmentWriter(std::ostream & _ofs) : ofs(_ofs) {}
 
-    void write(const Instance * inst) {
-        int len = inst->predicted_words.size();
-        for (int i = 0; i < len; ++ i) {
-            ofs << inst->predicted_words[i];
-            if (i+1==len) ofs << std::endl;
-            else ofs << "\t";
-        }
+  void write(const Instance * inst) {
+    int len = inst->predicted_words.size();
+    for (int i = 0; i < len; ++ i) {
+      ofs << inst->predicted_words[i];
+      if (i+1==len) ofs << std::endl;
+      else ofs << "\t";
     }
+  }
 
-    void debug(const Instance * inst, bool show_feat = false) {
-        int len = inst->size();
-        ofs << "_instance_debug_" << std::endl;
-        ofs << "FORMS: ";
-        for (int i = 0; i < len; ++ i) {
-            ofs << inst->forms[i] << "|";
-        }
-        ofs << std::endl;
+  void debug(const Instance * inst, bool show_feat = false) {
+    int len = inst->size();
+    ofs << "_instance_debug_" << std::endl;
+    ofs << "FORMS: ";
+    for (int i = 0; i < len; ++ i) {
+      ofs << inst->forms[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "TAGS: ";
-        for (int i = 0; i < inst->tags.size(); ++ i) {
-            ofs << inst->tags[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "TAGS: ";
+    for (int i = 0; i < inst->tags.size(); ++ i) {
+      ofs << inst->tags[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "TAGS(index): ";
-        for (int i = 0; i < inst->tagsidx.size(); ++ i) {
-            ofs << inst->tagsidx[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "TAGS(index): ";
+    for (int i = 0; i < inst->tagsidx.size(); ++ i) {
+      ofs << inst->tagsidx[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "PREDICTED TAGS: ";
-        for (int i = 0; i < inst->predicted_tags.size(); ++ i) {
-            ofs << inst->predicted_tags[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "PREDICTED TAGS: ";
+    for (int i = 0; i < inst->predicted_tags.size(); ++ i) {
+      ofs << inst->predicted_tags[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "PREDICTED TAGS(index): ";
-        for (int i = 0; i < inst->predicted_tagsidx.size(); ++ i) {
-            ofs << inst->predicted_tagsidx[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "PREDICTED TAGS(index): ";
+    for (int i = 0; i < inst->predicted_tagsidx.size(); ++ i) {
+      ofs << inst->predicted_tagsidx[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "WORDS: ";
-        for (int i = 0; i < inst->words.size(); ++ i) {
-            ofs << inst->words[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "WORDS: ";
+    for (int i = 0; i < inst->words.size(); ++ i) {
+      ofs << inst->words[i] << "|";
+    }
+    ofs << std::endl;
 
-        ofs << "PREDICTED WORDS: ";
-        for (int i = 0; i < inst->predicted_words.size(); ++ i) {
-            ofs << inst->predicted_words[i] << "|";
-        }
-        ofs << std::endl;
+    ofs << "PREDICTED WORDS: ";
+    for (int i = 0; i < inst->predicted_words.size(); ++ i) {
+      ofs << inst->predicted_words[i] << "|";
+    }
+    ofs << std::endl;
 
-        if (show_feat) {
-            ofs << "GOLD FEATURES: ";
-            inst->features.str(ofs);
-            ofs << std::endl;
+    if (show_feat) {
+      ofs << "GOLD FEATURES: ";
+      inst->features.str(ofs);
+      ofs << std::endl;
 
-            ofs << "PREDICTED FEATURES: ";
-            inst->predicted_features.str(ofs);
-            ofs << std::endl;
-        }
+      ofs << "PREDICTED FEATURES: ";
+      inst->predicted_features.str(ofs);
+      ofs << std::endl;
     }
+  }
 private:
-    std::ostream & ofs;
+  std::ostream & ofs;
 };
 
-}           //  end for namespace segmentor
-}           //  end for namespace ltp
-#endif      //  end for __LTP_SEGMENTOR_WRITER_H__
+}       //  end for namespace segmentor
+}       //  end for namespace ltp
+#endif    //  end for __LTP_SEGMENTOR_WRITER_H__