Merge pull request HIT-SCIR#92 from ruoshui1126/master

postagger memory optimization & segmentor comments
chivalrousS · Jan 19, 2015 · c33e98e · c33e98e
2 parents 884992c + 2061ed7
commit c33e98e
Show file tree

Hide file tree

Showing 22 changed files with 325 additions and 118 deletions.
diff --git a/src/postagger/decode_context.h b/src/postagger/decode_context.h
@@ -0,0 +1,49 @@
+#ifndef __LTP_POSTAGGER_DECODE_CONTEXT_H__
+#define __LTP_POSTAGGER_DECODE_CONTEXT_H__
+
+namespace ltp {
+namespace postagger {
+
+class DecodeContext {
+public:
+  //! the gold features.
+  math::SparseVec correct_features;
+  //! the predicted features.
+  math::SparseVec predicted_features;
+  //! the updated features.
+  math::SparseVec updated_features;
+  //! the feature cache.
+  math::Mat< math::FeatureVector * > uni_features;
+
+  DecodeContext() {}
+
+  ~DecodeContext() {
+    clear();
+  }
+
+  void clear() {
+    if (uni_features.total_size() > 0) {
+      int d1 = uni_features.nrows();
+      int d2 = uni_features.ncols();
+      for (int i = 0; i < d1; ++ i) {
+        if (uni_features[i][0]) {
+          uni_features[i][0]->clear();
+        }
+        for (int j = 0; j < d2; ++j) {
+          if (uni_features[i][j]) {
+            delete uni_features[i][j];
+          }
+        }
+      }
+    }
+
+    uni_features.dealloc();
+    correct_features.zero();
+    predicted_features.zero();
+  }// end clear
+};// end Decodecontext
+
+}// end postagger
+}// end ltp
+
+#endif // end for __LTP_POSTAGGER_DECODE_CONTEXT_H__
diff --git a/src/postagger/decoder.cpp b/src/postagger/decoder.cpp
@@ -5,9 +5,9 @@ namespace postagger {
 
 
 void
-Decoder::decode(Instance * inst) {
+Decoder::decode(Instance * inst, const ScoreMatrix* scm) {
   init_lattice(inst);
-  viterbi_decode(inst);
+  viterbi_decode(inst, scm);
   get_result(inst);
   free_lattice();
 }
@@ -19,9 +19,9 @@ Decoder::init_lattice(const Instance * inst) {
   lattice = NULL;
 }
 
-void Decoder::viterbi_decode_inner(const Instance * inst,int i,int l){
+void Decoder::viterbi_decode_inner(const Instance * inst,const ScoreMatrix* scm, int i,int l){
   if (i == 0) {
-    LatticeItem * item = new LatticeItem(i, l, inst->uni_scores[i][l], NULL);
+    LatticeItem * item = new LatticeItem(i, l, scm->uni_scores[i][l], NULL);
     lattice_insert(lattice[i][l], item);
   } else {
     for (int pl = 0; pl < L; ++ pl) {
@@ -32,20 +32,20 @@ void Decoder::viterbi_decode_inner(const Instance * inst,int i,int l){
         continue;
       }
 
-      score = inst->uni_scores[i][l] + inst->bi_scores[pl][l] + prev->score;
+      score = scm->uni_scores[i][l] + scm->bi_scores[pl][l] + prev->score;
       const LatticeItem * item = new LatticeItem(i, l, score, prev);
       lattice_insert(lattice[i][l], item);
     }
   }   //  end for if i != 0
 }
 
 void
-Decoder::viterbi_decode(const Instance * inst) {
+Decoder::viterbi_decode(const Instance * inst, const ScoreMatrix* scm) {
   int len = inst->size();
   for (int i = 0; i < len; ++ i) {
     for (int l = 0; l < L; ++ l) {
       if(inst->postag_constrain[i].get(l)) {
-        viterbi_decode_inner(inst,i,l);
+        viterbi_decode_inner(inst,scm,i,l);
       }
     }//end for l
   }//end for i

diff --git a/src/postagger/decoder.h b/src/postagger/decoder.h
@@ -5,6 +5,7 @@
 #include <map>
 #include <vector>
 #include "postagger/instance.h"
+#include "postagger/score_matrix.h"
 #include "utils/math/mat.h"
 
 namespace ltp {
@@ -35,11 +36,11 @@ class LatticeItem {
 class Decoder {
 public:
   Decoder (int _l) : L(_l) {}
-  void decode(Instance * inst);
+  void decode(Instance * inst, const ScoreMatrix* scm);
 private:
   void init_lattice(const Instance * inst);
-  void viterbi_decode_inner(const Instance * inst,int i,int l);
-  void viterbi_decode(const Instance * inst);
+  void viterbi_decode_inner(const Instance * inst,const ScoreMatrix* scm, int i,int l);
+  void viterbi_decode(const Instance * inst, const ScoreMatrix* scm);
   void get_result(Instance * inst);
   void free_lattice();
 

diff --git a/src/postagger/instance.h b/src/postagger/instance.h
@@ -17,21 +17,6 @@ class Instance {
   Instance() {}
 
   ~Instance() {
-    if (uni_features.total_size() > 0) {
-      int d1 = uni_features.nrows();
-      int d2 = uni_features.ncols();
-
-      for (int i = 0; i < d1; ++ i) {
-        if (uni_features[i][0]) {
-          uni_features[i][0]->clear();
-        }
-        for (int j = 0; j < d2; ++ j) {
-          if (uni_features[i][j]) {
-            delete uni_features[i][j];
-          }
-        }
-      }
-    }
   }
 
   inline size_t size() const {
@@ -69,13 +54,6 @@ class Instance {
 
   std::vector< Bitset >       postag_constrain;   /*< the postag constrain for decode */
 
-  math::SparseVec             features;           /*< the gold features */
-  math::SparseVec             predicted_features; /*< the predicted features */
-
-  math::Mat< math::FeatureVector *> uni_features;
-  math::Mat< double > uni_scores;
-  math::Mat< double > bi_scores;
-
   std::vector< std::vector< std::string> > chars;
 };
 

diff --git a/src/postagger/postag_dll.cpp b/src/postagger/postag_dll.cpp
@@ -61,12 +61,16 @@ class PostaggerWrapper : public ltp::postagger::Postagger {
       }
     }
 
-    ltp::postagger::Postagger::extract_features(inst);
-    ltp::postagger::Postagger::calculate_scores(inst, true);
-    deco.decode(inst);
+    ltp::postagger::DecodeContext * ctx = new ltp::postagger::DecodeContext;
+    ltp::postagger::ScoreMatrix* scm = new ltp::postagger::ScoreMatrix;
+    ltp::postagger::Postagger::extract_features(inst, ctx);
+    ltp::postagger::Postagger::calculate_scores(inst, ctx, true, scm);
+    deco.decode(inst, scm);
 
     ltp::postagger::Postagger::build_labels(inst, tags);
 
+    delete ctx;
+    delete scm;
     delete inst;
     return tags.size();
   }