diff --git a/.gitignore b/.gitignore index c41f82d27..2f3b23cda 100644 --- a/.gitignore +++ b/.gitignore @@ -27,17 +27,10 @@ bin/ tools/train/lgdpj tools/train/lgsrl tools/train/otcws +tools/train/otcws-customized tools/train/otpos tools/train/otner tools/train/maxent -examples/cws -examples/cws_cmdline -examples/multi_cws_cmdline -examples/pos -examples/pos_cmdline -examples/multi_pos_cmdline -examples/par -examples/ner ############### # data file # diff --git a/examples/cws b/examples/cws deleted file mode 100755 index 01177fc5f..000000000 Binary files a/examples/cws and /dev/null differ diff --git a/examples/ner b/examples/ner deleted file mode 100755 index 24d90ad03..000000000 Binary files a/examples/ner and /dev/null differ diff --git a/examples/par b/examples/par deleted file mode 100755 index 38c4d535e..000000000 Binary files a/examples/par and /dev/null differ diff --git a/examples/pos b/examples/pos deleted file mode 100755 index 2a0e919ed..000000000 Binary files a/examples/pos and /dev/null differ diff --git a/examples/srl b/examples/srl deleted file mode 100755 index e80a434c1..000000000 Binary files a/examples/srl and /dev/null differ diff --git a/src/framework/serializable.h b/src/framework/serializable.h new file mode 100644 index 000000000..41521cff5 --- /dev/null +++ b/src/framework/serializable.h @@ -0,0 +1,25 @@ +#ifndef __LTP_FRAMEWORK_SERIALIZABLE_H__ +#define __LTP_FRAMEWORK_SERIALIZABLE_H__ + +#include + +namespace ltp { +namespace framework { + +class Serializable { +protected: + void write_uint(std::ostream & out, unsigned int val) { + out.write(reinterpret_cast(&val), sizeof(unsigned int)); + } + + unsigned int read_uint(std::istream & in) { + char p[4]; + in.read(reinterpret_cast(p), sizeof(unsigned int)); + return *reinterpret_cast(p); + } +}; + +} // end for framework +} // end for ltp + +#endif // end for __LTP_FRAMEWORK_SERIALIZABLE_H__ diff --git a/src/ner/model.h b/src/ner/model.h index 7084889c8..fe9422082 100644 --- a/src/ner/model.h +++ b/src/ner/model.h @@ -1,6 +1,7 @@ #ifndef __LTP_NER_MODEL_H__ #define __LTP_NER_MODEL_H__ +#include "framework/serializable.h" #include "ner/featurespace.h" #include "ner/parameter.h" #include "utils/smartmap.hpp" @@ -8,9 +9,10 @@ namespace ltp { namespace ner { -using namespace ltp::utility; +namespace utils = ltp::utility; +namespace frame = ltp::framework; -class Model { +class Model : public frame::Serializable { public: Model(); ~Model(); @@ -38,21 +40,17 @@ class Model { */ bool load(std::istream & ifs); public: - IndexableSmartMap labels; - FeatureSpace space; - Parameters param; - - SmartMap cluster_lexicon; -private: - void write_uint(std::ostream & out, unsigned int val) { - out.write(reinterpret_cast(&val), sizeof(unsigned int)); - } + //! + utils::IndexableSmartMap labels; - unsigned int read_uint(std::istream & in) { - char p[4]; - in.read(reinterpret_cast(p), sizeof(unsigned int)); - return *reinterpret_cast(p); - } + //! + FeatureSpace space; + + //! + Parameters param; + + //! + utils::SmartMap cluster_lexicon; }; } // end for namespace ner diff --git a/src/parser/model.h b/src/parser/model.h index fb33bc3e5..b9d08ad8b 100644 --- a/src/parser/model.h +++ b/src/parser/model.h @@ -1,6 +1,7 @@ #ifndef __LTP_PARSER_MODEL_H__ #define __LTP_PARSER_MODEL_H__ +#include "framework/serializable.h" #include "parser/featurespace.h" #include "parser/parameters.h" #include "parser/options.h" @@ -8,7 +9,9 @@ namespace ltp { namespace parser { -class Model { +namespace frame = ltp::framework; + +class Model: public frame::Serializable { public: Model() : _dim(-1), @@ -74,17 +77,6 @@ class Model { * @return bool true on success, otherwise false */ bool load(istream & in); - -private: - void write_uint(ostream & out, unsigned int val) { - out.write(reinterpret_cast(&val), sizeof(unsigned int)); - } - - unsigned int read_uint(istream & in) { - char p[4]; - in.read(reinterpret_cast(p), sizeof(unsigned int)); - return *reinterpret_cast(p); - } }; // end for class model } // end for namespace parser diff --git a/src/postagger/model.h b/src/postagger/model.h index 89a8dd9c8..fa9c29a8b 100644 --- a/src/postagger/model.h +++ b/src/postagger/model.h @@ -1,18 +1,19 @@ #ifndef __LTP_POSTAGGER_MODEL_H__ #define __LTP_POSTAGGER_MODEL_H__ +#include "framework/serializable.h" #include "postagger/featurespace.h" #include "postagger/parameter.h" - #include "utils/smartmap.hpp" #include "utils/tinybitset.hpp" namespace ltp { namespace postagger { -using namespace ltp::utility; +namespace utils = ltp::utility; +namespace frame = ltp::framework; -class Model { +class Model: public frame::Serializable { public: Model(); ~Model(); @@ -40,22 +41,17 @@ class Model { */ bool load(std::istream & ifs); public: - IndexableSmartMap labels; - FeatureSpace space; - Parameters param; - //SmartMap internal_lexicon; - SmartMap external_lexicon; + //! The labels. + utils::IndexableSmartMap labels; -private: - void write_uint(std::ostream & out, unsigned int val) { - out.write(reinterpret_cast(&val), sizeof(unsigned int)); - } + //! The feature space. + FeatureSpace space; - unsigned int read_uint(std::istream & in) { - char p[4]; - in.read(reinterpret_cast(p), sizeof(unsigned int)); - return *reinterpret_cast(p); - } + //! The parameters. + Parameters param; + + //SmartMap internal_lexicon; + utils::SmartMap external_lexicon; }; } // end for namespace postagger diff --git a/src/segmentor/model.h b/src/segmentor/model.h index 476353fe0..6bb60675e 100644 --- a/src/segmentor/model.h +++ b/src/segmentor/model.h @@ -1,6 +1,7 @@ #ifndef __LTP_SEGMENTOR_MODEL_H__ #define __LTP_SEGMENTOR_MODEL_H__ +#include "framework/serializable.h" #include "segmentor/featurespace.h" #include "segmentor/parameter.h" #include "utils/smartmap.hpp" @@ -9,8 +10,9 @@ namespace ltp { namespace segmentor { namespace utils = ltp::utility; +namespace frame = ltp::framework; -class Model { +class Model: public frame::Serializable { public: Model(); ~Model(); @@ -28,7 +30,6 @@ class Model { * save the model to a output stream * * @param[out] ofs the output stream - * @param[in] full use to specify if dump full model. */ void save(std::ostream & ofs); @@ -40,25 +41,26 @@ class Model { bool load(std::istream & ifs); public: - int end_time; - bool full; - FeatureSpace space; - Parameters param; - utils::IndexableSmartMap labels; + //! The timestamp for the last training instance. + int end_time; - utils::SmartMap internal_lexicon; - utils::SmartMap external_lexicon; + //! Use to specified if dump the full model. + bool full; -private: - void write_uint(std::ostream & out, unsigned int val) { - out.write(reinterpret_cast(&val), sizeof(unsigned int)); - } + //! The feature space. + FeatureSpace space; - unsigned int read_uint(std::istream & in) { - char p[4]; - in.read(reinterpret_cast(p), sizeof(unsigned int)); - return *reinterpret_cast(p); - } + //! The parameter array. + Parameters param; + + //! The labels. + utils::IndexableSmartMap labels; + + //! The internal lexicon use to extract lexicon features. + utils::SmartMap internal_lexicon; + + //! The external lexicon use to extract lexicon features. + utils::SmartMap external_lexicon; }; } // end for namespace segmentor