support model s3 i/o

src/app/linear_method/model_evaluation.h

@@ -3,6 +3,15 @@
 #include "data/stream_reader.h"
 #include "util/evaluation.h"
 namespace PS {
+
+#if USE_S3
+bool s3file(const std::string& name);
+std::string s3Prefix(const std::string& path);
+std::string s3Bucket(const std::string& path);
+std::string s3FileUrl(const std::string& path);
+#endif // USE_S3
+
+
 namespace LM {
 
 class ModelEvaluation : public App {
@@ -22,13 +31,32 @@ void ModelEvaluation::Run() {
   auto model = searchFiles(conf_.model_input());
   NOTICE("find %d model files", model.file_size());
   for (int i = 0; i < model.file_size(); ++i) {
+#if USE_S3
+    std::ifstream in;
+    if (s3file(model.file(i))) {
+      // download file from s3
+      std::string cmd="curl -s -o model_file "+s3FileUrl(model.file(i));
+      LOG(INFO)<<cmd;
+      system(cmd.c_str());
+      in.open("model_file"); 
+    }
+    else {
+      in.open(model.file(i));
+    }
+#else
     std::ifstream in(model.file(i));
+#endif // USE_S3
     while (in.good()) {
       Key k; Real v;
       in >> k >> v;
       weight[k] = v;
     }
   }
+#if USE_S3
+ // remove local model after read done
+ std::string cmd="rm -rf model_file";
+ system(cmd.c_str());
+#endif // USE_S3
 
   NOTICE("load %lu model entries", weight.size());
 

src/parameter/kv_map.h

@@ -118,12 +118,12 @@ void KVMap<K,V,E,S>::WriteToFile(std::string file) {
 #if USE_S3
   if (s3file(s3_file)) {
     // upload model
-    std::string cmd = "curl -s -X PUT '"+s3FileUrl(s3_file)+"?Content-Length="
-    +std::to_string(File::size(file))+"&x-amz-acl=public-read' --data @"+file;
+    std::string cmd = "curl -s '"+s3FileUrl(s3_file)+"?Content-Length="
+    +std::to_string(File::size(file))+"&x-amz-acl=public-read' --upload-file "+file;
     LOG(INFO)<<cmd;
     system(cmd.c_str());
     // remove local model
-    cmd="rm -rf "+getPath(file);
+    cmd="rm -rf "+file;
     system(cmd.c_str());
   }
 #endif // USE_S3