use libcurl (vesoft-inc#4891)

* use libcurl * remove some useless curl * address some comment
L-shuai · Nov 17, 2022 · d59ee99 · d59ee99
1 parent 1aee962
commit d59ee99
Show file tree

Hide file tree

Showing 35 changed files with 233 additions and 175 deletions.
diff --git a/src/codec/test/CMakeLists.txt b/src/codec/test/CMakeLists.txt
@@ -19,7 +19,6 @@ set(CODEC_TEST_LIBS
     $<TARGET_OBJECTS:meta_client_obj>
     $<TARGET_OBJECTS:ws_obj>
     $<TARGET_OBJECTS:ws_common_obj>
-    $<TARGET_OBJECTS:http_client_obj>
     $<TARGET_OBJECTS:file_based_cluster_id_man_obj>
     $<TARGET_OBJECTS:base_obj>
     $<TARGET_OBJECTS:network_obj>

diff --git a/src/common/expression/test/CMakeLists.txt b/src/common/expression/test/CMakeLists.txt
@@ -25,7 +25,6 @@ set(expression_test_common_libs
     $<TARGET_OBJECTS:agg_function_manager_obj>
     $<TARGET_OBJECTS:meta_thrift_obj>
     $<TARGET_OBJECTS:graph_thrift_obj>
-    $<TARGET_OBJECTS:http_client_obj>
     $<TARGET_OBJECTS:storage_thrift_obj>
     $<TARGET_OBJECTS:meta_obj>
     $<TARGET_OBJECTS:meta_client_obj>

diff --git a/src/common/http/HttpClient.cpp b/src/common/http/HttpClient.cpp
@@ -1,99 +1,121 @@
-/* Copyright (c) 2019 vesoft inc. All rights reserved.
+/* Copyright (c) 2022 vesoft inc. All rights reserved.
  *
  * This source code is licensed under Apache 2.0 License.
  */
 
 #include "common/http/HttpClient.h"
 
-#include "common/process/ProcessUtils.h"
-
+#include "curl/curl.h"
 namespace nebula {
-namespace http {
-
-StatusOr<std::string> HttpClient::get(const std::string& path, const std::string& options) {
-  auto command = folly::stringPrintf("/usr/bin/curl %s \"%s\"", options.c_str(), path.c_str());
-  LOG(INFO) << "HTTP Get Command: " << command;
-  auto result = nebula::ProcessUtils::runCommand(command.c_str());
-  if (result.ok()) {
-    return result.value();
-  } else {
-    return Status::Error(folly::stringPrintf("Http Get Failed: %s", path.c_str()));
-  }
+
+CurlHandle::CurlHandle() {
+  curl_global_init(CURL_GLOBAL_ALL);
 }
 
-StatusOr<std::string> HttpClient::post(const std::string& path, const std::string& header) {
-  auto command =
-      folly::stringPrintf("/usr/bin/curl -X POST %s \"%s\"", header.c_str(), path.c_str());
-  LOG(INFO) << "HTTP POST Command: " << command;
-  auto result = nebula::ProcessUtils::runCommand(command.c_str());
-  if (result.ok()) {
-    return result.value();
-  } else {
-    return Status::Error(folly::stringPrintf("Http Post Failed: %s", path.c_str()));
-  }
+CurlHandle::~CurlHandle() {
+  curl_global_cleanup();
 }
 
-StatusOr<std::string> HttpClient::post(const std::string& path,
-                                       const std::unordered_map<std::string, std::string>& header) {
-  folly::dynamic mapData = folly::dynamic::object;
-  // Build a dynamic object from map
-  for (auto const& it : header) {
-    mapData[it.first] = it.second;
-  }
-  return post(path, mapData);
+CurlHandle* CurlHandle::instance() {
+  static CurlHandle handle;
+  return &handle;
+}
+
+HttpResponse HttpClient::get(const std::string& url) {
+  return HttpClient::get(url, {});
+}
+
+HttpResponse HttpClient::get(const std::string& url, const std::vector<std::string>& header) {
+  return sendRequest(url, "GET", header, "");
 }
 
-StatusOr<std::string> HttpClient::post(const std::string& path, const folly::dynamic& data) {
-  return sendRequest(path, data, "POST");
+HttpResponse HttpClient::post(const std::string& url,
+                              const std::vector<std::string>& header,
+                              const std::string& body) {
+  return sendRequest(url, "POST", header, body);
 }
 
-StatusOr<std::string> HttpClient::put(const std::string& path,
-                                      const std::unordered_map<std::string, std::string>& header) {
-  folly::dynamic mapData = folly::dynamic::object;
-  // Build a dynamic object from map
-  for (auto const& it : header) {
-    mapData[it.first] = it.second;
+HttpResponse HttpClient::delete_(const std::string& url, const std::vector<std::string>& header) {
+  return sendRequest(url, "DELETE", header, "");
+}
+
+HttpResponse HttpClient::put(const std::string& url,
+                             const std::vector<std::string>& header,
+                             const std::string& body) {
+  return sendRequest(url, "PUT", header, body);
+}
+
+HttpResponse HttpClient::sendRequest(const std::string& url,
+                                     const std::string& method,
+                                     const std::vector<std::string>& header,
+                                     const std::string& body) {
+  CurlHandle::instance();
+  HttpResponse resp;
+  CURL* curl = curl_easy_init();
+  CHECK(curl);
+  setUrl(curl, url);
+  setMethod(curl, method);
+  curl_slist* h = setHeaders(curl, header);
+  if (!body.empty()) {
+    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
+  }
+  setRespHeader(curl, resp.header);
+  setRespBody(curl, resp.body);
+  setTimeout(curl);
+  resp.curlCode = curl_easy_perform(curl);
+  if (resp.curlCode != 0) {
+    resp.curlMessage = std::string(curl_easy_strerror(resp.curlCode));
   }
-  return put(path, mapData);
-}
-
-StatusOr<std::string> HttpClient::put(const std::string& path, const std::string& header) {
-  auto command =
-      folly::stringPrintf("/usr/bin/curl -X PUT %s \"%s\"", header.c_str(), path.c_str());
-  LOG(INFO) << "HTTP PUT Command: " << command;
-  auto result = nebula::ProcessUtils::runCommand(command.c_str());
-  if (result.ok()) {
-    return result.value();
-  } else {
-    return Status::Error(folly::stringPrintf("Http Put Failed: %s", path.c_str()));
+  if (h) {
+    curl_slist_free_all(h);
   }
+  curl_easy_cleanup(curl);
+  return resp;
 }
 
-StatusOr<std::string> HttpClient::put(const std::string& path, const folly::dynamic& data) {
-  return sendRequest(path, data, "PUT");
+void HttpClient::setUrl(CURL* curl, const std::string& url) {
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
 }
 
-StatusOr<std::string> HttpClient::sendRequest(const std::string& path,
-                                              const folly::dynamic& data,
-                                              const std::string& reqType) {
-  std::string command;
-  if (data.empty()) {
-    command = folly::stringPrintf("curl -X %s -s \"%s\"", reqType.c_str(), path.c_str());
-  } else {
-    command =
-        folly::stringPrintf("curl -X %s -H \"Content-Type: application/json\" -d'%s' -s \"%s\"",
-                            reqType.c_str(),
-                            folly::toJson(data).c_str(),
-                            path.c_str());
+void HttpClient::setMethod(CURL* curl, const std::string& method) {
+  curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method.c_str());
+}
+curl_slist* HttpClient::setHeaders(CURL* curl, const std::vector<std::string>& headers) {
+  curl_slist* h = nullptr;
+  for (auto& header : headers) {
+    h = curl_slist_append(h, header.c_str());
   }
-  LOG(INFO) << folly::stringPrintf("HTTP %s Command: %s", reqType.c_str(), command.c_str());
-  auto result = nebula::ProcessUtils::runCommand(command.c_str());
-  if (result.ok()) {
-    return result.value();
-  } else {
-    return Status::Error(folly::stringPrintf("Http %s Failed: %s", reqType.c_str(), path.c_str()));
+  if (h) {
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, h);
   }
+  return h;
 }
 
-}  // namespace http
+void HttpClient::setRespHeader(CURL* curl, const std::string& header) {
+  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, onWriteData);
+  curl_easy_setopt(curl, CURLOPT_HEADERDATA, &header);
+}
+
+void HttpClient::setRespBody(CURL* curl, const std::string& body) {
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, onWriteData);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &body);
+}
+
+void HttpClient::setTimeout(CURL* curl) {
+  curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+  curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 1);
+  curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
+}
+
+size_t HttpClient::onWriteData(void* ptr, size_t size, size_t nmemb, void* stream) {
+  if (ptr == nullptr || size == 0) {
+    return 0;
+  }
+  CHECK(stream);
+  size_t realsize = size * nmemb;
+  std::string* buffer = static_cast<std::string*>(stream);
+  CHECK(buffer);
+  buffer->append(static_cast<char*>(ptr), realsize);
+  return realsize;
+}
 }  // namespace nebula
diff --git a/src/common/http/HttpClient.h b/src/common/http/HttpClient.h
@@ -1,47 +1,63 @@
-/* Copyright (c) 2019 vesoft inc. All rights reserved.
+/* Copyright (c) 2022 vesoft inc. All rights reserved.
  *
  * This source code is licensed under Apache 2.0 License.
  */
 
-#ifndef COMMON_HTTPCLIENT_H
-#define COMMON_HTTPCLIENT_H
+#ifndef COMMON_HTTP_HTTPCLIENT_H_
+#define COMMON_HTTP_HTTPCLIENT_H_
+#include <memory>
+#include <mutex>
+#include <string>
 
-#include "common/base/Base.h"
 #include "common/base/StatusOr.h"
+#include "curl/curl.h"
 
 namespace nebula {
-namespace http {
+
+class CurlHandle {
+ public:
+  static CurlHandle* instance();
+
+ private:
+  CurlHandle();
+  ~CurlHandle();
+};
+
+struct HttpResponse {
+  CURLcode curlCode;
+  std::string curlMessage;
+  std::string header;
+  std::string body;
+};
 
 class HttpClient {
  public:
-  HttpClient() = delete;
-
-  ~HttpClient() = default;
-  // Send a http GET request
-  static StatusOr<std::string> get(const std::string& path, const std::string& options = "-G");
-
-  // Send a http POST request
-  static StatusOr<std::string> post(const std::string& path, const std::string& header);
-  // Send a http POST request with a different function signature
-  static StatusOr<std::string> post(const std::string& path,
-                                    const std::unordered_map<std::string, std::string>& header);
-  static StatusOr<std::string> post(const std::string& path,
-                                    const folly::dynamic& data = folly::dynamic::object());
-
-  // Send a http PUT request
-  static StatusOr<std::string> put(const std::string& path, const std::string& header);
-  static StatusOr<std::string> put(const std::string& path,
-                                   const std::unordered_map<std::string, std::string>& header);
-  static StatusOr<std::string> put(const std::string& path,
-                                   const folly::dynamic& data = folly::dynamic::object());
-
- protected:
-  static StatusOr<std::string> sendRequest(const std::string& path,
-                                           const folly::dynamic& data,
-                                           const std::string& reqType);
+  static HttpResponse get(const std::string& url);
+  static HttpResponse get(const std::string& url, const std::vector<std::string>& headers);
+
+  static HttpResponse post(const std::string& url,
+                           const std::vector<std::string>& headers,
+                           const std::string& body);
+  static HttpResponse delete_(const std::string& url, const std::vector<std::string>& headers);
+  static HttpResponse put(const std::string& url,
+                          const std::vector<std::string>& headers,
+                          const std::string& body);
+
+ private:
+  static HttpResponse sendRequest(const std::string& url,
+                                  const std::string& method,
+                                  const std::vector<std::string>& headers,
+                                  const std::string& body);
+  static void setUrl(CURL* curl, const std::string& url);
+  static void setMethod(CURL* curl, const std::string& method);
+  static curl_slist* setHeaders(CURL* curl, const std::vector<std::string>& headers);
+
+  static void setRespBody(CURL* curl, const std::string& body);
+  static void setRespHeader(CURL* curl, const std::string& header);
+  static void setTimeout(CURL* curl);
+  static size_t onWriteData(void* ptr, size_t size, size_t nmemb, void* stream);
 };
 
-}  // namespace http
 }  // namespace nebula
 
-#endif  // COMMON_HTTPCLIENT_H
+#endif
diff --git a/src/common/http/test/CMakeLists.txt b/src/common/http/test/CMakeLists.txt
@@ -8,19 +8,20 @@ nebula_add_test(
     SOURCES
         HttpClientTest.cpp
     OBJECTS
+        $<TARGET_OBJECTS:base_obj>
         $<TARGET_OBJECTS:http_client_obj>
         $<TARGET_OBJECTS:ws_obj>
         $<TARGET_OBJECTS:ws_common_obj>
         $<TARGET_OBJECTS:process_obj>
         $<TARGET_OBJECTS:fs_obj>
         $<TARGET_OBJECTS:stats_obj>
         $<TARGET_OBJECTS:time_obj>
-        $<TARGET_OBJECTS:base_obj>
         $<TARGET_OBJECTS:version_obj>
         $<TARGET_OBJECTS:datatypes_obj>
         $<TARGET_OBJECTS:wkt_wkb_io_obj>
     LIBRARIES
         ${PROXYGEN_LIBRARIES}
         gtest
-        gtest_main
+        curl
+
 )
diff --git a/src/common/http/test/HttpClientTest.cpp b/src/common/http/test/HttpClientTest.cpp
@@ -69,16 +69,16 @@ TEST(HttpClient, get) {
   {
     auto url =
         folly::stringPrintf("http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, "/path");
-    auto result = HttpClient::get(url);
-    ASSERT_TRUE(result.ok());
-    ASSERT_EQ("HttpClientHandler successfully", result.value());
+    auto httpResp = HttpClient::get(url);
+    ASSERT_EQ(httpResp.curlCode, 0);
+    ASSERT_EQ("HttpClientHandler successfully", httpResp.body);
   }
   {
     auto url = folly::stringPrintf(
         "http://%s:%d%s", FLAGS_ws_ip.c_str(), FLAGS_ws_http_port, "/not_exist");
-    auto result = HttpClient::get(url);
-    ASSERT_TRUE(result.ok());
-    ASSERT_TRUE(result.value().empty());
+    auto httpResp = HttpClient::get(url);
+    ASSERT_EQ(httpResp.curlCode, 0);
+    ASSERT_TRUE(httpResp.body.empty());
   }
 }
 

diff --git a/src/common/id/test/CMakeLists.txt b/src/common/id/test/CMakeLists.txt
@@ -9,7 +9,6 @@ nebula_add_executable(
       $<TARGET_OBJECTS:base_obj>
       $<TARGET_OBJECTS:snowflake_obj>
       $<TARGET_OBJECTS:meta_client_obj>
-      $<TARGET_OBJECTS:http_client_obj>
       $<TARGET_OBJECTS:thrift_obj>
       $<TARGET_OBJECTS:meta_obj>
       $<TARGET_OBJECTS:ssl_obj>
@@ -49,7 +48,6 @@ nebula_add_test(
     $<TARGET_OBJECTS:base_obj>
     $<TARGET_OBJECTS:snowflake_obj>
     $<TARGET_OBJECTS:meta_client_obj>
-    $<TARGET_OBJECTS:http_client_obj>
     $<TARGET_OBJECTS:thrift_obj>
     $<TARGET_OBJECTS:meta_obj>
     $<TARGET_OBJECTS:ssl_obj>

diff --git a/src/common/plugin/fulltext/FTUtils.h b/src/common/plugin/fulltext/FTUtils.h
@@ -15,7 +15,7 @@
 #include "common/base/CommonMacro.h"
 #include "common/datatypes/HostAddr.h"
 
-#define CURL "/usr/bin/curl"
+#define CURL_COMMAND "/usr/bin/curl"
 #define XPUT " -XPUT"
 #define XPOST " -XPOST"
 #define XGET " -XGET"

diff --git a/src/common/plugin/fulltext/elasticsearch/ESGraphAdapter.cpp b/src/common/plugin/fulltext/elasticsearch/ESGraphAdapter.cpp
@@ -71,7 +71,7 @@ StatusOr<bool> ESGraphAdapter::fuzzy(const HttpClient& client,
 
 std::string ESGraphAdapter::header() const noexcept {
   std::stringstream os;
-  os << CURL << CURL_CONTENT_JSON;
+  os << CURL_COMMAND << CURL_CONTENT_JSON;
   return os.str();
 }
 
@@ -81,7 +81,7 @@ std::string ESGraphAdapter::header(const HttpClient& client,
   //    curl -H "Content-Type: application/json; charset=utf-8"
   //    -XGET http://127.0.0.1:9200/my_temp_index_3/_search?timeout=10ms
   std::stringstream os;
-  os << CURL << CURL_CONTENT_JSON << XGET;
+  os << CURL_COMMAND << CURL_CONTENT_JSON << XGET;
   os << client.toString() << item.index << "/_search?timeout=" << limit.timeout_ << "ms"
      << "\"";
   return os.str();