Code for communicating with another host via HTTP or HTTPS (ad-freibu…

…rg#823)

Add a new class HttpClient which can open HTTP and HTTPS connections to a given host, and which can then send GET or POST requests, receive the result synchronously, and write it to a (potentially very large) string. Uses Boost.Beast, just like for our HttpServer. Includes code to parse a URL into its components (protocol, host, port, target).
    
Add a combined test for our HttpServer and HttpClient (for a meaningful test one needs a server and a client). So far only tests the HTTP case because our HttpServer does not do HTTPS yet.
    
Use the occasion to rename the directory util/HttpServer to util/http because httpServer was a misnomer already before, which became even more apparent with the addition of HttpClient. This change required many small changes (in the includes and in the various CMakeLists.txt).

.gitignore

@@ -41,5 +41,7 @@ e2e_data/*
 *.swp
 
 # ctags generated tag file
+tags
 src/tags
 /build/
+sourcelist

CMakeLists.txt

@@ -97,6 +97,12 @@ find_package(Boost 1.74 COMPONENTS iostreams program_options REQUIRED)
 include_directories(${Boost_INCLUDE_DIR})
 
 
+######################################
+# SSL
+######################################
+find_package(OpenSSL REQUIRED)
+
+
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 # set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

Dockerfile

@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 FROM base as builder
 RUN apt-get update && apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
-RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev libssl-dev
 RUN apt install -y libboost1.74-dev libboost-program-options1.74-dev libboost-iostreams1.74-dev
 
 COPY . /app/
@@ -23,7 +23,7 @@ FROM base as runtime
 WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime make
-RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev libssl-dev
 RUN apt install -y libboost1.74-dev libboost-program-options1.74-dev libboost-iostreams1.74-dev
 
 ARG UID=1000

Dockerfiles/Dockerfile.Ubuntu18.04

@@ -19,7 +19,7 @@ RUN apt-get update
 
 RUN apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
 RUN apt install -y gcc-11 g++-11
-RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev libssl-dev
 RUN apt install -y libboost1.74-dev
 
 COPY . /app/
@@ -35,7 +35,7 @@ FROM base as runtime
 WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime
-RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev libssl-dev
 
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app

Dockerfiles/Dockerfile.Ubuntu20.04

@@ -13,7 +13,7 @@ RUN apt-get update
 FROM base as builder
 RUN apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
 RUN apt install -y gcc-11 g++-11
-RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev libssl-dev
 RUN apt install -y libboost1.74-dev
 
 COPY . /app/
@@ -29,7 +29,7 @@ FROM base as runtime
 WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime
-RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev libssl-dev
 
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app

src/engine/CMakeLists.txt

@@ -40,4 +40,4 @@ add_library(engine
         VariableToColumnMap.cpp)
 
 
-target_link_libraries(engine index parser sparqlExpressions httpServer SortPerformanceEstimator absl::flat_hash_set ${ICU_LIBRARIES} boost_iostreams)
+target_link_libraries(engine index parser sparqlExpressions http SortPerformanceEstimator absl::flat_hash_set ${ICU_LIBRARIES} boost_iostreams)

src/engine/Server.cpp

@@ -1,8 +1,8 @@
-// Copyright 2011, University of Freiburg,
-// Chair of Algorithms and Data Structures.
-// Author:
-//   2011-2017 Björn Buchhold (buchhold@informatik.uni-freiburg.de)
-//   2018-     Johannes Kalmbach (kalmbach@informatik.uni-freiburg.de)
+// Copyright 2011 - 2022, University of Freiburg
+// Chair of Algorithms and Data Structures
+// Authors: Björn Buchhold <[email protected]>
+//          Johannes Kalmbach <kalmbach@cs.uni-freiburg.de>
+//          Hannah Bast <bast@cs.uni-freiburg.de>
 
 #include "engine/Server.h"
 
@@ -13,6 +13,7 @@
 
 #include "engine/QueryPlanner.h"
 #include "util/BoostHelpers/AsyncWaitForFuture.h"
+
 template <typename T>
 using Awaitable = Server::Awaitable<T>;
 
@@ -398,7 +399,7 @@ Awaitable<void> Server::process(
   // TODO: The file server currently does not LOG much. For example, when a file
   // is not found, a corresponding response is returned to the requestion
   // client, but the log says nothing about it. The place to change this would
-  // be in `src/util/HttpServer/HttpUtils.h`.
+  // be in `src/util/http/HttpUtils.h`.
   LOG(INFO) << "Treating request target \"" << request.target() << "\""
             << " as a request for a file with that name" << std::endl;
   auto serveFileRequest = makeFileServer(

src/engine/Server.h

@@ -1,7 +1,7 @@
-// Copyright 2021, University of Freiburg,
-// Chair of Algorithms and Data Structures.
-// Author: Björn Buchhold <buchholb>
-// Author: Johannes Kalmbach<kalmbach@cs.uni-freiburg.de>
+// Copyright 2021 - 2022, University of Freiburg
+// Chair of Algorithms and Data Structures
+// Authors: Johannes Kalmbach<[email protected]>
+//          Hannah Bast <bast@cs.uni-freiburg.de>
 
 #pragma once
 
@@ -17,9 +17,9 @@
 #include "parser/ParseException.h"
 #include "parser/SparqlParser.h"
 #include "util/AllocatorWithLimit.h"
-#include "util/HttpServer/HttpServer.h"
-#include "util/HttpServer/streamable_body.h"
 #include "util/Timer.h"
+#include "util/http/HttpServer.h"
+#include "util/http/streamable_body.h"
 
 using nlohmann::json;
 using std::string;

src/util/BoostHelpers/AsyncWaitForFuture.h

@@ -8,7 +8,7 @@
 
 #include <future>
 
-#include "../HttpServer/beast.h"
+#include "util/http/beast.h"
 
 namespace ad_utility::asio_helpers {
 

src/util/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_subdirectory(HttpServer)
+add_subdirectory(http)
 add_library(util GeoSparqlHelpers.h GeoSparqlHelpers.cpp VisitMixin.h
         antlr/ANTLRErrorHandling.cpp antlr/ANTLRErrorHandling.h Conversions.cpp)
 target_link_libraries(util absl::strings)

src/util/CompressorStream.h

@@ -14,8 +14,8 @@
 #include <boost/iostreams/filtering_stream.hpp>
 #include <string>
 
-#include "./Generator.h"
-#include "./HttpServer/ContentEncodingHelper.h"
+#include "util/Generator.h"
+#include "util/http/ContentEncodingHelper.h"
 
 namespace ad_utility::streams {
 namespace io = boost::iostreams;

src/util/http/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_subdirectory(HttpParser)
+add_library(mediaTypes MediaTypes.h MediaTypes.cpp)
+target_compile_options(mediaTypes PUBLIC -Wno-attributes)
+target_link_libraries(mediaTypes ${ICU_LIBRARIES} absl::flat_hash_map)
+add_library(http HttpServer.h HttpClient.h HttpClient.cpp HttpUtils.h HttpUtils.cpp UrlParser.h UrlParser.cpp "HttpParser/AcceptHeaderQleverVisitor.h")
+
+target_link_libraries(http mediaTypes httpParser ${ICU_LIBRARIES} OpenSSL::SSL OpenSSL::Crypto)

src/util/HttpServer/ContentEncodingHelper.h → src/util/http/ContentEncodingHelper.h

@@ -1,83 +1,83 @@
-// Copyright 2022, University of Freiburg,
-// Chair of Algorithms and Data Structures.
-// Author: Robin Textor-Falconi ([email protected])
-
-#pragma once
-
-#include <absl/strings/ascii.h>
-#include <absl/strings/str_split.h>
-
-#include "./beast.h"
-
-namespace ad_utility::content_encoding {
-
-enum class CompressionMethod { NONE, DEFLATE, GZIP };
-
-namespace detail {
-
-constexpr std::string_view DEFLATE = "deflate";
-constexpr std::string_view GZIP = "gzip";
-
-inline CompressionMethod getCompressionMethodFromAcceptEncodingHeader(
-    std::vector<std::string_view> acceptedEncodings) {
-  const auto& contains = [&acceptedEncodings](std::string_view value) {
-    return std::find(acceptedEncodings.begin(), acceptedEncodings.end(),
-                     value) != acceptedEncodings.end();
-  };
-  if (contains(DEFLATE)) {
-    return CompressionMethod::DEFLATE;
-  } else if (contains(GZIP)) {
-    return CompressionMethod::GZIP;
-  }
-  return CompressionMethod::NONE;
-}
-}  // namespace detail
-
-using boost::beast::http::field;
-
-template <typename Body>
-CompressionMethod getCompressionMethodForRequest(
-    const boost::beast::http::request<Body>& request) {
-  auto acceptEncodingHeaders =
-      request.base().equal_range(field::accept_encoding);
-  std::vector<std::string_view> acceptedHeaders;
-  for (auto& current = acceptEncodingHeaders.first;
-       current != acceptEncodingHeaders.second; current++) {
-    std::string_view headerContent = current->value();
-    std::vector<std::string_view> tokens =
-        absl::StrSplit(headerContent, absl::ByChar(','));
-    for (auto token : tokens) {
-      acceptedHeaders.push_back(absl::StripAsciiWhitespace(token));
-    }
-  }
-  return detail::getCompressionMethodFromAcceptEncodingHeader(acceptedHeaders);
-}
-
-template <bool isRequest, typename Fields>
-void setContentEncodingHeaderForCompressionMethod(
-    CompressionMethod method,
-    boost::beast::http::header<isRequest, Fields>& header) {
-  if (method == CompressionMethod::DEFLATE) {
-    header.insert(field::content_encoding, detail::DEFLATE);
-  } else if (method == CompressionMethod::GZIP) {
-    header.insert(field::content_encoding, detail::GZIP);
-  }
-}
-
-inline std::ostream& operator<<(std::ostream& out,
-                                CompressionMethod compressionMethod) {
-  switch (compressionMethod) {
-    case CompressionMethod::NONE:
-      out << "CompressionMethod::NONE";
-      break;
-    case CompressionMethod::DEFLATE:
-      out << "CompressionMethod::DEFLATE";
-      break;
-    case CompressionMethod::GZIP:
-      out << "CompressionMethod::GZIP";
-      break;
-  }
-  return out;
-}
-
-}  // namespace ad_utility::content_encoding
+// Copyright 2022, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author: Robin Textor-Falconi ([email protected])
+
+#pragma once
+
+#include <absl/strings/ascii.h>
+#include <absl/strings/str_split.h>
+
+#include "util/http/beast.h"
+
+namespace ad_utility::content_encoding {
+
+enum class CompressionMethod { NONE, DEFLATE, GZIP };
+
+namespace detail {
+
+constexpr std::string_view DEFLATE = "deflate";
+constexpr std::string_view GZIP = "gzip";
+
+inline CompressionMethod getCompressionMethodFromAcceptEncodingHeader(
+    std::vector<std::string_view> acceptedEncodings) {
+  const auto& contains = [&acceptedEncodings](std::string_view value) {
+    return std::find(acceptedEncodings.begin(), acceptedEncodings.end(),
+                     value) != acceptedEncodings.end();
+  };
+  if (contains(DEFLATE)) {
+    return CompressionMethod::DEFLATE;
+  } else if (contains(GZIP)) {
+    return CompressionMethod::GZIP;
+  }
+  return CompressionMethod::NONE;
+}
+}  // namespace detail
+
+using boost::beast::http::field;
+
+template <typename Body>
+CompressionMethod getCompressionMethodForRequest(
+    const boost::beast::http::request<Body>& request) {
+  auto acceptEncodingHeaders =
+      request.base().equal_range(field::accept_encoding);
+  std::vector<std::string_view> acceptedHeaders;
+  for (auto& current = acceptEncodingHeaders.first;
+       current != acceptEncodingHeaders.second; current++) {
+    std::string_view headerContent = current->value();
+    std::vector<std::string_view> tokens =
+        absl::StrSplit(headerContent, absl::ByChar(','));
+    for (auto token : tokens) {
+      acceptedHeaders.push_back(absl::StripAsciiWhitespace(token));
+    }
+  }
+  return detail::getCompressionMethodFromAcceptEncodingHeader(acceptedHeaders);
+}
+
+template <bool isRequest, typename Fields>
+void setContentEncodingHeaderForCompressionMethod(
+    CompressionMethod method,
+    boost::beast::http::header<isRequest, Fields>& header) {
+  if (method == CompressionMethod::DEFLATE) {
+    header.insert(field::content_encoding, detail::DEFLATE);
+  } else if (method == CompressionMethod::GZIP) {
+    header.insert(field::content_encoding, detail::GZIP);
+  }
+}
+
+inline std::ostream& operator<<(std::ostream& out,
+                                CompressionMethod compressionMethod) {
+  switch (compressionMethod) {
+    case CompressionMethod::NONE:
+      out << "CompressionMethod::NONE";
+      break;
+    case CompressionMethod::DEFLATE:
+      out << "CompressionMethod::DEFLATE";
+      break;
+    case CompressionMethod::GZIP:
+      out << "CompressionMethod::GZIP";
+      break;
+  }
+  return out;
+}
+
+}  // namespace ad_utility::content_encoding