Add secondary HTTP storage

Fixes: ccache#858

.github/workflows/build.yaml

@@ -93,9 +93,9 @@ jobs:
 
             # Install ld.gold (binutils) and ld.lld on different runs.
             if [ "${{ matrix.config.os }}" = "ubuntu-18.04" ]; then
-              sudo apt-get install -y ninja-build elfutils libzstd-dev binutils
+              sudo apt-get install -y ninja-build elfutils libzstd-dev binutils python3
             else
-              sudo apt-get install -y ninja-build elfutils libzstd-dev lld
+              sudo apt-get install -y ninja-build elfutils libzstd-dev lld python3
             fi
 
             if [ "${{ matrix.config.compiler }}" = "gcc" ]; then

doc/MANUAL.adoc

@@ -796,6 +796,7 @@ Examples:
 +
 * `file:///shared/nfs/directory`
 * `file:///shared/nfs/one|read-only file:///shared/nfs/two`
+* `http://example.org/cache`
 
 [[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*)::
 
@@ -931,6 +932,27 @@ Optional attributes:
 * *update-mtime*: If *true*, update the modification time (mtime) of cache
   entries that are read. The default is *false*.
 
+=== HTTP storage backend
+
+URL format: `http://HOST[:PORT][/PATH]`
+
+This backend stores data in an HTTP compatible server. The required HTTP
+methods are `GET`, `PUT` and `DELETE`.
+
+Note that ccache will not perform any cleanup of the HTTP storage.
+
+Examples:
+
+* `http://localhost:8080/`
+* `http://example.org/cache`
+
+Known issues and limitations:
+
+* URLs containing IPv6 addresses like `http://[::1]/` are not supported
+* There are no HTTP timeouts implemented or configured
+* Authentication is not yet supported
+* HTTPS is not yet supported
+
 == Cache size management
 
 By default, ccache has a 5 GB limit on the total size of files in the cache and

src/ccache.hpp

@@ -28,6 +28,7 @@
 
 class Context;
 
+extern const char CCACHE_NAME[];
 extern const char CCACHE_VERSION[];
 
 using FindExecutableFunction =

src/storage/Storage.cpp

@@ -25,6 +25,7 @@
 #include <assertions.hpp>
 #include <fmtmacros.hpp>
 #include <storage/secondary/FileStorage.hpp>
+#include <storage/secondary/HttpStorage.hpp>
 #include <util/Tokenizer.hpp>
 #include <util/string_utils.hpp>
 
@@ -239,6 +240,9 @@ create_storage(const ParseStorageEntryResult& storage_entry)
   if (storage_entry.url.scheme() == "file") {
     return std::make_unique<secondary::FileStorage>(storage_entry.url,
                                                     storage_entry.attributes);
+  } else if (storage_entry.url.scheme() == "http") {
+    return std::make_unique<secondary::HttpStorage>(storage_entry.url,
+                                                    storage_entry.attributes);
   }
 
   return {};

src/storage/secondary/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(
   sources
   ${CMAKE_CURRENT_SOURCE_DIR}/FileStorage.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/HttpStorage.cpp
 )
 
 target_sources(ccache_lib PRIVATE ${sources})

src/storage/secondary/HttpStorage.cpp

@@ -0,0 +1,215 @@
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "HttpStorage.hpp"
+
+#include <Digest.hpp>
+#include <Logging.hpp>
+#include <Util.hpp>
+#include <ccache.hpp>
+#include <exceptions.hpp>
+#include <fmtmacros.hpp>
+
+#include <third_party/httplib.h>
+#include <third_party/nonstd/string_view.hpp>
+#include <third_party/url.hpp>
+
+namespace storage {
+namespace secondary {
+
+namespace {
+
+nonstd::string_view
+to_string(httplib::Error error)
+{
+  using httplib::Error;
+
+  switch (error) {
+  case Error::Success:
+    return "Success";
+  case Error::Connection:
+    return "Connection";
+  case Error::BindIPAddress:
+    return "BindIPAddress";
+  case Error::Read:
+    return "Read";
+  case Error::Write:
+    return "Write";
+  case Error::ExceedRedirectCount:
+    return "ExceedRedirectCount";
+  case Error::Canceled:
+    return "Canceled";
+  case Error::SSLConnection:
+    return "SSLConnection";
+  case Error::SSLLoadingCerts:
+    return "SSLLoadingCerts";
+  case Error::SSLServerVerification:
+    return "SSLServerVerification";
+  case Error::UnsupportedMultipartBoundaryChars:
+    return "UnsupportedMultipartBoundaryChars";
+  case Error::Compression:
+    return "Compression";
+  case Error::Unknown:
+    break;
+  }
+
+  return "Unknown";
+}
+
+int
+get_url_port(const Url& url)
+{
+  if (!url.port().empty()) {
+    return Util::parse_unsigned(url.port(), 1, 65535, "port");
+  }
+  if (url.scheme() == "http") {
+    return 80;
+  } else {
+    throw Error("Unknown scheme: {}", url.scheme());
+  }
+}
+
+std::string
+get_url_path(const Url& url)
+{
+  auto path = url.path();
+  if (path.empty() || path.back() != '/') {
+    path += '/';
+  }
+  return path;
+}
+
+} // namespace
+
+HttpStorage::HttpStorage(const Url& url, const AttributeMap&)
+  : m_url_path(get_url_path(url)),
+    m_http_client(
+      std::make_unique<httplib::Client>(url.host(), get_url_port(url)))
+{
+  m_http_client->set_default_headers(
+    {{"User-Agent", FMT("{}/{}", CCACHE_NAME, CCACHE_VERSION)}});
+  m_http_client->set_keep_alive(true);
+}
+
+HttpStorage::~HttpStorage() = default;
+
+nonstd::expected<nonstd::optional<std::string>, SecondaryStorage::Error>
+HttpStorage::get(const Digest& key)
+{
+  const auto url_path = get_entry_path(key);
+
+  const auto result = m_http_client->Get(url_path.c_str());
+
+  if (result.error() != httplib::Error::Success || !result) {
+    LOG("Failed to get {} from http storage: {} ({})",
+        url_path,
+        to_string(result.error()),
+        result.error());
+    return nonstd::make_unexpected(Error::error);
+  }
+
+  if (result->status < 200 || result->status >= 300) {
+    // Don't log failure if the entry doesn't exist.
+    return nonstd::nullopt;
+  }
+
+  return result->body;
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+HttpStorage::put(const Digest& key,
+                 const std::string& value,
+                 bool only_if_missing)
+{
+  const auto url_path = get_entry_path(key);
+
+  if (only_if_missing) {
+    const auto result = m_http_client->Head(url_path.c_str());
+
+    if (result.error() != httplib::Error::Success || !result) {
+      LOG("Failed to check for {} in http storage: {} ({})",
+          url_path,
+          to_string(result.error()),
+          result.error());
+      return nonstd::make_unexpected(Error::error);
+    }
+
+    if (result->status >= 200 && result->status < 300) {
+      LOG("Found entry {} already within http storage: status code: {}",
+          url_path,
+          result->status);
+      return false;
+    }
+  }
+
+  const auto content_type = "application/octet-stream";
+
+  const auto result = m_http_client->Put(
+    url_path.c_str(), value.data(), value.size(), content_type);
+
+  if (result.error() != httplib::Error::Success || !result) {
+    LOG("Failed to put {} to http storage: {} ({})",
+        url_path,
+        to_string(result.error()),
+        result.error());
+    return nonstd::make_unexpected(Error::error);
+  }
+
+  if (result->status < 200 || result->status >= 300) {
+    LOG("Failed to put {} to http storage: status code: {}",
+        url_path,
+        result->status);
+    return nonstd::make_unexpected(Error::error);
+  }
+
+  return true;
+}
+
+nonstd::expected<bool, SecondaryStorage::Error>
+HttpStorage::remove(const Digest& key)
+{
+  const auto url_path = get_entry_path(key);
+
+  const auto result = m_http_client->Delete(url_path.c_str());
+
+  if (result.error() != httplib::Error::Success || !result) {
+    LOG("Failed to delete {} from http storage: {} ({})",
+        url_path,
+        to_string(result.error()),
+        result.error());
+    return nonstd::make_unexpected(Error::error);
+  }
+
+  if (result->status < 200 || result->status >= 300) {
+    LOG("Failed to delete {} from http storage: status code: {}",
+        url_path,
+        result->status);
+    return nonstd::make_unexpected(Error::error);
+  }
+
+  return true;
+}
+
+std::string
+HttpStorage::get_entry_path(const Digest& key) const
+{
+  return m_url_path + key.to_string();
+}
+
+} // namespace secondary
+} // namespace storage

src/storage/secondary/HttpStorage.hpp

@@ -0,0 +1,56 @@
+// Copyright (C) 2021 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <storage/SecondaryStorage.hpp>
+#include <storage/types.hpp>
+
+#include <memory>
+#include <string>
+
+class Url;
+namespace httplib {
+class Client;
+}
+
+namespace storage {
+namespace secondary {
+
+class HttpStorage : public storage::SecondaryStorage
+{
+public:
+  HttpStorage(const Url& url, const AttributeMap& attributes);
+  ~HttpStorage() override;
+
+  nonstd::expected<nonstd::optional<std::string>, Error>
+  get(const Digest& key) override;
+  nonstd::expected<bool, Error> put(const Digest& key,
+                                    const std::string& value,
+                                    bool only_if_missing) override;
+  nonstd::expected<bool, Error> remove(const Digest& key) override;
+
+private:
+  const std::string m_url_path;
+  std::unique_ptr<httplib::Client> m_http_client;
+
+  std::string get_entry_path(const Digest& key) const;
+};
+
+} // namespace secondary
+} // namespace storage

test/CMakeLists.txt

@@ -53,6 +53,7 @@ addtest(readonly)
 addtest(readonly_direct)
 addtest(sanitize_blacklist)
 addtest(secondary_file)
+addtest(secondary_http)
 addtest(secondary_url)
 addtest(serialize_diagnostics)
 addtest(source_date_epoch)