Split UserDefinedSQLObjectsLoader to interface and implementation.

Author: vitlibar

programs/local/LocalServer.cpp

@@ -13,7 +13,6 @@
 #include <Interpreters/DatabaseCatalog.h>
 #include <base/getFQDNOrHostName.h>
 #include <Common/scope_guard_safe.h>
-#include <Interpreters/UserDefinedSQLObjectsLoader.h>
 #include <Interpreters/Session.h>
 #include <Access/AccessControl.h>
 #include <Common/Exception.h>
@@ -32,6 +31,7 @@
 #include <Parsers/IAST.h>
 #include <Parsers/ASTInsertQuery.h>
 #include <Common/ErrorHandlers.h>
+#include <Functions/UserDefined/IUserDefinedSQLObjectsLoader.h>
 #include <Functions/registerFunctions.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
@@ -602,21 +602,13 @@ void LocalServer::processConfig()
     global_context->setCurrentDatabase(default_database);
     applyCmdOptions(global_context);
 
-    bool enable_objects_loader = false;
-
     if (config().has("path"))
     {
         String path = global_context->getPath();
 
         /// Lock path directory before read
         status.emplace(fs::path(path) / "status", StatusFile::write_full_info);
 
-        LOG_DEBUG(log, "Loading user defined objects from {}", path);
-        Poco::File(path + "user_defined/").createDirectories();
-        UserDefinedSQLObjectsLoader::instance().loadObjects(global_context);
-        enable_objects_loader = true;
-        LOG_DEBUG(log, "Loaded user defined objects.");
-
         LOG_DEBUG(log, "Loading metadata from {}", path);
         loadMetadataSystem(global_context);
         attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
@@ -630,6 +622,9 @@ void LocalServer::processConfig()
             DatabaseCatalog::instance().loadDatabases();
         }
 
+        /// For ClickHouse local if path is not set the loader will be disabled.
+        global_context->getUserDefinedSQLObjectsLoader().loadObjects();
+
         LOG_DEBUG(log, "Loaded metadata.");
     }
     else if (!config().has("no-system-tables"))
@@ -639,9 +634,6 @@ void LocalServer::processConfig()
         attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
     }
 
-    /// Persist SQL user defined objects only if user_defined folder was created
-    UserDefinedSQLObjectsLoader::instance().enable(enable_objects_loader);
-
     server_display_name = config().getString("display_name", getFQDNOrHostName());
     prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
     std::map<String, String> prompt_substitutions{{"display_name", server_display_name}};

programs/server/Server.cpp

@@ -53,7 +53,6 @@
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/ProcessList.h>
 #include <Interpreters/loadMetadata.h>
-#include <Interpreters/UserDefinedSQLObjectsLoader.h>
 #include <Interpreters/JIT/CompiledExpressionCache.h>
 #include <Access/AccessControl.h>
 #include <Storages/StorageReplicatedMergeTree.h>
@@ -62,6 +61,7 @@
 #include <Storages/Cache/ExternalDataSourceCache.h>
 #include <Storages/Cache/registerRemoteFileMetadatas.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
+#include <Functions/UserDefined/IUserDefinedSQLObjectsLoader.h>
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Formats/registerFormats.h>
@@ -1007,12 +1007,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
         fs::create_directories(user_scripts_path);
     }
 
-    {
-        std::string user_defined_path = config().getString("user_defined_path", path / "user_defined/");
-        global_context->setUserDefinedPath(user_defined_path);
-        fs::create_directories(user_defined_path);
-    }
-
     /// top_level_domains_lists
     {
         const std::string & top_level_domains_path = config().getString("top_level_domains_path", path / "top_level_domains/");
@@ -1551,18 +1545,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
     /// system logs may copy global context.
     global_context->setCurrentDatabaseNameInGlobalContext(default_database);
 
-    LOG_INFO(log, "Loading user defined objects from {}", path_str);
-    try
-    {
-        UserDefinedSQLObjectsLoader::instance().loadObjects(global_context);
-    }
-    catch (...)
-    {
-        tryLogCurrentException(log, "Caught exception while loading user defined objects");
-        throw;
-    }
-    LOG_DEBUG(log, "Loaded user defined objects");
-
     LOG_INFO(log, "Loading metadata from {}", path_str);
 
     try
@@ -1590,6 +1572,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
         database_catalog.loadDatabases();
         /// After loading validate that default database exists
         database_catalog.assertDatabaseExists(default_database);
+        /// Load user-defined SQL functions.
+        global_context->getUserDefinedSQLObjectsLoader().loadObjects();
     }
     catch (...)
     {

src/CMakeLists.txt

@@ -284,6 +284,7 @@ add_object_library(clickhouse_processors_ttl Processors/TTL)
 add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Algorithms)
 add_object_library(clickhouse_processors_queryplan Processors/QueryPlan)
 add_object_library(clickhouse_processors_queryplan_optimizations Processors/QueryPlan/Optimizations)
+add_object_library(clickhouse_user_defined_functions Functions/UserDefined)
 
 if (TARGET ch_contrib::nuraft)
     add_object_library(clickhouse_coordination Coordination)

src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp

@@ -5,8 +5,8 @@
 
 #include <DataTypes/DataTypeFactory.h>
 
-#include <Interpreters/UserDefinedExecutableFunction.h>
-#include <Interpreters/UserDefinedExecutableFunctionFactory.h>
+#include <Functions/UserDefined/UserDefinedExecutableFunction.h>
+#include <Functions/UserDefined/UserDefinedExecutableFunctionFactory.h>
 #include <Functions/FunctionFactory.h>
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 

src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.h src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h

@@ -4,7 +4,7 @@
 
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/ExternalLoader.h>
-#include <Interpreters/UserDefinedExecutableFunction.h>
+#include <Functions/UserDefined/UserDefinedExecutableFunction.h>
 
 namespace DB
 {

src/Functions/UserDefined/IUserDefinedSQLObjectsLoader.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <base/types.h>
+
+
+namespace DB
+{
+class IAST;
+struct Settings;
+enum class UserDefinedSQLObjectType;
+
+/// Interface for a loader of user-defined SQL objects.
+/// Implementations: UserDefinedSQLLoaderFromDisk, UserDefinedSQLLoaderFromZooKeeper
+class IUserDefinedSQLObjectsLoader
+{
+public:
+    virtual ~IUserDefinedSQLObjectsLoader() = default;
+
+    /// Whether this loader can replicate SQL objects to another node.
+    virtual bool isReplicated() const { return false; }
+    virtual String getReplicationID() const { return ""; }
+
+    /// Loads all objects. Can be called once - if objects are already loaded the function does nothing.
+    virtual void loadObjects() = 0;
+
+    /// Stops watching.
+    virtual void stopWatching() {}
+
+    /// Immediately reloads all objects, throws an exception if failed.
+    virtual void reloadObjects() = 0;
+
+    /// Immediately reloads a specified object only.
+    virtual void reloadObject(UserDefinedSQLObjectType object_type, const String & object_name) = 0;
+
+    /// Stores an object (must be called only by UserDefinedSQLFunctionFactory::registerFunction).
+    virtual bool storeObject(
+        UserDefinedSQLObjectType object_type,
+        const String & object_name,
+        const IAST & create_object_query,
+        bool throw_if_exists,
+        bool replace_if_exists,
+        const Settings & settings) = 0;
+
+    /// Removes an object (must be called only by UserDefinedSQLFunctionFactory::unregisterFunction).
+    virtual bool removeObject(UserDefinedSQLObjectType object_type, const String & object_name, bool throw_if_not_exists) = 0;
+};
+}

src/Interpreters/UserDefinedExecutableFunction.cpp src/Functions/UserDefined/UserDefinedExecutableFunction.cpp

@@ -12,9 +12,9 @@
 
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
+#include <Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.h>
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Interpreters/convertFieldToType.h>
-#include <Interpreters/ExternalUserDefinedExecutableFunctionsLoader.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/castColumn.h>