[Hotfix] Add database filter to Server ExternalCatalog (#2414)

* [Hotfix] Add database filter to Server ExternalCatalog * [Hotfix] Add database filter to Server ExternalCatalog * Rename config database.filter-regular-expression to database-filter --------- Co-authored-by: baiyangtx <[email protected]>
apache · Dec 14, 2023 · 62ea8d2 · 62ea8d2
1 parent 95716bf
commit 62ea8d2
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 11 deletions.
diff --git a/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java b/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java
@@ -48,13 +48,8 @@ public class CatalogMetaProperties {
   // Deprecated from version v0.4.0, use KEY_WAREHOUSE
   @Deprecated public static final String KEY_WAREHOUSE_DIR = "warehouse.dir";
   public static final String KEY_WAREHOUSE = "warehouse";
-  /**
-   * @deprecated since 0.6.0, will be removed in 0.7.0; use {@link
-   *     CatalogMetaProperties#KEY_TABLE_FILTER} instead.
-   */
-  @Deprecated
-  public static final String KEY_DATABASE_FILTER_REGULAR_EXPRESSION =
-      "database.filter-regular-expression";
+
+  public static final String KEY_DATABASE_FILTER = "database-filter";
 
   public static final String KEY_TABLE_FILTER = "table-filter";
 

diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java
@@ -24,6 +24,7 @@ public class ExternalCatalog extends ServerCatalog {
   UnifiedCatalog unifiedCatalog;
   TableMetaStore tableMetaStore;
   private Pattern tableFilterPattern;
+  private Pattern databaseFilterPattern;
 
   protected ExternalCatalog(CatalogMeta metadata) {
     super(metadata);
@@ -32,6 +33,7 @@ protected ExternalCatalog(CatalogMeta metadata) {
         this.tableMetaStore.doAs(
             () -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap()));
     updateTableFilter(metadata);
+    updateDatabaseFilter(metadata);
   }
 
   public void syncTable(String database, String tableName, TableFormat format) {
@@ -58,6 +60,7 @@ public void updateMetadata(CatalogMeta metadata) {
     super.updateMetadata(metadata);
     this.tableMetaStore = CatalogUtil.buildMetaStore(metadata);
     this.unifiedCatalog.refresh();
+    updateDatabaseFilter(metadata);
     updateTableFilter(metadata);
   }
 
@@ -73,7 +76,14 @@ public boolean exist(String database, String tableName) {
 
   @Override
   public List<String> listDatabases() {
-    return doAs(() -> unifiedCatalog.listDatabases());
+    return doAs(
+        () ->
+            unifiedCatalog.listDatabases().stream()
+                .filter(
+                    database ->
+                        databaseFilterPattern == null
+                            || databaseFilterPattern.matcher(database).matches())
+                .collect(Collectors.toList()));
   }
 
   @Override
@@ -109,6 +119,16 @@ public AmoroTable<?> loadTable(String database, String tableName) {
     return doAs(() -> unifiedCatalog.loadTable(database, tableName));
   }
 
+  private void updateDatabaseFilter(CatalogMeta metadata) {
+    String databaseFilter =
+        metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_DATABASE_FILTER);
+    if (databaseFilter != null) {
+      databaseFilterPattern = Pattern.compile(databaseFilter);
+    } else {
+      databaseFilterPattern = null;
+    }
+  }
+
   private void updateTableFilter(CatalogMeta metadata) {
     String tableFilter =
         metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER);

diff --git a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java
@@ -97,9 +97,8 @@ public void initialize(String name, Map<String, String> properties, TableMetaSto
       cacheEnabled = false;
     }
     Pattern databaseFilterPattern = null;
-    if (properties.containsKey(CatalogMetaProperties.KEY_DATABASE_FILTER_REGULAR_EXPRESSION)) {
-      String databaseFilter =
-          properties.get(CatalogMetaProperties.KEY_DATABASE_FILTER_REGULAR_EXPRESSION);
+    if (properties.containsKey(CatalogMetaProperties.KEY_DATABASE_FILTER)) {
+      String databaseFilter = properties.get(CatalogMetaProperties.KEY_DATABASE_FILTER);
       databaseFilterPattern = Pattern.compile(databaseFilter);
     }
     Catalog catalog = buildIcebergCatalog(name, properties, metaStore.getConfiguration());

diff --git a/docs/admin-guides/managing-catalogs.md b/docs/admin-guides/managing-catalogs.md
@@ -53,6 +53,7 @@ You can create a catalog in the AMS frontend:
 Common properties include:
 - warehouse: Warehouse **must be configured** for ams/hadoop/glue catalog, as it determines where our database and table files should be placed
 - catalog-impl: when the metastore is **Custom**, an additional catalog-impl must be defined, and the user must put the jar package for the custom catalog implementation into the **{AMORO_HOME}/lib** directory, **and the service must be restarted to take effect**
+- database-filter: Configure a regular expression to filter databases in the catalog. If not set then all databases will be displayed in table menu.
 - table-filter: Configure a regular expression to filter tables in the catalog. The matching will be done in the format of `database.table`. For example, if it is set to `(A\.a)|(B\.b)`, it will ignore all tables except for table `a` in database `A` and table `b` in database `B`
 
 ### Configure table properties