Adding type parameter to tag list. (Netflix#310)

* Enhancing tag list with type paramter * Update iceberg to 0.5.1
kmccarp · Nov 20, 2018 · f36936a · f36936a
1 parent b9f540e
commit f36936a
Show file tree

Hide file tree

Showing 10 changed files with 233 additions and 49 deletions.
diff --git a/build.gradle b/build.gradle
@@ -150,9 +150,9 @@ configure(javaProjects) {
             dependency("org.elasticsearch.client:transport:5.4.1")
             dependency("net.snowflake:snowflake-jdbc:3.4.2")
             dependency("com.esotericsoftware.kryo:kryo:2.22")
-            dependency("com.github.Netflix.iceberg:iceberg-common:0.5.0")
-            dependency("com.github.Netflix.iceberg:iceberg-core:0.5.0")
-            dependency("com.github.Netflix.iceberg:iceberg-api:0.5.0")
+            dependency("com.github.Netflix.iceberg:iceberg-common:0.5.1")
+            dependency("com.github.Netflix.iceberg:iceberg-core:0.5.1")
+            dependency("com.github.Netflix.iceberg:iceberg-api:0.5.1")
         }
     }
 

diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/api/TagV1.java b/metacat-client/src/main/java/com/netflix/metacat/client/api/TagV1.java
@@ -62,6 +62,7 @@ public interface TagV1 {
      * @param sourceName   Prefix of the source name
      * @param databaseName Prefix of the database name
      * @param tableName    Prefix of the table name
+     * @param type         Qualified name type category, database, table
      * @return list of qualified names
      */
     @GET
@@ -78,7 +79,9 @@ List<QualifiedName> list(
         @QueryParam("databaseName")
             String databaseName,
         @QueryParam("tableName")
-            String tableName
+            String tableName,
+        @QueryParam("type")
+            QualifiedName.Type type
     );
 
     /**

diff --git a/...ommon-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TagService.java b/...ommon-server/src/main/java/com/netflix/metacat/common/server/usermetadata/TagService.java
@@ -28,6 +28,7 @@
  * Tag Service API.
  *
  * @author amajumdar
+ * @author zhenl
  */
 public interface TagService {
     /**
@@ -48,14 +49,16 @@ default Set<String> getTags() {
      * @param sourceName   catalog/source name
      * @param databaseName database name
      * @param tableName    table name
+     * @param type metacat qualified name type
      * @return list of qualified names of the items
      */
     default List<QualifiedName> list(
-        @Nullable Set<String> includeTags,
-        @Nullable Set<String> excludeTags,
-        @Nullable String sourceName,
-        @Nullable String databaseName,
-        @Nullable String tableName
+        @Nullable final Set<String> includeTags,
+        @Nullable final Set<String> excludeTags,
+        @Nullable final String sourceName,
+        @Nullable final String databaseName,
+        @Nullable final String tableName,
+        @Nullable final QualifiedName.Type type
     ) {
         return Collections.emptyList();
     }
@@ -70,10 +73,10 @@ default List<QualifiedName> list(
      * @return list of qualified names of the items
      */
     default List<QualifiedName> search(
-        @Nullable String tag,
-        @Nullable String sourceName,
-        @Nullable String databaseName,
-        @Nullable String tableName
+        @Nullable final String tag,
+        @Nullable final String sourceName,
+        @Nullable final String databaseName,
+        @Nullable final String tableName
     ) {
         return Collections.emptyList();
     }

diff --git a/metacat-common/src/main/java/com/netflix/metacat/common/QualifiedName.java b/metacat-common/src/main/java/com/netflix/metacat/common/QualifiedName.java
@@ -26,6 +26,7 @@
 
 import javax.annotation.Nullable;
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -326,6 +327,48 @@ public static String toWildCardString(
         return builder.toString();
     }
 
+    /**
+     * Change the qualified name query parameter to wildcard query string to allow source/database/table
+     * like queries. It uses '%' to represent the other field if not provided. e.g.
+     * query database like string is '%/database/%'
+     * query catalog and database like string is 'catalog/database/%'
+     *
+     * @param sourceName   source name
+     * @param databaseName database name
+     * @param tableName    table name
+     * @return query string
+     */
+    public static String qualifiedNameToWildCardQueryString(
+        @Nullable final String sourceName,
+        @Nullable final String databaseName,
+        @Nullable final String tableName
+    ) {
+        if (sourceName == null && databaseName == null && tableName == null) {
+            return null;
+        }
+        final StringBuilder builder = new StringBuilder();
+        if (!isNullOrEmpty(sourceName)) {
+            builder.append(sourceName);
+        } else {
+            builder.append('%');
+        }
+        if (isNullOrEmpty(databaseName) && isNullOrEmpty(tableName)) {
+            return builder.append('%').toString(); //query source level
+        }
+        if (!isNullOrEmpty(databaseName)) {
+            builder.append('/').append(databaseName);
+        } else {
+            builder.append("/%");
+        }
+        if (isNullOrEmpty(tableName)) {
+            return builder.append('%').toString(); //database level query
+        } else {
+            builder.append('/').append(tableName);
+        }
+        builder.append('%');
+        return builder.toString();
+    }
+
     /**
      * Get the catalog name.
      *
@@ -550,33 +593,76 @@ public String toString() {
         return qualifiedName;
     }
 
+    /**
+     * Checks if a CharSequence is empty ("") or null.
+     */
+    private static boolean isNullOrEmpty(@Nullable final CharSequence cs) {
+        return cs == null || cs.length() == 0;
+    }
+
     /**
      * Type of the connector resource.
      */
     public enum Type {
         /**
          * Catalog type.
          */
-        CATALOG,
+        CATALOG("^([^\\/]+)$"),
 
         /**
          * Database type.
          */
-        DATABASE,
+        DATABASE("^([^\\/]+)\\/([^\\/]+)$"),
 
         /**
          * Table type.
          */
-        TABLE,
+        TABLE("^([^\\/]+)\\/([^\\/]+)\\/([^\\/]+)$"),
 
         /**
          * Partition type.
          */
-        PARTITION,
+        PARTITION("^(.*)$"),
 
         /**
          * MView type.
          */
-        MVIEW
+        MVIEW("^([^\\/]+)\\/([^\\/]+)\\/([^\\/]+)\\/([^\\/]+)$");
+
+        private final String regexValue;
+
+        /**
+         * Constructor.
+         *
+         * @param value category value.
+         */
+        Type(final String value) {
+            this.regexValue = value;
+        }
+
+        /**
+         * get Regex Value.
+         * @return regex value
+         */
+        public String getRegexValue() {
+            return regexValue;
+        }
+
+        /**
+         * Type create from value.
+         *
+         * @param value string value
+         * @return Type object
+         */
+        public static Type fromValue(final String value) {
+            for (Type type : values()) {
+                if (type.name().equalsIgnoreCase(value)) {
+                    return type;
+                }
+            }
+            throw new IllegalArgumentException(
+                "Unknown enum type " + value + ", Allowed values are " + Arrays.toString(values()));
+        }
     }
+
 }
diff --git a/metacat-common/src/test/groovy/com/netflix/metacat/common/QualifiedNameSpec.groovy b/metacat-common/src/test/groovy/com/netflix/metacat/common/QualifiedNameSpec.groovy
@@ -237,4 +237,24 @@ class QualifiedNameSpec extends Specification {
         then:
         notThrown()
     }
+
+    def testQualifedNameToWildcardQueryString() {
+        when:
+        def result = QualifiedName.qualifiedNameToWildCardQueryString(sourceName, databaseName, tableName)
+        then:
+        result == ret
+        where:
+        sourceName  | databaseName | tableName | ret
+        null        | null         | null      | null
+        "prodhive"  | "database_1" | "abcd"    | "prodhive/database_1/abcd%"
+        "prodhive"  | ""           | ""        | "prodhive%"
+        "prodhive"  | null         | null      | "prodhive%"
+        "prodhive"  | "database_1" | ""        | "prodhive/database_1%"
+        "prodhive"  | "database_1" | null      | "prodhive/database_1%"
+        ""          | "database_1" | ""        | "%/database_1%"
+        null        | "database_1" | ""        | "%/database_1%"
+        null        | "database_1" | null      | "%/database_1%"
+        ""          | "database_1" | "abcd"    | "%/database_1/abcd%"
+        ""          | ""           | "abcd"    | "%/%/abcd%"
+    }
 }
diff --git a/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/...at-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy
@@ -1320,15 +1320,21 @@ class MetacatSmokeSpec extends Specification {
         def tableDto = api.getTable(catalogName, databaseName, tableName, true, true, false)
 
         def ret = tagApi.search('test_tag', null, null, null)
-        def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
-        def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null)
+        def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, null)
+        def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null, null)
+        def ret4 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, QualifiedName.Type.DATABASE)
+        def ret5 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, QualifiedName.Type.TABLE)
+        def ret6 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, QualifiedName.Type.CATALOG)
+
+        def ret7 = tagApi.list(['test_tag'] as Set<String>, null, null, databaseName, null, QualifiedName.Type.CATALOG)
+        def ret8 = tagApi.list(['test_tag'] as Set<String>, null, null, databaseName, null, QualifiedName.Type.DATABASE)
 
         tagApi.removeTags(TagRemoveRequestDto.builder().name(catalog).tags([]).deleteAll(true).build())
         tagApi.removeTags(TagRemoveRequestDto.builder().name(database).tags([]).deleteAll(true).build())
         tagApi.removeTags(TagRemoveRequestDto.builder().name(table).tags([]).deleteAll(true).build())
 
         def ret_new = tagApi.search('test_tag', null, null, null)
-        def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+        def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, null)
 
         then:
         metacatJson.convertValue(catalogDto.getDefinitionMetadata().get('tags'), Set.class) == tags as Set<String>
@@ -1337,7 +1343,12 @@ class MetacatSmokeSpec extends Specification {
 
         assert ret.size() == 3
         assert ret2.size() == 3
+        assert ret4.size() == 1 //only query database
+        assert ret5.size() == 1 //only query table
+        assert ret6.size() == 1 //only query catalog
         assert ret3.size() == 0
+        assert ret7.size() == 0 //limit to database
+        assert ret8.size() == 1 //limit to database
 
         assert ret_new.size() == 0
         assert ret2_new.size() == 0
@@ -1373,13 +1384,13 @@ class MetacatSmokeSpec extends Specification {
         tagApi.setTags(TagCreateRequestDto.builder().name(view).tags(tags).build())
 
         def ret = tagApi.search('test_tag', null, null, null)
-        def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
-        def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null)
+        def ret2 = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, null)
+        def ret3 = tagApi.list(['never_tag'] as Set<String>, tags as Set<String>, null, null, null, null)
 
         tagApi.removeTags(TagRemoveRequestDto.builder().name(view).tags([]).deleteAll(true).build())
 
         def ret_new = tagApi.search('test_tag', null, null, null)
-        def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null)
+        def ret2_new = tagApi.list(['test_tag'] as Set<String>, null, null, null, null, null)
 
         then:
         assert ret.size() == 1

diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/TagController.java b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/TagController.java
@@ -45,6 +45,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
+import org.springframework.web.bind.WebDataBinder;
+import org.springframework.web.bind.annotation.InitBinder;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -54,6 +56,7 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.annotation.Nullable;
+import java.beans.PropertyEditorSupport;
 import java.net.HttpURLConnection;
 import java.util.HashSet;
 import java.util.List;
@@ -143,6 +146,7 @@ public Set<String> getTags() {
      * @param sourceName   Prefix of the source name
      * @param databaseName Prefix of the database name
      * @param tableName    Prefix of the table name
+     * @param type         metacat qualifed name type
      * @return list of qualified names
      */
     @RequestMapping(
@@ -167,11 +171,20 @@ public List<QualifiedName> list(
         @ApiParam(value = "Prefix of the database name")
         @Nullable @RequestParam(name = "databaseName", required = false) final String databaseName,
         @ApiParam(value = "Prefix of the table name")
-        @Nullable @RequestParam(name = "tableName", required = false) final String tableName
+        @Nullable @RequestParam(name = "tableName", required = false) final String tableName,
+        @ApiParam(value = "Qualified name type")
+        @Nullable
+        @RequestParam(name = "type", required = false) final QualifiedName.Type type
     ) {
         return this.requestWrapper.processRequest(
             "TagV1Resource.list",
-            () -> this.tagService.list(includeTags, excludeTags, sourceName, databaseName, tableName)
+            () -> this.tagService.list(
+                includeTags,
+                excludeTags,
+                sourceName,
+                databaseName,
+                tableName,
+                type)
         );
     }
 
@@ -254,7 +267,7 @@ private Set<String> setResourceTags(@NonNull final TagCreateRequestDto tagCreate
             case CATALOG:
                 //catalog service will throw exception if not found
                 final CatalogDto catalogDto = this.catalogService.get(name, GetCatalogServiceParameters.builder()
-                        .includeDatabaseNames(false).includeUserMetadata(false).build());
+                    .includeDatabaseNames(false).includeUserMetadata(false).build());
                 if (catalogDto != null) {
                     return this.tagService.setTags(name, tags, true);
                 }
@@ -547,7 +560,7 @@ private void removeResourceTags(final TagRemoveRequestDto tagRemoveRequestDto) {
             case CATALOG:
                 //catalog service will throw exception if not found
                 final CatalogDto catalogDto = this.catalogService.get(name, GetCatalogServiceParameters.builder()
-                        .includeDatabaseNames(false).includeUserMetadata(false).build());
+                    .includeDatabaseNames(false).includeUserMetadata(false).build());
                 if (catalogDto != null) {
                     this.tagService.removeTags(name, tagRemoveRequestDto.getDeleteAll(),
                         new HashSet<>(tagRemoveRequestDto.getTags()), true);
@@ -622,6 +635,17 @@ private void removeResourceTags(final TagRemoveRequestDto tagRemoveRequestDto) {
             default:
                 throw new MetacatNotFoundException("Unsupported qualifiedName type {}" + name);
         }
+    }
 
+    @InitBinder
+    private void bindsCustomRequestParamType(final WebDataBinder dataBinder) {
+        dataBinder.registerCustomEditor(QualifiedName.Type.class, new QualifiedNameTypeConverter());
+    }
+
+    private static class QualifiedNameTypeConverter extends PropertyEditorSupport {
+        @Override
+        public void setAsText(final String text) throws IllegalArgumentException {
+            super.setValue(QualifiedName.Type.fromValue(text));
+        }
     }
 }
diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/services/MetadataService.java b/metacat-main/src/main/java/com/netflix/metacat/main/services/MetadataService.java
@@ -224,7 +224,8 @@ public boolean deleteDefinitionMetadata(final QualifiedName name, final boolean
      */
     public void cleanUpObsoleteTags() {
         log.info("Start deleting obsolete tags");
-        final List<QualifiedName> names = tagService.list(null, null, null, null, null);
+        final List<QualifiedName> names = tagService.list(null, null, null, null, null,
+            null);
         names.forEach(name -> {
             if (!name.isPartitionDefinition() && !name.isViewDefinition() && name.isTableDefinition()
                 && !tableService.exists(name)) {