feat(schema): support cross project type referencing (OpenSPG#119)

Co-authored-by: baifuyu <[email protected]>

python/knext/knext/schema/marklang/schema_ml.py

@@ -137,17 +137,18 @@ class SPGSchemaMarkLang:
     last_indent_level = 0
     namespace = None
     types = {}
+    defined_types = {}
 
     def __init__(self, filename):
         self.schema_file = filename
         self.current_line_num = 0
-        schema = SchemaClient()
-        thing = schema.query_spg_type("Thing")
+        self.schema = SchemaClient()
+        thing = self.schema.query_spg_type("Thing")
         for prop in thing.properties:
             self.entity_internal_property.add(prop)
             self.event_internal_property.add(prop)
             self.concept_internal_property.add(prop)
-        session = schema.create_session()
+        session = self.schema.create_session()
         for type_name in session.spg_types:
             spg_type = session.get(type_name)
             if session.get(type_name).spg_type_enum in [
@@ -245,6 +246,13 @@ def parse_type(self, expression):
             assert type_class in self.keyword_type, self.error_msg(
                 f"{type_class} is illegal, please define it before current line"
             )
+            assert (
+                type_name.startswith("STD.")
+                or "." not in type_name
+                or type_name.startswith(f"{self.namespace}.")
+            ), self.error_msg(
+                f"The name space of {type_name} does not belong to current project."
+            )
 
             spg_type = None
             if type_class == "EntityType":
@@ -453,7 +461,9 @@ def check_semantic_relation(self, predicate_name, predicate_class):
         predicate_class_ns = predicate_class
         if "." not in predicate_class:
             predicate_class_ns = f"{self.namespace}.{predicate_class}"
-        assert predicate_class_ns in self.types, self.error_msg(
+        assert (
+            predicate_class_ns in self.types or predicate_class_ns in self.defined_types
+        ), self.error_msg(
             f"{predicate_class} is illegal, please ensure that it appears in this schema."
         )
         object_type = self.types[predicate_class_ns]
@@ -550,32 +560,60 @@ def parse_predicate(self, expression):
                     f"{predicate_name} is a semantic predicate, please add the semantic prefix"
                 )
 
+        if (
+            "." in predicate_class
+            and predicate_class not in self.types
+            and predicate_class not in self.internal_type
+        ):
+            try:
+                cross_type = self.schema.query_spg_type(
+                    self.get_type_name_with_ns(predicate_class)
+                )
+                self.types[self.get_type_name_with_ns(predicate_class)] = cross_type
+            except Exception as e:
+                raise ValueError(
+                    self.error_msg(
+                        f"{predicate_class} is illegal, please ensure the name space or type name is correct."
+                    )
+                )
+
         assert (
             self.get_type_name_with_ns(predicate_class) in self.types
             or predicate_class in self.internal_type
+            or predicate_class in self.defined_types
         ), self.error_msg(
             f"{predicate_class} is illegal, please ensure that it appears in this schema."
         )
+
         assert predicate_name not in self.entity_internal_property, self.error_msg(
             f"property {predicate_name} is the default property of type"
         )
         if predicate_class not in self.internal_type:
-            predicate_type = self.types[self.get_type_name_with_ns(predicate_class)]
-            if predicate_type is not None:
-                if cur_type.spg_type_enum == SpgTypeEnum.Concept:
-                    assert (
-                        predicate_type.spg_type_enum == SpgTypeEnum.Concept
-                    ), self.error_msg(
-                        "Concept type only allow relationships that point to themselves"
-                    )
-                elif cur_type.spg_type_enum == SpgTypeEnum.Entity:
-                    assert (
-                        predicate_type.spg_type_enum != SpgTypeEnum.Event
-                    ), self.error_msg(
-                        "Relationships of entity types are not allowed to point to event types; "
-                        "instead, they are only permitted to point from event types to entity types, "
-                        "adhering to the principle of moving from dynamic to static."
-                    )
+            spg_type_enum = SpgTypeEnum.Entity
+            if self.get_type_name_with_ns(predicate_class) in self.types:
+                predicate_type = self.types[self.get_type_name_with_ns(predicate_class)]
+                spg_type_enum = predicate_type.spg_type_enum
+            elif predicate_class in self.defined_types:
+                spg_type_enum_txt = self.defined_types[predicate_class]
+                if spg_type_enum_txt == "EntityType":
+                    spg_type_enum = SpgTypeEnum.Entity
+                elif spg_type_enum_txt == "ConceptType":
+                    spg_type_enum = SpgTypeEnum.Concept
+                elif spg_type_enum_txt == "EventType":
+                    spg_type_enum = SpgTypeEnum.Event
+                elif spg_type_enum_txt == "StandardType":
+                    spg_type_enum = SpgTypeEnum.Standard
+
+            if cur_type.spg_type_enum == SpgTypeEnum.Concept:
+                assert spg_type_enum == SpgTypeEnum.Concept, self.error_msg(
+                    "Concept type only allow relationships that point to themselves"
+                )
+            elif cur_type.spg_type_enum == SpgTypeEnum.Entity:
+                assert spg_type_enum != SpgTypeEnum.Event, self.error_msg(
+                    "Relationships of entity types are not allowed to point to event types; "
+                    "instead, they are only permitted to point from event types to entity types, "
+                    "adhering to the principle of moving from dynamic to static."
+                )
 
         if self.parsing_register[RegisterUnit.Relation] is not None:
             assert (
@@ -631,6 +669,8 @@ def parse_predicate(self, expression):
                 name_zh=predicate_name_zh,
                 object_type_name=predicate_class,
             )
+            if predicate_class in self.types:
+                predicate.object_spg_type = self.types[predicate_class].spg_type_enum
             if (
                 self.parsing_register[RegisterUnit.Type].spg_type_enum
                 == SpgTypeEnum.Event
@@ -655,7 +695,10 @@ def parse_predicate(self, expression):
 
                         if "." not in subject_type:
                             subject_type = f"{self.namespace}.{predicate_class}"
-                        assert subject_type in self.types, self.error_msg(
+                        assert (
+                            subject_type in self.types
+                            or predicate_class in self.defined_types
+                        ), self.error_msg(
                             f"{predicate_class} is illegal, please ensure that it appears in this schema."
                         )
 
@@ -677,7 +720,10 @@ def parse_predicate(self, expression):
             assert not predicate_class.startswith("STD."), self.error_msg(
                 f"{predicate_class} is not allow appear in the definition of relation."
             )
-            assert predicate_class in self.types, self.error_msg(
+            assert (
+                predicate_class in self.types
+                or predicate_class.split(".")[1] in self.defined_types
+            ), self.error_msg(
                 f"{predicate_class} is illegal, please ensure that it appears in this schema."
             )
             assert (
@@ -692,6 +738,8 @@ def parse_predicate(self, expression):
             )
 
             predicate = Relation(name=predicate_name, object_type_name=predicate_class)
+            if predicate_class in self.types:
+                predicate.object_spg_type = self.types[predicate_class].spg_type_enum
             self.parsing_register[RegisterUnit.Type].add_relation(predicate)
             self.save_register(RegisterUnit.Relation, predicate)
         predicate.name_zh = predicate_name_zh
@@ -872,13 +920,34 @@ def complete_rule(self, rule):
 
         return rule.strip()
 
+    def preload_types(self, lines: list):
+        """
+        Pre analyze the script to obtain defined types
+        """
+
+        for line in lines:
+            type_match = re.match(
+                r"^([a-zA-Z0-9\.]+)\((\w+)\):\s*?([a-zA-Z0-9,]+)$", line
+            )
+            if type_match:
+                self.defined_types[type_match.group(1)] = type_match.group(3).strip()
+                continue
+            sub_type_match = re.match(
+                r"^([a-zA-Z0-9]+)\((\w+)\)\s*?->\s*?([a-zA-Z0-9\.]+):$", line
+            )
+            if sub_type_match:
+                self.defined_types[sub_type_match.group(1)] = type_match.group(
+                    3
+                ).strip()
+
     def load_script(self):
         """
         Load and then parse the script file
         """
 
         file = open(self.schema_file, "r", encoding="utf-8")
         lines = file.read().splitlines()
+        self.preload_types(lines)
         for line in lines:
             self.current_line_num += 1
             strip_line = line.strip()
@@ -1019,6 +1088,10 @@ def diff_and_sync(self, print_only):
 
         # generate the delete list of spg type
         for spg_type in session.spg_types:
+            if not spg_type.startswith("STD.") and not spg_type.startswith(
+                f"{self.namespace}."
+            ):
+                continue
             unique_id = session.spg_types[spg_type]._rest_model.ontology_id.unique_id
             if spg_type in self.internal_type and unique_id < 1000:
                 continue
@@ -1029,6 +1102,10 @@ def diff_and_sync(self, print_only):
 
         for spg_type in self.types:
             # generate the creation list of spg type
+            if not spg_type.startswith("STD.") and not spg_type.startswith(
+                f"{self.namespace}."
+            ):
+                continue
             if spg_type not in session.spg_types:
                 session.create_type(self.types[spg_type])
                 print(f"Create type: {spg_type}")
@@ -1263,7 +1340,11 @@ def export_schema_python(self, filename):
 
         spg_types = []
         for spg_type_name in sorted(session.spg_types):
-            if spg_type_name.startswith("STD.") or spg_type_name in self.internal_type:
+            if (
+                spg_type_name.startswith("STD.")
+                or not spg_type_name.startswith(self.namespace)
+                or spg_type_name in self.internal_type
+            ):
                 continue
 
             sub_properties = {}

reasoner/catalog/openspg-catalog/src/main/scala/com/antgroup/openspg/reasoner/catalog/impl/OpenSPGCatalog.scala

@@ -28,7 +28,7 @@ import com.antgroup.openspg.reasoner.lube.catalog.{AbstractConnection, Catalog,
 import com.antgroup.openspg.reasoner.lube.catalog.struct.{Edge, Field, Node}
 import com.antgroup.openspg.server.api.facade.ApiResponse
 import com.antgroup.openspg.server.api.facade.client.{ConceptFacade, SchemaFacade}
-import com.antgroup.openspg.server.api.facade.dto.schema.request.{ConceptRequest, ProjectSchemaRequest}
+import com.antgroup.openspg.server.api.facade.dto.schema.request.{ConceptRequest, ProjectSchemaRequest, SPGTypeRequest}
 import com.antgroup.openspg.server.api.http.client.{HttpConceptFacade, HttpSchemaFacade}
 import com.antgroup.openspg.server.api.http.client.util.{ConnectionInfo, HttpClientBootstrap}
 import org.apache.commons.collections4.CollectionUtils
@@ -115,8 +115,12 @@ class OpenSPGCatalog(val projectId: Long,
   private def toField(projectSchema: ProjectSchema,
                       spgType: BaseSPGType,
                       spgProperty: Property): Field = {
-    val propertyType = PropertySchemaOps
-      .stringToKgType2(projectSchema.getByRef(spgProperty.getObjectTypeRef))
+    var objectType = projectSchema.getByRef(spgProperty.getObjectTypeRef)
+    if (objectType == null) {
+      objectType = resultOf(spgSchemaFacade.querySPGType(
+        new SPGTypeRequest(spgProperty.getObjectTypeRef.getName)))
+    }
+    val propertyType = PropertySchemaOps.stringToKgType2(objectType)
     val rule = spgProperty.getLogicalRule
     val predicateName = spgProperty.getName
     if (rule != null && StringUtils.isNotBlank(rule.getContent)) {

server/core/schema/model/src/main/java/com/antgroup/openspg/core/schema/model/SchemaException.java

@@ -61,4 +61,10 @@ public static SchemaException propertyNotExist(String propertyName) {
   public static SchemaException relationNotExist(String relationName) {
     return new SchemaException("there is no relation with name={}", relationName);
   }
+
+  public static SchemaException existReference(String spgTypeName) {
+    return new SchemaException(
+        "{} is referenced by another project, so that it cannot be deleted until the reference is released",
+        spgTypeName);
+  }
 }

server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/check/PropertyChecker.java

@@ -137,7 +137,12 @@ protected void checkBasicInfo(
               "objectTypeRef.typeName of property/relation: %s can not be null",
               property.getBasicInfo().getName()));
     }
-    if (!context.containSpgType(property.getObjectTypeRef().getBaseSpgIdentifier())) {
+    if (!context.containSpgType(property.getObjectTypeRef().getBaseSpgIdentifier())
+        && property
+            .getObjectTypeRef()
+            .getBaseSpgIdentifier()
+            .getNamespace()
+            .equals(property.getSubjectTypeRef().getBaseSpgIdentifier().getNamespace())) {
       throw new IllegalArgumentException(
           String.format(
               "property/relation: %s depends on type: %s, but not exist",

server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/alter/stage/handler/OntologyIdHandler.java

@@ -173,7 +173,10 @@ private void setOntologyId(
       SPGTypeIdentifier objectTypeIdentifier = property.getObjectTypeRef().getBaseSpgIdentifier();
       BaseSPGType objectType = spgTypeMap.get(objectTypeIdentifier);
       if (null == objectType) {
-        throw SchemaException.spgTypeNotExist(objectTypeIdentifier.toString());
+        objectType = spgTypeService.querySPGTypeByIdentifier(objectTypeIdentifier);
+        if (null == objectType) {
+          throw SchemaException.spgTypeNotExist(objectTypeIdentifier.toString());
+        }
       }
       property.getObjectTypeRef().setOntologyId(objectType.getOntologyId());
     }

server/core/schema/service/src/main/java/com/antgroup/openspg/server/core/schema/service/predicate/impl/PropertyAdvancedConfigHandler.java

@@ -14,18 +14,26 @@
 package com.antgroup.openspg.server.core.schema.service.predicate.impl;
 
 import com.antgroup.openspg.core.schema.model.alter.AlterOperationEnum;
+import com.antgroup.openspg.core.schema.model.alter.AlterStatusEnum;
 import com.antgroup.openspg.core.schema.model.predicate.Property;
 import com.antgroup.openspg.core.schema.model.predicate.PropertyAdvancedConfig;
 import com.antgroup.openspg.core.schema.model.predicate.Relation;
 import com.antgroup.openspg.core.schema.model.predicate.SubProperty;
 import com.antgroup.openspg.core.schema.model.semantic.PredicateSemantic;
 import com.antgroup.openspg.core.schema.model.semantic.SPGOntologyEnum;
+import com.antgroup.openspg.core.schema.model.type.RefSourceEnum;
+import com.antgroup.openspg.core.schema.model.type.SPGTypeEnum;
+import com.antgroup.openspg.core.schema.model.type.SPGTypeRef;
 import com.antgroup.openspg.server.core.schema.service.predicate.SubPropertyService;
 import com.antgroup.openspg.server.core.schema.service.predicate.model.SimpleProperty;
 import com.antgroup.openspg.server.core.schema.service.predicate.repository.ConstraintRepository;
 import com.antgroup.openspg.server.core.schema.service.predicate.repository.PropertyRepository;
 import com.antgroup.openspg.server.core.schema.service.semantic.LogicalRuleService;
 import com.antgroup.openspg.server.core.schema.service.semantic.SemanticService;
+import com.antgroup.openspg.server.core.schema.service.type.model.ProjectOntologyRel;
+import com.antgroup.openspg.server.core.schema.service.type.model.SimpleSPGType;
+import com.antgroup.openspg.server.core.schema.service.type.repository.ProjectOntologyRelRepository;
+import com.antgroup.openspg.server.core.schema.service.type.repository.SPGTypeRepository;
 import com.google.common.collect.Lists;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -41,9 +49,12 @@ public class PropertyAdvancedConfigHandler {
   @Autowired private SubPropertyService subPropertyService;
   @Autowired private SemanticService semanticService;
   @Autowired private LogicalRuleService logicalRuleService;
+  @Autowired private ProjectOntologyRelRepository projectOntologyRelRepository;
+  @Autowired private SPGTypeRepository spgTypeRepository;
 
   public void alterAdvancedConfig(Property property, AlterOperationEnum alterOperation) {
     PropertyAdvancedConfig advancedConfig = property.getAdvancedConfig();
+    alterOntologyReference(property, alterOperation);
     switch (alterOperation) {
       case CREATE:
         this.createAdvancedConfig(advancedConfig);
@@ -167,4 +178,45 @@ private void updateSemantic(List<PredicateSemantic> semantics) {
             .collect(Collectors.toList());
     saveOrUpdateSemantics.forEach(e -> semanticService.saveOrUpdate(e));
   }
+
+  private void alterOntologyReference(Property property, AlterOperationEnum alterOperation) {
+    SPGTypeRef refType = property.getObjectTypeRef();
+    if (refType.getSpgTypeEnum() == SPGTypeEnum.BASIC_TYPE
+        || refType.getSpgTypeEnum() == SPGTypeEnum.STANDARD_TYPE
+        || property
+            .getSubjectTypeRef()
+            .getBaseSpgIdentifier()
+            .getNamespace()
+            .equals(refType.getBaseSpgIdentifier().getNamespace())) {
+      return;
+    }
+    if (refType.getUniqueId() == null) {
+      SimpleSPGType crossType = spgTypeRepository.queryByName(refType.getName());
+      refType.setOntologyId(crossType.getOntologyId());
+    }
+    ProjectOntologyRel reference =
+        projectOntologyRelRepository.queryByOntologyId(
+            refType.getUniqueId(), SPGOntologyEnum.TYPE, property.getProjectId());
+    switch (alterOperation) {
+      case CREATE:
+      case UPDATE:
+        if (reference == null) {
+          ProjectOntologyRel projectOntology =
+              new ProjectOntologyRel(
+                  null,
+                  property.getProjectId(),
+                  refType.getUniqueId(),
+                  SPGOntologyEnum.TYPE,
+                  1,
+                  AlterStatusEnum.ONLINE,
+                  RefSourceEnum.PROJECT);
+          projectOntologyRelRepository.save(projectOntology);
+        }
+        break;
+      case DELETE:
+        projectOntologyRelRepository.delete(refType.getUniqueId(), property.getProjectId());
+        break;
+      default:
+    }
+  }
 }