Merge pull request #102 from iai-group/feature/54-Create-endpoint-for…

…-NL-query

Implement endpoint /nl_processing

pkg_api/core/pkg_types.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import dataclasses
 import uuid
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Union
@@ -90,3 +91,15 @@ class PKGData:
     triple: Optional[Triple] = None
     preference: Optional[Preference] = None
     logging_data: Dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> Dict[str, Any]:
+        """Returns a dictionary representation of the PKGData."""
+        return {
+            "id": str(self.id),
+            "statement": self.statement,
+            "triple": dataclasses.asdict(self.triple) if self.triple else None,
+            "preference": (
+                dataclasses.asdict(self.preference) if self.preference else None
+            ),
+            "logging_data": self.logging_data,
+        }

pkg_api/server/__init__.py

@@ -3,12 +3,18 @@
 Resources give access to HTTP methods related to a PKG API feature.
 """
 
-from flask import Flask
+import importlib
+import os
+
+from flask import Config, Flask
 from flask_restful import Api
 
-from pkg_api.connector import DEFAULT_STORE_PATH
-from pkg_api.pkg import DEFAULT_VISUALIZATION_PATH
+from pkg_api.nl_to_pkg.annotators.three_step_annotator import (
+    ThreeStepStatementAnnotator,
+)
+from pkg_api.nl_to_pkg.nl_to_pkg import NLtoPKG
 from pkg_api.server.auth import AuthResource
+from pkg_api.server.config import DevelopmentConfig, TestingConfig
 from pkg_api.server.facts_management import PersonalFactsResource
 from pkg_api.server.models import db
 from pkg_api.server.nl_processing import NLResource
@@ -28,14 +34,13 @@ def create_app(testing: bool = False) -> Flask:
     app = Flask(__name__)
 
     if testing:
-        app.config["TESTING"] = True
-        app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///test.sqlite"
-        app.config["STORE_PATH"] = "tests/data/RDFStore"
-        app.config["VISUALIZATION_PATH"] = "tests/data/pkg_visualizations"
+        app.config.from_object(TestingConfig)
     else:
-        app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite"
-        app.config["STORE_PATH"] = DEFAULT_STORE_PATH
-        app.config["VISUALIZATION_PATH"] = DEFAULT_VISUALIZATION_PATH
+        app.config.from_object(DevelopmentConfig)
+
+    # Create storage directories
+    os.makedirs(app.config["STORE_PATH"], exist_ok=True)
+    os.makedirs(app.config["VISUALIZATION_PATH"], exist_ok=True)
 
     db.init_app(app)
 
@@ -49,6 +54,35 @@ def create_app(testing: bool = False) -> Flask:
     api.add_resource(ServiceManagementResource, "/service")
     api.add_resource(PersonalFactsResource, "/facts")
     api.add_resource(PKGExplorationResource, "/explore")
-    api.add_resource(NLResource, "/nl")
+    api.add_resource(
+        NLResource,
+        "/nl",
+        resource_class_kwargs={"nl_to_pkg": _init_nl_to_pkg(app.config)},
+    )
 
     return app
+
+
+def _init_nl_to_pkg(config: Config) -> NLtoPKG:
+    """Initializes the NL to PKG module.
+
+    Args:
+        config: Server configuration.
+
+    Returns:
+        NLtoPKG object.
+    """
+    annotator = ThreeStepStatementAnnotator(
+        prompt_paths=config["TS_ANNOTATOR_PROMPT_PATHS"],
+        config_path=config["TS_ANNOTATOR_CONFIG_PATH"],
+    )
+
+    # Create entity linker from config
+    entity_module, entity_class = config["ENTITY_LINKER_CONFIG"][
+        "class_path"
+    ].rsplit(".", maxsplit=1)
+    entity_cls_module = importlib.import_module(entity_module)
+    entity_cls = getattr(entity_cls_module, entity_class)
+    entity_linker = entity_cls(**config["ENTITY_LINKER_CONFIG"]["kwargs"])
+
+    return NLtoPKG(annotator, entity_linker)

pkg_api/server/config.py

@@ -0,0 +1,47 @@
+"""Define server configuration."""
+
+from pkg_api.nl_to_pkg.annotators.three_step_annotator import (
+    _DEFAULT_CONFIG_PATH as DEFAULT_3_STEP_CONFIG_PATH,
+)
+from pkg_api.nl_to_pkg.annotators.three_step_annotator import (
+    _DEFAULT_PROMPT_PATHS,
+)
+from pkg_api.nl_to_pkg.entity_linking.rel_entity_linking import (
+    _DEFAULT_API_URL,
+)
+from pkg_api.pkg import DEFAULT_VISUALIZATION_PATH
+
+
+class BaseConfig:
+    """Base configuration for the server."""
+
+    TESTING = False
+
+    # Three step annotator configuration
+    TS_ANNOTATOR_CONFIG_PATH = DEFAULT_3_STEP_CONFIG_PATH
+    TS_ANNOTATOR_PROMPT_PATHS = _DEFAULT_PROMPT_PATHS
+
+    # Entity linker configuration. Use REL by default.
+    ENTITY_LINKER_CONFIG = {
+        "class_path": "pkg_api.nl_to_pkg.entity_linking.rel_entity_linking."
+        "RELEntityLinker",
+        "kwargs": {"api_url": _DEFAULT_API_URL},
+    }
+
+
+class DevelopmentConfig(BaseConfig):
+    """Development configuration for the server."""
+
+    DEBUG = True
+    SQLALCHEMY_DATABASE_URI = "sqlite:///db.sqlite"
+    STORE_PATH = "data"
+    VISUALIZATION_PATH = DEFAULT_VISUALIZATION_PATH
+
+
+class TestingConfig(BaseConfig):
+    """Testing configuration for the server."""
+
+    TESTING = True
+    SQLALCHEMY_DATABASE_URI = "sqlite:///test.sqlite"
+    STORE_PATH = "tests/data/RDFStore"
+    VISUALIZATION_PATH = "tests/data/pkg_visualizations"

pkg_api/server/nl_processing.py

@@ -1,20 +1,73 @@
 """API Resource receiving NL input."""
 
-from typing import Dict, Tuple
+from typing import Any, Dict, Tuple
 
+from flask import request
 from flask_restful import Resource
 
+from pkg_api.core.intents import Intent
+from pkg_api.nl_to_pkg.nl_to_pkg import NLtoPKG
+from pkg_api.server.utils import open_pkg
+
 
 class NLResource(Resource):
-    def post(self) -> Tuple[Dict[str, str], int]:
+    def __init__(self, nl_to_pkg: NLtoPKG) -> None:
+        """Initializes the NL resource.
+
+        Args:
+            nl_to_pkg: NLtoPKG object.
+        """
+        self.nl_to_pkg = nl_to_pkg
+
+    def post(self) -> Tuple[Dict[str, Any], int]:
         """Processes the NL input to update the PKG.
 
         Note that the returned dictionary may contain additional fields based
         on the frontend's needs.
 
+        Raises:
+            KeyError: if there is missing information to open the user's PKG.
+
         Returns:
             A tuple with a dictionary containing a message, and the status code.
         """
-        # TODO: Implement this method following the approved pipeline.
-        # See issue: https://github.com/iai-group/pkg-api/issues/78
-        return {"message": "Not implemented."}, 400
+        data = request.json
+
+        try:
+            pkg = open_pkg(data)
+        except KeyError as e:
+            return {"message": e.args[0]}, 400
+
+        query = data.get("query", None)
+        if not query:
+            return {"message": "Missing query."}, 400
+
+        intent, statement_data = self.nl_to_pkg.annotate(query)
+        if intent == Intent.ADD:
+            pkg.add_statement(statement_data)
+            pkg.close()
+            return {
+                "message": "Statement added to your PKG.",
+                "annotation": statement_data.as_dict(),
+            }, 200
+        elif intent == Intent.GET:
+            statements = pkg.get_statements(
+                statement_data, triple_conditioned=True
+            )
+            return {
+                "message": "Statements retrieved from your PKG.",
+                "data": [s.as_dict() for s in statements],
+                "annotation": statement_data.as_dict(),
+            }, 200
+        elif intent == Intent.DELETE:
+            pkg.remove_statement(statement_data)
+            pkg.close()
+            return {
+                "message": "Statement was deleted if present.",
+                "annotation": statement_data.as_dict(),
+            }, 200
+
+        return {
+            "message": "The operation could not be performed. Please try to"
+            " rephrase your query."
+        }, 200

pkg_api/server/pkg_exploration.py

@@ -1,4 +1,5 @@
 """PKG Exploration Resource."""
+
 from typing import Any, Dict, Tuple
 
 from flask import request
@@ -18,7 +19,7 @@ def get(self) -> Tuple[Dict[str, Any], int]:
         try:
             pkg = open_pkg(data)
         except Exception as e:
-            return {"message": str(e)}, 400
+            return {"message": e.args[0]}, 400
 
         graph_img_path = pkg.visualize_graph()
         pkg.close()
@@ -39,7 +40,7 @@ def post(self) -> Tuple[Dict[str, Any], int]:
         try:
             pkg = open_pkg(data)
         except Exception as e:
-            return {"message": str(e)}, 400
+            return {"message": e.args[0]}, 400
 
         sparql_query = parse_query_request_data(data)
 

pkg_api/server/utils.py

@@ -1,4 +1,6 @@
 """Utility functions for the server."""
+
+import logging
 from typing import Any, Dict
 
 from flask import current_app
@@ -20,7 +22,9 @@ def open_pkg(data: Dict[str, str]) -> PKG:
     owner_uri = data.get("owner_uri", None)
     owner_username = data.get("owner_username", None)
     if owner_uri is None:
-        raise Exception("Missing owner URI")
+        e = KeyError("Missing owner URI")
+        logging.exception("Exception while opening the PKG", exc_info=e)
+        raise e
 
     store_path = current_app.config["STORE_PATH"]
     visualization_path = current_app.config["VISUALIZATION_PATH"]

pkg_api/utils.py

@@ -1,10 +1,11 @@
 """Utils methods for the API.
 
 The methods creates SPARQL queries to edit the PKG like adding/removing
-statements or preferences. See the PKG vocabulary for more information about
-the properties of statements and preferences.
+statements or preferences. See the PKG vocabulary for more information
+about the properties of statements and preferences.
 
-PKG vocabulary: https://iai-group.github.io/pkg-vocabulary/
+PKG vocabulary:
+https://iai-group.github.io/pkg-vocabulary/
 """
 
 import dataclasses
@@ -108,7 +109,7 @@ def _get_concept_representation(concept: Concept) -> str:
         else ""
     )
     representation = concept_template.format(
-        description=concept.description,
+        description=re.sub(r'"', r"\"", concept.description),
         related_entities=related_entities,
         broader_entities=broader_entities,
         narrower_entities=narrower_entities,
@@ -158,8 +159,9 @@ def _get_statement_representation(
     Returns:
         Representation of the statement.
     """
+    raw_statement = re.sub(r'"', r"\"", pkg_data.statement)
     statement = f"""{statement_node_id} a rdf:Statement ;
-        dc:description "{pkg_data.statement}" ; """
+        dc:description "{raw_statement}" ; """
 
     # Add triple annotation
     if pkg_data.triple is not None:

tests/nl_to_pkg/test_nl_to_pkg.py

@@ -68,17 +68,17 @@ def link_annotation_side_effect(*args, **kwargs):
 
 
 @pytest.fixture
-def nlp_to_pkg(
+def nl_to_pkg(
     statement_annotator_mock: Mock,
     entity_linker_mock: Mock,
 ) -> NLtoPKG:
     """Returns an NLtoPKG instance."""
     return NLtoPKG(statement_annotator_mock, entity_linker_mock)
 
 
-def test_annotate_success(statement: str, nlp_to_pkg: NLtoPKG) -> None:
+def test_annotate_success(statement: str, nl_to_pkg: NLtoPKG) -> None:
     """Tests that annotate returns the correct intent and annotations."""
-    intent, pkg_data = nlp_to_pkg.annotate(statement)
+    intent, pkg_data = nl_to_pkg.annotate(statement)
 
     assert intent == Intent.ADD
     assert pkg_data.triple is not None
@@ -87,7 +87,7 @@ def test_annotate_success(statement: str, nlp_to_pkg: NLtoPKG) -> None:
 
 def test_annotate_no_triple(
     statement: str,
-    nlp_to_pkg: NLtoPKG,
+    nl_to_pkg: NLtoPKG,
     statement_annotator_mock: Mock,
 ) -> None:
     """Tests that annotate returns the correct intent and annotations."""
@@ -99,17 +99,17 @@ def test_annotate_no_triple(
             triple=None,
         ),
     )
-    _, pkg_data = nlp_to_pkg.annotate(statement)
+    _, pkg_data = nl_to_pkg.annotate(statement)
 
     assert pkg_data.triple is None
     assert pkg_data.preference is None
 
 
 def test_annotate_with_preference_update(
-    statement: str, nlp_to_pkg: NLtoPKG
+    statement: str, nl_to_pkg: NLtoPKG
 ) -> None:
     """Tests that annotate returns the correct intent and annotations."""
-    _, pkg_data = nlp_to_pkg.annotate(statement)
+    _, pkg_data = nl_to_pkg.annotate(statement)
 
     assert pkg_data.preference is not None
     assert pkg_data.preference.topic == TripleElement("Object", "Linked Object")

tests/pkg_api/server/conftest.py

@@ -19,3 +19,7 @@ def client() -> Flask:
     yield client
     # Delete the test database
     os.remove(f"{app.instance_path}/test.sqlite")
+    # Delete turtle files
+    for file in os.listdir(f"{app.config['STORE_PATH']}"):
+        if file.endswith(".ttl"):
+            os.remove(f"{app.config['STORE_PATH']}/{file}")