add the source code to get visibility for debugging purpose

Dockerfile

@@ -7,17 +7,16 @@ RUN yum install -y epel-release &&\
     yum install -y gcc git python36-pip python36-requests httpd httpd-devel python36-devel &&\
     yum clean all
 
-COPY ./requirements.txt /
-
-RUN python3 -m pip install --upgrade pip>=10.0.0 &&\
-    pip3 install -r requirements.txt && rm requirements.txt
-
-COPY ./src /src
+RUN python3 -m pip install --upgrade pip>=10.0.0
 
 RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@${F8A_WORKER_VERSION}
 RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-auth.git@${F8A_AUTH_VERSION}
 
+COPY ./requirements.txt /
+RUN pip3 install -r requirements.txt && rm requirements.txt
+
 ADD scripts/entrypoint.sh /bin/entrypoint.sh
+COPY ./src /src
 
 RUN chmod 777 /bin/entrypoint.sh
 

requirements.in

@@ -13,3 +13,4 @@ radon
 boto3
 sentry-sdk
 requests-futures
+psycopg2

requirements.txt

@@ -2,13 +2,13 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements.txt requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 boto3==1.9.99
 botocore==1.12.99         # via boto3, s3transfer
-certifi==2018.8.24        # via requests
+certifi==2018.8.24        # via requests, sentry-sdk
 chardet==3.0.4            # via requests
 click==7.0                # via flask
 codecov==2.0.15
@@ -31,6 +31,7 @@ markupsafe==1.0           # via jinja2
 mccabe==0.6.1             # via flake8
 more-itertools==4.3.0     # via pytest
 pluggy==0.7.1             # via pytest
+psycopg2==2.7.7
 py==1.6.0                 # via pytest
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
@@ -42,8 +43,8 @@ radon==3.0.1
 requests-futures==1.0.0
 requests==2.19.1
 s3transfer==0.2.0         # via boto3
+sentry-sdk==0.7.2
 six==1.11.0               # via flask-cors, mando, more-itertools, pytest, python-dateutil
 sqlalchemy==1.2.12
-urllib3==1.23             # via botocore, requests
+urllib3==1.23             # via botocore, requests, sentry-sdk
 werkzeug==0.14.1          # via flask
-sentry-sdk==0.7.2

src/rest_api.py

@@ -7,7 +7,7 @@
 from flask_cors import CORS
 from utils import DatabaseIngestion, scan_repo, validate_request_data, \
     retrieve_worker_result, alert_user, GREMLIN_SERVER_URL_REST, _s3_helper, \
-    generate_comparison
+    generate_comparison, GraphPassThrough, PostgresPassThrough
 from f8a_worker.setup_celery import init_selinon
 from fabric8a_auth.auth import login_required, init_service_account_token
 from data_extractor import DataExtractor
@@ -29,6 +29,9 @@
 init_selinon()
 _session = FuturesSession(max_workers=3)
 
+gpt = GraphPassThrough()
+ppt = PostgresPassThrough()
+
 _SERVICE_HOST = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_HOST", "bayesian-data-importer")
 _SERVICE_PORT = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_PORT", "9192")
 _CVE_SYNC_ENDPOINT = "api/v1/sync_latest_non_cve_version"
@@ -435,6 +438,24 @@ def drop():  # pragma: no cover
     return flask.jsonify(resp_dict), 200
 
 
+@app.route('/api/v1/graph', methods=['POST'])
+def graph():
+    """Endpoint to get graph node properties."""
+    input_json = request.get_json()
+    response = gpt.fetch_nodes(input_json)
+
+    return flask.jsonify(response)
+
+
+@app.route('/api/v1/pgsql', methods=['POST'])
+def pgsql():
+    """Endpoint to get graph node properties."""
+    input_json = request.get_json()
+    response = ppt.fetch_records(input_json)
+
+    return flask.jsonify(response)
+
+
 @app.route('/api/v1/stacks-report/list/<frequency>', methods=['GET'])
 def list_stacks_reports(frequency='weekly'):
     """

src/utils.py

@@ -17,6 +17,8 @@
 import logging
 import boto3
 import json
+import re
+import psycopg2
 from botocore.exceptions import ClientError
 
 logger = logging.getLogger(__name__)
@@ -30,6 +32,75 @@
     port=os.environ.get("LICENSE_SERVICE_PORT"))
 
 
+def sanitize_text_for_query(text):
+    """
+    Sanitize text so it can used in queries.
+
+    :param text: string, text to sanitize
+    :return: sanitized text
+    """
+    if text is None:
+        return ''
+
+    if not isinstance(text, str):
+        raise ValueError(
+            'Invalid query text: expected string, got {t}'.format(t=type(text))
+        )
+
+    strict_check_words = ['drop', 'delete', 'update', 'remove', 'insert']
+    if re.compile('|'.join(strict_check_words), re.IGNORECASE).search(text):
+        raise ValueError('Only select queries are supported')
+
+    # remove newlines, quotes and backslash character
+    text = " ".join([l.strip() for l in text.split("\n")])
+    return text.strip()
+
+
+class GraphPassThrough:
+    """Graph database pass through handler."""
+
+    def fetch_nodes(self, data):
+        """Fetch node from graph database."""
+        if data and data.get('query'):
+            try:
+                query = sanitize_text_for_query(data['query'])
+                if query:
+                    payload = {'gremlin': query}
+                    resp = requests.post(url=GREMLIN_SERVER_URL_REST, json=payload)
+                    return {'data': resp.json()}
+            except (ValueError, Exception) as e:
+                return {'error': str(e)}
+        else:
+            return {'warning': 'Invalid payload. Check your payload once again'}
+
+
+class PostgresPassThrough:
+    """Postgres connection pass through session and cursor handler."""
+
+    def __init__(self):
+        """Initialize the connection to Postgres database using psycopg2 as a pass through."""
+        conn_string = "host='{host}' dbname='{dbname}' user='{user}' password='{password}'".\
+            format(host=os.getenv('PGBOUNCER_SERVICE_HOST', 'bayesian-pgbouncer'),
+                   dbname=os.getenv('POSTGRESQL_DATABASE', 'coreapi'),
+                   user=os.getenv('POSTGRESQL_USER', 'coreapi'),
+                   password=os.getenv('POSTGRESQL_PASSWORD', 'coreapi'))
+        self.conn = psycopg2.connect(conn_string)
+        self.cursor = self.conn.cursor()
+
+    def fetch_records(self, data):
+        """Fetch records from RDS database."""
+        if data and data.get('query'):
+            try:
+                query = sanitize_text_for_query(data['query'])
+                if query:
+                    self.cursor.execute(query)
+                    return {'data': self.cursor.fetchmany(10)}
+            except (ValueError, Exception) as e:
+                return {'error': str(e)}
+        else:
+            return {'warning': 'Invalid payload. Check your payload once again'}
+
+
 class Postgres:
     """Postgres utility class to create postgres connection session."""
 

tests/test_rest_api.py

@@ -442,3 +442,35 @@ def test_notify_user_endpoint_2(alert_user, client):
                        content_type='application/json')
 
     assert resp.status_code == 200
+
+
+@patch('src.rest_api.PostgresPassThrough.fetch_records',
+       return_value=[('id', '12345')])
+def test_pgsql_endpoint(_mock1, client):
+    """Test the /api/v1/pgsql endpoint."""
+    resp = client.post(api_route_for('pgsql'))
+    assert resp is not None
+
+
+graph_response = {
+    "data": {
+        "requestId": "96604945-a7ca-4f90-83c9-90b00b339fd2",
+        "result": {
+            "data": [],
+            "meta": {}
+        },
+        "status": {
+            "attributes": {},
+            "code": 200,
+            "message": ""
+        }
+    }
+}
+
+
+@patch('src.rest_api.GraphPassThrough.fetch_nodes',
+       return_value=graph_response)
+def test_graph_endpoint(_mock1, client):
+    """Test the /api/v1/graph endpoint."""
+    resp = client.post(api_route_for('graph'))
+    assert resp is not None

tests/test_utils.py

@@ -7,7 +7,8 @@
 from src.utils import (
     DatabaseIngestion, alert_user, fetch_public_key, get_session, get_session_retry,
     retrieve_worker_result, scan_repo, server_run_flow, validate_request_data,
-    fix_gremlin_output, generate_comparison, get_first_query_result, get_parser_from_ecosystem
+    fix_gremlin_output, generate_comparison, get_first_query_result, get_parser_from_ecosystem,
+    PostgresPassThrough, GraphPassThrough
 )
 
 from src.parsers.maven_parser import MavenParser
@@ -19,6 +20,8 @@
 import os
 import json
 
+ppt = PostgresPassThrough()
+gpt = GraphPassThrough()
 
 mocked_object_response = {'stacks_summary': {'total_average_response_time': '200ms'}}
 
@@ -346,3 +349,43 @@ def test_get_parser_from_ecosystem():
     assert get_parser_from_ecosystem("unknown") is None
     assert get_parser_from_ecosystem("maven").__name__ == MavenParser.__name__
     assert get_parser_from_ecosystem("npm").__name__ == NodeParser.__name__
+
+
+def test_fetch_records():
+    """Test the PostgresPassThrough fetch records module."""
+    query = "select id from worker_results limit 1;"
+    resp = ppt.fetch_records(data={})
+    assert resp['warning'] == 'Invalid payload. Check your payload once again'
+    resp = ppt.fetch_records(data={'query': {'query': ''}})
+    assert resp['error'] is not None
+    resp = ppt.fetch_records(data={'query': 'delete all from some_table;'})
+    assert resp['error'] is not None
+    data = {'query': query}
+    resp = ppt.fetch_records(data)
+    assert resp is not None
+
+
+graph_resp = {
+    "requestId": "5cc29849-8e9b-4b66-90d0-f2569dc962b9",
+    "status": {
+        "message": "",
+        "code": 200,
+        "attributes": {}
+    },
+    "result": {
+        "data": [],
+        "meta": {}
+    }
+}
+
+
+@patch("src.utils.requests.post", return_value=graph_resp)
+def test_fetch_nodes(_mock1):
+    """Test the GraphPassThrough fetch nodes module."""
+    resp = gpt.fetch_nodes(data={})
+    assert resp['warning'] == 'Invalid payload. Check your payload once again'
+    resp = gpt.fetch_nodes(data={"query": "g.V().has('foo','bar').drop()')"})
+    assert resp['error'] is not None
+    query = "g.V().has('name','foo').valueMap();"
+    resp = gpt.fetch_nodes(data={'query': query})
+    assert resp is not None