switching to client libraries

dlp/main.py

@@ -11,9 +11,58 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+#
+# EAC NOTES
+# - Set config
+# - On deploy, set service account if needed for permissions
+# - On deploy, gcfunction name must match entry function from source
+# - Build source will be zipped and copied as is to a GCS bucket
+# - Function build artifacts will live in separate bucket
+#  gcloud functions deploy redact_gcs --region us-central1 --service-account \
+#        <[email protected]> --trigger-bucket dlp-inbound --runtime python38
+#
+
 
 import sys
+import os
 from flask import escape
+from google.cloud import storage
+from google.cloud import dlp
+
+
+def redact_gcs2(event, context):
+    """
+    Pull image, redact, push image using client libraries
+    """
+    # CLIENT
+    stor = storage.Client()
+
+    # CONFIG
+    # NOTE  project = os.environ.get("GOOGLE_CLOUD_PROJECT")
+    # ENV variable only exists in GCE, not GCF runtime
+    project = 'sc-nlp'
+    output_bucket = "dlp-outbound"
+    output_file = "/tmp/" + event['name'] + "out"
+
+    #TODO
+    #input_bytes = event['name']
+
+    #output_file = redact_image_all_text2(input_bytes)
+    # write output_file to output_bucket
+
+
+def redact_image_all_text2(input_bytes):
+    # ALTERNATIVE CONSTRUCTUOR dlp = google.cloud.dlp_v2.DlpServiceClient()
+    dlp = dlp.Client()
+
+    # TODO
+    request_bytes = input_bytes
+    redacted_bytes = response[content]
+
+    return redacted_bytes
+
+
 
 def redact_gcs(event, context):
     """Background Cloud Function to be triggered by Cloud Storage.
@@ -29,11 +78,12 @@ def redact_gcs(event, context):
         None; the output is written to Stackdriver Logging
     """
 
-    import os
 
-
-    PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")
-    OUTPUT_BUCKET = "gs://dlp-outbound"
+    # CONFIG
+    output_bucket = "dlp-outbound"
+    #project = os.environ.get("GOOGLE_CLOUD_PROJECT")
+    # that ENV only exists in GCE, not GCF runtime
+    project = 'sc-nlp'
 
 
     #print('Event ID: {}'.format(context.event_id))
@@ -44,16 +94,21 @@ def redact_gcs(event, context):
     print('Created: {}'.format(event['timeCreated']))
     #print('Updated: {}'.format(event['updated']))
 
+
+    # DEBUG Get environment of worker env
+    print(os.environ)
+    print(os.system("whoami"))
+
     # Get Image
     pull_cmd = "gsutil cp gs://{}/{} .".format(event['bucket'], event['name'])
     os.system(pull_cmd)
 
     # Process image for DLP
-    out_file = event['name'] + 'out'
-    redact_image_all_text(PROJECT, event['name'], out_file)
+    output_file = "/tmp/" + event['name'] + "out"
+    redact_image_all_text(project, event['name'], output_file)
 
     # Push result to output bucket
-    push_cmd = "gsutil cp {} gs://{}/".format(out_file, OUT_BUCKET)
+    push_cmd = "gsutil cp {} gs://{}/".format(output_file, output_bucket)
     os.system(push_cmd)
 
 
@@ -112,8 +167,6 @@ def redact_image_all_text(
 
 
 
-
-
 # [START functions_http_content]
 def hello_content(request):
     """ Responds to an HTTP request using data from the request body parsed

dlp/requirements.txt

@@ -3,4 +3,3 @@ google-cloud-storage==1.39.0
 google-cloud-pubsub==2.6.0
 google-cloud-datastore==2.1.3
 google-cloud-bigquery==2.20.0
-gsutil

gcf/main.py

@@ -0,0 +1,66 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+# [START functions_helloworld_http]
+# [START functions_http_content]
+from flask import escape
+
+# [END functions_helloworld_http]
+# [END functions_http_content]
+
+
+# [START functions_helloworld_get]
+def hello_get(request):
+    """HTTP Cloud Function.
+    Args:
+        request (flask.Request): The request object.
+        <https://flask.palletsprojects.com/en/1.1.x/api/#incoming-request-data>
+    Returns:
+        The response text, or any set of values that can be turned into a
+        Response object using `make_response`
+        <https://flask.palletsprojects.com/en/1.1.x/api/#flask.make_response>.
+    Note:
+        For more information on how Flask integrates with Cloud
+        Functions, see the `Writing HTTP functions` page.
+        <https://cloud.google.com/functions/docs/writing/http#http_frameworks>
+    """
+    return 'Hello World!'
+# [END functions_helloworld_get]
+
+
+# [START functions_helloworld_http]
+def hello_http(request):
+    """HTTP Cloud Function.
+    Args:
+        request (flask.Request): The request object.
+        <https://flask.palletsprojects.com/en/1.1.x/api/#incoming-request-data>
+    Returns:
+        The response text, or any set of values that can be turned into a
+        Response object using `make_response`
+        <https://flask.palletsprojects.com/en/1.1.x/api/#flask.make_response>.
+    """
+    request_json = request.get_json(silent=True)
+    request_args = request.args
+
+    if request_json and 'name' in request_json:
+        name = request_json['name']
+    elif request_args and 'name' in request_args:
+        name = request_args['name']
+    else:
+        name = 'World'
+    return 'Hello {}!'.format(escape(name))
+# [END functions_helloworld_http]
+