Merge remote-tracking branch 'pso_dev/dev'

Makefile

@@ -17,7 +17,17 @@ SHELL := /usr/bin/env bash
 # The .PHONY directive tells make that this isn't a file target
 .PHONY: fmt
 fmt: ## Format files, including README
-	@python3 ./helpers/sort_lists.PY README.MD
+#	@python3 ./helpers/sort_lists.py README.MD
+	@$$SHELL ./helpers/format.sh
 
 help: ## Prints help for targets with comments
 	@grep -E '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "make \033[36m%- 30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: test
+test: ## Test if all files are properly formatted
+	@$$SHELL ./helpers/check_format.sh
+
+
+.PHONY: push_ci_image
+push_ci_image:
+	@cd cloudbuild && gcloud builds submit --project=cloud-eng-council --tag gcr.io/cloud-eng-council/make .

README.md

@@ -6,13 +6,13 @@ Common solutions and tools developed by Google Cloud's Professional Services tea
 ## Examples
 The examples folder contains example solutions across a variety of Google Cloud Platform products. Use these solutions as a reference for your own or extend them to fit your particular use case.
 
-
+* [Audio Content Profiling](examples/ml-audio-content-profiling) - A tool that builds a pipeline to scale the process of moderating audio files for inappropriate content using machine learning APIs.
 * [BigQuery Audit Log Dashboard](examples/bigquery-audit-log) - Solution to help audit BigQuery usage using Data Studio for visualization and a sample SQL script to query the back-end data source consisting of audit logs.
 * [BigQuery Billing Dashboard](examples/bigquery-billing-dashboard) - Solution to help displaying billing info using Data Studio for visualization and a sample SQL script to query the back-end billing export table in BigQuery.
 * [BigQuery Cross Project Slot Monitoring](examples/bigquery-cross-project-slot-monitoring) - Solution to help monitoring slot utilization across multiple projects, while breaking down allocation per project.
 * [BigQuery Group Sync For Row Level Access](examples/bigquery-row-access-groups) - Sample code to synchronize group membership from G Suite/Cloud Identity into BigQuery and join that with your data to control access at row level.
 * [BigQuery Pipeline Utility](tools/bqpipeline)  - Python utility class for defining data pipelines in BigQuery.
-* [Bigtable Dataflow Cyptocurrencies Exchange RealTime Example](examples/cryptorealtime) - Apache Beam example that reads from the Crypto Exchanges WebSocket API as Google Cloud Dataflow pipeline and saves the feed in Google Cloud Bigtable. Real time visualization and query examples from GCP Bigtable running on Flask server are included.
+* [Bigtable Dataflow Cryptocurrencies Exchange RealTime Example](examples/cryptorealtime) - Apache Beam example that reads from the Crypto Exchanges WebSocket API as Google Cloud Dataflow pipeline and saves the feed in Google Cloud Bigtable. Real time visualization and query examples from GCP Bigtable running on Flask server are included.
 * [Cloud Composer Examples](examples/cloud-composer-examples) - Examples of using Cloud Composer, GCP's managed Apache Airflow service.
 * [Cloud Function VM Delete Event Handler Example](examples/gcf-pubsub-vm-delete-event-handler) - Solution to automatically delete A records in Cloud DNS when a VM is deleted.  This solution implements a [Google Cloud Function][gcf] [Background Function][gcf-bg] triggered on `compute.instances.delete` events published through [Stackdriver Logs Export][logs-export].
 * [Cloud SQL Custom Metric](examples/cloud-sql-custom-metric) - An example of creating a Stackdriver custom metric monitoring Cloud SQL Private Services IP consumption.

cloudbuild.yaml

@@ -0,0 +1,3 @@
+steps:
+- name: 'gcr.io/cloud-eng-council/make'
+  args: ['test']

cloudbuild/Dockerfile

@@ -0,0 +1,13 @@
+# This Dockerfile builds the image used in Cloud Build CI to run 'make test'.
+# To push a new image, run 'gcloud builds submit --project=cloud-eng-council --tag gcr.io/cloud-eng-council/make .'
+# from this directory.
+
+FROM debian
+
+RUN apt-get update && apt-get install -y build-essential python python-pip python3 python3-pip
+RUN pip install yapf
+RUN pip3 install yapf
+
+RUN apt-get install -y golang
+
+ENTRYPOINT ["make"]

examples/bigquery-billing-dashboard/bigquery_billing_export.sql

@@ -57,8 +57,8 @@ SELECT
   TO_BASE64(MD5(billing_account_id)) AS billing_account_id,
   STRUCT(
     TO_BASE64(MD5(project.id)) AS id,
-    TO_BASE64MD5(project.name)) AS name,
-    TO_BASE64MD5(project.ancestry_numbers)) AS ancestry_numbers,
+    TO_BASE64(MD5(project.name)) AS name,
+    TO_BASE64(MD5(project.ancestry_numbers)) AS ancestry_numbers,
     project.labels AS labels
   ) AS project,
   billing.* EXCEPT(billing_account_id, project)

examples/dataflow-data-generator/README.md

@@ -67,7 +67,7 @@ taken was to generate a `REPEATED` `RECORD` (aka `ARRAY<STRUCT>`) and each recor
 will have between 0 and 3 elements in this array. 
 ie. 
 ```
---schema_file=gs://python-dataflow-examples/schemas/lineorder-schema.json
+--schema_file=gs://python-dataflow-example/schemas/lineorder-schema.json
 ```
 lineorder-schema.json:
 ```

examples/dataflow-data-generator/data-generator-pipeline/data_generator_pipeline.py

@@ -33,6 +33,7 @@
 write_n_line_file_to_gcs
 
 import avro.schema
+import fastavro
 import os
 
 from data_generator.CsvUtil import dict_to_csv
@@ -113,6 +114,7 @@ def run(argv=None):
 
     if data_args.avro_schema_file:
         avsc = avro.schema.parse(open(data_args.avro_schema_file, 'rb').read())
+        fastavro_avsc = fastavro.schema.load_schema(data_args.avro_schema_file)
 
         (rows
             # Need to convert time stamps from strings to timestamp-micros
@@ -123,7 +125,7 @@ def run(argv=None):
                     codec='null',
                     file_name_suffix='.avro',
                     use_fastavro=True,
-                    schema=avsc
+                    schema=fastavro_avsc
                 )
         )
 

examples/dataflow-data-generator/requirements.txt

@@ -12,4 +12,4 @@ numpy>=1.14.2
 pandas>=0.23.4
 scipy>=1.1.0
 six==1.10.0
-
+httplib2>=0.10.3

examples/dataproc-persistent-history-server/terraform/history-server.tf

@@ -43,7 +43,7 @@ resource "google_dataproc_cluster" "history-server" {
         "yarn:yarn.nodemanager.remote-app-log-dir"          = "gs://${var.history-bucket}/yarn/logs/"
         "yarn:yarn.log-aggregation.retain-seconds"          = "604800"
         "yarn:yarn.log.server.url"                          = "http://${var.history-server}-m:19888/jobhistory/logs"
-        "mapreduce.jobhistory.always-scan-user-dir"         = "true"
+        "mapred:mapreduce.jobhistory.always-scan-user-dir"  = "true"
         "mapred:mapreduce.jobhistory.address"               = "${var.history-server}-m:10020"
         "mapred:mapreduce.jobhistory.webapp.address"        = "${var.history-server}-m:19888"
         "mapred:mapreduce.jobhistory.done-dir"              = "gs://${var.history-bucket}/done-dir"