Bringing anuradha changes to resolve conflicts due to pr#653 (GoogleC…

…loudPlatform#655)

* Bringing anuradha changes to resolve conflicts due to pr#653

* fix license header year

tools/quota-monitoring-alerting/README.md

@@ -23,7 +23,7 @@ The architecture is built using Google Cloud managed services - Cloud Functions,
 - DataFlow is used to load data in BigQuery.
 - BigQuery is used to store data. 
 - Alert threshold will be applicable across all metrics. 
-- Alerts can be received by email and other channels.
+- Alerts can be received by Email, Mobile App, PagerDuty, SMS, Slack, Web Hooks and Pub/Sub. Cloud Monitoring custom log metric has been leveraged to create Alerts.
 - Easy to get started and deploy with Data Studio Dashboard. In addition to Data Studion, other visualization tools can be configured. 
 - The Data Studio report can be scheduled to be emailed to appropriate team for weekly/daily reporting.
 ## 3. Deployment Guide
@@ -122,10 +122,6 @@ Created service account [sa-quota-monitoring-project-1].
   - Cloud Functions Admin
 - Cloud Scheduler
   - Cloud Scheduler Admin
-- Dataflow
-  - Dataflow Admin
-  - Dataflow Worker
-  - Compute Storage Admin
 - Pub/Sub 
   - Pub/Sub Admin
 - Run Terraform
@@ -149,12 +145,6 @@ gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAcco
 
 gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudscheduler.admin" --condition=None
 
-gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/dataflow.admin" --condition=None
-
-gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/dataflow.worker" --condition=None
-
-gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.storageAdmin" --condition=None
-
 gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/pubsub.admin" --condition=None
 
 gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/iam.serviceAccountUser" --condition=None
@@ -309,9 +299,9 @@ gcloud iam service-accounts keys create CREDENTIALS_FILE.json \
 ```
 mkdir terraform
 cd terraform
-curl -o main.tf https://github.com/GoogleCloudPlatform/professional-services/tools/quota-monitoring-alerting/terraform/main.tf
-curl -o main.tf https://github.com/GoogleCloudPlatform/professional-services/tools/quota-monitoring-alerting/terraform/variables.tf
-curl -o main.tf https://github.com/GoogleCloudPlatform/professional-services/tools/quota-monitoring-alerting/terraform/terraform.tfvars
+gsutil cp gs://quota-monitoring-solution-demo-bucket/main.tf .
+gsutil cp gs://quota-monitoring-solution-demo-bucket/variables.tf .
+gsutil cp gs://quota-monitoring-solution-demo-bucket/terraform.tfvars .
 ```
 2. Verify that you have these 4 files in your local directory:
    - CREDENTIALS_FILE.json
@@ -322,7 +312,6 @@ curl -o main.tf https://github.com/GoogleCloudPlatform/professional-services/too
 1. Open terraform.tfvars file in your favourite editor and change values for the variable 
 2. Values for variable source_code_bucket_name, source_code_zip and source_code_notification_zip are for source code zip in the storage bucket. These are links to the Cloud Function source code. If you want to upgrade to latest code changes everytime you run 'terraform apply', change to this code source repository. DO NOT CHANGE if you do not want to recieve latest code changes while running 'terraform apply' everytime after deployment. 
 3. For region, use the same region as used for app engine in earlier steps.
-4. For Email notification, Create a Send Grid API Key [here](https://sendgrid.com/docs/ui/account-and-settings/api-keys/) and add in the terraform. Note: SendGrid may take a few days to create a key. This depends on the domain.Proceed with the rest of the configuration, add key once available and rerun terraform. 
 ```
 vi terraform.tfvars
 ```
@@ -361,9 +350,9 @@ vi terraform.tfvars
 <img src="img/ds_edit_data_source.png" align="center" />
 It will open the data source details
 <img src="img/ds_datasource_config_step_1.png" align="center" />
-6. In the query, replace BigQuery project, dataset id and table name
+6. In the panel, select BigQuery project, dataset id and table name
 <img src="img/ds_edit_data_source_big_query.png" align="center" />
-7. Verify the query by running in BigQuery Editor to make sure that query returns right results and there are no syntax errors:
+7. Verify the query by running in BigQuery Editor to make sure query returns right results and there are no syntax errors:
 Note: Replace BigQuery project id, dataset id and table name:
 
 ```
@@ -465,13 +454,12 @@ Quota monitoring reports can be scheduled from the Data Studio dashboard using 
 
 ## 4. What is Next? 
 1. Include quota metrics beyond compute
-2. Alerting (PagerDuty, ServiceNow, Slack)
-3. Graphs (Quota utilization over a period of time)
-4. Search project, folder, org, region
-5. Threshold configurable for each metric
+2. Graphs (Quota utilization over a period of time)
+3. Search project, folder, org, region
+4. Threshold configurable for each metric
 
 ## 5. Contact Us
-For any comments, issues or feedback, please reach out to us at pso-quota-framework@google.com
+For any comments, issues or feedback, please reach out to us at pso-quota-monitoring@google.com
 
 
 

tools/quota-monitoring-alerting/quota-notification/pom.xml

@@ -19,11 +19,6 @@
       <artifactId>maven-compiler-plugin</artifactId>
       <version>3.8.1</version>
     </dependency>
-    <dependency>
-      <groupId>com.sendgrid</groupId>
-      <artifactId>sendgrid-java</artifactId>
-      <version>4.1.2</version>
-    </dependency>
     <dependency>
       <groupId>com.google.cloud.functions</groupId>
       <artifactId>functions-framework-api</artifactId>
@@ -43,6 +38,11 @@
       <version>30.1-jre</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-logging</artifactId>
+      <version>2.2.0</version>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-bigquery</artifactId>

tools/quota-monitoring-alerting/quota-notification/src/main/java/functions/SendNotification.java

@@ -15,6 +15,7 @@
 */
 package functions;
 
+import com.google.cloud.MonitoredResource;
 import com.google.cloud.bigquery.BigQuery;
 import com.google.cloud.bigquery.BigQueryException;
 import com.google.cloud.bigquery.BigQueryOptions;
@@ -26,17 +27,14 @@
 import com.google.cloud.bigquery.TableResult;
 import com.google.cloud.functions.BackgroundFunction;
 import com.google.cloud.functions.Context;
-import com.sendgrid.Content;
-import com.sendgrid.Email;
-import com.sendgrid.Mail;
-import com.sendgrid.Method;
-import com.sendgrid.Personalization;
-import com.sendgrid.Request;
-import com.sendgrid.Response;
-import com.sendgrid.SendGrid;
+import com.google.cloud.logging.LogEntry;
+import com.google.cloud.logging.Logging;
+import com.google.cloud.logging.LoggingOptions;
+import com.google.cloud.logging.Payload.StringPayload;
+import com.google.cloud.logging.Severity;
 import functions.eventpojos.PubSubMessage;
-import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 import java.util.logging.Logger;
@@ -45,12 +43,9 @@
  * Cloud Function triggered by Pub/Sub topic to send notification
  * */
 public class SendNotification implements BackgroundFunction<PubSubMessage> {
-  private static final String FROM_EMAIL_ID = System.getenv("FROM_EMAIL_ID");
-  private static final String TO_EMAIL_IDS = System.getenv("TO_EMAIL_IDS");
   private static final String HOME_PROJECT = System.getenv("HOME_PROJECT");
   private static final String DATASET = System.getenv("ALERT_DATASET");
   private static final String TABLE = System.getenv("ALERT_TABLE");
-  private static final String SENDGRID_API_KEY = System.getenv("SENDGRID_API_KEY");
 
   private static final Logger logger = Logger.getLogger(SendNotification.class.getName());
 
@@ -60,8 +55,10 @@ public class SendNotification implements BackgroundFunction<PubSubMessage> {
   @Override
   public void accept(PubSubMessage message, Context context) {
     // logger.info(String.format(message.getEmailIds()));
+    logger.info("Successfully made it to sendNotification");
     List<String> alerts = browseAlertTable();
-    sendEmail(alerts);
+    logger.info("Successfully got data from alert table");
+    sendAlert(alerts);
     return;
   }
 
@@ -121,35 +118,24 @@ private static List<String> browseAlertTable() {
   }
 
   /*
-   * API to send Email using SendGrid API
+   * Logging API used to construct and send custom log containing quota data
+   * The logName triggers the alert policy
    * */
-  private static void sendEmail(List<String> alerts) {
-    Email from = new Email(FROM_EMAIL_ID);
-    String subject = "ALERT Usage of " + alerts.size() + " metrics is above threshold";
-    String[] toEmailIds = TO_EMAIL_IDS.split(",");
-    Email to = new Email(toEmailIds[0]);
-    Personalization p1 = new Personalization();
-    for (String email : toEmailIds) {
-      p1.addTo(new Email(email));
-    }
-    String alertsStr = String.join("\n", alerts);
-    Content content = new Content("text/plain", alertsStr);
-    Mail mail = new Mail(from, subject, to, content);
-    mail.addPersonalization(p1);
-
-    // SendGrid sg = new SendGrid(System.getenv("SENDGRID_API_KEY"));
-    SendGrid sg = new SendGrid(SENDGRID_API_KEY);
-    Request request = new Request();
-    try {
-      request.setMethod(Method.POST);
-      request.setEndpoint("mail/send");
-      request.setBody(mail.build());
-      Response response = sg.api(request);
-      logger.info(String.valueOf(response.getStatusCode()));
-      logger.info(String.valueOf(response.getBody()));
-      logger.info(String.valueOf(response.getHeaders()));
-    } catch (IOException ex) {
-      logger.severe(ex.getMessage());
-    }
+  private static void sendAlert(List<String> alerts) {
+    String logName = "quota-alerts";
+    String alertsStr = String.join("<br>", alerts);
+    String text = alertsStr;
+    logger.info(text);
+    LoggingOptions logging = LoggingOptions.getDefaultInstance();
+    Logging log_client = logging.getService();
+    LogEntry entry =
+        LogEntry.newBuilder(StringPayload.of(text))
+            .setSeverity(Severity.INFO)
+            .setLogName(logName)
+            .setResource(MonitoredResource.newBuilder("global").build())
+            .build();
+
+    // Writes the log entry asynchronously
+    log_client.write(Collections.singleton(entry));
   }
 }

tools/quota-monitoring-alerting/quota-scan/pom.xml

@@ -74,6 +74,10 @@
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-pubsub</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-bigquery</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.threeten</groupId>
       <artifactId>threetenbp</artifactId>