feat(metrics): adding total queries badge (Checkmarx#3370)

.github/scripts/metrics/get-metrics.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+import glob
+import json
+import os
+
+from tabulate import tabulate
+
+queries_basepath = 'assets/queries'
+queries_path = {
+    'cloudformation': os.path.join(queries_basepath, 'cloudFormation', '*'),
+    'openapi': os.path.join(queries_basepath, 'openAPI', '*'),
+    'ansible': os.path.join(queries_basepath, 'ansible', '**', '*'),
+    'k8s': os.path.join(queries_basepath, 'k8s', '*'),
+    'common': os.path.join(queries_basepath, 'common', '*'),
+    'dockerfile': os.path.join(queries_basepath, 'dockerfile', '*'),
+    'terraform': os.path.join(queries_basepath, 'terraform', '**', '*'),
+}
+samples_ext = {
+    'cloudformation': ['yaml', 'json'],
+    'openapi': ['yaml', 'json'],
+    'ansible': ['yaml'],
+    'k8s': ['yaml'],
+    'common': ['yaml', 'json', 'dockerfile', 'tf'],
+    'dockerfile': ['dockerfile'],
+    'terraform': ['tf'],
+}
+summary = {
+    'total': 0,
+}
+
+rego_summary = {
+    'total': 0
+}
+
+samples_summary = {
+    'total': 0
+}
+
+
+def queries_count(path):
+    rtn_count = 0
+    with open(path) as fp:
+        metadata_obj = json.load(fp)
+        if 'aggregation' in metadata_obj:
+            rtn_count = metadata_obj['aggregation']
+        else:
+            rtn_count = 1
+    return rtn_count
+
+
+for key, value in queries_path.items():
+    metadata_path = os.path.join(value, 'metadata.json')
+    platform_count = sum([queries_count(path)
+                         for path in glob.glob(metadata_path)])
+    summary[f'{key}_queries'] = platform_count
+    summary['total'] += platform_count
+
+    rego_path = os.path.join(value, 'query.rego')
+    rego_summary[f'{key}_rego'] = len([path for path in glob.glob(rego_path)])
+    rego_summary['total'] += len([path for path in glob.glob(rego_path)])
+
+    for ext in samples_ext[key]:
+        sample_path = os.path.join(value, 'test', f'*.{ext}')
+        ext_samples = len([path for path in glob.glob(sample_path)])
+        samples_summary[f'{key}_{ext}_samples'] = ext_samples
+        samples_summary['total'] += ext_samples
+
+print("::group::Queries Metrics")
+print(tabulate([[key, value] for key, value in summary.items()], headers=[
+      'Platform', 'Count'], tablefmt='orgtbl'))
+print("::endgroup::")
+print()
+print(f"::set-output name=total_queries::{summary['total']}")
+print()
+print("::group::Rego File Metrics")
+print(tabulate([[key, value] for key, value in rego_summary.items()], headers=[
+      'Platform', 'Count'], tablefmt='orgtbl'))
+print("::endgroup::")
+print()
+print(f"::set-output name=total_rego_files::{rego_summary['total']}")
+print()
+print("::group::Sample File Metrics")
+print(tabulate([[key, value] for key, value in samples_summary.items()], headers=[
+      'Samples', 'Count'], tablefmt='orgtbl'))
+print("::endgroup::")

.github/scripts/metrics/requirements.txt

@@ -0,0 +1 @@
+tabular

.github/workflows/go-ci-coverage.yaml

@@ -21,11 +21,15 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
+      - name: Instal CLI dependencies
+        run: sudo apt-get install tee -y
       - name: Run test metrics script
         id: testcov
         run: |
-          make test-coverage-report
-          cat coverage.html
+          make test-coverage-report | tee test-results
+          TOTAL_TESTS=$(cat test-results | grep -v TestQueriesContent/ | grep -v TestQueriesMetadata/ | grep -v TestQueries/ | grep PASS | wc -l)
+          echo "Total Tests :: ${TOTAL_TESTS}"
+          echo "::set-output name=total_tests::${TOTAL_TESTS}"
       - name: Generate badge
         run: |
           curl -L \

.github/workflows/go-ci-metrics.yaml

@@ -1,22 +1,53 @@
 name: go-ci-metrics
 
 on:
-  pull_request:
+  push:
     branches: [master]
-
+    paths:
+      - "assets/queries/"
 jobs:
   metrics:
     name: test-metrics
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Source
         uses: actions/[email protected]
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.16.x
-      - name: Install zsh
-        run: sudo apt install zsh
       - name: Run test metrics script
-        run: zsh .github/scripts/get-test-metrics.sh
-
+        id: metrics
+        run: |
+          pip3 install -r .github/scripts/metrics/requirements.txt
+          python3 .github/scripts/metrics/get-metrics.py
+      - name: Generate badge
+        run: |
+          curl -L \
+            https://img.shields.io/badge/Queries-${{ steps.metrics.outputs.total_queries }}-blue.svg > queries.svg
+          cat queries.svg
+      - uses: actions/upload-artifact@master
+        with:
+          name: ${{ runner.os }}-queries-badge-latest
+          path: queries.svg
+  publish:
+    name: publish-metrics
+    runs-on: ubuntu-latest
+    needs: metrics
+    steps:
+      - name: Checkout Source
+        uses: actions/[email protected]
+        with:
+          ref: gh-pages
+      - name: Configure git commit author
+        run: |
+          git config --global user.name "KICSBot"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Download Queries Badge SVG
+        uses: actions/download-artifact@master
+        with:
+          name: ${{ runner.os }}-queries-badge-latest
+          path: latest-metrics
+      - name: Generate badge
+        run: |
+          mv latest-metrics/coverage.svg queries.svg
+          git add queries.svg
+          git status
+          git commit -m 'chore(metrics): updating queries count badge'
+          git push origin gh-pages