Merge pull request #51 from esnet/sc-demo

More than 3 layers; Load layers from discreet URLs; Shore up tests; Bug Fixes

.github/workflows/ci.yml

@@ -1,19 +1,16 @@
 name: CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [
+  push,
+  pull_request
+]
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 
-      - name: Setup Node.js environment
+      - name: Setup Node.js environment for build and testing
         uses: actions/[email protected]
         with:
           node-version: "18.x"
@@ -43,9 +40,25 @@ jobs:
       - name: Install dependencies
         run: yarn install --frozen-lockfile
 
-      - name: Build and test frontend
+      - name: Build frontend
         run: yarn build
 
+      - name: Test components
+        run: make test:component
+
+      - name: Install E2E dependencies
+        run: |
+          npx playwright install --with-deps
+
+      - name: Setup E2E testing parameters
+        run: |
+          mkdir -p ./playwright/.auth
+          echo "{\"username\": \"admin\", \"password\": \"admin\" }" > ./playwright/.auth/credentials.json
+
+      - name: E2E testing
+        run: |
+          make test:e2e
+
       - name: Check for backend
         id: check-for-backend
         run: |

.gitignore

@@ -40,11 +40,18 @@ e2e-results/
 .idea
 .vscode
 .DS_store
+*.code-workspace
 
 # Playwright
 playwright*
 test-results
 tests-examples
 
+# webpack test junk
+test/dist/*
+
 # autogenerated by Makefile for e2e
 e2e/grafana-docker.json
+
+# artifacts from github
+.config/env

Makefile

@@ -5,7 +5,12 @@ BREW=$(which brew)
 CLI_TOOLS_PATH=~/work/cli-tools/stardust_map_topology
 PROJECT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
 CONTAINER_NAME=esnet-networkmap-panel
+PROXY_NAME=mitmproxy
+NETWORK_NAME=esnet-networkmap-e2e-net
 CONTAINER_ID=$(shell docker ps -f name=$(CONTAINER_NAME) -q)
+INSTANCES=$(shell docker ps --filter name=$(CONTAINER_NAME) --filter name=$(PROXY_NAME) -qa)
+NETWORKS=$(shell docker network ls --filter name=${NETWORK_NAME} -q)
+SPINUP_SLEEP_T=5
 
 .PHONY: prod
 prod:
@@ -32,24 +37,40 @@ restart:
 
 .PHONY: compose
 compose:
-	# start grafana docker instance and map project to plugin directory on instance
-	docker-compose up -d
-	# get instance info
+	# start grafana docker instances + network, and map project to plugin directory on instance
+	@if test "$(strip $(INSTANCES))" = ""; then docker -D compose up -d; else echo "Grafana instance found."; fi
+	# get instances info
+	sleep 2
 	docker inspect $(CONTAINER_NAME) > $(PROJECT_DIR)/e2e/grafana-docker.json
-
 .PHONY: test
 test: compose
+	@echo "Starting component tests..."
 	yarn test
+	yarn test:react
+	@echo "Waiting for container to spin up..."
+	@sleep $(SPINUP_SLEEP_T)
+	@echo "Starting E2E Tests..."
 	yarn e2e
 
 .PHONY: test\:component
 test\:component:
+	@echo "Starting component tests..."
 	yarn test
+	yarn test:react
 
 .PHONY: test\:e2e
 test\:e2e: compose
 	# run e2e tests
-	yarn e2e
+	@echo "Waiting for container to spin up..."
+	@sleep $(SPINUP_SLEEP_T)
+	@echo "Starting E2E Tests..."
+	yarn run e2e
+
+.PHONY: test\:ui
+test\:ui: compose
+	# run e2e tests, but with ui
+	@echo "Starting E2E tests in Playwright UI..."
+	yarn run e2e:ui
 
 .PHONY: testignore
 testignore:
@@ -76,3 +97,9 @@ push:
 
 .PHONY: publish
 publish: check_version prod testignore confirm push
+
+.PHONY: clean
+clean:
+	@rm -rf ${PROJECT_DIR}.config/env
+	@if test "$(strip $(INSTANCES))" = ""; then echo "No instances to cleanup."; else docker rm -v -f $(INSTANCES); fi
+	@if test "$(strip $(NETWORKS))" = ""; then echo "No network $(NETWORK_NAME) found to remove."; else docker network rm $(NETWORKS); fi

README.md

@@ -57,7 +57,7 @@ It also contains an example topology that will be used while setting up the Netw
 
 - Input your Google Sheets API key into the "API Key" input
 
-- Click [Test + Save] 
+- Click [Test + Save]
 
 ### 2. Setup a Test Dashboard
 
@@ -187,7 +187,7 @@ Leave this set to gray. It will help to show when we've correctly associated edg
 
 **Layer 1 Endpoint Identifier**
 
-This input specifies the attribute of "endpoint_identifiers" prop of each of the "edges" objects in your JSON topology. Each "endpoint_identifiers" entry in the JSON should have a key matching the value specified in this text box (although the assigned JSON value is an array of node names of length two, does not matter). The value for Grafana here should be left as the default "pops".
+This input specifies the attribute of "endpoint_identifiers" prop of each of the "edges" objects in your JSON topology. Each "endpoint_identifiers" entry in the JSON should have a key matching the value specified in this text box (although the assigned JSON value is an array of node names of length two, does not matter). The value for Grafana here should be left as the default "names".
 
 **Layer 1 Node Highlight Color**
 

demonstration/dashboard.json

@@ -89,35 +89,22 @@
         "initialViewStrategy": "static",
         "layers": [
           {
-            "color": "#49a7f25e",
-            "dashboardEdgeDstVar": "destination",
-            "dashboardEdgeDstVarL1": "dest",
+            "color": "grey",
+            "dashboardEdgeDstVar": "dest",
             "dashboardEdgeSrcVar": "source",
-            "dashboardEdgeSrcVarL1": "source",
             "dashboardNodeVar": "node",
             "dataFieldLabel": "Volume:",
             "dstField": "dst",
             "dstFieldLabel": "To:",
-            "edgeWidth": 2.5,
-            "endpointId": "pops",
+            "edgeWidth": 3,
+            "endpointId": "names",
             "inboundValueField": "in_bits",
             "legend": true,
-            "mapjson": "{\"edges\":[{\"name\":\"A--B\",\"meta\":{\"endpoint_identifiers\":{\"pops\":[\"A\",\"B\"]}},\"coordinates\":[[42.30619185993376,-106.00734642032997],[42.30619185993376,-92.30145333497663]],\"children\":[]},{\"name\":\"A--C\",\"meta\":{\"endpoint_identifiers\":{\"pops\":[\"A\",\"C\"]}},\"coordinates\":[[42.30619185993376,-106.00734642032997],[34.5952660320894,-99.10139879776413]],\"children\":[]},{\"name\":\"B--C\",\"meta\":{\"endpoint_identifiers\":{\"pops\":[\"B\",\"C\"]}},\"coordinates\":[[42.30619185993376,-92.30145333497663],[34.5952660320894,-99.10139879776413]],\"children\":[]}],\"nodes\":[{\"name\":\"A\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[42.30619185993376,-106.00734642032997]},{\"name\":\"B\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[42.30619185993376,-92.30145333497663]},{\"name\":\"C\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[34.5952660320894,-99.10139879776413]}]}",
+            "mapjson": "{\"edges\":[{\"name\":\"A--B\",\"meta\":{\"endpoint_identifiers\":{\"names\":[\"A\",\"B\"]}},\"coordinates\":[[42.30619185993376,-106.00734642032997],[42.30619185993376,-92.30145333497663]],\"children\":[]},{\"name\":\"A--C\",\"meta\":{\"endpoint_identifiers\":{\"names\":[\"A\",\"C\"]}},\"coordinates\":[[42.30619185993376,-106.00734642032997],[34.5952660320894,-99.10139879776413]],\"children\":[]},{\"name\":\"B--C\",\"meta\":{\"endpoint_identifiers\":{\"names\":[\"B\",\"C\"]}},\"coordinates\":[[42.30619185993376,-92.30145333497663],[34.5952660320894,-99.10139879776413]],\"children\":[]}],\"nodes\":[{\"name\":\"A\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[42.30619185993376,-106.00734642032997]},{\"name\":\"B\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[42.30619185993376,-92.30145333497663]},{\"name\":\"C\",\"meta\":{\"display_name\":\"\",\"svg\":\"\",\"template\":\"\"},\"coordinate\":[34.5952660320894,-99.10139879776413]}]}",
             "name": "Layer 1",
             "nodeHighlight": "dark-blue",
             "nodeNameMatchField": "Node",
-            "nodeThresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "red"
-                },
-                {
-                  "color": "dark-blue",
-                  "value": 1
-                }
-              ]
-            },
+            "nodeThresholds": "",
             "nodeValueField": "Up/Down (1: up, 0:down)",
             "nodeWidth": 5,
             "outboundValueField": "out_bits",
@@ -136,7 +123,7 @@
             "dataFieldLabel": "Volume:",
             "dstFieldLabel": "To:",
             "edgeWidth": 1.5,
-            "endpointId": "pops",
+            "endpointId": "names",
             "legend": false,
             "mapjson": "{\"edges\":[],\"nodes\":[]}",
             "name": "Layer 2",
@@ -154,7 +141,7 @@
             "dataFieldLabel": "Volume:",
             "dstFieldLabel": "To:",
             "edgeWidth": 3,
-            "endpointId": "pops",
+            "endpointId": "names",
             "legend": false,
             "mapjson": "{\"edges\":[],\"nodes\":[]}",
             "name": "Layer 3",

docker-compose.yaml

@@ -1,14 +1,39 @@
-version: '3.0'
-
 services:
+  mitmproxy:
+    image: mitmproxy/mitmproxy
+    container_name: mitmproxy
+    tty: true
+    ports:
+      - 8080:8080/tcp
+    command: mitmdump --mode transparent --modify-headers "/Access-Control-Allow-Origin/*" --modify-headers "/X-Remote-User/remote" --modify-headers "/X-Forwarded-Host/esnet-networkmap-panel:3000"
+    networks:
+      - esnet-networkmap-e2e-net
+
   grafana:
+    environment:
+      - GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource
     container_name: 'esnet-networkmap-panel'
     build:
       context: ./.config
+      dockerfile: Dockerfile
       args:
-        grafana_version: ${GRAFANA_VERSION:-9.3.8}
+        grafana_version: ${GRAFANA_VERSION:-10.4.0}
+    restart: always
     ports:
       - 3000:3000/tcp
     volumes:
       - ./dist:/var/lib/grafana/plugins/esnet-networkmap-panel
       - ./provisioning:/etc/grafana/provisioning
+    networks:
+      - esnet-networkmap-e2e-net
+
+networks:
+  esnet-networkmap-e2e-net:
+    name: esnet-networkmap-e2e-net
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        -
+          subnet: 192.168.2.0/24
+