From dc4652da1c92d544b6ffb70bb0f3820f591dd96a Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Wed, 13 Mar 2024 11:27:37 +0200 Subject: [PATCH 01/88] PMM-12977 adopt BM setup for PMM 3 --- pmm_psmdb-pbm_setup/Dockerfile | 23 ++++++++++--------- pmm_psmdb-pbm_setup/docker-compose-pmm.yaml | 6 ++--- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 14 +++++------ .../docker-compose-sharded.yaml | 20 ++++++++-------- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 240a81e..62fc3e7 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -52,20 +52,21 @@ RUN set -ex; \ yum -y install percona-backup-mongodb ; \ else \ yum -y install percona-backup-mongodb-${PBM_VERSION}.el9 ; \ - fi && \ - if [[ $PMM_CLIENT_VERSION == http* ]]; then \ - yum -y install pmm2-client && \ - curl -Lf -o /tmp/pmm2-client.tar.gz $PMM_CLIENT_VERSION && \ - cd /tmp && tar -xvf pmm2-client.tar.gz --transform 's,^/*[^/]*,/pmm2-client,S' && \ - cd /tmp/pmm2-client && ./install_tarball && cd ../ && rm -rf pmm2* ;\ - elif [[ "$PMM_CLIENT_VERSION" =~ latest|dev-latest ]]; then \ - yum -y install pmm2-client ; \ - elif [ $PMM_CLIENT_VERSION == "pmm2-rc" ]; then \ + fi + +RUN if [[ $PMM_CLIENT_VERSION == http* ]]; then \ + yum -y install pmm-client && \ + curl -Lf -o /tmp/pmm-client.tar.gz $PMM_CLIENT_VERSION && \ + cd /tmp && tar -xvf pmm-client.tar.gz --transform 's,^/*[^/]*,/pmm-client,S' && \ + cd /tmp/pmm-client && ./install_tarball && cd ../ && rm -rf pmm* ;\ + elif [[ "$PMM_CLIENT_VERSION" =~ 3-dev-latest|latest ]]; then \ + yum -y install pmm-client ; \ + elif [[ $PMM_CLIENT_VERSION =~ pmm3-rc ]]; then \ percona-release enable original testing \ yum update --refresh -y \ - yum -y install pmm2-client ; \ + yum -y install pmm-client ; \ else \ - yum -y install pmm2-client-${PMM_CLIENT_VERSION}-6.el9 ; \ + yum -y install pmm-client-${PMM_CLIENT_VERSION}-6.el9 ; \ fi && \ systemctl enable mongod && \ systemctl enable pbm-agent && \ diff --git a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml index 6d6b23f..ba782df 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml @@ -1,14 +1,14 @@ version: "3" services: pmm-server: - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} container_name: pmm-server environment: - "PMM_DEBUG=1" - "ENABLE_BACKUP_MANAGEMENT=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" networks: - qa-integration volumes: diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 637fb44..4495bb9 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -11,7 +11,7 @@ services: - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash rs101: @@ -36,7 +36,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -62,7 +62,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -88,7 +88,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -116,7 +116,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -142,7 +142,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -168,7 +168,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index d26b7b2..bf4fe47 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -10,7 +10,7 @@ services: - PMM_REPO=${PMM_REPO:-release} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash rs101: @@ -24,7 +24,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -44,7 +44,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -64,7 +64,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -84,7 +84,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -104,7 +104,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -124,7 +124,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -144,7 +144,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -164,7 +164,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -184,7 +184,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" From 04a7e8bf5e561ca93ea70b84613626ef57ea279e Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Wed, 13 Mar 2024 11:33:59 +0200 Subject: [PATCH 02/88] PMM-12977 update workflow --- .github/workflows/PMM_PSMDB_PBM.yml | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 8531a75..1bccd21 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -5,34 +5,40 @@ on: inputs: psmdb_version: description: "psmdb version" + default: "latest" required: false pbm_version: description: "pbm version" + default: "latest" required: false pmm_version: description: "pmm2-client version" + default: "3-dev-latest" required: false pmm_image: description: "pmm-server docker image" + default: "perconalab/pmm-server:3-dev-latest" required: false push: branches: - main + - -v3 pull_request: branches: - main + - v3 jobs: test_replica_set: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - uses: actions/checkout@v2 - name: test-rs @@ -44,10 +50,10 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - uses: actions/checkout@v2 - name: test-sharded @@ -59,10 +65,10 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - uses: KengoTODA/actions-setup-docker-compose@v1 with: From 9416fb43744d6cc40956ad075f649f510691de8b Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Wed, 13 Mar 2024 11:35:37 +0200 Subject: [PATCH 03/88] PMM-12977 remove using setup docker-compose action --- .github/workflows/PMM_PSMDB_PBM.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 1bccd21..8c2adf7 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -70,9 +70,6 @@ jobs: PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - - uses: KengoTODA/actions-setup-docker-compose@v1 - with: - version: '2.17.2' - uses: actions/checkout@v3 - name: test-auth run: | From 46ae79ba3cf5972f2d658be49349f5ff8940e8c9 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Wed, 13 Mar 2024 11:51:49 +0200 Subject: [PATCH 04/88] PMM-12977 fix tests --- pmm_psmdb-pbm_setup/docker-compose-pmm.yaml | 2 ++ pmm_psmdb-pbm_setup/test/test.py | 16 ++++++++++------ .../docker-compose-pmm-psmdb.yml | 10 +++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml index ba782df..0672c61 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml @@ -18,6 +18,8 @@ services: build: dockerfile: ./Dockerfile-testinfra context: . + environment: + PMM_SERVER_CONTAINER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} volumes: - ./test:/test - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index 77be1be..dd6e0e6 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -1,3 +1,5 @@ +import os + import requests import docker import pytest @@ -10,6 +12,8 @@ docker_rs103 = testinfra.get_host('docker://rs103') testinfra_hosts = ['docker://rs101','docker://rs102','docker://rs103'] +pytest.pmm_server_url = os.getenv('PMM_SERVER_CONTAINER_ADDRESS') + pytest.location_id = '' pytest.service_id = '' pytest.artifact_id = '' @@ -19,7 +23,7 @@ pytest.restore_id = '' def test_pmm_services(): - req = requests.post('https://pmm-server/v1/inventory/Services/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pytest.pmm_server_url}/v1/inventory/Services/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) print('\nGetting all mongodb services:') mongodb = req.json()['mongodb'] print(mongodb) @@ -43,7 +47,7 @@ def test_pmm_add_location(): 'bucket_name': 'bcp' } } - req = requests.post('https://pmm-server/v1/management/backup/Locations/Add',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Locations/Add",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) print('\nAdding new location:') print(req.json()) assert "location_id" in req.json()['location_id'] @@ -58,7 +62,7 @@ def test_pmm_logical_backup(): 'retries': 0, 'data_model': 'LOGICAL' } - req = requests.post('https://pmm-server/v1/management/backup/Backups/Start',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Backups/Start",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) print('\nCreating logical backup:') print(req.json()) assert "artifact_id" in req.json()['artifact_id'] @@ -68,7 +72,7 @@ def test_pmm_artifact(): backup_complete = False for i in range(600): done = False - req = requests.post('https://pmm-server/v1/management/backup/Artifacts/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Artifacts/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) assert req.json()['artifacts'] for artifact in req.json()['artifacts']: if artifact['artifact_id'] == pytest.artifact_id: @@ -105,7 +109,7 @@ def test_pmm_start_restore(): 'service_id': pytest.service_id, 'artifact_id': pytest.artifact_id } - req = requests.post('https://pmm-server/v1/management/backup/Backups/Restore',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Backups/Restore",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) print('\nRestoring logical backup:') print(req.json()) assert "restore_id" in req.json()['restore_id'] @@ -117,7 +121,7 @@ def test_pmm_restore(): restore_complete = False for i in range(600): done = False - req = requests.post('https://pmm-server/v1/management/backup/RestoreHistory/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post("https://{pytest.pmm_server_url}/v1/management/backup/RestoreHistory/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) assert req.json()['items'] for item in req.json()['items']: if item['restore_id'] == pytest.restore_id: diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 0fc4565..9834885 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -10,7 +10,7 @@ services: - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash test: @@ -34,7 +34,7 @@ services: - PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_USERNAME=admin - PMM_AGENT_SERVER_PASSWORD=admin - - PMM_AGENT_SERVER_ADDRESS=pmm-server:443 + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 entrypoint: - bash - -c @@ -62,12 +62,12 @@ services: pmm-server: container_name: pmm-server hostname: pmm-server - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} environment: - "PMM_DEBUG=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" volumes: - ./certs:/srv/nginx - 'pmm-server-data:/srv' From 6ae5063e1b121f0ed2666b6e71b8faf5957e9d20 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Wed, 13 Mar 2024 12:12:02 +0200 Subject: [PATCH 05/88] PMM-12977 fix tests --- pmm_psmdb-pbm_setup/test/test.py | 49 ++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index dd6e0e6..b408942 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -1,7 +1,6 @@ import os import requests -import docker import pytest import testinfra import time @@ -10,9 +9,9 @@ docker_rs101 = testinfra.get_host('docker://rs101') docker_rs102 = testinfra.get_host('docker://rs102') docker_rs103 = testinfra.get_host('docker://rs103') -testinfra_hosts = ['docker://rs101','docker://rs102','docker://rs103'] +testinfra_hosts = ['docker://rs101', 'docker://rs102', 'docker://rs103'] -pytest.pmm_server_url = os.getenv('PMM_SERVER_CONTAINER_ADDRESS') +pmm_server_url = os.getenv('PMM_SERVER_CONTAINER_ADDRESS') pytest.location_id = '' pytest.service_id = '' @@ -22,8 +21,10 @@ pytest.pbm_backup_name = '' pytest.restore_id = '' + def test_pmm_services(): - req = requests.post(f"https://{pytest.pmm_server_url}/v1/inventory/Services/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pmm_server_url}/v1/inventory/Services/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nGetting all mongodb services:') mongodb = req.json()['mongodb'] print(mongodb) @@ -32,27 +33,30 @@ def test_pmm_services(): for service in mongodb: assert "rs" or "mongos" in service['service_name'] if not "mongos" in service['service_name']: - pytest.service_id = service['service_id'] + pytest.service_id = service['service_id'] print('This service_id will be used in the next steps') print(pytest.service_id) + def test_pmm_add_location(): data = { 'name': 'test', 'description': 'test', 's3_config': { - 'endpoint': 'http://minio:9000', - 'access_key': 'minio1234', - 'secret_key': 'minio1234', - 'bucket_name': 'bcp' - } + 'endpoint': 'http://minio:9000', + 'access_key': 'minio1234', + 'secret_key': 'minio1234', + 'bucket_name': 'bcp' } - req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Locations/Add",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Locations/Add", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nAdding new location:') print(req.json()) assert "location_id" in req.json()['location_id'] pytest.location_id = req.json()['location_id'] + def test_pmm_logical_backup(): data = { 'service_id': pytest.service_id, @@ -61,18 +65,21 @@ def test_pmm_logical_backup(): 'description': 'test', 'retries': 0, 'data_model': 'LOGICAL' - } - req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Backups/Start",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Start", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nCreating logical backup:') print(req.json()) assert "artifact_id" in req.json()['artifact_id'] pytest.artifact_id = req.json()['artifact_id'] + def test_pmm_artifact(): backup_complete = False for i in range(600): done = False - req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Artifacts/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Artifacts/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['artifacts'] for artifact in req.json()['artifacts']: if artifact['artifact_id'] == pytest.artifact_id: @@ -84,7 +91,7 @@ def test_pmm_artifact(): print(artifact) pytest.artifact_pbm_meta = artifact['metadata_list'][0]['pbm_metadata']['name'] if "is_sharded_cluster" in artifact: - pytest.artifact_is_sharded = artifact['is_sharded_cluster'] + pytest.artifact_is_sharded = artifact['is_sharded_cluster'] break if done: backup_complete = True @@ -93,6 +100,7 @@ def test_pmm_artifact(): time.sleep(1) assert backup_complete + def test_pbm_artifact(): status = docker_rs101.check_output('pbm status --out json') parsed_status = json.loads(status) @@ -102,26 +110,30 @@ def test_pbm_artifact(): assert parsed_status['backups']['snapshot'][0]['status'] == "done" pytest.pbm_backup_name = parsed_status['backups']['snapshot'][0]['name'] + def test_pmm_start_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") data = { 'service_id': pytest.service_id, 'artifact_id': pytest.artifact_id - } - req = requests.post(f"https://{pytest.pmm_server_url}/v1/management/backup/Backups/Restore",json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Restore", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nRestoring logical backup:') print(req.json()) assert "restore_id" in req.json()['restore_id'] pytest.restore_id = req.json()['restore_id'] + def test_pmm_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") restore_complete = False for i in range(600): done = False - req = requests.post("https://{pytest.pmm_server_url}/v1/management/backup/RestoreHistory/List",json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post("https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['items'] for item in req.json()['items']: if item['restore_id'] == pytest.restore_id: @@ -139,6 +151,7 @@ def test_pmm_restore(): time.sleep(1) assert restore_complete + def test_pbm_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") From 4bc5155a777560472e74bddc1f9510cddc731e65 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 13:28:50 +0200 Subject: [PATCH 06/88] PMM-12977 fix tests --- pmm_psmdb-pbm_setup/test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index b408942..a748fa8 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -132,7 +132,7 @@ def test_pmm_restore(): restore_complete = False for i in range(600): done = False - req = requests.post("https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['items'] for item in req.json()['items']: From de6c4b0021004f23ecde0030e40f1a79ad64a499 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 17:35:37 +0200 Subject: [PATCH 07/88] PMM-12977 fix auth tests --- .../docker-compose-pmm-psmdb.yml | 5 ++ pmm_psmdb_diffauth_setup/test/test.py | 85 ++++++++++--------- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 9834885..b02000d 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -63,6 +63,11 @@ services: container_name: pmm-server hostname: pmm-server image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} + healthcheck: + test: [ "CMD", "curl", "-f", "http://127.0.0.1:8080/ping" ] + interval: 3s + timeout: 2s + retries: 20 environment: - "PMM_DEBUG=1" ports: diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index 0598add..e787542 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -2,17 +2,16 @@ import pytest import testinfra import re -import datetime import time import os import json -import requests -env_vars = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY','AWS_USERNAME'] +env_vars = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_USERNAME'] client = docker.from_env() docker_pmm_client = testinfra.get_host('docker://psmdb-server') + def run_test(add_db_command): try: docker_pmm_client.check_output('pmm-admin remove mongodb psmdb-server', timeout=30) @@ -26,67 +25,75 @@ def run_test(add_db_command): pmm_admin_list = json.loads(docker_pmm_client.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: - if agent['agent_type'] == 'MONGODB_EXPORTER': - agent_id = agent['agent_id'] - agent_port = agent['port'] - break + if agent['agent_type'] == 'MONGODB_EXPORTER': + agent_id = agent['agent_id'] + agent_port = agent['port'] + break - url = f'http://psmdb-server:{agent_port}/metrics' + url = f'http://localhost:{agent_port}/metrics' try: - response = requests.get(url, auth=('pmm', agent_id), timeout=5) - assert response.status_code == 200, f"Request for metrics failed with status code {response.status_code}" + response = docker_pmm_client.check_output(f"curl --request GET --url {url} --header 'Content-Type: " + f"application/json' --user 'pmm:{agent_id}'") pattern = r'mongodb_up (\d+)' - result = re.search(pattern, response.text) + result = re.search(pattern, response) assert result is not None, "MongoDB related data isn't exported" - except requests.exceptions.ConnectionError: + except AssertionError: pytest.fail(f"Connection to {url} failed") def test_simple_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" '\ - '--host psmdb-server --port 27017') + run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ''--host ' + 'psmdb-server --port 27017') + def test_simple_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" '\ - '--host psmdb-server --port 27017 '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ - '--cluster=mycluster') + run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ' + '--host psmdb-server --port 27017 ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' + '--cluster=mycluster') + def test_x509_auth(): - run_test('pmm-admin add mongodb psmdb-server --host=psmdb-server --port 27017 '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ - '--authentication-mechanism=MONGODB-X509 --authentication-database=\'$external\' '\ + run_test('pmm-admin add mongodb psmdb-server --host=psmdb-server --port 27017 ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' + '--authentication-mechanism=MONGODB-X509 --authentication-database=\'$external\' ' '--cluster=mycluster') + def test_ldap_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=PLAIN --authentication-database=\'$external\' '\ + run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' '--cluster=mycluster') + def test_ldap_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=PLAIN --authentication-database=\'$external\' '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ + run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' '--cluster=mycluster') + @pytest.mark.skipif( any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username='+ os.environ.get('AWS_ACCESS_KEY_ID') +' '\ - '--password='+ os.environ.get('AWS_SECRET_ACCESS_KEY') +' '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' '\ - '--cluster=mycluster') + run_test('pmm-admin add mongodb psmdb-server --username=' + os.environ.get('AWS_ACCESS_KEY_ID') + ' ' \ + '--password=' + os.environ.get( + 'AWS_SECRET_ACCESS_KEY') + ' ' \ + '--host=psmdb-server --port 27017 ' \ + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' \ + '--cluster=mycluster') + @pytest.mark.skipif( any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username='+ os.environ.get('AWS_ACCESS_KEY_ID') +' '\ - '--password='+ os.environ.get('AWS_SECRET_ACCESS_KEY') +' '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ - '--cluster=mycluster') + run_test('pmm-admin add mongodb psmdb-server --username=' + os.environ.get('AWS_ACCESS_KEY_ID') + ' ' \ + '--password=' + os.environ.get( + 'AWS_SECRET_ACCESS_KEY') + ' ' \ + '--host=psmdb-server --port 27017 ' \ + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' \ + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' \ + '--cluster=mycluster') From a589db9d2b7778db49eab0fd20f96047a9410791 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 17:47:28 +0200 Subject: [PATCH 08/88] PMM-12977 formatting --- pmm_psmdb_diffauth_setup/test/test.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index e787542..8ac14ba 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -40,6 +40,7 @@ def run_test(add_db_command): except AssertionError: pytest.fail(f"Connection to {url} failed") + def test_simple_auth_wo_tls(): run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ''--host ' 'psmdb-server --port 27017') @@ -78,22 +79,20 @@ def test_ldap_auth_tls(): any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=' + os.environ.get('AWS_ACCESS_KEY_ID') + ' ' \ - '--password=' + os.environ.get( - 'AWS_SECRET_ACCESS_KEY') + ' ' \ - '--host=psmdb-server --port 27017 ' \ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' \ - '--cluster=mycluster') + run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' + '--cluster=mycluster') @pytest.mark.skipif( any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=' + os.environ.get('AWS_ACCESS_KEY_ID') + ' ' \ - '--password=' + os.environ.get( - 'AWS_SECRET_ACCESS_KEY') + ' ' \ - '--host=psmdb-server --port 27017 ' \ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' \ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' \ - '--cluster=mycluster') + run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' + '--cluster=mycluster') From 93bf3c9a70a0aa20d45111414940b2afcabb18bf Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 18:03:17 +0200 Subject: [PATCH 09/88] PMM-12977 fix sharded cluster docker-compose --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index bf4fe47..bd603fa 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -237,14 +237,14 @@ services: /bin/sh -c " sleep 5; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " pmm-server: - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} container_name: pmm-server environment: - "PMM_DEBUG=1" - "ENABLE_BACKUP_MANAGEMENT=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" networks: - test-network volumes: From 6e2818bbad00f5edf913c05602d56bc06a0a27c2 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 18:11:55 +0200 Subject: [PATCH 10/88] PMM-12977 fix sharded cluster docker-compose --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index bd603fa..7574f7f 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -7,7 +7,7 @@ services: context: . args: - REPO=${REPO:-testing} - - PMM_REPO=${PMM_REPO:-release} + - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} From 12017baef7db5d23a163633f2c11fba005748bed Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 19 Mar 2024 18:21:43 +0200 Subject: [PATCH 11/88] PMM-12977 fix sharded cluster docker-compose --- pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 7574f7f..574520d 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -254,6 +254,8 @@ services: build: dockerfile: ./Dockerfile-testinfra context: . + environment: + PMM_SERVER_CONTAINER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} volumes: - ./test:/test - /var/run/docker.sock:/var/run/docker.sock:ro From 33f76ca8893520dea20b9af3cf5249596c1250ea Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 21 Mar 2024 15:27:11 +0200 Subject: [PATCH 12/88] PMM-7 consider arbiter --- pmm_psmdb-pbm_setup/configure-agents.sh | 11 ++++++++++- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 11 ++++++++++- pmm_psmdb-pbm_setup/start-rs-only.sh | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index c28ed66..457dc1c 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -15,6 +15,11 @@ do docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + + echo "stop pbm agent for arbiter node" + if [[ $node == "rs103" ]]; then + docker-compose -f docker-compose-rs.yaml exec -T $node systemctl stop pbm-agent + fi done echo echo "configuring pmm agents" @@ -23,7 +28,11 @@ for node in $nodes do echo "congiguring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + if [[ $node == "rs103" ]]; then + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 + else + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + fi done echo echo "adding some data" diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 75019b1..c44a3ff 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -15,6 +15,11 @@ do docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + + if [[ $node == "rs203" ]]; then + echo "stop pbm agent for arbiter node rs203" + docker-compose -f docker-compose-rs.yaml exec -T $node systemctl stop pbm-agent + fi done echo echo "configuring pmm agents" @@ -23,6 +28,10 @@ for node in $nodes do echo "configuring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --cluster=replicaset1 --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + if [[ $node == "rs203" ]]; then + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 + else + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + fi done echo diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index b21c460..0481f93 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -18,7 +18,7 @@ echo echo "waiting 30 seconds for replica set members to start" sleep 30 echo -bash -x ./configure-replset.sh +bash -x ./configure-psa.sh bash -x ./configure-agents.sh if [ $profile = "extra" ]; then From ec542f4559d5af547eb0eeea292c0ffee25df528 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Mon, 25 Mar 2024 16:12:53 +0200 Subject: [PATCH 13/88] PMM-7 support psa and pss --- pmm_psmdb-pbm_setup/configure-agents.sh | 13 ++- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 18 +-- pmm_psmdb-pbm_setup/configure-extra-psa.sh | 103 ++++++++++++++++++ pmm_psmdb-pbm_setup/configure-psa.sh | 2 +- pmm_psmdb-pbm_setup/start-rs-only.sh | 14 ++- pmm_psmdb-pbm_setup/start-rs.sh | 8 +- 6 files changed, 139 insertions(+), 19 deletions(-) create mode 100755 pmm_psmdb-pbm_setup/configure-extra-psa.sh diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 457dc1c..4f647b3 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -5,6 +5,7 @@ pmm_mongo_user=${PMM_MONGO_USER:-pmm} pmm_mongo_user_pass=${PMM_MONGO_USER_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} +mongo_setup_type=${MONGO_SETUP_TYPE:-pss} echo echo "configuring pbm agents" @@ -15,12 +16,12 @@ do docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent - - echo "stop pbm agent for arbiter node" - if [[ $node == "rs103" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl stop pbm-agent - fi done + +if [[ $mongo_setup_type == "psa" ]]; then + echo "stop pbm agent for arbiter node" + docker-compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent +fi echo echo "configuring pmm agents" nodes="rs101 rs102 rs103" @@ -28,7 +29,7 @@ for node in $nodes do echo "congiguring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - if [[ $node == "rs103" ]]; then + if [[ $mongo_setup_type == "psa" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 else docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index c44a3ff..014bf13 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -5,6 +5,7 @@ pmm_mongo_user=${PMM_MONGO_USER:-pmm} pmm_mongo_user_pass=${PMM_MONGO_USER_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} +mongo_setup_type=${MONGO_SETUP_TYPE:-pss} echo echo "configuring pbm agents" @@ -15,12 +16,12 @@ do docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent - - if [[ $node == "rs203" ]]; then - echo "stop pbm agent for arbiter node rs203" - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl stop pbm-agent - fi done + +if [[ $mongo_setup_type == "psa" ]]; then + echo "stop pbm agent for arbiter node rs203" + docker-compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent +fi echo echo "configuring pmm agents" nodes="rs201 rs202 rs203" @@ -28,10 +29,9 @@ for node in $nodes do echo "configuring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - if [[ $node == "rs203" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 + if [[ $mongo_setup_type == "psa" ]]; then + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 $node 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 fi done -echo diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh new file mode 100755 index 0000000..d667f6b --- /dev/null +++ b/pmm_psmdb-pbm_setup/configure-extra-psa.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -e + +pmm_mongo_user=${PMM_MONGO_USER:-pmm} +pmm_mongo_user_pass=${PMM_MONGO_USER_PASS:-pmmpass} +pbm_user=${PBM_USER:-pbm} +pbm_pass=${PBM_PASS:-pbmpass} + +echo +echo "configuring extra PSA replicaset with members priorities" +docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF + config = { + "_id" : "rs", + "members" : [ + { + "_id" : 0, + "host" : "rs201:27017", + "priority": 2 + }, + { + "_id" : 1, + "host" : "rs202:27017", + "priority": 1 + }, + { + "_id" : 2, + "host" : "rs203:27017", + "arbiterOnly": true + } + ] + }; + rs.initiate(config); +EOF +echo +sleep 60 +echo +echo "configuring root user on primary" +docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); +EOF +echo +echo "configuring pbm and pmm roles" +docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +db.getSiblingDB("admin").createRole({ + "role": "pbmAnyAction", + "privileges": [{ + "resource": { "anyResource": true }, + "actions": [ "anyAction" ] + }], + "roles": [] +}); +db.getSiblingDB("admin").createRole({ + role: "explainRole", + privileges: [{ + resource: { + db: "", + collection: "" + }, + actions: [ + "listIndexes", + "listCollections", + "dbStats", + "dbHash", + "collStats", + "find" + ] + }], + roles:[] +}); +EOF +echo +echo "creating pbm user" +docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +db.getSiblingDB("admin").createUser({ + user: "${pbm_user}", + pwd: "${pbm_pass}", + "roles" : [ + { "db" : "admin", "role" : "readWrite", "collection": "" }, + { "db" : "admin", "role" : "backup" }, + { "db" : "admin", "role" : "clusterMonitor" }, + { "db" : "admin", "role" : "restore" }, + { "db" : "admin", "role" : "pbmAnyAction" } + ] +}); +EOF +echo +echo "creating pmm user" +docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +db.getSiblingDB("admin").createUser({ + user: "${pmm_mongo_user}", + pwd: "${pmm_mongo_user_pass}", + roles: [ + { role: "explainRole", db: "admin" }, + { role: "clusterMonitor", db: "admin" }, + { role: "read", db: "local" }, + { "db" : "admin", "role" : "readWrite", "collection": "" }, + { "db" : "admin", "role" : "backup" }, + { "db" : "admin", "role" : "clusterMonitor" }, + { "db" : "admin", "role" : "restore" }, + { "db" : "admin", "role" : "pbmAnyAction" } + ] +}); +EOF diff --git a/pmm_psmdb-pbm_setup/configure-psa.sh b/pmm_psmdb-pbm_setup/configure-psa.sh index 9ba3e17..879adb1 100755 --- a/pmm_psmdb-pbm_setup/configure-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-psa.sh @@ -7,7 +7,7 @@ pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} echo -echo "configuring replicaset with members priorities" +echo "configuring PSA replicaset with members priorities" docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index 0481f93..209d3ed 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -2,6 +2,7 @@ set -e profile=${COMPOSE_PROFILES:-classic} +mongo_setup_type=${MONGO_SETUP_TYPE:-pss} docker network create qa-integration || true docker network create pmm-qa || true @@ -10,6 +11,7 @@ docker network create pmm2-upgrade-tests_pmm-network || true docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} +export MONGO_SETUP_TYPE=${mongo_setup_type} docker-compose -f docker-compose-rs.yaml down -v --remove-orphans docker-compose -f docker-compose-rs.yaml build --no-cache @@ -18,10 +20,18 @@ echo echo "waiting 30 seconds for replica set members to start" sleep 30 echo -bash -x ./configure-psa.sh +if [ $mongo_setup_type == "pss" ]; then + bash -e ./configure-replset.sh +else + bash -e ./configure-psa.sh +fi bash -x ./configure-agents.sh if [ $profile = "extra" ]; then - bash -x ./configure-extra-replset.sh + if [ $mongo_setup_type == "pss" ]; then + bash -x ./configure-extra-replset.sh + else + bash -x ./configure-extra-psa.sh + fi bash -x ./configure-extra-agents.sh fi diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index 527003e..7958b87 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -3,6 +3,7 @@ set -e pmm_server_admin_pass=${ADMIN_PASSWORD:-password} profile=${COMPOSE_PROFILES:-classic} +mongo_setup_type=${}MONGO_SETUP_TYPE:-pss} docker network create qa-integration || true docker network create pmm-qa || true @@ -11,6 +12,7 @@ docker network create pmm2-upgrade-tests_pmm-network || true docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} +export MONGO_SETUP_TYPE=${mongo_setup_type} docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml build @@ -24,7 +26,11 @@ echo "restarting pmm-server" docker-compose -f docker-compose-pmm.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 -bash -e ./configure-replset.sh +if [ $mongo_setup_type == "pss" ]; then + bash -e ./configure-replset.sh +else + bash -e ./configure-psa.sh +fi bash -e ./configure-agents.sh tests=${TESTS:-yes} if [ $tests != "no" ]; then From 42d716ad5e424e6036d82441d2693c7a659b4f2f Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 26 Mar 2024 13:15:51 +0200 Subject: [PATCH 14/88] PMM-7 fix psa --- pmm_psmdb-pbm_setup/configure-agents.sh | 2 +- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 4f647b3..cc30ff2 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,7 +29,7 @@ for node in $nodes do echo "congiguring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - if [[ $mongo_setup_type == "psa" ]]; then + if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 else docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 014bf13..56ad77a 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -29,7 +29,7 @@ for node in $nodes do echo "configuring pmm agent on $node" docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup - if [[ $mongo_setup_type == "psa" ]]; then + if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 $node 127.0.0.1:27017 else docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 From d1704f629155d0fe33b9e1ddab1aca2c4bbb4874 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:24:44 +0530 Subject: [PATCH 15/88] PMM Framework (#37) 1) Move pmm-frameowrk files to qa-integration branch 2) Add new pmm-framework python script wrapper 3) MySQL + PGSM + PSMDB Support added 4) Client setup for V3 only --- pmm_psmdb-pbm_setup/Dockerfile | 2 +- pmm_qa/client_container_ms_setup.sh | 112 +++++ pmm_qa/client_container_proxysql_setup.sh | 10 + pmm_qa/client_container_ps_setup.sh | 80 ++++ pmm_qa/client_container_pxc_setup.sh | 83 ++++ pmm_qa/haproxy_setup.sh | 30 ++ pmm_qa/haproxy_setup.yml | 73 ++++ pmm_qa/ms_pmm_setup.yml | 71 ++++ pmm_qa/pdpgsql_pgsm_setup.yml | 81 ++++ pmm_qa/pg_stat_monitor_setup.sh | 116 +++++ pmm_qa/pg_stat_statements_setup.sh | 92 ++++ pmm_qa/pgsm_run_queries.sh | 10 + pmm_qa/pgsql-vacuum.sh | 51 +++ pmm_qa/pgsql_pgss_setup.yml | 73 ++++ pmm_qa/pgstatmonitor_metrics_queries.sh | 13 + pmm_qa/pmm-framework.py | 494 ++++++++++++++++++++++ pmm_qa/pmm3-client-setup.sh | 129 ++++++ pmm_qa/product_version_download_helper | 26 ++ pmm_qa/prometheus.base.yml | 23 + pmm_qa/ps_pmm_setup.yml | 70 +++ pmm_qa/psmdb_setup.sh | 116 +++++ pmm_qa/psmdb_setup.yml | 81 ++++ pmm_qa/pxc_proxysql_setup.sh | 66 +++ pmm_qa/pxc_proxysql_setup.yml | 115 +++++ 24 files changed, 2016 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/client_container_ms_setup.sh create mode 100644 pmm_qa/client_container_proxysql_setup.sh create mode 100644 pmm_qa/client_container_ps_setup.sh create mode 100644 pmm_qa/client_container_pxc_setup.sh create mode 100644 pmm_qa/haproxy_setup.sh create mode 100644 pmm_qa/haproxy_setup.yml create mode 100644 pmm_qa/ms_pmm_setup.yml create mode 100644 pmm_qa/pdpgsql_pgsm_setup.yml create mode 100644 pmm_qa/pg_stat_monitor_setup.sh create mode 100644 pmm_qa/pg_stat_statements_setup.sh create mode 100644 pmm_qa/pgsm_run_queries.sh create mode 100644 pmm_qa/pgsql-vacuum.sh create mode 100644 pmm_qa/pgsql_pgss_setup.yml create mode 100644 pmm_qa/pgstatmonitor_metrics_queries.sh create mode 100755 pmm_qa/pmm-framework.py create mode 100755 pmm_qa/pmm3-client-setup.sh create mode 100644 pmm_qa/product_version_download_helper create mode 100644 pmm_qa/prometheus.base.yml create mode 100644 pmm_qa/ps_pmm_setup.yml create mode 100644 pmm_qa/psmdb_setup.sh create mode 100644 pmm_qa/psmdb_setup.yml create mode 100644 pmm_qa/pxc_proxysql_setup.sh create mode 100644 pmm_qa/pxc_proxysql_setup.yml diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index a2eac5b..bca3144 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -52,7 +52,7 @@ RUN set -ex; \ yum -y install percona-backup-mongodb ; \ else \ yum -y install percona-backup-mongodb-${PBM_VERSION}.el9 ; \ - fi + fi RUN if [[ $PMM_CLIENT_VERSION == http* ]]; then \ yum -y install pmm-client && \ diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh new file mode 100644 index 0000000..b11fd42 --- /dev/null +++ b/pmm_qa/client_container_ms_setup.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$number_of_nodes" ] +then + export number_of_nodes=1 +fi + +if [ -z "$ms_version" ] +then + export ms_version="8.0.33" +fi + +if [ -z "$ms_tarball" ] +then + export ms_tarball="https://dev.mysql.com/get/Downloads/MySQL-8/mysql-8.0.33-linux-glibc2.28-x86_64.tar.gz" +fi + +if [ -z "$query_source" ] +then + export query_source=perfschema +fi + +touch sysbench_prepare.txt +touch sysbench_run.txt + +## Setup DB deployer +curl -L -s https://bit.ly/dbdeployer | bash || true + +### Get the tarball +wget ${ms_tarball} +mkdir ~/ms${ms_version} || true +mkdir /tmp || true +chmod 1777 /tmp || true + +## Deploy DB deployer +export tar_ball_name=$(ls mysql-*) +dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ms${ms_version} --overwrite +export db_version_sandbox=$(ls ~/ms${ms_version}) +export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) + +if [[ $number_of_nodes == 1 ]];then + if [[ ! -z $group_replication ]]; then + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + else + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + fi + fi + if [[ ! -z $group_replication ]]; then + for j in `seq 1 3`;do + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + fi + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-group-replication-node + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ms-prod --cluster=ms-prod-cluster --replication-set=ms-repl ms-group-replication-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + node_port=$(($node_port + 1)) + sleep 20 + done + else + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-single + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ms-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port + fi +else + dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + for j in `seq 1 $number_of_nodes`; do + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + fi + if [ $(( ${j} % 2 )) -eq 0 ]; then + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ms-prod --cluster=ms-prod-cluster --replication-set=ms-repl2 ms-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + else + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ms-dev --cluster=ms-dev-cluster --replication-set=ms-repl1 ms-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + fi + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-multiple-node + node_port=$(($node_port + 1)) + sleep 20 + done +fi + +## Start Running Load +~/sandboxes/${db_sandbox}/sysbench_ready prepare > sysbench_prepare.txt 2>&1 & +sleep 120 +~/sandboxes/${db_sandbox}/sysbench_ready run > sysbench_run.txt 2>&1 & diff --git a/pmm_qa/client_container_proxysql_setup.sh b/pmm_qa/client_container_proxysql_setup.sh new file mode 100644 index 0000000..c3e864a --- /dev/null +++ b/pmm_qa/client_container_proxysql_setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +mysql -A -uroot -S /home/pxc/PXC/node1/socket.sock -e "drop database if exists sbtest;create database sbtest;" +mysql -A -uroot -S /home/pxc/PXC/node1/socket.sock -e "GRANT ALL PRIVILEGES ON sbtest.* TO 'proxysql_user'@'127.%';" + +## Start Running Load +sysbench /usr/share/sysbench/oltp_insert.lua --mysql-db=sbtest --mysql-user=proxysql_user --mysql-host=127.0.0.1 --mysql-port=6033 --mysql-password=passw0rd --db-driver=mysql --threads=1 --tables=10 --table-size=1000 prepare > sysbench_run_node1_prepare.txt 2>&1 & +sleep 20 +sysbench /usr/share/sysbench/oltp_read_only.lua --mysql-db=sbtest --mysql-user=proxysql_user --mysql-host=127.0.0.1 --mysql-port=6033 --mysql-password=passw0rd --db-driver=mysql --threads=1 --tables=10 --table-size=1000 --time=12000 run > sysbench_run_node1_read_only.txt 2>&1 & +sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-db=sbtest --mysql-user=proxysql_user --mysql-host=127.0.0.1 --mysql-port=6033 --mysql-password=passw0rd --db-driver=mysql --threads=1 --tables=10 --table-size=1000 --time=0 run > sysbench_run_node1_read_write.txt 2>&1 & diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh new file mode 100644 index 0000000..e719b72 --- /dev/null +++ b/pmm_qa/client_container_ps_setup.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$number_of_nodes" ] +then + export number_of_nodes=3 +fi + +if [ -z "$ps_version" ] +then + export ps_version=8 +fi + +if [ -z "$ps_tarball" ] +then + export ps_tarball=https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.29-21/binary/tarball/Percona-Server-8.0.29-21-Linux.x86_64.glibc2.17-minimal.tar.gz +fi + +if [ -z "$query_source" ] +then + export query_source=slowlog +fi + +export PS_PORT=3307 +export PS_USER=msandbox +export PS_PASSWORD=msandbox +touch sysbench_prepare.txt +touch sysbench_run.txt + +## Setup DB deployer +curl -L -s https://bit.ly/dbdeployer | bash || true + +### Get the tarball +wget ${ps_tarball} +mkdir ~/ps${ps_version} || true +mkdir /tmp || true +chmod 1777 /tmp || true + +## Deploy DB deployer +export tar_ball_name=$(ls Percona-Server*) +dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ps${ps_version} --flavor=percona +export db_version_sandbox=$(ls ~/ps${ps_version}) +dbdeployer deploy single ${db_version_sandbox} --port=${PS_PORT} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 + +export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') +export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) +if [ "$query_source" == "slowlog" ]; then + ### enable Slow log + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_rate_limit=1;" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_slave_statements=ON;" + pmm-admin add mysql --username=${PS_USER} --password=${PS_PASSWORD} --port=3307 --query-source=slowlog mysql_client_slowlog_${SERVICE_RANDOM_NUMBER} +else + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL innodb_monitor_enable=all;" + if echo "$ps_version" | grep '5.7'; then + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL query_response_time_stats=ON;" + fi + mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + pmm-admin add mysql --username=${PS_USER} --password=${PS_PASSWORD} --port=3307 --query-source=perfschema mysql_client_perfschema_${SERVICE_RANDOM_NUMBER} +fi + +## Start Running Load +~/sandboxes/${db_sandbox}/sysbench_ready prepare > sysbench_prepare.txt 2>&1 & +sleep 120 +~/sandboxes/${db_sandbox}/sysbench_ready run > sysbench_run.txt 2>&1 & diff --git a/pmm_qa/client_container_pxc_setup.sh b/pmm_qa/client_container_pxc_setup.sh new file mode 100644 index 0000000..a84ef87 --- /dev/null +++ b/pmm_qa/client_container_pxc_setup.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$number_of_nodes" ] +then + export number_of_nodes=3 +fi + +if [ -z "$pxc_version" ] +then + export pxc_version=8 +fi + +if [ -z "$pxc_tarball" ] +then + export pxc_tarball=https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-80/Percona-XtraDB-Cluster-8.0.32/binary/tarball/Percona-XtraDB-Cluster_8.0.32-24.1_Linux.x86_64.glibc2.34-minimal.tar.gz +fi + +if [ -z "$query_source" ] +then + export query_source=perfschema +fi + +if [ -z "$pxc_dev_cluster" ] +then + export pxc_dev_cluster=pxc-dev-cluster +fi + +whoami +cd ~ +wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/pxc-tests/pxc-startup.sh +sed -i 's/log-output=none/log-output=file/g' pxc-startup.sh +## bug https://bugs.mysql.com/bug.php?id=90553 workaround +sed -i 's+${MID} --datadir+${MID} --socket=\\${node}/socket.sock --port=\\${RBASE1} --datadir+g' pxc-startup.sh + +## Download right PXC version +if echo "$pxc_version" | grep '8'; then + sed -i 's+wsrep_node_incoming_address=$ADDR+wsrep_node_incoming_address=$ADDR:$RBASE1+g' pxc-startup.sh +fi + +curl ${pxc_tarball} -o Percona-XtraDB-Cluster.tar.gz +sleep 10 +tar -xzf Percona-XtraDB-Cluster.tar.gz +sleep 10 +rm -r Percona-XtraDB-Cluster.tar.gz +mv Percona-XtraDB-Cluster* PXC +cd PXC + +## start PXC +bash ../pxc-startup.sh +bash ./start_pxc $number_of_nodes +touch sysbench_run_node1_prepare.txt +touch sysbench_run_node1_read_write.txt +touch sysbench_run_node1_read_only.txt + +### enable slow log +if [ "$query_source" == "slowlog" ]; then + for j in `seq 1 ${number_of_nodes}`; + do + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL slow_query_log='ON';" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL long_query_time=0;" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_rate_limit=1;" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_verbosity='full';" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_rate_type='query';" + done +fi + +bin/mysql -A -uroot -S/home/pxc/PXC/node1/socket.sock -e "create user 'admin'@'%' identified with mysql_native_password by 'admin';" +bin/mysql -A -uroot -S/home/pxc/PXC/node1/socket.sock -e "grant all on *.* to 'admin'@'%';" + +export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) +for j in `seq 1 ${number_of_nodes}`;do + pmm-admin add mysql --query-source=${query_source} --username=admin --password=admin --host=127.0.0.1 --port=$(cat /home/pxc/PXC/node$j.cnf | grep port | awk -F"=" '{print $2}') --environment=pxc-dev --cluster=${pxc_dev_cluster} --replication-set=pxc-repl pxc_node__${j}_${SERVICE_RANDOM_NUMBER} +done diff --git a/pmm_qa/haproxy_setup.sh b/pmm_qa/haproxy_setup.sh new file mode 100644 index 0000000..6aff623 --- /dev/null +++ b/pmm_qa/haproxy_setup.sh @@ -0,0 +1,30 @@ +#!/bin/sh + + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$metrics_mode" ] +then + export metrics_mode=push +fi + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl git gnupg2 lsb-release +apt-get -y install -y git ca-certificates gcc libc6-dev liblua5.3-dev libpcre3-dev libssl-dev libsystemd-dev make wget zlib1g-dev + +## Get Haproxy +git clone https://github.com/haproxy/haproxy.git +cd haproxy +make TARGET=linux-glibc USE_LUA=1 USE_OPENSSL=1 USE_PCRE=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_PROMEX=1 +make install-bin +cp /usr/local/sbin/haproxy /usr/sbin/haproxy diff --git a/pmm_qa/haproxy_setup.yml b/pmm_qa/haproxy_setup.yml new file mode 100644 index 0000000..d106838 --- /dev/null +++ b/pmm_qa/haproxy_setup.yml @@ -0,0 +1,73 @@ +--- +# This playbook does following: +# enables Percona testing repository + +- hosts: all + vars: + haproxy_container: "{{ lookup('vars', 'extra_haproxy_container', default=lookup('env','HAPROXY_CONTAINER') | default('HAPROXY', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ haproxy_container }}" | grep -q . && docker stop {{ haproxy_container }} && docker rm -fv {{ haproxy_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for HAPROXY + shell: > + sudo docker run -d -p 42100:42100 --name={{ haproxy_container }} + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker haproxy_container + shell: "{{ item }}" + with_items: + - docker cp ./haproxy_setup.sh {{ haproxy_container }}:/ + - docker cp ./haproxy.cfg {{ haproxy_container }}:/ + - docker exec {{ haproxy_container }} apt-get update + - docker exec {{ haproxy_container }} apt-get -y install wget curl git gnupg2 lsb-release + + - name: Setup haproxy for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh > ~/setup_haproxy.log + + - name: Install pmm2-client on the haproxy_container + shell: "{{ item }}" + with_items: + - docker exec {{ haproxy_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ haproxy_container }} + - docker exec {{ haproxy_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Wait for Haproxy building to be finished and binary available + shell: docker exec {{ haproxy_container }} ls -la /usr/sbin/haproxy + retries: 6 + delay: 10 + + - name: Start haproxy for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ haproxy_container }} haproxy -f haproxy.cfg -D + + - name: Remove HAPROXY for monitoring from PMM if already exist with same service name + shell: "{{ item }}" + with_items: + - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove haproxy {{ haproxy_container }}_service' + ignore_errors: true + + - name: Add HAPROXY for Monitoring to PMM via pmm-admin add haproxy command + shell: "{{ item }}" + with_items: + - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add haproxy --listen-port=42100 --environment=haproxy {{ haproxy_container }}_service' + + - name: Run ping to FE of HAPROXY to monitore the load balancing + shell: while true; do curl http://localhost:42100/; sleep 10; done >/dev/null 2>&1 & diff --git a/pmm_qa/ms_pmm_setup.yml b/pmm_qa/ms_pmm_setup.yml new file mode 100644 index 0000000..158a5a8 --- /dev/null +++ b/pmm_qa/ms_pmm_setup.yml @@ -0,0 +1,71 @@ +--- +# This playbook does following: +# enables Percona testing repository + +- hosts: all + become: true + become_method: sudo + vars: + ms_version: "{{ lookup('vars', 'extra_ms_version', default=lookup('env','MS_VERSION') | default('8.0', true) ) }}" + ms_tarball: "{{ lookup('vars', 'extra_ms_tarball', default=lookup('env','MS_TARBALL') | default('', true) ) }}" + ms_container: "{{ lookup('vars', 'extra_ms_container', default=lookup('env','MS_CONTAINER') | default('ms_container', true) ) }}" + query_source: "{{ lookup('vars', 'extra_query_source', default=lookup('env','QUERY_SOURCE') | default('slowlog', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','MS_NODES') | default('1', true) ) }}" + group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION')) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ ms_container }}" | grep -q . && docker stop {{ ms_container }} && docker rm -fv {{ ms_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for MySQL + shell: > + docker run -d --name={{ ms_container }} + phusion/baseimage:focal-1.2.0 + + - name: Copy all required Artifacts to the docker ms_container + shell: "{{ item }}" + with_items: + - docker exec {{ ms_container }} mkdir -p artifacts + - docker cp ./client_container_ms_setup.sh {{ ms_container }}:/ + + - name: Get Product tarball URL based on the version + shell: cat product_version_download_helper | grep -v pxc | grep mysql | grep "\-{{ ms_version }}" | head -1 | awk -F'# ' '{print $3}' + register: tarball + when: lookup('env', 'MS_TARBALL') == '' + + - name: Set Product URL if environment or paramater are not defined + set_fact: + ms_tarball: "{{tarball.stdout | default(ms_tarball,true)}}" + + - name: Setup libraries required inside the container + shell: "{{ item }}" + with_items: + - docker exec {{ ms_container }} apt-get update + - docker exec {{ ms_container }} apt-get -y install wget curl git gnupg2 lsb-release libncurses5 + - docker exec {{ ms_container }} apt-get -y install libaio1 libaio-dev libnuma-dev socat libtirpc-dev + - docker exec {{ ms_container }} apt install -y sysbench + + - name: Install pmm2-client on the ms_container + shell: "{{ item }}" + with_items: + - docker exec {{ ms_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ ms_container }} + - docker exec {{ ms_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Execute Setup script inside the MySQL ms_container + shell: "{{ item }}" + with_items: + - docker exec {{ ms_container }} bash -xe ./client_container_ms_setup.sh --ms_version {{ ms_version }} --ms_tarball {{ ms_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ms_{{ ms_version }}.log + diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml new file mode 100644 index 0000000..9170d1d --- /dev/null +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -0,0 +1,81 @@ +--- +# This playbook does following: +# enables Percona testing repository + +- hosts: all + become: true + become_method: sudo + vars: + pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('14', true) ) }}" + pdpgsql_pgsm_container: "{{ lookup('vars', 'extra_pdpgsql_pgsm_container', default=lookup('env','PDPGSQL_PGSM_CONTAINER') | default('pdpgsql_pgsm', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('1.1.1', true) ) }}" + pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" + use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ pdpgsql_pgsm_container }}" | grep -q . && docker stop {{ pdpgsql_pgsm_container }} && docker rm -fv {{ pdpgsql_pgsm_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PostgreSQL + shell: > + sudo docker run -d --name={{ pdpgsql_pgsm_container }} + -p 5447:5432 + phusion/baseimage:focal-1.2.0 + + - name: Copy all required Artifacts to the docker pdpgsql_pgsm_container + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} mkdir -p artifacts + - docker cp ./pg_stat_monitor_setup.sh {{ pdpgsql_pgsm_container }}:/ + - docker cp ./pgsm_run_queries.sh {{ pdpgsql_pgsm_container }}:/ + + - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution PPG --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log + + - name: Install pmm2-client on the pdpgsql_pgsm_container + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ pdpgsql_pgsm_container }} + - docker exec {{ pdpgsql_pgsm_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + + - name: Remove pdpgsql service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove postgresql {{ pdpgsql_pgsm_container }}_service__{{ random_number }}' + ignore_errors: true + + - name: Add pdpgsql_pgsm for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ pdpgsql_pgsm_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor {{ pdpgsql_pgsm_container }}_service_{{ random_number }}' + + - name: Add pdpgsql_pgsm for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ pdpgsql_pgsm_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor --socket=/var/run/postgresql socket_{{ pdpgsql_pgsm_container }}_service_{{ random_number }}' + + - name: Run queries for generating data + shell: "{{ item }}" + with_items: + - docker exec {{ pdpgsql_pgsm_container }} bash ./pgsm_run_queries.sh & diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh new file mode 100644 index 0000000..88526ef --- /dev/null +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -0,0 +1,116 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +# If postgres server version is not provided then it will default to version 14. +if [ -z "$pgsql_version" ] +then + export pgsql_version=14 +fi + +# If branch/tag is not provided then it will default to main branch +if [ -z "$pgstat_monitor_branch" ] +then + export pgstat_monitor_branch="1.1.1" +fi + +# If repo is not provided then it will default to percona PGSM repository +if [ -z "$pgstat_monitor_repo" ] +then + export pgstat_monitor_repo=percona/pg_stat_monitor +fi + +# If distribution is not provided then it will default to percona distribution 'PPG' +# For PG community distribution please use 'PGDG' +if [ -z "$distribution" ] +then + export distribution=PPG +fi + +# Need to add a user postgres either here or in Dockerfile +cd /home +mkdir postgres +useradd postgres +chown -R postgres:postgres postgres +cd postgres + +# Install the dependencies +apt-get update +apt-get -y install wget curl git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl wget +sleep 10 + +# Install the PG server from selected distribution +if [[ $distribution == "PGDG" ]]; +then + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + apt update + apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} +else + wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb + dpkg -i percona-release_latest.generic_all.deb + percona-release setup ppg-${pgsql_version} + apt-get -y update + apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all +fi + +sleep 10 +sed -i 's/\(host\s*all\s*all\s*127.0.0.1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(host\s*all\s*all\s*::1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*postgres.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*all.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +service postgresql restart + +sleep 10 +chown -R postgres:postgres /var/lib/postgresql/${pgsql_version}/main +chmod 0700 -R /var/lib/postgresql/${pgsql_version}/main +sed -i "s/#listen_addresses.*/listen_addresses = '*'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/${pgsql_version}/main/pg_hba.conf + +sleep 10 +service postgresql restart + +export PATH="/usr/lib/postgresql/${pgsql_version}/bin:$PATH" +echo $PATH +cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin + +# Clone PGSM repo and move to /home/postgres/pg_stat_monitor dir +cd /home/postgres +git clone --depth 1 --branch ${pgstat_monitor_branch} https://github.com/${pgstat_monitor_repo} +chown -R postgres:postgres pg_stat_monitor +cd pg_stat_monitor + +# Build PGSM +make USE_PGXS=1 + +# Install built PGSM library into server +make USE_PGXS=1 install + +# Stop server and edit postgresql.conf to load PGSM library with required configurations +service postgresql stop +echo "shared_preload_libraries = 'pg_stat_monitor'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "track_io_timing=ON" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "max_connections=1000" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf + +# Create init.sql file required by PMM +echo "CREATE DATABASE sbtest1;" >> /home/postgres/init.sql +echo "CREATE DATABASE sbtest2;" >> /home/postgres/init.sql +echo "CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD 'pmm';" >> /home/postgres/init.sql +echo "GRANT pg_monitor TO pmm;" >> /home/postgres/init.sql +echo "ALTER USER postgres PASSWORD 'pass+this';" >> /home/postgres/init.sql + +# Start server, run init.sql and Create extension PGSM +service postgresql start +su postgres bash -c 'psql -f /home/postgres/init.sql' +su postgres bash -c 'psql -c "CREATE DATABASE contrib_regression;"' +su postgres bash -c 'psql -U postgres -c "CREATE EXTENSION pg_stat_monitor;"' diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh new file mode 100644 index 0000000..b6c8b84 --- /dev/null +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -0,0 +1,92 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +# If postgres server version is not provided then it will default to version 14. +if [ -z "$pgsql_version" ] +then + export pgsql_version=14 +fi + +# If distribution is not provided then it will default to percona distribution 'PPG' +# For PG community distribution please use 'PGDG' +if [ -z "$distribution" ] +then + export distribution=PGDG +fi + +# Need to add a user postgres either here or in Dockerfile +cd /home +mkdir postgres +useradd postgres +chown -R postgres:postgres postgres +cd postgres + +# Install the dependencies +apt-get update +apt-get -y install wget curl git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl wget +sleep 10 + +# Install the PG server from selected distribution +if [[ $distribution == "PGDG" ]]; +then + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + apt update + apt -y install postgresql-${pgsql_version} postgresql-server-dev-${pgsql_version} postgresql-contrib-${pgsql_version} +else + wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb + dpkg -i percona-release_latest.generic_all.deb + percona-release setup ppg-${pgsql_version} + apt-get -y update + apt-get -y install percona-postgresql-${pgsql_version} percona-postgresql-contrib percona-postgresql-server-dev-all +fi + +sleep 10 +sed -i 's/\(host\s*all\s*all\s*127.0.0.1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(host\s*all\s*all\s*::1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*postgres.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*all.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +service postgresql restart + +sleep 10 +chown -R postgres:postgres /var/lib/postgresql/${pgsql_version}/main +chmod 0700 -R /var/lib/postgresql/${pgsql_version}/main +sed -i "s/#listen_addresses.*/listen_addresses = '*'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/${pgsql_version}/main/pg_hba.conf + +sleep 10 +service postgresql restart + +export PATH="/usr/lib/postgresql/${pgsql_version}/bin:$PATH" +echo $PATH +cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin + +# Stop server and edit postgresql.conf to load pg_stat_sstatement library with required configurations +service postgresql stop +echo "shared_preload_libraries = 'pg_stat_statements'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "track_io_timing=ON" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "pg_stat_statements.track=all" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf + +# Create init.sql file required by PMM +echo "CREATE DATABASE sbtest1;" >> /home/postgres/init.sql +echo "CREATE DATABASE sbtest2;" >> /home/postgres/init.sql +echo "CREATE USER pmm WITH PASSWORD 'pmm';" >> /home/postgres/init.sql +echo "GRANT pg_monitor TO pmm;" >> /home/postgres/init.sql +echo "ALTER USER postgres PASSWORD 'pass+this';" >> /home/postgres/init.sql + +# Start server, run init.sql and Create extension PGSM +service postgresql start +su postgres bash -c 'psql -f /home/postgres/init.sql' +su postgres bash -c 'psql -c "CREATE DATABASE contrib_regression;"' +su postgres bash -c 'psql -U postgres -c "CREATE EXTENSION pg_stat_statements;"' diff --git a/pmm_qa/pgsm_run_queries.sh b/pmm_qa/pgsm_run_queries.sh new file mode 100644 index 0000000..0afaa2b --- /dev/null +++ b/pmm_qa/pgsm_run_queries.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +## Running Queries +wget https://raw.githubusercontent.com/percona/pmm-agent/main/testqueries/postgres/pg_stat_monitor_load.sql +while true +do + su postgres bash -c 'psql -d contrib_regression -f pg_stat_monitor_load.sql' + su postgres bash -c 'psql -d sbtest1 -f pg_stat_monitor_load.sql' + sleep 30 +done diff --git a/pmm_qa/pgsql-vacuum.sh b/pmm_qa/pgsql-vacuum.sh new file mode 100644 index 0000000..184bed4 --- /dev/null +++ b/pmm_qa/pgsql-vacuum.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +export DOCKER_CONTAINER_NAME=pgsql_vacuum_db +export PGSQL_VACUUM_VERSION="latest" +echo "Setting up Postgres for vacuum monitoring" +if [ ! -z $@ ]; then + PGSQL_VACUUM_VERSION=$1 +fi +docker stop pgsql_vacuum_db || true +docker rm pgsql_vacuum_db || true +docker run --name pgsql_vacuum_db -p 7432:5432 -e POSTGRES_PASSWORD=YIn7620U1SUc -d postgres:$PGSQL_VACUUM_VERSION \ + -c shared_preload_libraries='pg_stat_statements' -c pg_stat_statements.max=10000 -c pg_stat_statements.track=all +sleep 20 +# --network pmm-qa \ +docker exec pgsql_vacuum_db apt-get update +docker exec pgsql_vacuum_db apt-get install -y wget unzip +docker exec pgsql_vacuum_db wget https://www.postgresqltutorial.com/wp-content/uploads/2019/05/dvdrental.zip +docker exec pgsql_vacuum_db unzip dvdrental.zip +docker exec pgsql_vacuum_db psql -U postgres -c "CREATE EXTENSION pg_stat_statements;" +docker exec pgsql_vacuum_db psql -U postgres -c 'create database dvdrental;' +docker exec pgsql_vacuum_db pg_restore -U postgres -d dvdrental dvdrental.tar + +## Prepare Data with 1000 tables and each table having around 10k records +## Get 10000 Rows into Testing Table, Import the sample Database +rm dvdrental.tar.xz || true +rm dvdrental.sql || true +wget https://github.com/percona/pmm-qa/raw/PMM-10244-2/pmm-tests/postgres/SampleDB/dvdrental.tar.xz +tar -xvf dvdrental.tar.xz ## only works on Linux/Mac based OS +docker cp dvdrental.sql pgsql_vacuum_db:/ +docker exec pgsql_vacuum_db psql -d dvdrental -f dvdrental.sql -U postgres + +pmm-admin add postgresql --username=postgres --password=YIn7620U1SUc pgsql_vacuum_db localhost:7432 + +## Update & Delete tables using a while loop with sleep +j=0 +while [ $j -lt 3 ] +do + export LENGTH=$(shuf -i 100-120 -n 1) + export LENGTH_NEW=$(shuf -i 100-120 -n 1) + export TABLE=$(shuf -i 1-1000 -n 1) + export COUNT=$(docker exec pgsql_vacuum_db psql -U postgres -d dvdrental -c "select count(*) from film_testing_${TABLE} where length=${LENGTH};" | tail -3 | head -1 | xargs) + docker exec pgsql_vacuum_db psql -U postgres -d dvdrental -c "delete from film_testing_${TABLE} where length=${LENGTH};" + i=0 + while [ "$i" -le ${COUNT} ]; do + docker exec pgsql_vacuum_db psql -U postgres -d dvdrental -c "insert into film_testing_${TABLE} values (${i}, 'title for ${i}', 'Description for ${i}', ${LENGTH});" + i=$(( i + 1 )) + done + docker exec pgsql_vacuum_db psql -U postgres -d dvdrental -c "update film_testing_${TABLE} set length=${LENGTH_NEW} where length=${LENGTH};" + sleep 5 + j=$(( j + 1 )) +done diff --git a/pmm_qa/pgsql_pgss_setup.yml b/pmm_qa/pgsql_pgss_setup.yml new file mode 100644 index 0000000..d1c3f0e --- /dev/null +++ b/pmm_qa/pgsql_pgss_setup.yml @@ -0,0 +1,73 @@ +--- +# This playbook does following: +# install postgres with pgstatement enabled for QAN and PMM Integration testing + +- hosts: all + become: true + become_method: sudo + vars: + pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('14', true) ) }}" + pgsql_pgss_container: "{{ lookup('vars', 'extra_pgsql_pgss_container', default=lookup('env','PGSQL_PGSS_CONTAINER') | default('pgsql_pgss', true) ) }}" + pgsql_pgss_port: "{{ lookup('vars', 'extra_pgsql_port', default=lookup('env','PGSQL_PGSS_PORT') | default('5438', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ pgsql_pgss_container }}" | grep -q . && docker stop {{ pgsql_pgss_container }} && docker rm -fv {{ pgsql_pgss_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PostgreSQL + shell: > + sudo docker run -d --name={{ pgsql_pgss_container }} + -p {{ pgsql_pgss_port }}:5432 + phusion/baseimage:focal-1.2.0 + + - name: Copy all required Artifacts to the docker pgsql_pgss_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} mkdir -p artifacts + - docker cp ./pg_stat_statements_setup.sh {{ pgsql_pgss_container }}:/ + - docker cp ./pgsm_run_queries.sh {{ pgsql_pgss_container }}:/ + + - name: Execute Setup script inside the pgsql pgsql_pgss_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} bash -xe ./pg_stat_statements_setup.sh --pgsql_version {{ pgsql_version }} > setup_pgsql_pgss_{{ pgsql_version }}.log + + - name: Install pmm2-client on the pgsql_pgss_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ pgsql_pgss_container }} + - docker exec {{ pgsql_pgss_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + + - name: Remove pgsql service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove postgresql {{ pgsql_pgss_container }}_service_{{ random_number }}' + ignore_errors: true + + - name: Add pgsql_pgss for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ pgsql_pgss_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatements {{ pgsql_pgss_container }}_service_{{ random_number }}' + + - name: Run queries for generating data, using the same script for pgsm + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_pgss_container }} bash ./pgsm_run_queries.sh & diff --git a/pmm_qa/pgstatmonitor_metrics_queries.sh b/pmm_qa/pgstatmonitor_metrics_queries.sh new file mode 100644 index 0000000..c208da4 --- /dev/null +++ b/pmm_qa/pgstatmonitor_metrics_queries.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export PDPGSQL_CONTAINER=$1 +db=1 +docker exec ${PDPGSQL_CONTAINER} mkdir /tmp/sql +wget https://raw.githubusercontent.com/percona/pmm-agent/main/testqueries/postgres/pg_stat_monitor_load.sql +docker exec ${PDPGSQL_CONTAINER} bash -c "psql -h localhost -U postgres -c 'create database test1'" +docker cp pg_stat_monitor_load.sql ${PDPGSQL_CONTAINER}:/tmp/sql +while true +do + docker exec -u postgres ${PDPGSQL_CONTAINER} psql test1 postgres -f /tmp/sql/pg_stat_monitor_load.sql + sleep 30 +done diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py new file mode 100755 index 0000000..db3e3f5 --- /dev/null +++ b/pmm_qa/pmm-framework.py @@ -0,0 +1,494 @@ +#! /usr/bin/python3 +import subprocess +import argparse +import os +import sys + +# Database configurations +database_configs = { + "PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, + "MYSQL": { + "versions": ["8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": ""} + }, + "PS": { + "versions": ["5.7", "8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} + }, + "PGSQL": { + "versions": ["11", "12", "13", "14", "15", "16"], + "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + }, + "PDPGSQL": { + "versions": ["11", "12", "13", "14", "15", "16"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + }, +} + +def run_ansible_playbook(playbook_filename, env_vars, args): + # Install Ansible + try: + subprocess.run(['sudo', 'yum', 'install', 'ansible', '-y']) + except Exception as e: + print(f"Error installing Ansible: {e}") + exit(1) + + # Get Script Dir + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + playbook_path = script_dir + "/" + playbook_filename + + # Build the commands to execute the playbook + command = ["ansible-playbook", f"{playbook_path}", f'-e os_type=linux', f'--connection=local', + f'-l localhost', f'-i localhost,'] + + if args.verbose: + print(f'Options set after considering defaults: {env_vars}') + + try: + subprocess.run(command, env=env_vars, check=True) + except subprocess.CalledProcessError as e: + print(f"Error executing Ansible {command}: {e}") + exit(1) + + +def get_running_container_name(): + container_name = "pmm-server" + try: + # Run 'docker ps' to get a list of running containers + output = subprocess.check_output(['docker', 'ps', '--format', 'table {{.ID}}\t{{.Image}}\t{{.Names}}']) + # Split the output into a list of container + containers = output.strip().decode('utf-8').split('\n')[1:] + # Check each line for the docker image name + for line in containers: + # Extract the image name + image_info = line.split('\t')[0] + info_parts = image_info.split()[2:] + # Check if the container is in the list of running containers + # and establish N/W connection with it. + if container_name in info_parts: + subprocess.run(['docker', 'network', 'create', 'pmm-qa']) + subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name]) + return container_name + + except subprocess.CalledProcessError: + # Handle the case where the 'docker ps' command fails + return None + + return None + + +def get_value(key, db_type, args, db_config): + # Check if the variable exists in the environment + env_value = os.environ.get(key) + if env_value is not None: + return env_value + + # Only for client_version we accept global command line argument + if key == "CLIENT_VERSION" and args.client_version is not None: + return args.client_version + + # Check if the variable exists in the args config + config_value = db_config.get(key) + if config_value is not None: + return config_value + + # Fall back to default configs value or empty '' + return database_configs[db_type]["configurations"].get(key, '') + + +def setup_ps(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + ps_version = os.getenv('PS_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PS_NODES': '1', + 'PS_VERSION': ps_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PS_CONTAINER': 'pdmysql_pmm_' + str(ps_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'PS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'ps_pmm_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +def setup_mysql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running.., Exiting") + exit() + + # Gather Version details + ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Check Setup Types + setup_type = '' + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": + setup_type = 1 + + # Define environment variables for playbook + env_vars = { + 'GROUP_REPLICATION': f'{setup_type}', + 'MS_NODES': '3' if setup_type else '1', + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'ms_pmm_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + pdpgsql_version = os.getenv('PDPGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PGSTAT_MONITOR_BRANCH': 'main', + 'PDPGSQL_VERSION': pdpgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PDPGSQL_PGSM_CONTAINER': 'pdpgsql_pgsm_pmm_' + str(pdpgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'pdpgsql_pgsm_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +def setup_pgsql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + pgsql_version = os.getenv('PGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PGSQL_VERSION': pgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'pgsql_pgss_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +def execute_docker_compose(compose_filename, commands, env_vars, args): + # Setup n/w used by compose setup + subprocess.run(['docker', 'network', 'create', 'qa-integration']) + subprocess.run(['docker', 'network', 'create', 'pmm-ui-tests_pmm-network']) + subprocess.run(['docker', 'network', 'create', 'pmm2-upgrade-tests_pmm-network']) + subprocess.run(['docker', 'network', 'create', 'pmm2-ui-tests_pmm-network']) + + # Get Script Dir + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + compose_path = script_dir + "/../pmm_psmdb-pbm_setup/" + compose_filename + + # Set environment variables if provided + if env_vars: + for key, value in env_vars.items(): + os.environ[key] = value + + if args.verbose: + print(f'Options set after considering defaults: {env_vars}') + + for command, options in commands.items(): + # Build the Docker Compose command + docker_compose_cmd = ['docker-compose', '-f', compose_path, command] + + # Add options if provided + if options: + docker_compose_cmd.extend(options) + + # Execute Docker Compose + try: + subprocess.run(docker_compose_cmd, check=True) + print(f"Docker Compose {command} executed successfully!") + except subprocess.CalledProcessError as e: + print(f"Error executing Docker Compose {command}: {e}") + exit(1) + + +def execute_shell_scripts(shell_scripts, env_vars, args): + # Get script directory + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + shell_scripts_path = script_dir + "/../pmm_psmdb-pbm_setup/" + + # Get the original working directory + original_dir = os.getcwd() + + if args.verbose: + print(f'Options set after considering defaults: {env_vars}') + + # Set environment variables if provided + if env_vars: + for key, value in env_vars.items(): + os.environ[key] = value + + # Execute each shell script + for script in shell_scripts: + try: + # Change directory to where the script is located + os.chdir(shell_scripts_path) + subprocess.run(['bash', script], check=True) + print(f"Shell script '{script}' executed successfully!") + except subprocess.CalledProcessError as e: + print(f"Error executing shell script '{script} at path {shell_scripts_path}': {e}") + exit(1) + finally: + # Return to the original working directory + os.chdir(original_dir) + + +# Temporary method for Sharding Setup. +def mongo_sharding_setup(script_filename, args): + # Get script directory + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + scripts_path = script_dir + "/../pmm_psmdb-pbm_setup/" + + # Temporary shell script filename + shell_file_path = scripts_path + script_filename + + # Temporary docker compose filename + compose_filename = f'docker-compose-sharded-no-server.yaml' + compose_file_path = scripts_path + compose_filename + + no_server = True + # Add workaround (copy files) till sharding only support is ready. + try: + if no_server: + # Search & Replace content in the temporary compose files + subprocess.run( + ['cp', f'{scripts_path}docker-compose-sharded.yaml', f'{compose_file_path}']) + admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' + subprocess.run(['sed', '-i', f's/password/{admin_password}/g', f'{compose_file_path}']) + subprocess.run(['sed', '-i', '/- test-network/a\\ - pmm-qa', f'{compose_file_path}']) + subprocess.run(['sed', '-i', '/driver: bridge/a\\ pmm-qa:\\n name: pmm-qa\\n external: true', + f'{compose_file_path}']) + subprocess.run( + ['sed', '-i', '/^ pmm-server:/,/^$/{/^ test:/!d}', f'{compose_file_path}']) + with open(f'{compose_file_path}', 'a') as f: + subprocess.run(['echo', ' backups: null'], stdout=f, text=True, check=True) + + # Search replace content in the temporary shell files + subprocess.run(['cp', f'{scripts_path}start-sharded.sh', f'{shell_file_path}']) + subprocess.run(['sed', '-i', '/echo "configuring pmm-server/,/sleep 30/d', + f'{shell_file_path}']) + subprocess.run(['sed', '-i', f's/docker-compose-sharded.yaml/{compose_filename}/g', + f'{shell_file_path}']) + except subprocess.CalledProcessError as e: + print(f"Error occurred: {e}") + + +def setup_psmdb(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running...Exiting") + exit(1) + + # Gather Version details + psmdb_version = os.getenv('PSMDB_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PSMDB_VERSION': psmdb_version, + 'PMM_SERVER_CONTAINER_ADDRESS': f'{args.pmm_server_ip}:8443' or f'{container_name}:8443' or '127.0.0.1:443', + 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), + 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), + 'TESTS': 'no', + 'CLEANUP': 'no' + } + + compose_filename = '' + shell_scripts = [] + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss": + # Docker Compose filename + compose_filename = 'docker-compose-rs.yaml' + # Shell script names + shell_scripts = ['configure-replset.sh', 'configure-agents.sh'] + + # If profile is extra, include additional shell scripts + if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": + shell_scripts.append('configure-extra-replset.sh') + shell_scripts.append('configure-extra-agents.sh') + elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "psa": + # Docker Compose filename + compose_filename = 'docker-compose-rs.yaml' + # Shell script names + shell_scripts = ['configure-psa.sh', 'configure-agents.sh'] + + # If profile is extra, include additional shell scripts + if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": + shell_scripts.append('configure-extra-psa.sh') + shell_scripts.append('configure-extra-agents.sh') + elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": + # Shell script names + shell_scripts = [f'start-sharded-no-server.sh'] + mongo_sharding_setup(shell_scripts[0], args) + + # Define commands for compose file setup + commands = { + 'down': ['-v', '--remove-orphans'], # Cleanup containers + 'build': ['--no-cache'], # Build containers + 'up': ['-d'], # Start containers + } + # Call the function to run the Compose files + if not compose_filename == '': + execute_docker_compose(compose_filename, commands, env_vars, args) + + # Execute shell scripts + if not shell_scripts == []: + execute_shell_scripts(shell_scripts, env_vars, args) + + +# Function to set up a databases based on choice +def setup_database(db_type, db_version=None, db_config=None, args=None): + if args.verbose: + if db_version: + print(f"Setting up {db_type} version {db_version}", end=" ") + else: + print(f"Setting up {db_type}", end=" ") + + if db_config: + print(f"with configuration: {db_config}") + else: + print() + + if db_type == 'MYSQL': + setup_mysql(db_type, db_version, db_config, args) + elif db_type == 'PS': + setup_ps(db_type, db_version, db_config, args) + elif db_type == 'PGSQL': + setup_pgsql(db_type, db_version, db_config, args) + elif db_type == 'PDPGSQL': + setup_pdpgsql(db_type, db_version, db_config, args) + elif db_type == 'PSMDB': + setup_psmdb(db_type, db_version, db_config, args) + else: + print(f"Database type {db_type} is not recognised, Exiting...") + exit(1) + + +# Main +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases') + # Add arguments + parser.add_argument("--database", action='append', nargs=1, + metavar='db_name[,=version][,option1=value1,option2=value2,...]', + help="(e.g: " + "--database mysql=5.7,QUERY_SOURCE=perfschema,SETUP_TYPE=gr,CLIENT_VERSION=3-dev-latest " + "--database pdpgsql=16,USE_SOCKET=1,CLIENT_VERSION=3.0.0 " + "--database psmdb=latest,SETUP_TYPE=psa,CLIENT_VERSION=3.0.0)") + parser.add_argument("--pmm-server-ip", nargs='?', help='PMM Server IP to connect', default='pmm-server') + parser.add_argument("--pmm-server-password", nargs='?', help='PMM Server password') + parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') + parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') + args = parser.parse_args() + + # Parse arguments + try: + for db in args.database: + db_parts = db[0].split(',') + configs = db_parts[0:] if len(db_parts) > 1 else db[0:] + db_type = None + db_version = None + db_config = {} + + if configs: + for config in configs: + if "=" in config: + key, value = config.split('=') + else: + key, value = config, None + + # Convert all arguments/options only to uppercase + key = key.upper() + + try: + if key in database_configs: + db_type = key + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using default value") + elif key in database_configs[db_type]["configurations"]: + db_config[key] = value + else: + if args.verbose: + print(f"Option {key} is not recognised, will be using default option") + continue + except KeyError: + print(f"Option {key} is not recognised, Please check and try again") + parser.print_help() + exit(1) + # Set up the specified databases + setup_database(db_type, db_version, db_config, args) + except argparse.ArgumentError as e: + print(f"Option is not recognised:", e) + parser.print_help() + exit(1) + except Exception as e: + print("An unexpected error occurred:", e) + parser.print_help() + exit(1) diff --git a/pmm_qa/pmm3-client-setup.sh b/pmm_qa/pmm3-client-setup.sh new file mode 100755 index 0000000..be28321 --- /dev/null +++ b/pmm_qa/pmm3-client-setup.sh @@ -0,0 +1,129 @@ +#!/bin/sh + +echo "start installing pmm-agent" + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$admin_password" ]; then + export admin_password=admin +fi + +if [ -z "$pmm_server_ip" ]; then + export pmm_server_ip=127.0.0.1 +fi + +if [ -z "$client_version" ]; then + export client_version=dev-latest +fi + +if [ -z "$install_client" ]; then + export install_client=yes +fi + +if [ -z "$metrics_mode" ]; then + export metrics_mode=auto +fi + +if [ -z "$use_metrics_mode" ]; then + export use_metrics_mode=yes +fi + +if [ ! -z "$upgrade" ]; then + upgrade="-u" +fi + +port=8443 +if [[ "$pmm_server_ip" == "127.0.0.1" ]]; then + port=443 +fi + +apt-get update +apt-get install -y wget gnupg2 libtinfo-dev libnuma-dev mysql-client postgresql-client +wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb +dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb +apt-get update +export PMM_AGENT_SETUP_NODE_NAME=client_container_$(echo $((1 + $RANDOM % 9999))) +mv -v /artifacts/* . + +if [[ "$client_version" == "3-dev-latest" ]]; then + percona-release enable-only original experimental + apt-get update + apt-get -y install pmm-client +fi + +if [[ "$client_version" == "pmm3-rc" ]]; then + percona-release enable-only original testing + apt-get update + apt-get -y install pmm-client +fi + +if [[ "$client_version" == "pmm3-latest" ]]; then + apt-get -y install pmm-client + apt-get -y update + percona-release enable-only original experimental +fi + +## Only supported for debian based systems for now +if [[ "$client_version" == 3* ]]; then + ## Note: only experimental packages are present atm. + wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/experimental/p/pmm-client/pmm-client_${client_version}-6.$(lsb_release -sc)_amd64.deb + dpkg -i pmm-client.deb +fi + +## Default Binary path +path="/usr/local/percona/pmm"; +## As export PATH is not working link the paths +ln -sf ${path}/bin/pmm-admin /usr/local/bin/pmm-admin +ln -sf ${path}/bin/pmm-agent /usr/local/bin/pmm-agent + +if [[ "$client_version" == http* ]]; then + if [[ "$install_client" == "yes" ]]; then + wget -O pmm-client.tar.gz --progress=dot:giga "${client_version}" + fi + tar -zxpf pmm-client.tar.gz + rm -r pmm-client.tar.gz + PMM_CLIENT=`ls -1td pmm-client* 2>/dev/null | grep -v ".tar" | grep -v ".sh" | head -n1` + echo ${PMM_CLIENT} + rm -rf pmm-client + mv ${PMM_CLIENT} pmm-client + rm -rf /usr/local/bin/pmm-client + mv -f pmm-client /usr/local/bin + pushd /usr/local/bin/pmm-client + ## only setting up all binaries in default path /usr/local/percona/pmm + bash -x ./install_tarball ${upgrade} + pwd + popd + pmm-admin --version +fi + +## Check if we are upgrading or attempting fresh install. +if [[ -z "$upgrade" ]]; then + if [[ "$use_metrics_mode" == "yes" ]]; then + echo "setup pmm-agent" + pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=${pmm_server_ip}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password=${admin_password} + else + echo "setup pmm-agent" + pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=${pmm_server_ip}:${port} --server-insecure-tls --server-username=admin --server-password=${admin_password} + fi + sleep 10 + pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > pmm-agent.log 2>&1 & + sleep 10 +else + pid=`ps -ef | grep pmm-agent | grep -v grep | awk -F ' ' '{print $2}'` + if [[ ! -z "$pid" ]]; then + kill -9 $pid + echo "killed old agent, restarting agent...." + pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > pmm-agent.log 2>&1 & + sleep 10 + fi +fi +echo "pmm-admin status" +pmm-admin status diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper new file mode 100644 index 0000000..72398a7 --- /dev/null +++ b/pmm_qa/product_version_download_helper @@ -0,0 +1,26 @@ +##ProductName ##Version ##Tarball/Package Link + +pxc # 5.7.43 # https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-57/Percona-XtraDB-Cluster-5.7.43/binary/tarball/Percona-XtraDB-Cluster-5.7.43-rel47-65.1.Linux.x86_64.glibc2.17-minimal.tar.gz + +pxc # 8.0.33 # https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-80/Percona-XtraDB-Cluster-8.0.33/binary/tarball/Percona-XtraDB-Cluster_8.0.33-25.1_Linux.x86_64.glibc2.17-minimal.tar.gz + +psmdb # 4.4.16 # https://downloads.percona.com/downloads/percona-server-mongodb-4.4/percona-server-mongodb-4.4.16-16/binary/tarball/percona-server-mongodb-4.4.16-16-x86_64.glibc2.17-minimal.tar.gz + +psmdb # 4.2.23 # https://downloads.percona.com/downloads/TESTING/psmdb-4.2.23/percona-server-mongodb-4.2.23-23-x86_64.glibc2.17-minimal.tar.gz + +psmdb # 4.2.22 # https://downloads.percona.com/downloads/percona-server-mongodb-4.2/percona-server-mongodb-4.2.22-22/binary/tarball/percona-server-mongodb-4.2.22-22-x86_64.glibc2.17-minimal.tar.gz + +psmdb # 5.0.11 # https://downloads.percona.com/downloads/percona-server-mongodb-LATEST/percona-server-mongodb-5.0.11-10/binary/tarball/percona-server-mongodb-5.0.11-10-x86_64.glibc2.17-minimal.tar.gz + +psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mongodb-6.0/percona-distribution-mongodb-6.0.12/binary/tarball/percona-server-mongodb-6.0.12-9-x86_64.glibc2.17-minimal.tar.gz + +psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz + +ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz + +ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz + +ms # 8.0.33 # https://dev.mysql.com/get/Downloads/MySQL-8/mysql-8.0.33-linux-glibc2.28-x86_64.tar.gz + +proxysql # 2.5.3 # https://downloads.percona.com/downloads/proxysql2/proxysql2-2.5.3/binary/debian/focal/x86_64/proxysql2_2.5.3-1.1.focal_amd64.deb + diff --git a/pmm_qa/prometheus.base.yml b/pmm_qa/prometheus.base.yml new file mode 100644 index 0000000..7f69fc8 --- /dev/null +++ b/pmm_qa/prometheus.base.yml @@ -0,0 +1,23 @@ +scrape_configs: +- job_name: blackbox80 + params: + module: + - http_2xx + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /probe + scheme: http + + static_configs: + - targets: + - http://www.alfaexploit.com/archive + + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - source_labels: [__param_target] + target_label: node_name + - target_label: __address__ + replacement: pmm.alfaexploit.com:9115 diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml new file mode 100644 index 0000000..419429b --- /dev/null +++ b/pmm_qa/ps_pmm_setup.yml @@ -0,0 +1,70 @@ +--- +# This playbook does following: +# enables Percona testing repository + +- hosts: all + become: true + become_method: sudo + vars: + ps_version: "{{ lookup('vars', 'extra_ps_version', default=lookup('env','PS_VERSION') | default('8.0', true) ) }}" + ps_tarball: "{{ lookup('vars', 'extra_ps_tarball', default=lookup('env','PS_TARBALL') | default('', true) ) }}" + ps_container: "{{ lookup('vars', 'extra_ps_container', default=lookup('env','PS_CONTAINER') | default('ps_container', true) ) }}" + query_source: "{{ lookup('vars', 'extra_query_source', default=lookup('env','QUERY_SOURCE') | default('slowlog', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_NODES') | default('3', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ ps_container }}" | grep -q . && docker stop {{ ps_container }} && docker rm -fv {{ ps_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for ps and Proxysql + shell: > + docker run -d --name={{ ps_container }} + phusion/baseimage:jammy-1.0.1 + + - name: Copy all required Artifacts to the docker ps_container + shell: "{{ item }}" + with_items: + - docker exec {{ ps_container }} mkdir -p artifacts + - docker cp ./client_container_ps_setup.sh {{ ps_container }}:/ + + - name: Get Product tarball URL based on the version + shell: cat product_version_download_helper | grep -v pxc | grep ps | grep "\-{{ ps_version }}" | head -1 | awk -F'# ' '{print $3}' + register: tarball + when: lookup('env', 'PS_TARBALL') == '' + + - name: Set Product URL if environment or paramater are not defined + set_fact: + ps_tarball: "{{tarball.stdout | default(ps_tarball,true)}}" + + - name: Setup libraries required inside the container + shell: "{{ item }}" + with_items: + - docker exec {{ ps_container }} apt-get update + - docker exec {{ ps_container }} apt-get -y install wget curl git gnupg2 lsb-release + - docker exec {{ ps_container }} apt-get -y install libaio1 libaio-dev libnuma-dev socat + - docker exec {{ ps_container }} apt install -y sysbench + + - name: Install pmm2-client on the ps_container + shell: "{{ item }}" + with_items: + - docker exec {{ ps_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ ps_container }} + - docker exec {{ ps_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Execute Setup script inside the ps ps_container + shell: "{{ item }}" + with_items: + - docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} > setup_ps_{{ ps_version }}.log + diff --git a/pmm_qa/psmdb_setup.sh b/pmm_qa/psmdb_setup.sh new file mode 100644 index 0000000..ca7e62c --- /dev/null +++ b/pmm_qa/psmdb_setup.sh @@ -0,0 +1,116 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=4.4 +fi + +if [ -z "$mongdb_setup" ]; then + export mongdb_setup=replica +fi + +if [ -z "$metrics_mode" ]; then + export metrics_mode=push +fi + +# Mongo user credtials for the replicat set cluster +export user="dba" +export pwd="test1234" + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl jq git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl +sleep 10 + +wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh +chmod +x mongo_startup.sh +export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999))) + +### Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) +psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) + +echo "Downloading ${psmdb_latest} ..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep percona-server-mongodb) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} psmdb_${mongodb_version} + +# TODO: refactor if to match range of versions 6.0+ +if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" ]]; then + ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally + echo "Downloading mongosh ..." + mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh) + wget -O mongosh.tar.gz ${mongosh_link} + tar -xvf mongosh.tar.gz + mv percona-mongodb-mongosh* mongosh + cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo + rm mongosh.tar.gz +fi + +if [ "$mongodb_setup" == "sharded" ]; then + bash ./mongo_startup.sh -s -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=./psmdb_${mongodb_version}/bin + pmm-admin add mongodb --cluster mongodb_node_cluster --environment=mongodb_shraded_node mongodb_shraded_node_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27027 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27028 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27029 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27018 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27019 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27020 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:28018 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:28019 + sleep 2 + pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:28020 + sleep 20 + #./nodes/cl_mongos.sh mongodb_user_setup.js +fi + +if [ "$mongodb_setup" == "replica" ]; then + bash ./mongo_startup.sh -r -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=./psmdb_${mongodb_version}/bin + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1 || true; pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2 || true; pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3 || true; pmm-admin add mongodb --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +#Arbiter setup with Auth enabled (keyfile) +if [ "$mongodb_setup" == "arbiter" ]; then + bash ./mongo_startup.sh -x -r -a -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=./psmdb_${mongodb_version}/bin + sleep 20 + pmm-admin remove mongodb mongodb_rs2_1 || true; pmm-admin add mongodb --cluster mongodb_node_cluster2 --replication-set=rs2 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --debug --username=${user} --password=${pwd} 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs2_2 || true; pmm-admin add mongodb --cluster mongodb_node_cluster2 --replication-set=rs2 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --debug --username=${user} --password=${pwd} 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs2_3 || true; pmm-admin add mongodb --cluster mongodb_node_cluster2 --replication-set=rs2 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + + +if [ "$mongodb_setup" == "regular" ]; then + bash ./mongo_startup.sh -m -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=./psmdb_${mongodb_version}/bin + pmm-admin add mongodb --cluster mongodb_node_cluster --environment=mongodb_single_node mongodb_rs_single_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017 + sleep 20 +fi +rm percona_server_mongodb.tar.gz* diff --git a/pmm_qa/psmdb_setup.yml b/pmm_qa/psmdb_setup.yml new file mode 100644 index 0000000..8dd03d4 --- /dev/null +++ b/pmm_qa/psmdb_setup.yml @@ -0,0 +1,81 @@ +--- + +- hosts: all + vars: + psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}" + psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}" + psmdb_setup: "{{ lookup('vars', 'extra_psmdb_setup', default=lookup('env','PSMDB_SETUP') | default('regular', true) ) }}" + psmdb_container: "{{ lookup('vars', 'extra_psmdb_container', default=lookup('env','PSMDB_CONTAINER') | default('psmdb', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ psmdb_container }}" | grep -q . && docker stop {{ psmdb_container }} && docker rm -fv {{ psmdb_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PSMDB + shell: > + docker run -d --name={{ psmdb_container }} + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker psmdb_container + shell: "{{ item }}" + with_items: + - docker cp ./psmdb_setup.sh {{ psmdb_container }}:/ + - docker exec {{ psmdb_container }} apt-get update + - docker exec {{ psmdb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq + + - name: Install pmm2-client on the psmdb_container + shell: "{{ item }}" + with_items: + - docker exec {{ psmdb_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker exec {{ psmdb_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/mongodb_user_setup.js + - docker network connect pmm-qa {{ psmdb_container }} + - docker exec {{ psmdb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Setup psmdb for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ psmdb_container }} bash -x ./psmdb_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > ~/setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + + - name: Setup Load Running Docker Container + shell: "{{ item }}" + with_items: + - rm -r ~/psmdb_{{ psmdb_version }} || true; mkdir ~/psmdb_{{ psmdb_version }} + - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/Dockerfile" + - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/mongodb_query.php" + - docker build --tag php-db ~/psmdb_{{ psmdb_version }}/ > ~/docker-build_mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }}.log || true + - docker rm mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} || true + - docker run --rm --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db composer require mongodb/mongodb || true + + - name: Run User setup script when Running Regular MongoDB setup + shell: docker exec {{ psmdb_container }} ./nodes/cl.sh mongodb_user_setup.js + when: psmdb_setup == "regular" + + - name: Run load on Replica Set Master + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27018 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + when: psmdb_setup == "replica" or psmdb_setup == "arbiter" + + - name: Run load on Regular + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + when: psmdb_setup == "regular" + + - name: Run load on Sharded Clusters Master + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27019 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + when: psmdb_setup == "sharded" diff --git a/pmm_qa/pxc_proxysql_setup.sh b/pmm_qa/pxc_proxysql_setup.sh new file mode 100644 index 0000000..11c211a --- /dev/null +++ b/pmm_qa/pxc_proxysql_setup.sh @@ -0,0 +1,66 @@ +#!/bin/bash +export number_of_nodes=$1 +export pxc_version=$2 +export query_source=$3 + +sudo yum install -y socat +wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/pxc-tests/pxc-startup.sh +sed -i 's/log-output=none/log-output=file/g' pxc-startup.sh +## bug https://bugs.mysql.com/bug.php?id=90553 workaround +sed -i 's+${MID} --datadir+${MID} --socket=\\${node}/socket.sock --port=\\${RBASE1} --datadir+g' pxc-startup.sh + +## Download right PXC version +if [ "$pxc_version" == "5.7" ]; then + wget https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-57/Percona-XtraDB-Cluster-5.7.34-31.51/binary/tarball/Percona-XtraDB-Cluster-5.7.34-rel37-51.1.Linux.x86_64.glibc2.12-minimal.tar.gz + sudo yum install -y percona-xtrabackup-24 +fi +if [ "$pxc_version" == "8.0" ]; then + sed -i 's+wsrep_node_incoming_address=$ADDR+wsrep_node_incoming_address=$ADDR:$RBASE1+g' pxc-startup.sh + wget https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-LATEST/Percona-XtraDB-Cluster-8.0.27/binary/tarball/Percona-XtraDB-Cluster_8.0.27-18.1_Linux.x86_64.glibc2.17-minimal.tar.gz +fi +tar -xzf Percona-XtraDB-Cluster* +rm -r Percona-XtraDB-Cluster*.tar.gz +mv Percona-XtraDB-Cluster* PXC +cd PXC + +## start PXC +bash ../pxc-startup.sh +bash ./start_pxc $number_of_nodes +touch sysbench_run_node1_prepare.txt +touch sysbench_run_node1_read_write.txt +touch sysbench_run_node1_read_only.txt + +## Install proxysql2 +sudo yum install -y proxysql2 + +### enable slow log +if [ "$query_source" == "slowlog" ]; then + for j in `seq 1 ${number_of_nodes}`; + do + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL slow_query_log='ON';" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL long_query_time=0;" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_rate_limit=1;" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_verbosity='full';" + bin/mysql -A -uroot -Snode$j/socket.sock -e "SET GLOBAL log_slow_rate_type='query';" + done +fi + +bin/mysql -A -uroot -Snode1/socket.sock -e "create user admin@localhost identified with mysql_native_password by 'admin';" +bin/mysql -A -uroot -Snode1/socket.sock -e "grant all on *.* to admin@localhost;" +bin/mysql -A -uroot -Snode1/socket.sock -e "create user sysbench@'%' identified with mysql_native_password by 'test';" +bin/mysql -A -uroot -Snode1/socket.sock -e "grant all on *.* to sysbench@'%';" +bin/mysql -A -uroot -Snode1/socket.sock -e "drop database if exists sbtest;create database sbtest;" + +### update proxysql configuration use, correct port +export node1_port=$(cat node1.cnf | grep port | awk -F"=" '{print $2}') +sudo sed -i "s/3306/${node1_port}/" /etc/proxysql-admin.cnf + +sudo service proxysql start +sleep 20 +sudo proxysql-admin -e + +## Start Running Load +#sysbench /usr/share/sysbench/oltp_insert.lua --mysql-db=sbtest --mysql-user=sysbench --mysql-socket=node1/socket.sock --mysql-password=test --db-driver=mysql --threads=5 --tables=10 --table-size=1000 prepare > sysbench_run_node1_prepare.txt 2>&1 & +#sleep 20 +#sysbench /usr/share/sysbench/oltp_read_only.lua --mysql-db=sbtest --mysql-user=sysbench --mysql-socket=node1/socket.sock --mysql-password=test --db-driver=mysql --threads=5 --tables=10 --table-size=1000 --time=12000 run > sysbench_run_node1_read_only.txt 2>&1 & +#sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-db=sbtest --mysql-user=sysbench --mysql-socket=node1/socket.sock --mysql-password=test --db-driver=mysql --threads=5 --tables=10 --table-size=1000 --time=12000 run > sysbench_run_node1_read_write.txt 2>&1 & diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml new file mode 100644 index 0000000..9fa5a12 --- /dev/null +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -0,0 +1,115 @@ +# This playbook does following: +# enables Percona testing repository + +- hosts: all + become: true + become_method: sudo + vars: + pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8', true) ) }}" + pxc_tarball: "{{ lookup('vars', 'extra_pxc_tarball', default=lookup('env','PXC_TARBALL') | default('', true) ) }}" + proxysql_version: "{{ lookup('vars', 'extra_proxysql_version', default=lookup('env','PROXYSQL_VERSION') | default('2', true) ) }}" + proxysql_package: "{{ lookup('vars', 'extra_proxysql_package', default=lookup('env','PROXYSQL_PACKAGE') | default('', true) ) }}" + pxc_container: "{{ lookup('vars', 'extra_pxc_container', default=lookup('env','PXC_CONTAINER') | default('pxc_container', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('host.docker.internal', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PXC_NODES') | default('3', true) ) }}" + pxc_dev_cluster: "{{ lookup('vars', 'extra_pxc_cluster_name', default=lookup('env','PXC_CLUSTER_NAME') | default('pxc-dev-cluster', true) ) }}" + query_source: "{{ lookup('vars', 'extra_query_source', default=lookup('env','QUERY_SOURCE') | default('perfschema', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ pxc_container }}" | grep -q . && docker stop {{ pxc_container }} && docker rm -fv {{ pxc_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PXC and Proxysql + shell: > + docker run -d --name={{ pxc_container }} + phusion/baseimage:jammy-1.0.1 + + - name: Copy all required Artifacts to the docker pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} mkdir -p artifacts + - docker cp ./client_container_pxc_setup.sh {{ pxc_container }}:/ + - docker cp ./client_container_proxysql_setup.sh {{ pxc_container }}:/ + + - name: Get PXC Product tarball URL based on the version + shell: cat product_version_download_helper | grep pxc | grep "\-{{ pxc_version }}" | head -1 | awk -F'# ' '{print $3}' + register: tarball + when: lookup('env', 'PXC_TARBALL') == '' + + - name: Set PXC Product URL if environment or paramater are not defined + set_fact: + pxc_tarball: "{{tarball.stdout | default(pxc_tarball,true)}}" + + - name: Get ProxySQL Product package URL based on the version + shell: cat product_version_download_helper | grep proxysql | grep "\-{{ proxysql_version }}" | head -1 | awk -F'# ' '{print $3}' + register: package + when: lookup('env', 'PROXYSQL_PACKAGE') == '' + + - name: Set ProxySQL Product URL if environment or paramater are not defined + set_fact: + proxysql_package: "{{package.stdout | default(proxysql_package,true)}}" + + - name : Get filename from URL for ProxySQL + shell: basename {{ proxysql_package }} + register: file_name + + - name: Execute User & Proxysql Setup inside the PXC pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} apt-get update + - docker exec {{ pxc_container }} apt-get -y install wget curl git gnupg2 lsb-release debconf-utils + - docker exec {{ pxc_container }} apt-get -y install libaio1 libaio-dev libnuma-dev socat + - docker exec {{ pxc_container }} adduser --disabled-password --gecos "" pxc + - docker exec {{ pxc_container }} wget {{ proxysql_package }} + - docker exec {{ pxc_container }} dpkg -i {{ file_name.stdout }} + - docker exec {{ pxc_container }} apt install -y sysbench + + - name: Install pmm2-client on the pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ pxc_container }} + - docker exec {{ pxc_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + - docker exec {{ pxc_container }} usermod -aG pmm-agent pxc + + - name: Execute User & Proxysql Setup inside the PXC pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} percona-release enable-only tools release + - docker exec {{ pxc_container }} apt-get update + - docker exec {{ pxc_container }} apt-get install -y percona-xtrabackup-24 + when: pxc_version == "5.7" + + - name: Execute Setup script inside the PXC pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} chmod 1777 /tmp + - docker exec --user pxc {{ pxc_container }} bash -xe ./client_container_pxc_setup.sh --pxc_version {{ pxc_version }} --pxc_tarball {{ pxc_tarball }} --number_of_nodes {{ number_of_nodes }} --pxc_dev_cluster {{ pxc_dev_cluster }} --query_source {{query_source}} > setup_pxc_{{ pxc_version }}.log + + - name: Start ProxySQL inside the PXC extra_pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} bash -c 'sed -i s#3306#'"\$(grep 'port' /home/pxc/PXC/node1.cnf | cut -d= -f2)"'# /etc/proxysql.cnf' + - docker exec {{ pxc_container }} proxysql -c /etc/proxysql.cnf + - docker exec {{ pxc_container }} sleep 20 + - docker exec {{ pxc_container }} bash -c 'sed -i s#3306#'"\$(grep 'port' /home/pxc/PXC/node1.cnf | cut -d= -f2)"'# /etc/proxysql-admin.cnf' + - docker exec {{ pxc_container }} proxysql-admin --config-file=/etc/proxysql-admin.cnf --enable + - docker exec {{ pxc_container }} sleep 20 + - docker exec {{ pxc_container }} pmm-admin add proxysql --username=admin --password=admin --service-name=my-new-proxysql_{{ pxc_container }} --host=127.0.0.1 --port=6032 + + - name: Execute Service, Sysbench Script inside the PXC pxc_container + shell: "{{ item }}" + with_items: + - docker exec {{ pxc_container }} bash -xe ./client_container_proxysql_setup.sh > setup_proxy_{{ pxc_version }}.log + From dc3cc98509983bd793040d4a852f2807da171f58 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 4 Apr 2024 22:51:36 +0530 Subject: [PATCH 16/88] PMM7 Fix IP issue (#42) Fix IP/Port issue and Updates --- .../docker-compose-sharded.yaml | 36 +++++++++---------- pmm_psmdb-pbm_setup/start-rs.sh | 2 +- pmm_qa/pmm-framework.py | 8 ++--- pmm_qa/pmm3-client-setup.sh | 6 ++-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index e7fed62..418d903 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -24,9 +24,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs101 hostname: rs101 @@ -44,9 +44,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs102 hostname: rs102 @@ -64,9 +64,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs103 hostname: rs103 @@ -84,9 +84,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs201 hostname: rs201 @@ -104,9 +104,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs202 hostname: rs202 @@ -124,9 +124,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs203 hostname: rs203 @@ -144,9 +144,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rscfg01 hostname: rscfg01 @@ -164,9 +164,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rscfg02 hostname: rscfg02 @@ -184,9 +184,9 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" + - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" - "PMM_AGENT_SERVER_USERNAME=admin" - - "PMM_AGENT_SERVER_PASSWORD=password" + - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rscfg03 hostname: rscfg03 diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index 7958b87..cddf62f 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -3,7 +3,7 @@ set -e pmm_server_admin_pass=${ADMIN_PASSWORD:-password} profile=${COMPOSE_PROFILES:-classic} -mongo_setup_type=${}MONGO_SETUP_TYPE:-pss} +mongo_setup_type=${MONGO_SETUP_TYPE:-pss} docker network create qa-integration || true docker network create pmm-qa || true diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index db3e3f5..1f8f28b 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -1,4 +1,4 @@ -#! /usr/bin/python3 +#! /usr/bin/python3 -E import subprocess import argparse import os @@ -117,7 +117,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': '1', 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'pdmysql_pmm_' + str(ps_version), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version), 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'PS_TARBALL': get_value('TARBALL', db_type, args, db_config), @@ -348,7 +348,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Define environment variables for playbook env_vars = { 'PSMDB_VERSION': psmdb_version, - 'PMM_SERVER_CONTAINER_ADDRESS': f'{args.pmm_server_ip}:8443' or f'{container_name}:8443' or '127.0.0.1:443', + 'PMM_SERVER_CONTAINER_ADDRESS': f'{args.pmm_server_ip}:443' or f'{container_name}:8443' or '127.0.0.1:443', 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), @@ -438,7 +438,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): "--database mysql=5.7,QUERY_SOURCE=perfschema,SETUP_TYPE=gr,CLIENT_VERSION=3-dev-latest " "--database pdpgsql=16,USE_SOCKET=1,CLIENT_VERSION=3.0.0 " "--database psmdb=latest,SETUP_TYPE=psa,CLIENT_VERSION=3.0.0)") - parser.add_argument("--pmm-server-ip", nargs='?', help='PMM Server IP to connect', default='pmm-server') + parser.add_argument("--pmm-server-ip", nargs='?', help='PMM Server IP to connect') parser.add_argument("--pmm-server-password", nargs='?', help='PMM Server password') parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') diff --git a/pmm_qa/pmm3-client-setup.sh b/pmm_qa/pmm3-client-setup.sh index be28321..84df692 100755 --- a/pmm_qa/pmm3-client-setup.sh +++ b/pmm_qa/pmm3-client-setup.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash echo "start installing pmm-agent" @@ -41,7 +41,7 @@ if [ ! -z "$upgrade" ]; then fi port=8443 -if [[ "$pmm_server_ip" == "127.0.0.1" ]]; then +if [[ ! "$pmm_server_ip" =~ ^"pmm-server"$ ]]; then port=443 fi @@ -72,7 +72,7 @@ if [[ "$client_version" == "pmm3-latest" ]]; then fi ## Only supported for debian based systems for now -if [[ "$client_version" == 3* ]]; then +if [[ "$client_version" =~ ^"3." ]]; then ## Note: only experimental packages are present atm. wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/experimental/p/pmm-client/pmm-client_${client_version}-6.$(lsb_release -sc)_amd64.deb dpkg -i pmm-client.deb From 7ce8bd538bdc2770d4f33c4d67c6946780a5bf38 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Fri, 5 Apr 2024 14:09:38 +0300 Subject: [PATCH 17/88] PMM-7 use ansible-runner lib, create Makefile with init target, add requirements.txt --- pmm_qa/Makefile | 11 +++++++++++ pmm_qa/pmm-framework.py | 32 +++++++++++++------------------- pmm_qa/requirements.txt | 10 ++++++++++ 3 files changed, 34 insertions(+), 19 deletions(-) create mode 100644 pmm_qa/Makefile create mode 100644 pmm_qa/requirements.txt diff --git a/pmm_qa/Makefile b/pmm_qa/Makefile new file mode 100644 index 0000000..9e36f6f --- /dev/null +++ b/pmm_qa/Makefile @@ -0,0 +1,11 @@ +default: help + +help: ## Display this help message. + @echo "Please use \`make \` where is one of:" + @grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \ + awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}' + +init: ## Create venv and install dependencies + python3 -m venv pmm_framework + . pmm_framework/bin/activate + python3 -m pip install -r requirements.txt diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 1f8f28b..2f2a812 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -3,6 +3,7 @@ import argparse import os import sys +import ansible_runner # Database configurations database_configs = { @@ -30,31 +31,24 @@ }, } -def run_ansible_playbook(playbook_filename, env_vars, args): - # Install Ansible - try: - subprocess.run(['sudo', 'yum', 'install', 'ansible', '-y']) - except Exception as e: - print(f"Error installing Ansible: {e}") - exit(1) +def run_ansible_playbook(playbook_filename, env_vars, args): # Get Script Dir script_path = os.path.abspath(sys.argv[0]) script_dir = os.path.dirname(script_path) playbook_path = script_dir + "/" + playbook_filename - # Build the commands to execute the playbook - command = ["ansible-playbook", f"{playbook_path}", f'-e os_type=linux', f'--connection=local', - f'-l localhost', f'-i localhost,'] - - if args.verbose: - print(f'Options set after considering defaults: {env_vars}') - - try: - subprocess.run(command, env=env_vars, check=True) - except subprocess.CalledProcessError as e: - print(f"Error executing Ansible {command}: {e}") - exit(1) + print(script_dir) + print(playbook_filename) + + r = ansible_runner.run( + private_data_dir=script_dir, + playbook=playbook_path, + inventory='127.0.0.1', + cmdline='-l localhost, --connection=local', + envvars=env_vars + ) + print("{} playbook execution {}".format(playbook_filename, r.status)) def get_running_container_name(): diff --git a/pmm_qa/requirements.txt b/pmm_qa/requirements.txt new file mode 100644 index 0000000..aab8888 --- /dev/null +++ b/pmm_qa/requirements.txt @@ -0,0 +1,10 @@ +ansible-runner==2.3.6 +docutils==0.20.1 +lockfile==0.12.2 +packaging==24.0 +pexpect==4.9.0 +ptyprocess==0.7.0 +python-daemon==3.0.1 +PyYAML==6.0.1 +setuptools==69.2.0 +six==1.16.0 From be80d60a5ada31a0e3f4350f3c2031a10e557389 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Fri, 5 Apr 2024 15:40:55 +0300 Subject: [PATCH 18/88] PMM-7 review changes --- pmm_qa/Makefile | 8 ++++++++ pmm_qa/pmm-framework.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pmm_qa/Makefile b/pmm_qa/Makefile index 9e36f6f..b3e6c97 100644 --- a/pmm_qa/Makefile +++ b/pmm_qa/Makefile @@ -9,3 +9,11 @@ init: ## Create venv and install dependencies python3 -m venv pmm_framework . pmm_framework/bin/activate python3 -m pip install -r requirements.txt + +init-ubuntu: ## Create venv and install dependencies for ubuntu + sudo apt update + sudo apt install -y python3.10-venv + sudo python3 -m venv pmm_framework + . pmm_framework/bin/activate + sudo python3 -m pip install -r requirements.txt + diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2f2a812..d1dc786 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -38,8 +38,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): script_dir = os.path.dirname(script_path) playbook_path = script_dir + "/" + playbook_filename - print(script_dir) - print(playbook_filename) + if args.verbose: + print(f'Options set after considering defaults: {env_vars}') r = ansible_runner.run( private_data_dir=script_dir, @@ -48,7 +48,7 @@ def run_ansible_playbook(playbook_filename, env_vars, args): cmdline='-l localhost, --connection=local', envvars=env_vars ) - print("{} playbook execution {}".format(playbook_filename, r.status)) + print(f'{playbook_filename} playbook execution {r.status}') def get_running_container_name(): From 1cd52ad69c994e94d18bb415e6a71a2085221391 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Mon, 8 Apr 2024 12:18:34 +0300 Subject: [PATCH 19/88] PMM-7 update requirements --- pmm_qa/requirements.txt | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pmm_qa/requirements.txt b/pmm_qa/requirements.txt index aab8888..a221365 100644 --- a/pmm_qa/requirements.txt +++ b/pmm_qa/requirements.txt @@ -1,10 +1 @@ -ansible-runner==2.3.6 -docutils==0.20.1 -lockfile==0.12.2 -packaging==24.0 -pexpect==4.9.0 -ptyprocess==0.7.0 -python-daemon==3.0.1 -PyYAML==6.0.1 -setuptools==69.2.0 -six==1.16.0 +ansible-runner==2.3.2 From 51281d1f07df03bce9dd6eeaa31f5dc2c7702471 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 9 Apr 2024 09:47:47 +0300 Subject: [PATCH 20/88] PMM-7 remove python from script, update env vars --- pmm_qa/pmm-framework.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index d1dc786..0f34199 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -1,4 +1,3 @@ -#! /usr/bin/python3 -E import subprocess import argparse import os @@ -116,7 +115,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'PS_TARBALL': get_value('TARBALL', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -152,7 +151,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -181,7 +180,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -209,7 +208,7 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename From f7297e915dba0bdbc1a0c634822382d7c9f75ba7 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 9 Apr 2024 14:42:30 +0300 Subject: [PATCH 21/88] PMM-7 update server address logic --- pmm_qa/pmm-framework.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 0f34199..fc4c120 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -338,10 +338,20 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or db_version or database_configs[db_type]["versions"][-1] + # Handle port address for external or internal address + server_hostname = container_name + port = 8443 + + if args.pmm_server_ip: + port = 443 + server_hostname = args.pmm_server_ip + + server_address = f'{server_hostname}:{port}' + # Define environment variables for playbook env_vars = { 'PSMDB_VERSION': psmdb_version, - 'PMM_SERVER_CONTAINER_ADDRESS': f'{args.pmm_server_ip}:443' or f'{container_name}:8443' or '127.0.0.1:443', + 'PMM_SERVER_CONTAINER_ADDRESS': server_address, 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), From a83aac5899c4eaa3a9ebd26572ef081e1f7ea53b Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Tue, 9 Apr 2024 15:48:23 +0300 Subject: [PATCH 22/88] PMM-7 rely on start-rs-only --- pmm_qa/pmm-framework.py | 81 +++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index fc4c120..0a21eac 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -362,45 +362,48 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): } compose_filename = '' - shell_scripts = [] - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss": - # Docker Compose filename - compose_filename = 'docker-compose-rs.yaml' - # Shell script names - shell_scripts = ['configure-replset.sh', 'configure-agents.sh'] - - # If profile is extra, include additional shell scripts - if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - shell_scripts.append('configure-extra-replset.sh') - shell_scripts.append('configure-extra-agents.sh') - elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "psa": - # Docker Compose filename - compose_filename = 'docker-compose-rs.yaml' - # Shell script names - shell_scripts = ['configure-psa.sh', 'configure-agents.sh'] - - # If profile is extra, include additional shell scripts - if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - shell_scripts.append('configure-extra-psa.sh') - shell_scripts.append('configure-extra-agents.sh') - elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": - # Shell script names - shell_scripts = [f'start-sharded-no-server.sh'] - mongo_sharding_setup(shell_scripts[0], args) - - # Define commands for compose file setup - commands = { - 'down': ['-v', '--remove-orphans'], # Cleanup containers - 'build': ['--no-cache'], # Build containers - 'up': ['-d'], # Start containers - } - # Call the function to run the Compose files - if not compose_filename == '': - execute_docker_compose(compose_filename, commands, env_vars, args) - - # Execute shell scripts - if not shell_scripts == []: - execute_shell_scripts(shell_scripts, env_vars, args) + shell_scripts = ['start-rs-only.sh'] + + execute_shell_scripts(shell_scripts, env_vars, args) + + # if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss": + # # Docker Compose filename + # compose_filename = 'docker-compose-rs.yaml' + # # Shell script names + # shell_scripts = ['configure-replset.sh', 'configure-agents.sh'] + # + # # If profile is extra, include additional shell scripts + # if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": + # shell_scripts.append('configure-extra-replset.sh') + # shell_scripts.append('configure-extra-agents.sh') + # elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "psa": + # # Docker Compose filename + # compose_filename = 'docker-compose-rs.yaml' + # # Shell script names + # shell_scripts = ['configure-psa.sh', 'configure-agents.sh'] + # + # # If profile is extra, include additional shell scripts + # if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": + # shell_scripts.append('configure-extra-psa.sh') + # shell_scripts.append('configure-extra-agents.sh') + # elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": + # # Shell script names + # shell_scripts = [f'start-sharded-no-server.sh'] + # mongo_sharding_setup(shell_scripts[0], args) + # + # # Define commands for compose file setup + # commands = { + # 'down': ['-v', '--remove-orphans'], # Cleanup containers + # 'build': ['--no-cache'], # Build containers + # 'up': ['-d'], # Start containers + # } + # # Call the function to run the Compose files + # if not compose_filename == '': + # execute_docker_compose(compose_filename, commands, env_vars, args) + # + # # Execute shell scripts + # if not shell_scripts == []: + # execute_shell_scripts(shell_scripts, env_vars, args) # Function to set up a databases based on choice From 142d7406b6766d64d45fd163f9b8673d1da21c18 Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 17:58:23 +0300 Subject: [PATCH 23/88] PMM-7 return exit code if ansible fails --- pmm_qa/pmm-framework.py | 58 +++++++++++------------------------------ 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 0a21eac..890ee49 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -47,8 +47,12 @@ def run_ansible_playbook(playbook_filename, env_vars, args): cmdline='-l localhost, --connection=local', envvars=env_vars ) + print(f'{playbook_filename} playbook execution {r.status}') + if r.rc != 0: + exit(1) + def get_running_container_name(): container_name = "pmm-server" @@ -361,49 +365,17 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): 'CLEANUP': 'no' } - compose_filename = '' - shell_scripts = ['start-rs-only.sh'] - - execute_shell_scripts(shell_scripts, env_vars, args) - - # if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss": - # # Docker Compose filename - # compose_filename = 'docker-compose-rs.yaml' - # # Shell script names - # shell_scripts = ['configure-replset.sh', 'configure-agents.sh'] - # - # # If profile is extra, include additional shell scripts - # if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - # shell_scripts.append('configure-extra-replset.sh') - # shell_scripts.append('configure-extra-agents.sh') - # elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "psa": - # # Docker Compose filename - # compose_filename = 'docker-compose-rs.yaml' - # # Shell script names - # shell_scripts = ['configure-psa.sh', 'configure-agents.sh'] - # - # # If profile is extra, include additional shell scripts - # if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - # shell_scripts.append('configure-extra-psa.sh') - # shell_scripts.append('configure-extra-agents.sh') - # elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": - # # Shell script names - # shell_scripts = [f'start-sharded-no-server.sh'] - # mongo_sharding_setup(shell_scripts[0], args) - # - # # Define commands for compose file setup - # commands = { - # 'down': ['-v', '--remove-orphans'], # Cleanup containers - # 'build': ['--no-cache'], # Build containers - # 'up': ['-d'], # Start containers - # } - # # Call the function to run the Compose files - # if not compose_filename == '': - # execute_docker_compose(compose_filename, commands, env_vars, args) - # - # # Execute shell scripts - # if not shell_scripts == []: - # execute_shell_scripts(shell_scripts, env_vars, args) + shell_scripts = [] + + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("pss" or "psa"): + shell_scripts = ['start-rs-only.sh'] + elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": + shell_scripts = [f'start-sharded-no-server.sh'] + mongo_sharding_setup(shell_scripts[0], args) + + # Execute shell scripts + if not shell_scripts == []: + execute_shell_scripts(shell_scripts, env_vars, args) # Function to set up a databases based on choice From 2398dd50a14271b0373922f2f7d34243e141984e Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 18:25:03 +0300 Subject: [PATCH 24/88] PMM-7 add random service names to mongo services --- pmm_psmdb-pbm_setup/configure-agents.sh | 7 ++++--- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 7 ++++--- pmm_psmdb-pbm_setup/start-sharded.sh | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index cc30ff2..eabb34b 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -24,15 +24,16 @@ if [[ $mongo_setup_type == "psa" ]]; then fi echo echo "configuring pmm agents" +random_number=$RANDOM nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs $node 127.0.0.1:27017 + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 56ad77a..096f982 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -24,14 +24,15 @@ if [[ $mongo_setup_type == "psa" ]]; then fi echo echo "configuring pmm agents" +random_number=$RANDOM nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 $node 127.0.0.1:27017 + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} $node 127.0.0.1:27017 + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index e60b8e3..de602fa 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -235,11 +235,11 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} $node 127.0.0.1:27017 + docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} + docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos mongos:27017 +docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" docker-compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 From 45f0010bb439af8842e3ff117ba4eed62ce2cc2c Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 19:01:39 +0300 Subject: [PATCH 25/88] PMM-7 add random service names to mongo services --- pmm_psmdb-pbm_setup/configure-agents.sh | 2 +- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 2 +- pmm_psmdb-pbm_setup/start-sharded.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index eabb34b..9b2d17f 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,7 +29,7 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 096f982..82c68f4 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -29,7 +29,7 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index de602fa..717d84b 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -235,7 +235,7 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup --node-name=${node}_${random_number} + docker-compose -f docker-compose-sharded.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" From afaa23288941e130c268a2103dc3323550ad78dd Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 19:14:44 +0300 Subject: [PATCH 26/88] PMM-7 add random service names to mongo services --- pmm_psmdb-pbm_setup/configure-agents.sh | 3 ++- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 3 ++- pmm_psmdb-pbm_setup/start-sharded.sh | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 9b2d17f..75ce273 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,7 +29,8 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup + export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 82c68f4..7cfff57 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -29,7 +29,8 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup + export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 717d84b..a66ca48 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -230,12 +230,14 @@ do done echo echo "configuring pmm agents" +random_number=$RANDOM nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pmm agent on $node" + export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T $node PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} pmm-agent setup + docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" From 2b91a36f8032552786c7e751747c8c426d56c80e Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 21:10:41 +0300 Subject: [PATCH 27/88] PMM-7 add random service names to mongo services --- pmm_psmdb-pbm_setup/configure-agents.sh | 2 +- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 3 ++- pmm_psmdb-pbm_setup/start-sharded.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 75ce273..56ecd07 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,7 +29,7 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 7cfff57..893432e 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -30,7 +30,8 @@ for node in $nodes do echo "configuring pmm agent on $node" export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} + docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index a66ca48..d3e3da1 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -235,8 +235,8 @@ nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pmm agent on $node" - export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} rs=$(echo $node | awk -F "0" '{print $1}') + docker-compose -f docker-compose-sharded.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done From 1c5125d0b71cb408dc4fdc1725642bf1ca31da3d Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Thu, 11 Apr 2024 21:26:53 +0300 Subject: [PATCH 28/88] PMM-7 add random service names to mongo services --- pmm_psmdb-pbm_setup/configure-agents.sh | 3 +-- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 4 +--- pmm_psmdb-pbm_setup/start-sharded.sh | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 56ecd07..dd4514c 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,8 +29,7 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 893432e..bb01b00 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -29,9 +29,7 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} - docker-compose -f docker-compose-rs.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index d3e3da1..91673d3 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -236,8 +236,7 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T $node export PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-agent setup + docker-compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" From 33a1d8ba066825599c6ae8105cc7b33927b6ebe4 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 12 Apr 2024 18:17:02 +0530 Subject: [PATCH 29/88] Updates on PXC/ProxySQL and GR for PS. (#44) * Updates on PXC/ProxySQL and GR for PS. * Updates and PSMDB Cleanup * Update add debug info for shell script execution --- pmm_qa/client_container_ps_setup.sh | 107 +++++++++++++--- pmm_qa/pmm-framework.py | 184 ++++++++++++++-------------- pmm_qa/ps_pmm_setup.yml | 3 +- 3 files changed, 186 insertions(+), 108 deletions(-) diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index e719b72..2c74470 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -53,25 +53,96 @@ dbdeployer deploy single ${db_version_sandbox} --port=${PS_PORT} --sandbox-binar export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) -if [ "$query_source" == "slowlog" ]; then - ### enable Slow log - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL slow_query_log='ON';" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL long_query_time=0;" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_rate_limit=1;" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_admin_statements=ON;" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL log_slow_slave_statements=ON;" - pmm-admin add mysql --username=${PS_USER} --password=${PS_PASSWORD} --port=3307 --query-source=slowlog mysql_client_slowlog_${SERVICE_RANDOM_NUMBER} + +if [[ $number_of_nodes == 1 ]];then + if [[ ! -z $group_replication ]]; then + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + else + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_rate_limit=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + else + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + if echo "$ps_version" | grep '5.7'; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL query_response_time_stats=ON;" + fi + fi + fi + if [[ ! -z $group_replication ]]; then + for j in `seq 1 3`;do + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_rate_limit=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + else + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + if echo "$ps_version" | grep '5.7'; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL query_response_time_stats=ON;" + fi + fi + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-group-replication-node + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ms-prod --cluster=ps-prod-cluster --replication-set=ps-repl ps-group-replication-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + node_port=$(($node_port + 1)) + sleep 20 + done + else + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-single + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ps-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port + fi else - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL innodb_monitor_enable=all;" - if echo "$ps_version" | grep '5.7'; then - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "SET GLOBAL query_response_time_stats=ON;" - fi - mysql -h 127.0.0.1 -u ${PS_USER} -p${PS_PASSWORD} --port $PS_PORT -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" - pmm-admin add mysql --username=${PS_USER} --password=${PS_PASSWORD} --port=3307 --query-source=perfschema mysql_client_perfschema_${SERVICE_RANDOM_NUMBER} + dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 + export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') + node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + for j in `seq 1 $number_of_nodes`; do + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + if [[ "${query_source}" == "slowlog" ]]; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_rate_limit=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + else + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + if echo "$ps_version" | grep '5.7'; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL query_response_time_stats=ON;" + fi + fi + if [ $(( ${j} % 2 )) -eq 0 ]; then + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-prod --cluster=ps-prod-cluster --replication-set=ps-repl2 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + else + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + fi + #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-multiple-node + node_port=$(($node_port + 1)) + sleep 20 + done fi ## Start Running Load diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 1f8f28b..cf97d40 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -18,7 +18,7 @@ }, "PS": { "versions": ["5.7", "8.0"], - "configurations": {"QUERY_SOURCE": "perfschema", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, "PGSQL": { "versions": ["11", "12", "13", "14", "15", "16"], @@ -28,15 +28,18 @@ "versions": ["11", "12", "13", "14", "15", "16"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, + "PXC": { + "versions": ["7", "8"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""} + }, + "PROXYSQL": { + "versions": ["2"], + "configurations": {"PACKAGE": ""} + }, } + def run_ansible_playbook(playbook_filename, env_vars, args): - # Install Ansible - try: - subprocess.run(['sudo', 'yum', 'install', 'ansible', '-y']) - except Exception as e: - print(f"Error installing Ansible: {e}") - exit(1) # Get Script Dir script_path = os.path.abspath(sys.argv[0]) @@ -109,12 +112,18 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): print(f"Check if PMM Server is Up and Running..Exiting") exit() + # Check Setup Types + setup_type = '' + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("group_repilication" or "gr"): + setup_type = '1' + # Gather Version details ps_version = os.getenv('PS_VERSION') or db_version or database_configs[db_type]["versions"][-1] # Define environment variables for playbook env_vars = { - 'PS_NODES': '1', + 'GROUP_REPLICATION': f'{setup_type}', + 'PS_NODES': '1' if isinstance(setup_type, str) and len(setup_type) == 0 else '3', 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'PS_CONTAINER': 'ps_pmm_' + str(ps_version), @@ -122,7 +131,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'PS_TARBALL': get_value('TARBALL', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -144,13 +153,13 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check Setup Types setup_type = '' - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": - setup_type = 1 + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("group_repilication" or "gr"): + setup_type = '1' # Define environment variables for playbook env_vars = { 'GROUP_REPLICATION': f'{setup_type}', - 'MS_NODES': '3' if setup_type else '1', + 'MS_NODES': '1' if isinstance(setup_type, str) and len(setup_type) == 0 else '3', 'MS_VERSION': ms_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), @@ -158,7 +167,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -187,7 +196,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -215,7 +224,7 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -225,43 +234,6 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) -def execute_docker_compose(compose_filename, commands, env_vars, args): - # Setup n/w used by compose setup - subprocess.run(['docker', 'network', 'create', 'qa-integration']) - subprocess.run(['docker', 'network', 'create', 'pmm-ui-tests_pmm-network']) - subprocess.run(['docker', 'network', 'create', 'pmm2-upgrade-tests_pmm-network']) - subprocess.run(['docker', 'network', 'create', 'pmm2-ui-tests_pmm-network']) - - # Get Script Dir - script_path = os.path.abspath(sys.argv[0]) - script_dir = os.path.dirname(script_path) - compose_path = script_dir + "/../pmm_psmdb-pbm_setup/" + compose_filename - - # Set environment variables if provided - if env_vars: - for key, value in env_vars.items(): - os.environ[key] = value - - if args.verbose: - print(f'Options set after considering defaults: {env_vars}') - - for command, options in commands.items(): - # Build the Docker Compose command - docker_compose_cmd = ['docker-compose', '-f', compose_path, command] - - # Add options if provided - if options: - docker_compose_cmd.extend(options) - - # Execute Docker Compose - try: - subprocess.run(docker_compose_cmd, check=True) - print(f"Docker Compose {command} executed successfully!") - except subprocess.CalledProcessError as e: - print(f"Error executing Docker Compose {command}: {e}") - exit(1) - - def execute_shell_scripts(shell_scripts, env_vars, args): # Get script directory script_path = os.path.abspath(sys.argv[0]) @@ -284,11 +256,27 @@ def execute_shell_scripts(shell_scripts, env_vars, args): try: # Change directory to where the script is located os.chdir(shell_scripts_path) - subprocess.run(['bash', script], check=True) - print(f"Shell script '{script}' executed successfully!") - except subprocess.CalledProcessError as e: - print(f"Error executing shell script '{script} at path {shell_scripts_path}': {e}") - exit(1) + process = subprocess.Popen(['bash', script], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Read output streams asynchronously + while process.poll() is None: # Check if the subprocess is still running + # Read from stdout + for stdout_line in process.stdout: + if stdout_line: + print(stdout_line.decode('utf-8').strip()) + + # Read from stderr + for stderr_line in process.stderr: + if stderr_line: + print(stderr_line.decode('utf-8').strip()) + + # Get the return code of the process + return_code = process.returncode + if return_code == 0: + print(f"Shell script '{script}' executed successfully.") + else: + print(f"Shell script '{script}' failed with return code: {return_code}!") + except Exception as e: + print("Unexpected error occurred:", e) finally: # Return to the original working directory os.chdir(original_dir) @@ -308,6 +296,9 @@ def mongo_sharding_setup(script_filename, args): compose_filename = f'docker-compose-sharded-no-server.yaml' compose_file_path = scripts_path + compose_filename + # Create pmm-qa n/w used in workaround + subprocess.run(['docker', 'network', 'create', 'pmm-qa']) + no_server = True # Add workaround (copy files) till sharding only support is ready. try: @@ -345,10 +336,18 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Gather Version details psmdb_version = os.getenv('PSMDB_VERSION') or db_version or database_configs[db_type]["versions"][-1] + # Handle port address for external or internal address + server_hostname = container_name + port = 8443 + if args.pmm_server_ip: + port = 443 + server_hostname = args.pmm_server_ip + server_address = f'{server_hostname}:{port}' + # Define environment variables for playbook env_vars = { 'PSMDB_VERSION': psmdb_version, - 'PMM_SERVER_CONTAINER_ADDRESS': f'{args.pmm_server_ip}:443' or f'{container_name}:8443' or '127.0.0.1:443', + 'PMM_SERVER_CONTAINER_ADDRESS': server_address, 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), @@ -358,49 +357,54 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): 'CLEANUP': 'no' } - compose_filename = '' shell_scripts = [] - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss": - # Docker Compose filename - compose_filename = 'docker-compose-rs.yaml' + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("pss" or "psa"): # Shell script names - shell_scripts = ['configure-replset.sh', 'configure-agents.sh'] - - # If profile is extra, include additional shell scripts - if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - shell_scripts.append('configure-extra-replset.sh') - shell_scripts.append('configure-extra-agents.sh') - elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "psa": - # Docker Compose filename - compose_filename = 'docker-compose-rs.yaml' - # Shell script names - shell_scripts = ['configure-psa.sh', 'configure-agents.sh'] - - # If profile is extra, include additional shell scripts - if get_value('COMPOSE_PROFILES', db_type, args, db_config).lower() == "extra": - shell_scripts.append('configure-extra-psa.sh') - shell_scripts.append('configure-extra-agents.sh') + shell_scripts = ['start-rs-only.sh'] elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": # Shell script names shell_scripts = [f'start-sharded-no-server.sh'] mongo_sharding_setup(shell_scripts[0], args) - # Define commands for compose file setup - commands = { - 'down': ['-v', '--remove-orphans'], # Cleanup containers - 'build': ['--no-cache'], # Build containers - 'up': ['-d'], # Start containers - } - # Call the function to run the Compose files - if not compose_filename == '': - execute_docker_compose(compose_filename, commands, env_vars, args) - # Execute shell scripts if not shell_scripts == []: execute_shell_scripts(shell_scripts, env_vars, args) -# Function to set up a databases based on choice +def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + pxc_version = os.getenv('PXC_VERSION') or db_version or database_configs[db_type]["versions"][-1] + proxysql_version = os.getenv('PROXYSQL_VERSION') or db_version or database_configs["PROXYSQL"]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PXC_NODES': '3', + 'PXC_VERSION': pxc_version, + 'PROXYSQL_VERSION': proxysql_version, + 'PXC_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'PROXYSQL_PACKAGE': get_value('PACKAGE', 'PROXYSQL', args, db_config), + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PXC_CONTAINER': 'pxc_proxysql_pmm_' + str(pxc_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'pxc_proxysql_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +# Set up databases based on arguments received def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: if db_version: @@ -423,6 +427,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_pdpgsql(db_type, db_version, db_config, args) elif db_type == 'PSMDB': setup_psmdb(db_type, db_version, db_config, args) + elif db_type == 'PXC': + setup_pxc_proxysql(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index 419429b..01d322e 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -15,6 +15,7 @@ admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_NODES') | default('3', true) ) }}" + group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION')) }}" tasks: - name: cleanup container for client and DB setup @@ -66,5 +67,5 @@ - name: Execute Setup script inside the ps ps_container shell: "{{ item }}" with_items: - - docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} > setup_ps_{{ ps_version }}.log + - docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ps_{{ ps_version }}.log From 5dfc47d2480c65f1e77ba03f6e20f6802234aa5d Mon Sep 17 00:00:00 2001 From: yurkovychv Date: Mon, 15 Apr 2024 16:05:19 +0300 Subject: [PATCH 30/88] PMM-7 update init-ubuntu --- pmm_qa/Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pmm_qa/Makefile b/pmm_qa/Makefile index b3e6c97..888b96b 100644 --- a/pmm_qa/Makefile +++ b/pmm_qa/Makefile @@ -11,9 +11,11 @@ init: ## Create venv and install dependencies python3 -m pip install -r requirements.txt init-ubuntu: ## Create venv and install dependencies for ubuntu - sudo apt update - sudo apt install -y python3.10-venv - sudo python3 -m venv pmm_framework - . pmm_framework/bin/activate - sudo python3 -m pip install -r requirements.txt + sudo chown $(whoami) -R . + apt install -y python3.10-venv + python3 -m venv virtenv + . virtenv/bin/activate + python --version + which python + python -m pip install -r requirements.txt From 2a1e7acaa1c2923fc61bfe87e88b5568b89d0836 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:30:06 +0530 Subject: [PATCH 31/88] Updates for HAProxy and External exporters (#45) * Updates for HAPROXY, Redis and NodeExporter Setup --- pmm_qa/external_setup.sh | 49 ++++++++++++++++++ pmm_qa/external_setup.yml | 101 ++++++++++++++++++++++++++++++++++++++ pmm_qa/haproxy.cfg | 18 +++++++ pmm_qa/haproxy_setup.yml | 12 +++-- pmm_qa/pmm-framework.py | 97 +++++++++++++++++++++++++++++++----- 5 files changed, 260 insertions(+), 17 deletions(-) create mode 100644 pmm_qa/external_setup.sh create mode 100644 pmm_qa/external_setup.yml create mode 100644 pmm_qa/haproxy.cfg diff --git a/pmm_qa/external_setup.sh b/pmm_qa/external_setup.sh new file mode 100644 index 0000000..b2199d5 --- /dev/null +++ b/pmm_qa/external_setup.sh @@ -0,0 +1,49 @@ +#!/bin/sh + + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$metrics_mode" ] +then + export metrics_mode=push +fi + +if [ -z "$setup_version" ] +then + export setup_version="1.14.0" +fi + +if [ -z "$setup_type" ] +then + export setup_type=redis +fi + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl git gnupg2 lsb-release +apt-get -y install -y git ca-certificates gcc libc6-dev liblua5.3-dev libpcre3-dev libssl-dev libsystemd-dev make wget zlib1g-dev + +if [[ "$setup_type" == "redis" ]]; then + wget https://github.com/oliver006/redis_exporter/releases/download/v${setup_version}/redis_exporter-v${setup_version}.linux-386.tar.gz + tar -xvf redis_exporter-v${setup_version}.linux-386.tar.gz + sleep 10 + rm redis_exporter*.tar.gz + mv redis_exporter-* redis_exporter || exit +elif [[ "$setup_type" == "nodeprocess" ]]; then + wget https://github.com/ncabatoff/process-exporter/releases/download/v${setup_version}/process-exporter-${setup_version}.linux-amd64.tar.gz + tar -xvf process-exporter-${setup_version}.linux-amd64.tar.gz || exit + sleep 10 + rm process-exporter*.tar.gz + mv process-exporter-* process-exporter || exit +fi + + diff --git a/pmm_qa/external_setup.yml b/pmm_qa/external_setup.yml new file mode 100644 index 0000000..fa1c607 --- /dev/null +++ b/pmm_qa/external_setup.yml @@ -0,0 +1,101 @@ +--- +# This playbook does following: +# enables Percona testing repository + +- hosts: all + vars: + external_container: "{{ lookup('vars', 'extra_haproxy_container', default=lookup('env','EXTERNAL_CONTAINER') | default('EXTERNAL', true) ) }}" + redis_version: "{{ lookup('vars', 'extra_redis_version', default=lookup('env','REDIS_EXPORTER_VERSION') | default('0.7.5', true) ) }}" + nodeprocess_version: "{{ lookup('vars', 'extra_nodeprocess_version', default=lookup('env','NODE_PROCESS_EXPORTER_VERSION') | default('0.7.5', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ external_container }}" | grep -q . && docker stop {{ external_container }} && docker rm -fv {{ external_container }} + docker stop redis_container + docker rm -fv redis_container + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for External Exporters + shell: > + sudo docker run -d -p 42100:42100 --name={{ external_container }} + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker external_container + shell: "{{ item }}" + with_items: + - docker cp ./external_setup.sh {{ external_container }}:/ + - docker exec {{ external_container }} apt-get update + - docker exec {{ external_container }} apt-get -y install wget curl git gnupg2 lsb-release + + - name: Run Redis Container in Host + shell: "{{ item }}" + with_items: + - docker run --name redis_container -d -p 6379:6379 redis '--requirepass oFukiBRg7GujAJXq3tmd' + - docker network connect pmm-qa redis_container + + - name: Setup External Exporters for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -x ./external_setup.sh > setup_external.log + + - name: Install pmm2-client on the external_container + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh + - docker network connect pmm-qa {{ external_container }} + - docker exec {{ external_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Execute Setup script inside the External container for Redis + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -xe ./external_setup.sh --setup_type redis --setup_version {{ redis_version }} > setup_external_redis.log + delay: 10 + + - name: Start Redis for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} nohup bash -c './redis_exporter/redis_exporter -redis.password=oFukiBRg7GujAJXq3tmd -redis.addr=redis_container:6379 -web.listen-address=:42200 > redis.log 2>&1 &' + + - name: Execute Setup script inside the External container for Process Exporter + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -xe ./external_setup.sh --setup_type nodeprocess --setup_version {{ nodeprocess_version }} > setup_external_nodeprocess.log + + - name: Start Node Process Exporter for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} nohup bash -c './process-exporter/process-exporter > process-exporter.log 2>&1 &' + + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + + - name: Remove EXTERNAL process for monitoring from PMM if already exist with same service name + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove external redis_service_{{ random_number }}' + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove external nodeprocess_service_{{ random_number }}' + ignore_errors: true + + - name: Add Redis for Monitoring to PMM via pmm-admin add external command + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add external --listen-port=42200 --group="redis" --service-name=redis_external_service_{{ random_number }}' + + - name: Add Node Process for Monitoring to PMM via pmm-admin add external command + shell: "{{ item }}" + with_items: + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ external_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add external --group=processes --listen-port=9256 --service-name=nodeprocess_service_{{ random_number }}' diff --git a/pmm_qa/haproxy.cfg b/pmm_qa/haproxy.cfg new file mode 100644 index 0000000..956b06c --- /dev/null +++ b/pmm_qa/haproxy.cfg @@ -0,0 +1,18 @@ +frontend stats + bind *:42100 + stats enable + stats uri /stats + stats refresh 10s + mode http + timeout client 60s + http-request use-service prometheus-exporter if { path /metrics } + default_backend allservers + +backend allservers + timeout connect 10s + timeout server 100s + mode http + server server2222 127.0.0.1:2222 + server server3333 127.0.0.1:3333 + server server4444 127.0.0.1:4444 + server server5555 127.0.0.1:5555 diff --git a/pmm_qa/haproxy_setup.yml b/pmm_qa/haproxy_setup.yml index d106838..38d65b6 100644 --- a/pmm_qa/haproxy_setup.yml +++ b/pmm_qa/haproxy_setup.yml @@ -38,7 +38,7 @@ - name: Setup haproxy for monitoring shell: "{{ item }}" with_items: - - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh > ~/setup_haproxy.log + - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh > setup_haproxy.log - name: Install pmm2-client on the haproxy_container shell: "{{ item }}" @@ -57,17 +57,21 @@ with_items: - docker exec {{ haproxy_container }} haproxy -f haproxy.cfg -D + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + - name: Remove HAPROXY for monitoring from PMM if already exist with same service name shell: "{{ item }}" with_items: - - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove haproxy {{ haproxy_container }}_service' + - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove haproxy {{ haproxy_container }}_service_{{ random_number }}' ignore_errors: true - name: Add HAPROXY for Monitoring to PMM via pmm-admin add haproxy command shell: "{{ item }}" with_items: - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' - - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add haproxy --listen-port=42100 --environment=haproxy {{ haproxy_container }}_service' + - docker exec {{ haproxy_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add haproxy --listen-port=42100 --environment=haproxy {{ haproxy_container }}_service_{{ random_number }}' - - name: Run ping to FE of HAPROXY to monitore the load balancing + - name: Run ping to FE of HAPROXY to monitor the load balancing shell: while true; do curl http://localhost:42100/; sleep 10; done >/dev/null 2>&1 & diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index e1fa644..c82b166 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -18,7 +18,8 @@ }, "PS": { "versions": ["5.7", "8.0"], - "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": ""} }, "PGSQL": { "versions": ["11", "12", "13", "14", "15", "16"], @@ -36,6 +37,19 @@ "versions": ["2"], "configurations": {"PACKAGE": ""} }, + "HAPROXY": { + "versions": [""], + "configurations": {"CLIENT_VERSION": "3-dev-latest"} + }, + "EXTERNAL": { + "REDIS": { + "versions": ["1.14.0", "1.58.0"], + }, + "NODEPROCESS": { + "versions": ["0.7.5", "0.7.10"], + }, + "configurations": {"CLIENT_VERSION": "3-dev-latest"} + }, } @@ -117,7 +131,7 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): # Check Setup Types setup_type = '' - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("group_repilication" or "gr"): + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": setup_type = '1' # Gather Version details @@ -156,7 +170,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Check Setup Types setup_type = '' - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("group_repilication" or "gr"): + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": setup_type = '1' # Define environment variables for playbook @@ -237,6 +251,58 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) +def setup_haproxy(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Define environment variables for playbook + env_vars = { + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'HAPROXY_CONTAINER': 'haproxy_pmm', + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'haproxy_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + +def setup_external(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + redis_version = os.getenv('REDIS_VERSION') or db_version or database_configs["EXTERNAL"]["REDIS"]["versions"][-1] + nodeprocess_version = os.getenv('NODE_PROCESS_VERSION') or db_version or database_configs["EXTERNAL"]["NODEPROCESS"]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'REDIS_EXPORTER_VERSION': redis_version, + 'NODE_PROCESS_EXPORTER_VERSION': nodeprocess_version, + 'EXTERNAL_CONTAINER': 'external_pmm', + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'external_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + def execute_shell_scripts(shell_scripts, env_vars, args): # Get script directory script_path = os.path.abspath(sys.argv[0]) @@ -363,8 +429,8 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): } shell_scripts = [] - - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == ("pss" or "psa"): + if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss" or "psa": + # Shell script names shell_scripts = ['start-rs-only.sh'] elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": shell_scripts = [f'start-sharded-no-server.sh'] @@ -433,6 +499,10 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_psmdb(db_type, db_version, db_config, args) elif db_type == 'PXC': setup_pxc_proxysql(db_type, db_version, db_config, args) + elif db_type == 'HAPROXY': + setup_haproxy(db_type, db_version, db_config, args) + elif db_type == 'EXTERNAL': + setup_external(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) @@ -476,20 +546,21 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): try: if key in database_configs: db_type = key - if value in database_configs[db_type]["versions"]: - db_version = value - else: - if args.verbose: - print( - f"Value {value} is not recognised for Option {key}, will be using default value") + if "versions" in database_configs[db_type]: + if value in database_configs[db_type]["versions"]: + db_version = value + else: + if args.verbose: + print( + f"Value {value} is not recognised for Option {key}, will be using default value") elif key in database_configs[db_type]["configurations"]: db_config[key] = value else: if args.verbose: print(f"Option {key} is not recognised, will be using default option") continue - except KeyError: - print(f"Option {key} is not recognised, Please check and try again") + except KeyError as e: + print(f"Option {key} is not recognised with error {e}, Please check and try again") parser.print_help() exit(1) # Set up the specified databases From 5b9636819f3c9bbd3511295fbaa580a6cea783f9 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 19 Apr 2024 14:13:07 +0530 Subject: [PATCH 32/88] Updates fixe for PS, MS for Replication and GR (#47) --- pmm_qa/client_container_ms_setup.sh | 2 +- pmm_qa/client_container_ps_setup.sh | 13 +++++++---- pmm_qa/ms_pmm_setup.yml | 2 +- pmm_qa/pmm-framework.py | 36 ++++++++++++++++++----------- pmm_qa/ps_pmm_setup.yml | 2 +- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index b11fd42..7d37efb 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -48,7 +48,7 @@ dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ms${ms_version} --overwrit export db_version_sandbox=$(ls ~/ms${ms_version}) export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) -if [[ $number_of_nodes == 1 ]];then +if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index 2c74470..b43550a 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -54,7 +54,7 @@ dbdeployer deploy single ${db_version_sandbox} --port=${PS_PORT} --sandbox-binar export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) -if [[ $number_of_nodes == 1 ]];then +if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') @@ -72,6 +72,7 @@ if [[ $number_of_nodes == 1 ]];then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" else + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if echo "$ps_version" | grep '5.7'; then @@ -92,8 +93,9 @@ if [[ $number_of_nodes == 1 ]];then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" else - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if echo "$ps_version" | grep '5.7'; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" @@ -124,8 +126,9 @@ else mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" else - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if echo "$ps_version" | grep '5.7'; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" diff --git a/pmm_qa/ms_pmm_setup.yml b/pmm_qa/ms_pmm_setup.yml index 158a5a8..088e49b 100644 --- a/pmm_qa/ms_pmm_setup.yml +++ b/pmm_qa/ms_pmm_setup.yml @@ -15,7 +15,7 @@ admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','MS_NODES') | default('1', true) ) }}" - group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION')) }}" + group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION') | default('', true) ) }}" tasks: - name: cleanup container for client and DB setup diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c82b166..44ae16d 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -53,7 +53,6 @@ } - def run_ansible_playbook(playbook_filename, env_vars, args): # Get Script Dir script_path = os.path.abspath(sys.argv[0]) @@ -130,17 +129,21 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): exit() # Check Setup Types - setup_type = '' - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": - setup_type = '1' + setup_type = None + no_of_nodes = 1 + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() + if setup_type_value in ("group_replication", "gr"): + setup_type = 1 + elif setup_type_value in ("replication", "replica"): + setup_type = None + no_of_nodes = 2 # Gather Version details ps_version = os.getenv('PS_VERSION') or db_version or database_configs[db_type]["versions"][-1] - # Define environment variables for playbook env_vars = { - 'GROUP_REPLICATION': f'{setup_type}', - 'PS_NODES': '1' if isinstance(setup_type, str) and len(setup_type) == 0 else '3', + 'GROUP_REPLICATION': setup_type, + 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'PS_CONTAINER': 'ps_pmm_' + str(ps_version), @@ -169,14 +172,20 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] # Check Setup Types - setup_type = '' - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "group_repilication" or "gr": - setup_type = '1' + # Check Setup Types + setup_type = None + no_of_nodes = 1 + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() + if setup_type_value in ("group_replication", "gr"): + setup_type = 1 + elif setup_type_value in ("replication", "replica"): + setup_type = None + no_of_nodes = 2 # Define environment variables for playbook env_vars = { - 'GROUP_REPLICATION': f'{setup_type}', - 'MS_NODES': '1' if isinstance(setup_type, str) and len(setup_type) == 0 else '3', + 'GROUP_REPLICATION': setup_type, + 'MS_NODES': no_of_nodes, 'MS_VERSION': ms_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), @@ -283,7 +292,8 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): # Gather Version details redis_version = os.getenv('REDIS_VERSION') or db_version or database_configs["EXTERNAL"]["REDIS"]["versions"][-1] - nodeprocess_version = os.getenv('NODE_PROCESS_VERSION') or db_version or database_configs["EXTERNAL"]["NODEPROCESS"]["versions"][-1] + nodeprocess_version = os.getenv('NODE_PROCESS_VERSION') or db_version or \ + database_configs["EXTERNAL"]["NODEPROCESS"]["versions"][-1] # Define environment variables for playbook env_vars = { diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index 01d322e..97e6311 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -15,7 +15,7 @@ admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_NODES') | default('3', true) ) }}" - group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION')) }}" + group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION') | default('', true) ) }}" tasks: - name: cleanup container for client and DB setup From 17eac4c0998c9c578ad8c6d334e5c581ce6a22d5 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:45:06 +0530 Subject: [PATCH 33/88] Updates fix for PS, MS single instance setup (#49) * Updates fix for PS, MS single instance setup --- pmm_qa/pmm-framework.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 44ae16d..49a5d81 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -129,13 +129,14 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): exit() # Check Setup Types - setup_type = None + setup_type = '' no_of_nodes = 1 setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() if setup_type_value in ("group_replication", "gr"): setup_type = 1 + no_of_nodes = 1 elif setup_type_value in ("replication", "replica"): - setup_type = None + setup_type = '' no_of_nodes = 2 # Gather Version details @@ -172,14 +173,14 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] # Check Setup Types - # Check Setup Types - setup_type = None + setup_type = '' no_of_nodes = 1 setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() if setup_type_value in ("group_replication", "gr"): setup_type = 1 + no_of_nodes = 1 elif setup_type_value in ("replication", "replica"): - setup_type = None + setup_type = '' no_of_nodes = 2 # Define environment variables for playbook @@ -354,6 +355,7 @@ def execute_shell_scripts(shell_scripts, env_vars, args): print(f"Shell script '{script}' executed successfully.") else: print(f"Shell script '{script}' failed with return code: {return_code}!") + exit(return_code) except Exception as e: print("Unexpected error occurred:", e) finally: From 7b50f6483b9547e1b5ddfb400365f50567e01d1e Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:35:48 +0300 Subject: [PATCH 34/88] PMM-13027 dot for PMM Agent node name (#46) * PMM-13027 dot for PMM Agent node name * PMM-13027 fix external and haproxy * PMM-13027 external setup ports --- pmm_psmdb-pbm_setup/configure-agents.sh | 2 +- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 2 +- pmm_qa/external_setup.yml | 2 +- pmm_qa/pmm-framework.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index dd4514c..97e2a56 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -29,7 +29,7 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index bb01b00..1839ceb 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -29,7 +29,7 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else diff --git a/pmm_qa/external_setup.yml b/pmm_qa/external_setup.yml index fa1c607..4084d44 100644 --- a/pmm_qa/external_setup.yml +++ b/pmm_qa/external_setup.yml @@ -28,7 +28,7 @@ - name: Prepare Container for External Exporters shell: > - sudo docker run -d -p 42100:42100 --name={{ external_container }} + sudo docker run -d -p 43100:43100 --name={{ external_container }} phusion/baseimage:focal-1.1.0 - name: Copy all required Artifacts to the docker external_container diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 49a5d81..058fc48 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -274,7 +274,7 @@ def setup_haproxy(db_type, db_version=None, db_config=None, args=None): 'HAPROXY_CONTAINER': 'haproxy_pmm', 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename @@ -304,7 +304,7 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): 'EXTERNAL_CONTAINER': 'external_pmm', 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('ADMIN_PASSWORD') or 'v3' + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename From 6af896d0579774931e2140f3862ff901c1688af5 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Fri, 3 May 2024 11:17:23 +0300 Subject: [PATCH 35/88] PMM-13086 setup improvements and fixes for BM (#53) --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 12 ++++++------ pmm_qa/pdpgsql_pgsm_setup.yml | 2 +- pmm_qa/pgsql_pgss_setup.yml | 2 +- pmm_qa/pmm-framework.py | 2 +- pmm_qa/ps_pmm_setup.yml | 2 +- pmm_qa/pxc_proxysql_setup.yml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index e188aa2..56790fb 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -32,7 +32,7 @@ services: - ./conf/mongod-rs:/etc/mongod - ./conf/datagen:/etc/datagen:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 @@ -58,7 +58,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 @@ -84,7 +84,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 @@ -112,7 +112,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 @@ -138,7 +138,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 @@ -164,7 +164,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw - - /tmp/backup_data:/tmp/backup_data + - /tmp/backup_data:/tmp/backup_data:rw privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 9170d1d..73c4e4e 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -40,6 +40,7 @@ - docker exec {{ pdpgsql_pgsm_container }} mkdir -p artifacts - docker cp ./pg_stat_monitor_setup.sh {{ pdpgsql_pgsm_container }}:/ - docker cp ./pgsm_run_queries.sh {{ pdpgsql_pgsm_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ pdpgsql_pgsm_container }}:/ - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container shell: "{{ item }}" @@ -49,7 +50,6 @@ - name: Install pmm2-client on the pdpgsql_pgsm_container shell: "{{ item }}" with_items: - - docker exec {{ pdpgsql_pgsm_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ pdpgsql_pgsm_container }} - docker exec {{ pdpgsql_pgsm_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/pgsql_pgss_setup.yml b/pmm_qa/pgsql_pgss_setup.yml index d1c3f0e..bd95163 100644 --- a/pmm_qa/pgsql_pgss_setup.yml +++ b/pmm_qa/pgsql_pgss_setup.yml @@ -38,6 +38,7 @@ - docker exec {{ pgsql_pgss_container }} mkdir -p artifacts - docker cp ./pg_stat_statements_setup.sh {{ pgsql_pgss_container }}:/ - docker cp ./pgsm_run_queries.sh {{ pgsql_pgss_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ pgsql_pgss_container }}:/ - name: Execute Setup script inside the pgsql pgsql_pgss_container shell: "{{ item }}" @@ -47,7 +48,6 @@ - name: Install pmm2-client on the pgsql_pgss_container shell: "{{ item }}" with_items: - - docker exec {{ pgsql_pgss_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ pgsql_pgss_container }} - docker exec {{ pgsql_pgss_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 058fc48..b690298 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -354,7 +354,7 @@ def execute_shell_scripts(shell_scripts, env_vars, args): if return_code == 0: print(f"Shell script '{script}' executed successfully.") else: - print(f"Shell script '{script}' failed with return code: {return_code}!") + print(f"Shell script '{script}' failed with return code: {return_code}! \n {process.stderr}") exit(return_code) except Exception as e: print("Unexpected error occurred:", e) diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index 97e6311..a8988ae 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -39,6 +39,7 @@ with_items: - docker exec {{ ps_container }} mkdir -p artifacts - docker cp ./client_container_ps_setup.sh {{ ps_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ ps_container }}:/ - name: Get Product tarball URL based on the version shell: cat product_version_download_helper | grep -v pxc | grep ps | grep "\-{{ ps_version }}" | head -1 | awk -F'# ' '{print $3}' @@ -60,7 +61,6 @@ - name: Install pmm2-client on the ps_container shell: "{{ item }}" with_items: - - docker exec {{ ps_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ ps_container }} - docker exec {{ ps_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index 9fa5a12..d9626c6 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -41,6 +41,7 @@ - docker exec {{ pxc_container }} mkdir -p artifacts - docker cp ./client_container_pxc_setup.sh {{ pxc_container }}:/ - docker cp ./client_container_proxysql_setup.sh {{ pxc_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ pxc_container }}:/ - name: Get PXC Product tarball URL based on the version shell: cat product_version_download_helper | grep pxc | grep "\-{{ pxc_version }}" | head -1 | awk -F'# ' '{print $3}' @@ -78,7 +79,6 @@ - name: Install pmm2-client on the pxc_container shell: "{{ item }}" with_items: - - docker exec {{ pxc_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ pxc_container }} - docker exec {{ pxc_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no - docker exec {{ pxc_container }} usermod -aG pmm-agent pxc From 8eb1f1e4ba5e6829f04ef65f16e3078a0beffcfd Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 3 May 2024 23:00:35 +0530 Subject: [PATCH 36/88] Fixes for CLI Tests 1) exposing ports for databases 2) added dockerclients setup for dockers only. 3) updates (#52) Fixes for CLI Tests 1) exposing ports for databases 2) added dockerclients setup for dockers only. 3) updates (handling PSMDB Versions, improvements) --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 1 + .../docker-compose-sharded.yaml | 3 + pmm_qa/client_container_ms_setup.sh | 3 +- pmm_qa/client_container_ps_setup.sh | 3 +- pmm_qa/docker-compose-clients.yaml | 64 ++++++++++++++ pmm_qa/ms_pmm_setup.yml | 2 + pmm_qa/pmm-framework.py | 88 +++++++++++++++---- pmm_qa/ps_pmm_setup.yml | 1 + pmm_qa/pxc_proxysql_setup.yml | 9 +- pmm_qa/requirements.txt | 1 + pmm_qa/setup_docker_client_images.sh | 16 ++++ 11 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 pmm_qa/docker-compose-clients.yaml create mode 100755 pmm_qa/setup_docker_client_images.sh diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 56790fb..4b4451d 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -33,6 +33,7 @@ services: - ./conf/datagen:/etc/datagen:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw + - /tmp/mongodb:/tmp privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 418d903..2949bad 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -21,6 +21,7 @@ services: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs1:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /tmp/mongodb:/tmp privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" @@ -30,6 +31,8 @@ services: - "PMM_AGENT_SERVER_INSECURE_TLS=1" container_name: rs101 hostname: rs101 + ports: + - "27027:27017" networks: - test-network diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index 7d37efb..f97d8d2 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -43,6 +43,7 @@ mkdir /tmp || true chmod 1777 /tmp || true ## Deploy DB deployer +export MS_PORT=3308 export tar_ball_name=$(ls mysql-*) dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ms${ms_version} --overwrite export db_version_sandbox=$(ls ~/ms${ms_version}) @@ -55,7 +56,7 @@ if [[ "$number_of_nodes" == 1 ]];then node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --port=$MS_PORT --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index b43550a..c0cd024 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -49,7 +49,6 @@ chmod 1777 /tmp || true export tar_ball_name=$(ls Percona-Server*) dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ps${ps_version} --flavor=percona export db_version_sandbox=$(ls ~/ps${ps_version}) -dbdeployer deploy single ${db_version_sandbox} --port=${PS_PORT} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) @@ -61,7 +60,7 @@ if [[ "$number_of_nodes" == 1 ]];then node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" diff --git a/pmm_qa/docker-compose-clients.yaml b/pmm_qa/docker-compose-clients.yaml new file mode 100644 index 0000000..b783074 --- /dev/null +++ b/pmm_qa/docker-compose-clients.yaml @@ -0,0 +1,64 @@ +version: '3.7' + +services: + pmm-server: + container_name: pmm-server-1 + image: ${DOCKER_VERSION:-perconalab/pmm-server:3-dev-latest} + ports: + - "2553:8443" + - "8085:8080" + environment: + - PMM_DEBUG=1 + networks: + - docker-client-check + + pmm-client: + container_name: pmm-client-1 + image: ${CLIENT_DOCKER_VERSION:-perconalab/pmm-client:3-dev-latest} + environment: + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 + - PMM_AGENT_SERVER_USERNAME=admin + - PMM_AGENT_SERVER_PASSWORD=admin + - PMM_AGENT_SERVER_INSECURE_TLS=1 + - PMM_AGENT_SETUP=1 + - PMM_AGENT_SETUP_FORCE=1 + - PMM_AGENT_CONFIG_FILE=/usr/local/percona/pmm/config/pmm-agent.yaml + depends_on: + pmm-server: + condition: service_healthy + networks: + - docker-client-check + + ps: + container_name: ps-1 + image: percona:8.0 + environment: + - MYSQL_ROOT_PASSWORD=root + - MYSQL_USER=pmm + - MYSQL_PASSWORD=pmm-pass + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 + networks: + - docker-client-check + + mongodb: + container_name: psmdb-1 + image: percona/percona-server-mongodb:7.0 + environment: + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 + - MONGO_INITDB_ROOT_USERNAME=pmm + - MONGO_INITDB_ROOT_PASSWORD=pmm-pass + networks: + - docker-client-check + + postgres: + container_name: pdpgsql-1 + image: percona/percona-distribution-postgresql:16 + environment: + - POSTGRES_USER=pmm + - POSTGRES_PASSWORD=pmm-pass + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 + networks: + - docker-client-check + +networks: + docker-client-check: diff --git a/pmm_qa/ms_pmm_setup.yml b/pmm_qa/ms_pmm_setup.yml index 088e49b..cff597e 100644 --- a/pmm_qa/ms_pmm_setup.yml +++ b/pmm_qa/ms_pmm_setup.yml @@ -32,6 +32,8 @@ - name: Prepare Container for MySQL shell: > docker run -d --name={{ ms_container }} + -p 3318:3308 + -v /tmp:/tmp phusion/baseimage:focal-1.2.0 - name: Copy all required Artifacts to the docker ms_container diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b690298..5c91c69 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -3,6 +3,8 @@ import os import sys import ansible_runner +import requests +import re # Database configurations database_configs = { @@ -30,7 +32,7 @@ "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "PXC": { - "versions": ["7", "8"], + "versions": ["5.7", "8.0"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""} }, "PROXYSQL": { @@ -50,6 +52,8 @@ }, "configurations": {"CLIENT_VERSION": "3-dev-latest"} }, + "DOCKERCLIENTS": { + }, } @@ -86,13 +90,20 @@ def get_running_container_name(): # Check each line for the docker image name for line in containers: # Extract the image name - image_info = line.split('\t')[0] - info_parts = image_info.split()[2:] + info_parts = line.split('\t')[0] + image_info = info_parts.split()[2:] # Check if the container is in the list of running containers # and establish N/W connection with it. - if container_name in info_parts: - subprocess.run(['docker', 'network', 'create', 'pmm-qa']) - subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name]) + if container_name in image_info: + # Check if pmm-qa n/w exists and already connected to pmm-server n/w. + result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True, text=True) + if result.returncode != 0: + subprocess.run(['docker', 'network', 'create', 'pmm-qa']) + subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name]) + else: + networks = result.stdout + if container_name not in networks: + subprocess.run(['docker', 'network', 'connect', 'pmm-qa', container_name]) return container_name except subprocess.CalledProcessError: @@ -314,11 +325,11 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) -def execute_shell_scripts(shell_scripts, env_vars, args): +def execute_shell_scripts(shell_scripts, scripts_path, env_vars, args): # Get script directory script_path = os.path.abspath(sys.argv[0]) script_dir = os.path.dirname(script_path) - shell_scripts_path = script_dir + "/../pmm_psmdb-pbm_setup/" + shell_scripts_path = script_dir + scripts_path # Get the original working directory original_dir = os.getcwd() @@ -378,7 +389,9 @@ def mongo_sharding_setup(script_filename, args): compose_file_path = scripts_path + compose_filename # Create pmm-qa n/w used in workaround - subprocess.run(['docker', 'network', 'create', 'pmm-qa']) + result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) + if not result: + subprocess.run(['docker', 'network', 'create', 'pmm-qa']) no_server = True # Add workaround (copy files) till sharding only support is ready. @@ -407,6 +420,33 @@ def mongo_sharding_setup(script_filename, args): print(f"Error occurred: {e}") +def get_latest_psmdb_version(psmdb_version): + if psmdb_version == "latest": + return psmdb_version + # Define the data to be sent in the POST request + data = { + 'version': f'percona-server-mongodb-{psmdb_version}' + } + + # Make the POST request + response = requests.post('https://www.percona.com/products-api.php', data=data) + + # Extract the version number using regular expression + version_number = re.findall(r'value="([^"]*)"', response.text) + + if version_number: + # Sort the version numbers and extract the latest one + latest_version = sorted(version_number, key=lambda x: tuple(map(int, x.split('-')[-1].split('.'))))[-1] + + # Extract the full version number + major_version = latest_version.split('-')[3].strip() # Trim spaces + minor_version = latest_version.split('-')[4].strip() # Trim spaces + + return f'{major_version}-{minor_version}' + else: + return None + + def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -415,7 +455,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): exit(1) # Gather Version details - psmdb_version = os.getenv('PSMDB_VERSION') or db_version or database_configs[db_type]["versions"][-1] + psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name @@ -441,16 +481,18 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): } shell_scripts = [] - if get_value('SETUP_TYPE', db_type, args, db_config).lower() == "pss" or "psa": - # Shell script names + scripts_path = "/../pmm_psmdb-pbm_setup/" + setup_type = get_value('SETUP_TYPE', db_type, args, db_config).lower() + + if setup_type in ("pss", "psa"): shell_scripts = ['start-rs-only.sh'] - elif get_value('SETUP_TYPE', db_type, args, db_config).lower() == "shards": - shell_scripts = [f'start-sharded-no-server.sh'] + elif setup_type in ("shards", "sharding"): + shell_scripts = ['start-sharded-no-server.sh'] mongo_sharding_setup(shell_scripts[0], args) # Execute shell scripts if not shell_scripts == []: - execute_shell_scripts(shell_scripts, env_vars, args) + execute_shell_scripts(shell_scripts, scripts_path, env_vars, args) def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): @@ -462,7 +504,7 @@ def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): # Gather Version details pxc_version = os.getenv('PXC_VERSION') or db_version or database_configs[db_type]["versions"][-1] - proxysql_version = os.getenv('PROXYSQL_VERSION') or db_version or database_configs["PROXYSQL"]["versions"][-1] + proxysql_version = os.getenv('PROXYSQL_VERSION') or database_configs["PROXYSQL"]["versions"][-1] # Define environment variables for playbook env_vars = { @@ -486,6 +528,18 @@ def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) +def setup_dockerclients(db_type, db_version=None, db_config=None, args=None): + # Define environment variables for shell script + env_vars = {} + + # Shell script filename + shell_scripts = ['setup_docker_client_images.sh'] + shell_scripts_path = '' + + # Call the function to run the Ansible playbook + execute_shell_scripts(shell_scripts, shell_scripts_path, env_vars, args) + + # Set up databases based on arguments received def setup_database(db_type, db_version=None, db_config=None, args=None): if args.verbose: @@ -515,6 +569,8 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_haproxy(db_type, db_version, db_config, args) elif db_type == 'EXTERNAL': setup_external(db_type, db_version, db_config, args) + elif db_type == 'DOCKERCLIENTS': + setup_dockerclients(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index a8988ae..967a189 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -32,6 +32,7 @@ - name: Prepare Container for ps and Proxysql shell: > docker run -d --name={{ ps_container }} + -p 3317:3307 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker ps_container diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index d9626c6..9a416f1 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -5,7 +5,7 @@ become: true become_method: sudo vars: - pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8', true) ) }}" + pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8.0', true) ) }}" pxc_tarball: "{{ lookup('vars', 'extra_pxc_tarball', default=lookup('env','PXC_TARBALL') | default('', true) ) }}" proxysql_version: "{{ lookup('vars', 'extra_proxysql_version', default=lookup('env','PROXYSQL_VERSION') | default('2', true) ) }}" proxysql_package: "{{ lookup('vars', 'extra_proxysql_package', default=lookup('env','PROXYSQL_PACKAGE') | default('', true) ) }}" @@ -33,6 +33,7 @@ - name: Prepare Container for PXC and Proxysql shell: > docker run -d --name={{ pxc_container }} + -p 6033:6033 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pxc_container @@ -97,6 +98,10 @@ - docker exec {{ pxc_container }} chmod 1777 /tmp - docker exec --user pxc {{ pxc_container }} bash -xe ./client_container_pxc_setup.sh --pxc_version {{ pxc_version }} --pxc_tarball {{ pxc_tarball }} --number_of_nodes {{ number_of_nodes }} --pxc_dev_cluster {{ pxc_dev_cluster }} --query_source {{query_source}} > setup_pxc_{{ pxc_version }}.log + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + - name: Start ProxySQL inside the PXC extra_pxc_container shell: "{{ item }}" with_items: @@ -106,7 +111,7 @@ - docker exec {{ pxc_container }} bash -c 'sed -i s#3306#'"\$(grep 'port' /home/pxc/PXC/node1.cnf | cut -d= -f2)"'# /etc/proxysql-admin.cnf' - docker exec {{ pxc_container }} proxysql-admin --config-file=/etc/proxysql-admin.cnf --enable - docker exec {{ pxc_container }} sleep 20 - - docker exec {{ pxc_container }} pmm-admin add proxysql --username=admin --password=admin --service-name=my-new-proxysql_{{ pxc_container }} --host=127.0.0.1 --port=6032 + - docker exec {{ pxc_container }} pmm-admin add proxysql --username=proxysql_user --password=passw0rd --service-name=my-new-proxysql_{{ pxc_container }}_{{ random_number }} --host=127.0.0.1 --port=6033 - name: Execute Service, Sysbench Script inside the PXC pxc_container shell: "{{ item }}" diff --git a/pmm_qa/requirements.txt b/pmm_qa/requirements.txt index a221365..4566316 100644 --- a/pmm_qa/requirements.txt +++ b/pmm_qa/requirements.txt @@ -1 +1,2 @@ ansible-runner==2.3.2 +requests==2.26.0 diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh new file mode 100755 index 0000000..f76d873 --- /dev/null +++ b/pmm_qa/setup_docker_client_images.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -xe + +echo +echo "Configuring Multiple Docker Images with PMM Server and Client" +echo "Please wait...." +docker network create docker-client-check || true +docker-compose -f docker-compose-clients.yaml down -v --remove-orphans +docker-compose -f docker-compose-clients.yaml build --no-cache +docker-compose -f docker-compose-clients.yaml up -d +sleep 20 +echo "Adding DB Clients to PMM Server" +docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true +docker exec pmm-client-1 pmm-admin add postgresql --query-source=pgstatements --username=pmm --password=pmm-pass --service-name=pdpgsql-1 --host=pdpgsql-1 --port=5432 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true +docker exec pmm-client-1 pmm-admin add mongodb --username=pmm --password=pmm-pass --service-name=mongodb-7.0 --host=psmdb-1 --port=27017 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true + From 9b92ba05deb7f0298968d1c7811aaa21d7c09e20 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 6 May 2024 12:56:04 +0530 Subject: [PATCH 37/88] Updates for Base Image changes (#54) * Updates for Base Image changes fixs and improvements --- pmm_qa/external_setup.yml | 4 ++-- pmm_qa/haproxy_setup.yml | 4 ++-- pmm_qa/ms_pmm_setup.yml | 4 ++-- pmm_qa/pdpgsql_pgsm_setup.yml | 2 +- pmm_qa/pg_stat_monitor_setup.sh | 3 ++- pmm_qa/pg_stat_statements_setup.sh | 2 +- pmm_qa/pgsql_pgss_setup.yml | 2 +- pmm_qa/pmm-framework.py | 15 +++++++++------ pmm_qa/pmm3-client-setup.sh | 2 +- pmm_qa/psmdb_setup.yml | 4 ++-- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pmm_qa/external_setup.yml b/pmm_qa/external_setup.yml index 4084d44..456e636 100644 --- a/pmm_qa/external_setup.yml +++ b/pmm_qa/external_setup.yml @@ -29,12 +29,13 @@ - name: Prepare Container for External Exporters shell: > sudo docker run -d -p 43100:43100 --name={{ external_container }} - phusion/baseimage:focal-1.1.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker external_container shell: "{{ item }}" with_items: - docker cp ./external_setup.sh {{ external_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ external_container }}:/ - docker exec {{ external_container }} apt-get update - docker exec {{ external_container }} apt-get -y install wget curl git gnupg2 lsb-release @@ -52,7 +53,6 @@ - name: Install pmm2-client on the external_container shell: "{{ item }}" with_items: - - docker exec {{ external_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ external_container }} - docker exec {{ external_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/haproxy_setup.yml b/pmm_qa/haproxy_setup.yml index 38d65b6..6f92b78 100644 --- a/pmm_qa/haproxy_setup.yml +++ b/pmm_qa/haproxy_setup.yml @@ -25,13 +25,14 @@ - name: Prepare Container for HAPROXY shell: > sudo docker run -d -p 42100:42100 --name={{ haproxy_container }} - phusion/baseimage:focal-1.1.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker haproxy_container shell: "{{ item }}" with_items: - docker cp ./haproxy_setup.sh {{ haproxy_container }}:/ - docker cp ./haproxy.cfg {{ haproxy_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ haproxy_container }}:/ - docker exec {{ haproxy_container }} apt-get update - docker exec {{ haproxy_container }} apt-get -y install wget curl git gnupg2 lsb-release @@ -43,7 +44,6 @@ - name: Install pmm2-client on the haproxy_container shell: "{{ item }}" with_items: - - docker exec {{ haproxy_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ haproxy_container }} - docker exec {{ haproxy_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/ms_pmm_setup.yml b/pmm_qa/ms_pmm_setup.yml index cff597e..26951a9 100644 --- a/pmm_qa/ms_pmm_setup.yml +++ b/pmm_qa/ms_pmm_setup.yml @@ -34,13 +34,14 @@ docker run -d --name={{ ms_container }} -p 3318:3308 -v /tmp:/tmp - phusion/baseimage:focal-1.2.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker ms_container shell: "{{ item }}" with_items: - docker exec {{ ms_container }} mkdir -p artifacts - docker cp ./client_container_ms_setup.sh {{ ms_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ ms_container }}:/ - name: Get Product tarball URL based on the version shell: cat product_version_download_helper | grep -v pxc | grep mysql | grep "\-{{ ms_version }}" | head -1 | awk -F'# ' '{print $3}' @@ -62,7 +63,6 @@ - name: Install pmm2-client on the ms_container shell: "{{ item }}" with_items: - - docker exec {{ ms_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker network connect pmm-qa {{ ms_container }} - docker exec {{ ms_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 73c4e4e..6fc4998 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -32,7 +32,7 @@ shell: > sudo docker run -d --name={{ pdpgsql_pgsm_container }} -p 5447:5432 - phusion/baseimage:focal-1.2.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pdpgsql_pgsm_container shell: "{{ item }}" diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index 88526ef..d2adb9f 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -45,7 +45,7 @@ cd postgres # Install the dependencies apt-get update apt-get -y install wget curl git gnupg2 lsb-release -apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl wget +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev bison make flex libipc-run-perl wget sleep 10 # Install the PG server from selected distribution @@ -101,6 +101,7 @@ echo "shared_preload_libraries = 'pg_stat_monitor'" >> /etc/postgresql/${pgsql_v echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "track_io_timing=ON" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "max_connections=1000" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "listen_addresses = '*'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf # Create init.sql file required by PMM echo "CREATE DATABASE sbtest1;" >> /home/postgres/init.sql diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh index b6c8b84..ed05a5f 100644 --- a/pmm_qa/pg_stat_statements_setup.sh +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -33,7 +33,7 @@ cd postgres # Install the dependencies apt-get update apt-get -y install wget curl git gnupg2 lsb-release -apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl wget +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev bison make flex libipc-run-perl wget sleep 10 # Install the PG server from selected distribution diff --git a/pmm_qa/pgsql_pgss_setup.yml b/pmm_qa/pgsql_pgss_setup.yml index bd95163..d3a414f 100644 --- a/pmm_qa/pgsql_pgss_setup.yml +++ b/pmm_qa/pgsql_pgss_setup.yml @@ -30,7 +30,7 @@ shell: > sudo docker run -d --name={{ pgsql_pgss_container }} -p {{ pgsql_pgss_port }}:5432 - phusion/baseimage:focal-1.2.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pgsql_pgss_container shell: "{{ item }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5c91c69..d8adc4b 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -64,7 +64,7 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook_path = script_dir + "/" + playbook_filename if args.verbose: - print(f'Options set after considering defaults: {env_vars}') + print(f'Options set after considering Defaults: {env_vars}') r = ansible_runner.run( private_data_dir=script_dir, @@ -81,7 +81,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): def get_running_container_name(): - container_name = "pmm-server" + container_image_name = "pmm-server" + container_name = '' try: # Run 'docker ps' to get a list of running containers output = subprocess.check_output(['docker', 'ps', '--format', 'table {{.ID}}\t{{.Image}}\t{{.Names}}']) @@ -91,11 +92,13 @@ def get_running_container_name(): for line in containers: # Extract the image name info_parts = line.split('\t')[0] - image_info = info_parts.split()[2:] + image_info = info_parts.split()[1] # Check if the container is in the list of running containers # and establish N/W connection with it. - if container_name in image_info: - # Check if pmm-qa n/w exists and already connected to pmm-server n/w. + if container_image_name in image_info: + container_name = info_parts.split()[2] + # Check if pmm-qa n/w exists and already connected to running container n/w + # if not connect it. result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True, text=True) if result.returncode != 0: subprocess.run(['docker', 'network', 'create', 'pmm-qa']) @@ -620,7 +623,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): else: if args.verbose: print( - f"Value {value} is not recognised for Option {key}, will be using default value") + f"Value {value} is not recognised for Option {key}, will be using Default value") elif key in database_configs[db_type]["configurations"]: db_config[key] = value else: diff --git a/pmm_qa/pmm3-client-setup.sh b/pmm_qa/pmm3-client-setup.sh index 84df692..9060a1f 100755 --- a/pmm_qa/pmm3-client-setup.sh +++ b/pmm_qa/pmm3-client-setup.sh @@ -41,7 +41,7 @@ if [ ! -z "$upgrade" ]; then fi port=8443 -if [[ ! "$pmm_server_ip" =~ ^"pmm-server"$ ]]; then +if [[ "$pmm_server_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then port=443 fi diff --git a/pmm_qa/psmdb_setup.yml b/pmm_qa/psmdb_setup.yml index 8dd03d4..50f9652 100644 --- a/pmm_qa/psmdb_setup.yml +++ b/pmm_qa/psmdb_setup.yml @@ -26,19 +26,19 @@ - name: Prepare Container for PSMDB shell: > docker run -d --name={{ psmdb_container }} - phusion/baseimage:focal-1.1.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker psmdb_container shell: "{{ item }}" with_items: - docker cp ./psmdb_setup.sh {{ psmdb_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ psmdb_container }}:/ - docker exec {{ psmdb_container }} apt-get update - docker exec {{ psmdb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq - name: Install pmm2-client on the psmdb_container shell: "{{ item }}" with_items: - - docker exec {{ psmdb_container }} wget https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{ pmm_qa_branch }}/pmm_qa/pmm3-client-setup.sh - docker exec {{ psmdb_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/mongodb_user_setup.js - docker network connect pmm-qa {{ psmdb_container }} - docker exec {{ psmdb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no From d85d6eef4bac73e5d78afa447c5dbe7502797624 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Tue, 7 May 2024 18:22:28 +0300 Subject: [PATCH 38/88] PMM-7 fix framework in GH worker (#55) * PMM-13086 fix calling shell scripts with framework * PMM-13086 log error if failed --- pmm_psmdb-pbm_setup/start-rs-only.sh | 4 +-- pmm_qa/pmm-framework.py | 48 +++++++++++++--------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index 209d3ed..df21e30 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -17,8 +17,8 @@ docker-compose -f docker-compose-rs.yaml down -v --remove-orphans docker-compose -f docker-compose-rs.yaml build --no-cache docker-compose -f docker-compose-rs.yaml up -d echo -echo "waiting 30 seconds for replica set members to start" -sleep 30 +echo "waiting 60 seconds for replica set members to start" +sleep 60 echo if [ $mongo_setup_type == "pss" ]; then bash -e ./configure-replset.sh diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index d8adc4b..73c4e41 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -328,11 +328,10 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) -def execute_shell_scripts(shell_scripts, scripts_path, env_vars, args): +def execute_shell_scripts(shell_scripts, project_relative_scripts_dir, env_vars, args): # Get script directory - script_path = os.path.abspath(sys.argv[0]) - script_dir = os.path.dirname(script_path) - shell_scripts_path = script_dir + scripts_path + current_directory = os.getcwd() + shell_scripts_path = os.path.abspath(os.path.join(current_directory, os.pardir, project_relative_scripts_dir)) # Get the original working directory original_dir = os.getcwd() @@ -347,32 +346,29 @@ def execute_shell_scripts(shell_scripts, scripts_path, env_vars, args): # Execute each shell script for script in shell_scripts: + result = '' try: + print(f'running script {script}') # Change directory to where the script is located os.chdir(shell_scripts_path) - process = subprocess.Popen(['bash', script], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # Read output streams asynchronously - while process.poll() is None: # Check if the subprocess is still running - # Read from stdout - for stdout_line in process.stdout: - if stdout_line: - print(stdout_line.decode('utf-8').strip()) - - # Read from stderr - for stderr_line in process.stderr: - if stderr_line: - print(stderr_line.decode('utf-8').strip()) - + print(f'changed directory {os.getcwd()}') + result = subprocess.run(['bash', script], capture_output=True, text=True, check=True) + print("Output:") + print(result.stdout) + except subprocess.CalledProcessError as e: + print("Error:") + print(e.stderr) + except Exception as e: + print("Unexpected error occurred:", e) + finally: # Get the return code of the process - return_code = process.returncode + return_code = result.returncode if return_code == 0: print(f"Shell script '{script}' executed successfully.") else: - print(f"Shell script '{script}' failed with return code: {return_code}! \n {process.stderr}") + print(f"Shell script '{script}' failed with return code: {return_code}! \n {result.stderr}") exit(return_code) - except Exception as e: - print("Unexpected error occurred:", e) - finally: + # Return to the original working directory os.chdir(original_dir) @@ -484,7 +480,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): } shell_scripts = [] - scripts_path = "/../pmm_psmdb-pbm_setup/" + scripts_folder = "pmm_psmdb-pbm_setup" setup_type = get_value('SETUP_TYPE', db_type, args, db_config).lower() if setup_type in ("pss", "psa"): @@ -495,7 +491,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): # Execute shell scripts if not shell_scripts == []: - execute_shell_scripts(shell_scripts, scripts_path, env_vars, args) + execute_shell_scripts(shell_scripts, scripts_folder, env_vars, args) def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): @@ -537,9 +533,9 @@ def setup_dockerclients(db_type, db_version=None, db_config=None, args=None): # Shell script filename shell_scripts = ['setup_docker_client_images.sh'] - shell_scripts_path = '' + shell_scripts_path = 'pmm_qa' - # Call the function to run the Ansible playbook + # Call the function to run the setup_docker_client_images script execute_shell_scripts(shell_scripts, shell_scripts_path, env_vars, args) From 55eca0a5fd6b574b80b0c46e2584659da17d1b97 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 14 May 2024 13:22:11 +0530 Subject: [PATCH 39/88] Updates for PXC when client tar ball install (#58) --- pmm_qa/product_version_download_helper | 2 +- pmm_qa/pxc_proxysql_setup.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index 72398a7..d41657f 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -22,5 +22,5 @@ ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona ms # 8.0.33 # https://dev.mysql.com/get/Downloads/MySQL-8/mysql-8.0.33-linux-glibc2.28-x86_64.tar.gz -proxysql # 2.5.3 # https://downloads.percona.com/downloads/proxysql2/proxysql2-2.5.3/binary/debian/focal/x86_64/proxysql2_2.5.3-1.1.focal_amd64.deb +proxysql # 2.6.2 # https://downloads.percona.com/downloads/proxysql2/proxysql2-2.6.2/binary/debian/jammy/x86_64/proxysql2_2.6.2-1.1.jammy_amd64.deb diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index 9a416f1..1f40c9d 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -82,7 +82,12 @@ with_items: - docker network connect pmm-qa {{ pxc_container }} - docker exec {{ pxc_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Configure PXC for PMM Client if non-tarball install + shell: "{{ item }}" + with_items: - docker exec {{ pxc_container }} usermod -aG pmm-agent pxc + when: "'http' not in client_version" - name: Execute User & Proxysql Setup inside the PXC pxc_container shell: "{{ item }}" From b234eed8651c782e52b803f667bcc3d9d1c167f2 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Tue, 14 May 2024 15:28:42 +0300 Subject: [PATCH 40/88] PMM-7 fix psmdb setup (#56) * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup * PMM-7 fix psmdb setup --- pmm_psmdb-pbm_setup/Dockerfile | 10 +++++----- pmm_qa/pmm-framework.py | 17 +++++------------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index bca3144..d6f8df2 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -48,22 +48,22 @@ RUN set -ex; \ percona-server-mongodb-server-${PSMDB_VERSION}.el9 \ percona-server-mongodb-mongos-${PSMDB_VERSION}.el9 ; \ fi && \ - if [ $PBM_VERSION == "latest" ]; then \ + if [ "$PBM_VERSION" == "latest" ]; then \ yum -y install percona-backup-mongodb ; \ else \ yum -y install percona-backup-mongodb-${PBM_VERSION}.el9 ; \ fi -RUN if [[ $PMM_CLIENT_VERSION == http* ]]; then \ +RUN if [[ "$PMM_CLIENT_VERSION" == http* ]]; then \ yum -y install pmm-client && \ curl -Lf -o /tmp/pmm-client.tar.gz $PMM_CLIENT_VERSION && \ cd /tmp && tar -xvf pmm-client.tar.gz --transform 's,^/*[^/]*,/pmm-client,S' && \ cd /tmp/pmm-client && ./install_tarball && cd ../ && rm -rf pmm* ;\ elif [[ "$PMM_CLIENT_VERSION" =~ 3-dev-latest|latest ]]; then \ yum -y install pmm-client ; \ - elif [[ $PMM_CLIENT_VERSION =~ pmm3-rc ]]; then \ - percona-release enable original testing \ - yum update --refresh -y \ + elif [[ "$PMM_CLIENT_VERSION" = "pmm3-rc" ]]; then \ + percona-release enable original testing && \ + yum update --refresh -y && \ yum -y install pmm-client ; \ else \ yum -y install pmm-client-${PMM_CLIENT_VERSION}-6.el9 ; \ diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 73c4e41..a7a1602 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -346,7 +346,7 @@ def execute_shell_scripts(shell_scripts, project_relative_scripts_dir, env_vars, # Execute each shell script for script in shell_scripts: - result = '' + result: subprocess.CompletedProcess try: print(f'running script {script}') # Change directory to where the script is located @@ -355,20 +355,13 @@ def execute_shell_scripts(shell_scripts, project_relative_scripts_dir, env_vars, result = subprocess.run(['bash', script], capture_output=True, text=True, check=True) print("Output:") print(result.stdout) + print(f"Shell script '{script}' executed successfully.") except subprocess.CalledProcessError as e: - print("Error:") - print(e.stderr) + print(f"Shell script '{script}' failed with return code: {e.returncode}! \n {e.stderr} \n Output: \n {e.stdout} ") + exit(e.returncode) except Exception as e: print("Unexpected error occurred:", e) finally: - # Get the return code of the process - return_code = result.returncode - if return_code == 0: - print(f"Shell script '{script}' executed successfully.") - else: - print(f"Shell script '{script}' failed with return code: {return_code}! \n {result.stderr}") - exit(return_code) - # Return to the original working directory os.chdir(original_dir) @@ -639,4 +632,4 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): except Exception as e: print("An unexpected error occurred:", e) parser.print_help() - exit(1) + exit(1) \ No newline at end of file From 16497100127b24fad02c58a9f5b77409a1c119b2 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 15 May 2024 13:27:56 +0530 Subject: [PATCH 41/88] framework proxysql fix (#59) * Fix ProxySQL to use admin port for PMM --- pmm_qa/pxc_proxysql_setup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index 1f40c9d..967c5d0 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -33,7 +33,7 @@ - name: Prepare Container for PXC and Proxysql shell: > docker run -d --name={{ pxc_container }} - -p 6033:6033 + -p 6032:6032 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pxc_container @@ -116,8 +116,8 @@ - docker exec {{ pxc_container }} bash -c 'sed -i s#3306#'"\$(grep 'port' /home/pxc/PXC/node1.cnf | cut -d= -f2)"'# /etc/proxysql-admin.cnf' - docker exec {{ pxc_container }} proxysql-admin --config-file=/etc/proxysql-admin.cnf --enable - docker exec {{ pxc_container }} sleep 20 - - docker exec {{ pxc_container }} pmm-admin add proxysql --username=proxysql_user --password=passw0rd --service-name=my-new-proxysql_{{ pxc_container }}_{{ random_number }} --host=127.0.0.1 --port=6033 - + - docker exec {{ pxc_container }} pmm-admin add proxysql --username=admin --password=admin --service-name=my-new-proxysql_{{ pxc_container }}_{{ random_number }} --host=127.0.0.1 --port=6032 + - name: Execute Service, Sysbench Script inside the PXC pxc_container shell: "{{ item }}" with_items: From 9083d5932722dd9dea87f0176e790d068637de0b Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 28 May 2024 12:52:40 +0530 Subject: [PATCH 42/88] Fix innodb metrics for upstream (#60) * Fix innodb metrics for upstream --- pmm_qa/client_container_ms_setup.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index f97d8d2..9e06872 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -55,11 +55,17 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" else dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --port=$MS_PORT --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if [[ "${query_source}" == "slowlog" ]]; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" @@ -88,8 +94,11 @@ else dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" for j in `seq 1 $number_of_nodes`; do - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" if [[ "${query_source}" == "slowlog" ]]; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" From 12d4b4ae8ecc4379e16a8bfeac881081ad5d5ba7 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 28 May 2024 15:04:19 +0530 Subject: [PATCH 43/88] Updates upstream dosent contain userstat table (#61) --- pmm_qa/client_container_ms_setup.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index 9e06872..dc5323d 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -55,7 +55,6 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" else @@ -63,7 +62,6 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if [[ "${query_source}" == "slowlog" ]]; then @@ -95,7 +93,6 @@ else export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" for j in `seq 1 $number_of_nodes`; do From ce86bc928ff75745d51ae18b123d40cfcd1aa3eb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 10 Jun 2024 22:11:23 +0200 Subject: [PATCH 44/88] PMM-7: Add root user to the ps setup --- pmm_qa/client_container_ps_setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index c0cd024..8090e33 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -59,11 +59,13 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" else dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" if [[ "${query_source}" == "slowlog" ]]; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" From 336e843e58b82154773fa7233efadce833f75546 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 11 Jun 2024 09:14:08 +0200 Subject: [PATCH 45/88] PMM-7: test dbdeployer --- pmm_qa/client_container_ps_setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index 8090e33..1d2aa09 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -55,13 +55,13 @@ export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then - dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force --custom-role-extra='WITH GRANT OPTION' export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force --custom-role-extra='WITH GRANT OPTION' export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" From bd2fb7fd3307e492dbec71a5daf5911219a5efbf Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 11 Jun 2024 18:07:17 +0530 Subject: [PATCH 46/88] Add TLS files and fixs from main(tls, psmdb) setups (#63) * Add TLS setup, and fixs from main * Updates Enable pmm-agent logging --- pmm_psmdb-pbm_setup/Dockerfile | 1 + pmm_qa/psmdb_setup.yml | 18 +++- pmm_qa/tls-ssl-setup/create_certs.sh | 22 +++++ .../mongodb/mongodb_ssl_setup.sh | 72 ++++++++++++++ pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml | 80 ++++++++++++++++ pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh | 93 +++++++++++++++++++ pmm_qa/tls-ssl-setup/mysql_tls_setup.yml | 84 +++++++++++++++++ pmm_qa/tls-ssl-setup/postgres/init.sql | 8 ++ pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh | 53 +++++++++++ pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml | 83 +++++++++++++++++ 10 files changed, 509 insertions(+), 5 deletions(-) create mode 100644 pmm_qa/tls-ssl-setup/create_certs.sh create mode 100644 pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh create mode 100644 pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml create mode 100644 pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh create mode 100644 pmm_qa/tls-ssl-setup/mysql_tls_setup.yml create mode 100644 pmm_qa/tls-ssl-setup/postgres/init.sql create mode 100644 pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh create mode 100644 pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index d6f8df2..244f1fd 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -71,6 +71,7 @@ RUN if [[ "$PMM_CLIENT_VERSION" == http* ]]; then \ sed 's/Type=forking/#Type=forking/' -i /usr/lib/systemd/system/mongod.service && \ systemctl enable mongod && \ systemctl enable pbm-agent && \ + sed -i '/ExecStart/a StandardError=file:/var/log/pmm-agent.log' /usr/lib/systemd/system/pmm-agent.service && \ systemctl enable pmm-agent && \ curl -Lf -o /tmp/mgodatagen.tar.gz https://github.com/feliixx/mgodatagen/releases/download/v0.11.2/mgodatagen_0.11.2_Linux_x86_64.tar.gz && \ tar -xf /tmp/mgodatagen.tar.gz -C /usr/bin && \ diff --git a/pmm_qa/psmdb_setup.yml b/pmm_qa/psmdb_setup.yml index 50f9652..c27e60c 100644 --- a/pmm_qa/psmdb_setup.yml +++ b/pmm_qa/psmdb_setup.yml @@ -1,6 +1,8 @@ --- - hosts: all + become: true + become_method: sudo vars: psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}" psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}" @@ -46,7 +48,7 @@ - name: Setup psmdb for monitoring shell: "{{ item }}" with_items: - - docker exec {{ psmdb_container }} bash -x ./psmdb_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > ~/setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + - docker exec {{ psmdb_container }} bash -x ./psmdb_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log - name: Setup Load Running Docker Container shell: "{{ item }}" @@ -62,20 +64,26 @@ shell: docker exec {{ psmdb_container }} ./nodes/cl.sh mongodb_user_setup.js when: psmdb_setup == "regular" - - name: Run load on Replica Set Master + - name: Run load on Replica Set Master(PSS) shell: "{{ item }}" with_items: - - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27018 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27018 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log when: psmdb_setup == "replica" or psmdb_setup == "arbiter" + - name: Run load on Replica Set Master(PSA) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27018 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 MONGODB_USER="dba" -e MONGODB_PASSWORD="test1234" --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + when: psmdb_setup == "arbiter" + - name: Run load on Regular shell: "{{ item }}" with_items: - - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log when: psmdb_setup == "regular" - name: Run load on Sharded Clusters Master shell: "{{ item }}" with_items: - - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27019 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27019 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log when: psmdb_setup == "sharded" diff --git a/pmm_qa/tls-ssl-setup/create_certs.sh b/pmm_qa/tls-ssl-setup/create_certs.sh new file mode 100644 index 0000000..dcb6d94 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/create_certs.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +export PWD=$(pwd) +export HOST=localhost +mkdir -p certificates +pushd certificates +echo -e "\n=== Generating SSL certificates in ${PWD} ===" +# Generate self signed root CA cert +openssl req -nodes -x509 -newkey rsa:4096 -keyout ca.key -out ca.crt -subj "/C=US/ST=California/L=San Francisco/O=Percona/OU=root/CN=${HOST}/emailAddress=test@percona.com" +# Generate server cert to be signed +openssl req -nodes -newkey rsa:4096 -keyout server.key -out server.csr -subj "/C=US/ST=California/L=San Francisco/O=Percona/OU=server/CN=${HOST}/emailAddress=test@percona.com" +# Sign server sert +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt +# Create server PEM file +cat server.key server.crt > server.pem +# Generate client cert to be signed +openssl req -nodes -newkey rsa:4096 -keyout client.key -out client.csr -subj "/C=US/ST=California/L=San Francisco/O=Percona/OU=client/CN=${HOST}/emailAddress=test@percona.com" +# Sign the client cert +openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -out client.crt +# Create client PEM file +cat client.key client.crt > client.pem +popd diff --git a/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh new file mode 100644 index 0000000..fe35024 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh @@ -0,0 +1,72 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=4.4 +fi + +apt-get update +apt-get -y install wget curl git jq +wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb +dpkg -i percona-release_latest.generic_all.deb +wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh +chmod +x mongo_startup.sh +wget https://raw.githubusercontent.com/percona/pmm-qa/main/pmm-tests/mongodb_user_setup.js + +### Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) +psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) + +echo "Downloading ${psmdb_latest} ..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz +rm percona_server_mongodb.tar.gz* +mv percona-server-mongodb-${mongodb_version}.* psmdb_${mongodb_version} + +# TODO: refactor if to match range of versions 6.0+ +if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" ]]; then + ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally + echo "Downloading mongosh ..." + mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh) + wget -O mongosh.tar.gz ${mongosh_link} + tar -xvf mongosh.tar.gz + mv percona-mongodb-mongosh* mongosh + cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo + rm mongosh.tar.gz +fi + +bash ./mongo_startup.sh -m --ssl -x -e wiredTiger --mongodExtra="--profile 2 --slowms 1 --bind_ip_all" --b=/psmdb_${mongodb_version}/bin +sleep 20 +/nodes/cl.sh mongodb_user_setup.js +cat > add_new_ssl_user.js < + docker ps -a --filter "name={{ mongodb_ssl_container }}" | grep -q . && docker stop {{ mongodb_ssl_container }} && docker rm -fv {{ mongodb_ssl_container }} + ignore_errors: true + tags: + - cleanup + - name: delete network if exist + shell: docker network rm "{{ mongodb_ssl_container }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ mongodb_ssl_container }}_network" + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for mongodb ssl container + shell: > + docker run -d --name={{ mongodb_ssl_container }} + --network "{{ mongodb_ssl_container }}_network" + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} mkdir -p artifacts + - docker cp ./mongodb/mongodb_ssl_setup.sh {{ mongodb_ssl_container }}:/ + + - name: Execute Setup script inside the mongodb mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -xe ./mongodb_ssl_setup.sh --mongodb_version {{ mongodb_version }} > mongodb/setup_mongodb_ssl_{{ mongodb_version }}.log + + - name: Install pmm2-client on the mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh + - docker network connect pmm-qa {{ mongodb_ssl_container }} + - docker exec {{ mongodb_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Add pmm-admin binary to path when tar ball installation + shell: docker exec {{ mongodb_ssl_container }} echo "export PATH=$PATH:/pmm2-client/bin" > setup_path.sh + when: '"http" in client_version' + + - name: Remove mongodb service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove mongodb {{ mongodb_ssl_container }}_service' + ignore_errors: true + + - name: Add mongodb_ssl for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mongodb --tls --tls-skip-verify --authentication-mechanism=MONGODB-X509 --authentication-database=$external --tls-certificate-key-file=/nodes/certificates/client.pem --tls-certificate-key-file-password=/nodes/certificates/client.key --tls-ca-file=/nodes/certificates/ca.crt {{ mongodb_ssl_container }}_ssl_service' + + - name: Get client cert Files on host + shell: "{{ item }}" + with_items: + - mkdir -p mongodb/{{ mongodb_version }} || true + - docker exec {{ mongodb_ssl_container }} cat /nodes/certificates/ca.crt > mongodb/{{ mongodb_version }}/ca.crt + - docker exec {{ mongodb_ssl_container }} cat /nodes/certificates/client.key > mongodb/{{ mongodb_version }}/client.key + - docker exec {{ mongodb_ssl_container }} cat /nodes/certificates/client.pem > mongodb/{{ mongodb_version }}/client.pem diff --git a/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh new file mode 100644 index 0000000..388f176 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh @@ -0,0 +1,93 @@ +#!/bin/sh + + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$mysql_version" ] +then + export mysql_version=8.0 +fi + +apt-get update +apt-get -y install wget curl git +wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb +dpkg -i percona-release_latest.generic_all.deb +sleep 10 +if [ "$mysql_version" == "8.0" ]; then + percona-release setup ps80 + sleep 10 + DEBIAN_FRONTEND=noninteractive apt-get -y install percona-server-server sysbench sysbench-tpcc bc screen +cat > /etc/mysql/my.cnf << EOF +[mysqld] +innodb_buffer_pool_size=256M +innodb_buffer_pool_instances=1 +innodb_log_file_size=1G +innodb_flush_method=O_DIRECT +innodb_numa_interleave=1 +innodb_flush_neighbors=0 +log_bin +server_id=1 +binlog_expire_logs_seconds=600 +log_output=file +slow_query_log=ON +long_query_time=0 +log_slow_rate_limit=1 +log_slow_rate_type=query +log_slow_verbosity=full +log_slow_admin_statements=ON +log_slow_slave_statements=ON +slow_query_log_always_write_time=1 +slow_query_log_use_global_control=all +innodb_monitor_enable=all +userstat=1 +bind-address=0.0.0.0 +require_secure_transport=ON +EOF + +fi + +if [ "$mysql_version" == "5.7" ]; then + percona-release setup ps57 + sleep 10 + DEBIAN_FRONTEND=noninteractive apt-get -y install percona-server-server-5.7 +cat > /etc/mysql/my.cnf << EOF +[mysqld] +innodb_buffer_pool_size=256M +innodb_buffer_pool_instances=1 +innodb_log_file_size=1G +innodb_flush_method=O_DIRECT +innodb_numa_interleave=1 +innodb_flush_neighbors=0 +log_bin +server_id=1 +expire_logs_days=1 +log_output=file +slow_query_log=ON +long_query_time=0 +log_slow_rate_limit=1 +log_slow_rate_type=query +log_slow_verbosity=full +log_slow_admin_statements=ON +log_slow_slave_statements=ON +slow_query_log_always_write_time=1 +slow_query_log_use_global_control=all +innodb_monitor_enable=all +userstat=1 +bind-address=0.0.0.0 +require_secure_transport=ON +EOF + +fi +service mysql restart +mysql -e "create user pmm@'%' identified by \"pmm\"" +mysql -e "grant all on *.* to pmm@'%'" +mysql -e "CREATE USER 'pmm_tls'@'%' REQUIRE X509" +service mysql restart diff --git a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml new file mode 100644 index 0000000..66ce332 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml @@ -0,0 +1,84 @@ +--- +# This playbook does following: +# enables Percona testing repository +# Install Percona Server at Version 8.0.25 +# Install all required tools for backups in compatible version + +- hosts: all + become: true + become_method: sudo + vars: + mysql_version: "{{ lookup('vars', 'extra_mysql_version', default=lookup('env','MYSQL_VERSION') | default('8.0', true) ) }}" + mysql_ssl_container: "{{ lookup('vars', 'extra_mysql_ssl_container', default=lookup('env','MYSQL_SSL_CONTAINER') | default('mysql_ssl', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: Cleanup Docker container for client and DB setup + shell: > + docker ps -a --filter "name={{ mysql_ssl_container }}" | grep -q . && docker stop {{ mysql_ssl_container }} && docker rm -fv {{ mysql_ssl_container }} + ignore_errors: true + tags: + - cleanup + - name: delete network if exist + shell: docker network rm "{{ mysql_ssl_container }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ mysql_ssl_container }}_network" + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for mysql ssl container + shell: > + docker run -d --name={{ mysql_ssl_container }} + --network "{{ mysql_ssl_container }}_network" + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} mkdir -p artifacts + - docker cp ./mysql/mysql_ssl_setup.sh {{ mysql_ssl_container }}:/ + + - name: Execute Setup script inside the mysql mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -xe ./mysql_ssl_setup.sh --mysql_version {{ mysql_version }} > mysql/setup_mysql_ssl_{{ mysql_version }}.log + + - name: Install pmm2-client on the mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh + - docker network connect pmm-qa {{ mysql_ssl_container }} + - docker exec {{ mysql_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Add pmm-admin binary to path when tar ball installation + shell: docker exec {{ mysql_ssl_container }} echo "export PATH=$PATH:/pmm2-client/bin" > setup_path.sh + when: '"http" in client_version' + + - name: Remove mysql service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove mysql {{ mysql_ssl_container }}_service' + ignore_errors: true + + - name: Add mysql_ssl for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mysql --username=pmm --password=pmm --query-source=perfschema --tls --tls-skip-verify --tls-ca=/var/lib/mysql/ca.pem --tls-cert=/var/lib/mysql/client-cert.pem --tls-key=/var/lib/mysql/client-key.pem {{ mysql_ssl_container }}_ssl_service' + + - name: Get client cert Files on host + shell: "{{ item }}" + with_items: + - mkdir -p mysql/{{ mysql_version }} || true + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/ca.pem > mysql/{{ mysql_version }}/ca.pem + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/client-key.pem > mysql/{{ mysql_version }}/client-key.pem + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/client-cert.pem > mysql/{{ mysql_version }}/client-cert.pem diff --git a/pmm_qa/tls-ssl-setup/postgres/init.sql b/pmm_qa/tls-ssl-setup/postgres/init.sql new file mode 100644 index 0000000..e11a91a --- /dev/null +++ b/pmm_qa/tls-ssl-setup/postgres/init.sql @@ -0,0 +1,8 @@ +CREATE DATABASE sbtest1; +CREATE DATABASE sbtest2; +CREATE USER pmm WITH PASSWORD 'pmm'; +GRANT pg_monitor TO pmm; +CREATE EXTENSION pg_stat_statements; +ALTER SYSTEM SET shared_preload_libraries TO 'pg_stat_statements'; +ALTER SYSTEM SET track_activity_query_size=2048; +ALTER SYSTEM SET track_io_timing=ON; diff --git a/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh b/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh new file mode 100644 index 0000000..3bd841e --- /dev/null +++ b/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh @@ -0,0 +1,53 @@ +#!/bin/sh + + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + + shift +done + +if [ -z "$pgsql_version" ] +then + export pgsql_version=13 +fi + +apt-get update +apt-get -y install wget curl git +wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb +dpkg -i percona-release_latest.generic_all.deb +percona-release setup ppg${pgsql_version} +sleep 10 +pushd artifacts +bash -x create_certs.sh +popd +sleep 10 +pwd +apt -y install percona-postgresql-${pgsql_version} +apt -y install percona-postgresql-contrib +sleep 10 +sed -i 's/\(host\s*all\s*all\s*127.0.0.1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(host\s*all\s*all\s*::1.*\) md5/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*postgres.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sed -i 's/\(local\s*all\s*all.*\) peer/\1 trust/g' /etc/postgresql/${pgsql_version}/main/pg_hba.conf +service postgresql restart +sleep 10 +cp -a ./artifacts/certificates/. /var/lib/postgresql/${pgsql_version}/main/ +ls -la ./artifacts/certificates/ +chown -R postgres:postgres /var/lib/postgresql/${pgsql_version}/main +chmod 0700 -R /var/lib/postgresql/${pgsql_version}/main +sed -i "s/ssl_cert_file.*/ssl_cert_file = 'server.crt'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +sed -i "s/#listen_addresses.*/listen_addresses = '*'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +sed -i "s/ssl_key_file.*/ssl_key_file = 'server.key'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +sed -i "s/ssl_ca_file.*/ssl_ca_file = 'ca.crt'/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +sed -i "s/#ssl_prefer_server_ciphers.*/ssl_prefer_server_ciphers = on/g" /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "hostssl all all 0.0.0.0/0 md5" >> /etc/postgresql/${pgsql_version}/main/pg_hba.conf +echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/${pgsql_version}/main/pg_hba.conf +sleep 10 +service postgresql restart +su postgres bash -c 'psql -f init.sql' +service postgresql restart diff --git a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml new file mode 100644 index 0000000..712496e --- /dev/null +++ b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml @@ -0,0 +1,83 @@ +--- +# This playbook does following: +# enables Percona testing repository +# Install Percona Server at Version 8.0.25 +# Install all required tools for backups in compatible version + +- hosts: all + become: true + become_method: sudo + vars: + pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('13', true) ) }}" + pgsql_ssl_container: "{{ lookup('vars', 'extra_pgsql_ssl_container', default=lookup('env','PGSQL_SSL_CONTAINER') | default('pgsql_ssl', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ pgsql_ssl_container }}" | grep -q . && docker stop {{ pgsql_ssl_container }} && docker rm -fv {{ pgsql_ssl_container }} + ignore_errors: true + tags: + - cleanup + - name: delete network if exist + shell: docker network rm "{{ pgsql_ssl_container }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ pgsql_ssl_container }}_network" + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PostgreSQL + shell: > + docker run -d --name={{ pgsql_ssl_container }} + --network "{{ pgsql_ssl_container }}_network" + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker pgsql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_ssl_container }} mkdir -p artifacts + - docker cp ./create_certs.sh {{ pgsql_ssl_container }}:/artifacts/ + - docker cp ./postgres/setup_pgsql.sh {{ pgsql_ssl_container }}:/ + - docker cp ./postgres/init.sql {{ pgsql_ssl_container }}:/ + + - name: Execute Setup script inside the pgsql pgsql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_ssl_container }} bash -xe ./setup_pgsql.sh --pgsql_version {{ pgsql_version }} > postgres/setup_pgsql_{{ pgsql_version }}.log + + - name: Install pmm2-client on the pgsql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh + - docker network connect pmm-qa {{ pgsql_ssl_container }} + - docker exec {{ pgsql_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Remove pgsql service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove postgresql {{ pgsql_ssl_container }}_ssl_service' + ignore_errors: true + + - name: Add pgsql_ssl for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source="pgstatements" --tls --tls-ca-file=./certificates/ca.crt --tls-cert-file=./certificates/client.crt --tls-key-file=./certificates/client.pem {{ pgsql_ssl_container }}_ssl_service' + # - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile; pmm-admin add postgresql --socket=/var/run/postgresql postgresql_socket' + + - name: Get client cert Files on host + shell: "{{ item }}" + with_items: + - mkdir -p postgres/{{ pgsql_version }} + - docker exec {{ pgsql_ssl_container }} cat ./certificates/ca.crt > postgres/{{ pgsql_version }}/ca.crt + - docker exec {{ pgsql_ssl_container }} cat ./certificates/client.crt > postgres/{{ pgsql_version }}/client.crt + - docker exec {{ pgsql_ssl_container }} cat ./certificates/client.pem > postgres/{{ pgsql_version }}/client.pem From 6906c288637b4c4e507fae2062f788c7b608b04a Mon Sep 17 00:00:00 2001 From: Saikumar Date: Tue, 11 Jun 2024 18:48:50 +0530 Subject: [PATCH 47/88] Updates root user creds --- pmm_qa/client_container_ms_setup.sh | 3 +++ pmm_qa/client_container_ps_setup.sh | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index dc5323d..292377d 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -55,6 +55,7 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" else @@ -62,6 +63,7 @@ if [[ "$number_of_nodes" == 1 ]];then export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if [[ "${query_source}" == "slowlog" ]]; then @@ -93,6 +95,7 @@ else export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" for j in `seq 1 $number_of_nodes`; do diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index 1d2aa09..0b3f84f 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -55,13 +55,13 @@ export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then - dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force --custom-role-extra='WITH GRANT OPTION' + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force --custom-role-extra='WITH GRANT OPTION' + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" @@ -120,6 +120,7 @@ else node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` for j in `seq 1 $number_of_nodes`; do mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" if [[ "${query_source}" == "slowlog" ]]; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" From 31e5873b29a45e9c168230432a270ff31b6f9d60 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 13 Jun 2024 12:15:08 +0530 Subject: [PATCH 48/88] Add query plan fpr ppg_stat_monitor (#62) --- pmm_qa/pg_stat_monitor_setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index d2adb9f..c3f5d6c 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -102,6 +102,7 @@ echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/ echo "track_io_timing=ON" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "max_connections=1000" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "listen_addresses = '*'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf +echo "pg_stat_monitor.pgsm_enable_query_plan = 'yes'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf # Create init.sql file required by PMM echo "CREATE DATABASE sbtest1;" >> /home/postgres/init.sql From d69490f13cdb2dfc872fdfb9c9d942a6903e7903 Mon Sep 17 00:00:00 2001 From: Puneet Kala Date: Fri, 28 Jun 2024 18:06:25 +0530 Subject: [PATCH 49/88] PMM-7 SSL mysql base support (#48) * PMM-7 SSL mysql base support * Updates for MySQL, PGSQL and Mongo SSL setups --- pmm_psmdb_diffauth_setup/test-auth.sh | 4 + pmm_qa/pmm-framework.py | 177 +++++++++++++++++- pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh | 2 +- pmm_qa/tls-ssl-setup/mysql_tls_setup.yml | 12 +- pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml | 14 +- pmm_qa/tls-ssl-setup/setup_mysql | 84 +++++++++ 6 files changed, 280 insertions(+), 13 deletions(-) create mode 100644 pmm_qa/tls-ssl-setup/setup_mysql diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 3abe7a9..a8933a5 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -65,6 +65,10 @@ while [ $i -le 3 ]; do sleep 1 done +#Add Mongo Service +random_number=$RANDOM +docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster + tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a7a1602..8eb9ac8 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -13,6 +13,11 @@ "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, + "SSL_PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, "MYSQL": { "versions": ["8.0"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", @@ -23,6 +28,11 @@ "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, + "SSL_MYSQL": { + "versions": ["5.7", "8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": ""} + }, "PGSQL": { "versions": ["11", "12", "13", "14", "15", "16"], "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} @@ -31,6 +41,10 @@ "versions": ["11", "12", "13", "14", "15", "16"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, + "SSL_PDPGSQL": { + "versions": ["11", "12", "13", "14", "15", "16"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + }, "PXC": { "versions": ["5.7", "8.0"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""} @@ -218,6 +232,37 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) +def setup_ssl_mysql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Check Setup Types + setup_type = None + no_of_nodes = 1 + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() + + # Gather Version details + ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] + # Define environment variables for playbook + env_vars = { + 'MYSQL_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MYSQL_SSL_CONTAINER': 'mysql_ssl_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'tls-ssl-setup/mysql_tls_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -247,6 +292,35 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) +def setup_ssl_pdpgsql(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + pdpgsql_version = os.getenv('PDPGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PGSTAT_MONITOR_BRANCH': 'main', + 'PGSQL_VERSION': pdpgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PGSQL_SSL_CONTAINER': 'pdpgsql_pgsm_ssl_' + str(pdpgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'tls-ssl-setup/postgresql_tls_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + def setup_pgsql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -357,7 +431,8 @@ def execute_shell_scripts(shell_scripts, project_relative_scripts_dir, env_vars, print(result.stdout) print(f"Shell script '{script}' executed successfully.") except subprocess.CalledProcessError as e: - print(f"Shell script '{script}' failed with return code: {e.returncode}! \n {e.stderr} \n Output: \n {e.stdout} ") + print( + f"Shell script '{script}' failed with return code: {e.returncode}! \n {e.stderr} \n Output: \n {e.stdout} ") exit(e.returncode) except Exception as e: print("Unexpected error occurred:", e) @@ -447,7 +522,8 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): exit(1) # Gather Version details - psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or database_configs[db_type]["versions"][-1] + psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ + database_configs[db_type]["versions"][-1] # Handle port address for external or internal address server_hostname = container_name @@ -487,6 +563,95 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): execute_shell_scripts(shell_scripts, scripts_folder, env_vars, args) +# Temporary method for Mongo SSL Setup. +def mongo_ssl_setup(script_filename, args): + # Get script directory + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + scripts_path = script_dir + "/../pmm_psmdb_diffauth_setup/" + + # Temporary shell script filename + shellscript_file_path = scripts_path + script_filename + + # Temporary docker compose filename + compose_filename = f'docker-compose-psmdb.yml' + compose_file_path = scripts_path + compose_filename + + # Create pmm-qa n/w used in workaround + result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) + if not result: + subprocess.run(['docker', 'network', 'create', 'pmm-qa']) + + no_server = True + # Add workaround (copy files) till sharding only support is ready. + try: + if no_server: + # Search & Replace content in the temporary compose files + subprocess.run( + ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) + admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' + subprocess.run(['sed', '-i', f's/password/{admin_password}/g', f'{compose_file_path}']) + subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) + subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', + f'{compose_file_path}']) + subprocess.run(['sed', '-i', + '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', + f'{compose_file_path}']) + subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) + + # Search replace content in-line in shell file + subprocess.run(['sed', '-i', f's/pmm-agent setup 2/pmm-agent setup --server-insecure-tls 2/g', + f'{shellscript_file_path}']) + subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', + f'{shellscript_file_path}']) + except subprocess.CalledProcessError as e: + print(f"Error occurred: {e}") + + +def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running...Exiting") + exit(1) + + # Gather Version details + psmdb_version = os.getenv('PSMDB_VERSION') or get_latest_psmdb_version(db_version) or \ + database_configs[db_type]["versions"][-1] + + # Handle port address for external or internal address + server_hostname = container_name + port = 8443 + + if args.pmm_server_ip: + port = 443 + server_hostname = args.pmm_server_ip + + server_address = f'{server_hostname}:{port}' + + # Define environment variables for playbook + env_vars = { + 'PSMDB_VERSION': psmdb_version, + 'PMM_SERVER_CONTAINER_ADDRESS': server_address, + 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), + 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), + 'TESTS': 'no', + 'CLEANUP': 'no' + } + + scripts_folder = "pmm_psmdb_diffauth_setup" + + shell_scripts = ['test-auth.sh'] + mongo_ssl_setup(shell_scripts[0], args) + + # Execute shell scripts + if not shell_scripts == []: + execute_shell_scripts(shell_scripts, scripts_folder, env_vars, args) + + def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -563,6 +728,12 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_external(db_type, db_version, db_config, args) elif db_type == 'DOCKERCLIENTS': setup_dockerclients(db_type, db_version, db_config, args) + elif db_type == 'SSL_MYSQL': + setup_ssl_mysql(db_type, db_version, db_config, args) + elif db_type == 'SSL_PDPGSQL': + setup_ssl_pdpgsql(db_type, db_version, db_config, args) + elif db_type == 'SSL_PSMDB': + setup_ssl_psmdb(db_type, db_version, db_config, args) else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) @@ -632,4 +803,4 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): except Exception as e: print("An unexpected error occurred:", e) parser.print_help() - exit(1) \ No newline at end of file + exit(1) diff --git a/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh index 388f176..d31899f 100644 --- a/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh +++ b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh @@ -24,7 +24,7 @@ sleep 10 if [ "$mysql_version" == "8.0" ]; then percona-release setup ps80 sleep 10 - DEBIAN_FRONTEND=noninteractive apt-get -y install percona-server-server sysbench sysbench-tpcc bc screen + DEBIAN_FRONTEND=noninteractive apt-get -y install percona-server-server sysbench bc screen cat > /etc/mysql/my.cnf << EOF [mysqld] innodb_buffer_pool_size=256M diff --git a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml index 66ce332..2f19f18 100644 --- a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml @@ -39,13 +39,14 @@ shell: > docker run -d --name={{ mysql_ssl_container }} --network "{{ mysql_ssl_container }}_network" - phusion/baseimage:focal-1.1.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker mysql_ssl_container shell: "{{ item }}" with_items: - docker exec {{ mysql_ssl_container }} mkdir -p artifacts - docker cp ./mysql/mysql_ssl_setup.sh {{ mysql_ssl_container }}:/ + - docker cp ../pmm3-client-setup.sh {{ mysql_ssl_container }}:/ - name: Execute Setup script inside the mysql mysql_ssl_container shell: "{{ item }}" @@ -55,9 +56,8 @@ - name: Install pmm2-client on the mysql_ssl_container shell: "{{ item }}" with_items: - - docker exec {{ mysql_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh - docker network connect pmm-qa {{ mysql_ssl_container }} - - docker exec {{ mysql_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + - docker exec {{ mysql_ssl_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no - name: Add pmm-admin binary to path when tar ball installation shell: docker exec {{ mysql_ssl_container }} echo "export PATH=$PATH:/pmm2-client/bin" > setup_path.sh @@ -69,11 +69,15 @@ - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove mysql {{ mysql_ssl_container }}_service' ignore_errors: true + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" + - name: Add mysql_ssl for monitoring shell: "{{ item }}" with_items: - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' - - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mysql --username=pmm --password=pmm --query-source=perfschema --tls --tls-skip-verify --tls-ca=/var/lib/mysql/ca.pem --tls-cert=/var/lib/mysql/client-cert.pem --tls-key=/var/lib/mysql/client-key.pem {{ mysql_ssl_container }}_ssl_service' + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mysql --username=pmm --password=pmm --query-source=perfschema --tls --tls-skip-verify --tls-ca=/var/lib/mysql/ca.pem --tls-cert=/var/lib/mysql/client-cert.pem --tls-key=/var/lib/mysql/client-key.pem {{ mysql_ssl_container }}_ssl_service_{{ random_number }}' - name: Get client cert Files on host shell: "{{ item }}" diff --git a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml index 712496e..c689f91 100644 --- a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml @@ -39,7 +39,7 @@ shell: > docker run -d --name={{ pgsql_ssl_container }} --network "{{ pgsql_ssl_container }}_network" - phusion/baseimage:focal-1.1.0 + phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pgsql_ssl_container shell: "{{ item }}" @@ -48,6 +48,7 @@ - docker cp ./create_certs.sh {{ pgsql_ssl_container }}:/artifacts/ - docker cp ./postgres/setup_pgsql.sh {{ pgsql_ssl_container }}:/ - docker cp ./postgres/init.sql {{ pgsql_ssl_container }}:/ + - docker cp ../pmm3-client-setup.sh {{ pgsql_ssl_container }}:/ - name: Execute Setup script inside the pgsql pgsql_ssl_container shell: "{{ item }}" @@ -57,21 +58,24 @@ - name: Install pmm2-client on the pgsql_ssl_container shell: "{{ item }}" with_items: - - docker exec {{ pgsql_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh - docker network connect pmm-qa {{ pgsql_ssl_container }} - - docker exec {{ pgsql_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + - docker exec {{ pgsql_ssl_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no - name: Remove pgsql service if already added previously shell: "{{ item }}" with_items: - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove postgresql {{ pgsql_ssl_container }}_ssl_service' - ignore_errors: true + ignore_errors: true + + - name: Set Random Number Fact + set_fact: + random_number: "{{ (10000 | random) | int }}" - name: Add pgsql_ssl for monitoring shell: "{{ item }}" with_items: - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' - - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source="pgstatements" --tls --tls-ca-file=./certificates/ca.crt --tls-cert-file=./certificates/client.crt --tls-key-file=./certificates/client.pem {{ pgsql_ssl_container }}_ssl_service' + - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add postgresql --username=pmm --password=pmm --query-source="pgstatements" --tls --tls-ca-file=./certificates/ca.crt --tls-cert-file=./certificates/client.crt --tls-key-file=./certificates/client.pem {{ pgsql_ssl_container }}_ssl_service{{ random_number }}' # - docker exec {{ pgsql_ssl_container }} bash -c 'source ~/.bash_profile; pmm-admin add postgresql --socket=/var/run/postgresql postgresql_socket' - name: Get client cert Files on host diff --git a/pmm_qa/tls-ssl-setup/setup_mysql b/pmm_qa/tls-ssl-setup/setup_mysql new file mode 100644 index 0000000..66ce332 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/setup_mysql @@ -0,0 +1,84 @@ +--- +# This playbook does following: +# enables Percona testing repository +# Install Percona Server at Version 8.0.25 +# Install all required tools for backups in compatible version + +- hosts: all + become: true + become_method: sudo + vars: + mysql_version: "{{ lookup('vars', 'extra_mysql_version', default=lookup('env','MYSQL_VERSION') | default('8.0', true) ) }}" + mysql_ssl_container: "{{ lookup('vars', 'extra_mysql_ssl_container', default=lookup('env','MYSQL_SSL_CONTAINER') | default('mysql_ssl', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: Cleanup Docker container for client and DB setup + shell: > + docker ps -a --filter "name={{ mysql_ssl_container }}" | grep -q . && docker stop {{ mysql_ssl_container }} && docker rm -fv {{ mysql_ssl_container }} + ignore_errors: true + tags: + - cleanup + - name: delete network if exist + shell: docker network rm "{{ mysql_ssl_container }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ mysql_ssl_container }}_network" + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for mysql ssl container + shell: > + docker run -d --name={{ mysql_ssl_container }} + --network "{{ mysql_ssl_container }}_network" + phusion/baseimage:focal-1.1.0 + + - name: Copy all required Artifacts to the docker mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} mkdir -p artifacts + - docker cp ./mysql/mysql_ssl_setup.sh {{ mysql_ssl_container }}:/ + + - name: Execute Setup script inside the mysql mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -xe ./mysql_ssl_setup.sh --mysql_version {{ mysql_version }} > mysql/setup_mysql_ssl_{{ mysql_version }}.log + + - name: Install pmm2-client on the mysql_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} wget https://raw.githubusercontent.com/percona/pmm-qa/{{ pmm_qa_branch }}/pmm-tests/pmm2-client-setup.sh + - docker network connect pmm-qa {{ mysql_ssl_container }} + - docker exec {{ mysql_ssl_container }} bash -x ./pmm2-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Add pmm-admin binary to path when tar ball installation + shell: docker exec {{ mysql_ssl_container }} echo "export PATH=$PATH:/pmm2-client/bin" > setup_path.sh + when: '"http" in client_version' + + - name: Remove mysql service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove mysql {{ mysql_ssl_container }}_service' + ignore_errors: true + + - name: Add mysql_ssl for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ mysql_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mysql --username=pmm --password=pmm --query-source=perfschema --tls --tls-skip-verify --tls-ca=/var/lib/mysql/ca.pem --tls-cert=/var/lib/mysql/client-cert.pem --tls-key=/var/lib/mysql/client-key.pem {{ mysql_ssl_container }}_ssl_service' + + - name: Get client cert Files on host + shell: "{{ item }}" + with_items: + - mkdir -p mysql/{{ mysql_version }} || true + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/ca.pem > mysql/{{ mysql_version }}/ca.pem + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/client-key.pem > mysql/{{ mysql_version }}/client-key.pem + - docker exec {{ mysql_ssl_container }} cat /var/lib/mysql/client-cert.pem > mysql/{{ mysql_version }}/client-cert.pem From 99f91018263653204e93ca065d95111e0933c1f8 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:31:43 +0530 Subject: [PATCH 50/88] Updates for Help Options (#57) --- pmm_qa/pmm-framework.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 8eb9ac8..a9ce849 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -67,6 +67,7 @@ "configurations": {"CLIENT_VERSION": "3-dev-latest"} }, "DOCKERCLIENTS": { + "configurations": {} # Empty dictionary for consistency }, } @@ -741,14 +742,20 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): # Main if __name__ == "__main__": - parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases') + parser = argparse.ArgumentParser(description='PMM Framework Script to setup Multiple Databases', + usage=argparse.SUPPRESS) + # Add subparsers for database types + subparsers = parser.add_subparsers(dest='database_type', help='Choose database type above') + + # Add subparser for each database type dynamically + for db_type, options in database_configs.items(): + db_parser = subparsers.add_parser(db_type.lower()) + for config, value in options['configurations'].items(): + db_parser.add_argument(f'{config}',metavar='', help=f'{config} for {db_type} (default: {value})') + # Add arguments parser.add_argument("--database", action='append', nargs=1, - metavar='db_name[,=version][,option1=value1,option2=value2,...]', - help="(e.g: " - "--database mysql=5.7,QUERY_SOURCE=perfschema,SETUP_TYPE=gr,CLIENT_VERSION=3-dev-latest " - "--database pdpgsql=16,USE_SOCKET=1,CLIENT_VERSION=3.0.0 " - "--database psmdb=latest,SETUP_TYPE=psa,CLIENT_VERSION=3.0.0)") + metavar='db_name[,=version][,option1=value1,option2=value2,...]') parser.add_argument("--pmm-server-ip", nargs='?', help='PMM Server IP to connect') parser.add_argument("--pmm-server-password", nargs='?', help='PMM Server password') parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') From 87b6aadf7e3e4d33c1a3f1daa36a6c0220ebd41c Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 8 Jul 2024 09:59:07 +0530 Subject: [PATCH 51/88] Updates for pmm3-client repo changes v3 (#69) * Updates for pmm3-client repo changes v3 --- pmm_psmdb-pbm_setup/Dockerfile | 4 ++-- pmm_qa/pmm3-client-setup.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 244f1fd..35c1550 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -34,7 +34,7 @@ RUN set -ex; \ rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY; \ percona-release enable pbm ${REPO} && \ percona-release enable ${PSMDB_REPO} ${REPO} && \ - percona-release enable original ${PMM_REPO} && \ + percona-release enable pmm3-client ${PMM_REPO} && \ yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ yum update --refresh -y && \ if [ $PSMDB_VERSION == "latest" ]; then \ @@ -62,7 +62,7 @@ RUN if [[ "$PMM_CLIENT_VERSION" == http* ]]; then \ elif [[ "$PMM_CLIENT_VERSION" =~ 3-dev-latest|latest ]]; then \ yum -y install pmm-client ; \ elif [[ "$PMM_CLIENT_VERSION" = "pmm3-rc" ]]; then \ - percona-release enable original testing && \ + percona-release enable pmm3-client testing && \ yum update --refresh -y && \ yum -y install pmm-client ; \ else \ diff --git a/pmm_qa/pmm3-client-setup.sh b/pmm_qa/pmm3-client-setup.sh index 9060a1f..abc906a 100755 --- a/pmm_qa/pmm3-client-setup.sh +++ b/pmm_qa/pmm3-client-setup.sh @@ -54,13 +54,13 @@ export PMM_AGENT_SETUP_NODE_NAME=client_container_$(echo $((1 + $RANDOM % 9999)) mv -v /artifacts/* . if [[ "$client_version" == "3-dev-latest" ]]; then - percona-release enable-only original experimental + percona-release enable-only pmm3-client experimental apt-get update apt-get -y install pmm-client fi if [[ "$client_version" == "pmm3-rc" ]]; then - percona-release enable-only original testing + percona-release enable-only pmm3-client testing apt-get update apt-get -y install pmm-client fi @@ -68,7 +68,7 @@ fi if [[ "$client_version" == "pmm3-latest" ]]; then apt-get -y install pmm-client apt-get -y update - percona-release enable-only original experimental + percona-release enable-only pmm3-client experimental fi ## Only supported for debian based systems for now @@ -120,8 +120,8 @@ else pid=`ps -ef | grep pmm-agent | grep -v grep | awk -F ' ' '{print $2}'` if [[ ! -z "$pid" ]]; then kill -9 $pid - echo "killed old agent, restarting agent...." - pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > pmm-agent.log 2>&1 & + echo "Killing and restarting pmm agent...." + pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml >> pmm-agent.log 2>&1 & sleep 10 fi fi From 12ab6713795b061e260fce4f59d91ae0d4eef665 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 12 Jul 2024 15:22:30 +0530 Subject: [PATCH 52/88] Updates for Dockerfile for changed keys(v3) (#70) * Updates for Dockerfile for changed keys --- pmm_psmdb-pbm_setup/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 35c1550..0e54e18 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -21,12 +21,13 @@ CMD ["/usr/sbin/init"] RUN set -ex; \ export GNUPGHOME="$(mktemp -d)"; \ export PERCONA_TELEMETRY_URL="https://check-dev.percona.com/v1/telemetry/GenericReport"; \ + yum install -y gnupg2; \ if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-60 ; fi ; \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 99DB70FAE1D7CE227FB6488205B555B38483C65D 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1; \ - gpg --batch --export --armor 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A > ${GNUPGHOME}/RPM-GPG-KEY-Percona; \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1; \ + gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \ gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ gpg --batch --export --armor 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 > ${GNUPGHOME}/RPM-GPG-KEY-EPEL-9; \ - rpmkeys --import ${GNUPGHOME}/RPM-GPG-KEY-Percona ${GNUPGHOME}/RPM-GPG-KEY-centosofficial ${GNUPGHOME}/RPM-GPG-KEY-EPEL-9; \ + rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-centosofficial ${GNUPGHOME}/RPM-GPG-KEY-EPEL-9; \ curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \ rpmkeys --checksig /tmp/percona-release.rpm; \ rpm -i /tmp/percona-release.rpm; \ From cd45f4c25bdb8f236c9ab7adf9e4455b2a50c478 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:04:22 +0530 Subject: [PATCH 53/88] Updates for release repo ssl dependencies (#68) --- pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh | 2 +- pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh | 2 +- pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh index fe35024..76da342 100644 --- a/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh +++ b/pmm_qa/tls-ssl-setup/mongodb/mongodb_ssl_setup.sh @@ -13,7 +13,7 @@ if [ -z "$mongodb_version" ]; then fi apt-get update -apt-get -y install wget curl git jq +apt-get -y install wget curl git jq gnupg2 lsb-release wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb dpkg -i percona-release_latest.generic_all.deb wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh diff --git a/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh index d31899f..7e9278c 100644 --- a/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh +++ b/pmm_qa/tls-ssl-setup/mysql/mysql_ssl_setup.sh @@ -17,7 +17,7 @@ then fi apt-get update -apt-get -y install wget curl git +apt-get -y install wget curl git gnupg2 lsb-release wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb dpkg -i percona-release_latest.generic_all.deb sleep 10 diff --git a/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh b/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh index 3bd841e..5c2d70f 100644 --- a/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh +++ b/pmm_qa/tls-ssl-setup/postgres/setup_pgsql.sh @@ -17,7 +17,7 @@ then fi apt-get update -apt-get -y install wget curl git +apt-get -y install wget curl git gnupg2 lsb-release wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb dpkg -i percona-release_latest.generic_all.deb percona-release setup ppg${pgsql_version} From f74b18ab51db96bd02000197a3820096543a281c Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 17 Jul 2024 22:47:44 +0530 Subject: [PATCH 54/88] PMM3 psmdb agent ports fix (#72) * Updates for PSMDB SSL setup port fix --- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 4 +++- pmm_psmdb_diffauth_setup/test-auth.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 3b8d7f1..aad57ae 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -35,6 +35,8 @@ services: - PMM_AGENT_SERVER_USERNAME=admin - PMM_AGENT_SERVER_PASSWORD=admin - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 + - PMM_AGENT_PORTS_MIN=43000 + - PMM_AGENT_PORTS_MAX=43020 entrypoint: - bash - -c @@ -48,7 +50,7 @@ services: - 'psmdb-server-data:/data/db' privileged: true ports: - - "42000-42020:42000-42020" + - "43000-43020:43000-43020" healthcheck: test: | test $$(mongo --quiet --eval "try { rs.initiate().ok } catch (_) {} rs.status().ok") -eq 1 diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index a8933a5..a758ef9 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -37,6 +37,7 @@ cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; #Start setup +docker-compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans docker-compose -f docker-compose-pmm-psmdb.yml build docker-compose -f docker-compose-pmm-psmdb.yml up -d From 12f00788f99bcbc58f263062b8ecbf37e71aabef Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 31 Jul 2024 08:01:38 +0530 Subject: [PATCH 55/88] PMM-13240 PMM3 integration setup fixs (#73) * Updates for PMM v3 server and client ports --- .github/workflows/PMM_PDPGSQL.yaml | 114 ++++++++++----------- .github/workflows/PMM_PROXYSQL.yaml | 99 ------------------- .github/workflows/PMM_PS.yaml | 98 ++++++++++++------- .github/workflows/PMM_PSMDB_PBM.yml | 2 +- .github/workflows/PMM_PXC.yaml | 99 ------------------- .github/workflows/PMM_PXC_PROXYSQL.yaml | 125 ++++++++++++++++++++++++ 6 files changed, 245 insertions(+), 292 deletions(-) delete mode 100644 .github/workflows/PMM_PROXYSQL.yaml delete mode 100644 .github/workflows/PMM_PXC.yaml create mode 100644 .github/workflows/PMM_PXC_PROXYSQL.yaml diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index 2499a8b..e457459 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -3,86 +3,88 @@ on: workflow_dispatch: inputs: pdpgsql_version: - description: "PDPGSQL Docker hub example perconalab/percona-distribution-postgresql:16.0 latest available Image" + description: "PDPGSQL Version available Image versions on github, example: 14, 15, 16" required: true - pmm_image: - description: "pmm_image, example: perconalab/pmm-server:dev-latest" + default: '16' + type: string + pmm_qa_branch: + description: "Branch for qa-integration to checkout" required: false - repo: - description: "Percona Release Repo defaults to Experiemental example: experimental" + type: string + pmm_ui_tests_branch: + description: "Branch for PMM-UI tests to checkout" required: false + type: string + pmm_server_image: + description: "pmm_image, example: perconalab/pmm-server:3-dev-latest" + default: 'perconalab/pmm-server:3-dev-latest' + required: false + type: string + pmm_client_version: + description: 'PMM Client version (3-dev-latest|pmm3-rc|x.xx.x|https...)' + default: '3-dev-latest' + required: false + type: string push: branches: - - main + - v3 pull_request: branches: - - main + - v3 jobs: PMM_PDPGSQL_TEST: runs-on: ubuntu-latest timeout-minutes: 20 env: - PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || 'perconalab/percona-distribution-postgresql:16.0' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} - REPO: ${{ github.event.inputs.repo || 'experimental' }} - + ADMIN_PASSWORD: 'admin' + PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '16' }} + PMM_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} + PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} + PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} + PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_client_version || '3-dev-latest' }} + steps: - uses: actions/checkout@v2 - - name: Concatenate values to environment file - run: | - echo "PMM_QA_REPO_URL=https://github.com/percona/pmm-qa/" >> $GITHUB_ENV - echo "PMM_QA_REPO_BRANCH=main" >> $GITHUB_ENV - - - name: Install Ansible and update the node - run: sudo apt-get update -y && sudo apt-get install ansible -y - - - name: Install Percona Release Package - run: wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb && sudo dpkg -i percona-release_latest.generic_all.deb - - - name: Install Enable percona-release experimental for dev-latest installation of pmm2-client - if: ${{ inputs.repo == 'experimental' }} - run: sudo percona-release enable-only original experimental + - name: Run the PMM Server container + run: docker run --detach --restart always -p 443:8443 -p 8081:8080 --name pmm-server ${{ env.PMM_IMAGE }} - - name: Install Enable percona-release testing for rc installation of pmm2-client - if: ${{ inputs.repo == 'testing' }} - run: sudo percona-release enable-only original testing - - - name: Install PMM Client - run: sudo apt update -y && sudo apt-get upgrade -y && sudo apt-get install -y pmm2-client + - name: Checkout qa-integration repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_QA_BRANCH }} + repository: Percona-Lab/qa-integration + path: ./qa-integration - - name: Delete if the Repo already checked out - run: sudo rm -r pmm-qa || true + - name: Export path to qa-integration repo + working-directory: qa-integration + run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV - - name: Clone the PMM_QA repo - run: git clone -b ${{ env.PMM_QA_REPO_BRANCH }} ${{ env.PMM_QA_REPO_URL }} - - - name: Give write perimssion to the bash script - run: chmod +x ./pmm-tests/pmm-framework.sh - working-directory: pmm-qa - - - name: Run the pmm container in backgrounds - run: docker run --detach --restart always -p 443:443 -p 80:80 --name pmm-server ${{ env.PMM_IMAGE }} - - - name: Wait before pmm-admin config command - run: sleep 120 - - - name: setup pmm-admin config - run: sudo pmm-admin config --server-insecure-tls --server-url=https://admin:admin@localhost:443 + - name: Run Setup for E2E Tests + working-directory: qa-integration/pmm_qa + run: | + mkdir -m 777 -p /tmp/backup_data + python3 -m venv virtenv + . virtenv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }} - - name: Run for PDPGSQL tests - run: sudo ./pmm-framework.sh --pdpgsql-version=16 --pmm2 --download --addclient=pdpgsql,1 - working-directory: pmm-qa/pmm-tests - - - name: Checkout the repo and install node packages - run: git clone https://github.com/percona/pmm-ui-tests.git && cd pmm-ui-tests && npm ci + - name: Checkout PMM UI tests + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_UI_BRANCH }} + repository: percona/pmm-ui-tests + path: ./pmm-ui-tests - name: Install npx dependencies - run: sudo npx playwright install-deps && npx codeceptjs def pr.codecept.js + run: npm ci && npx playwright install --with-deps && npx codeceptjs def pr.codecept.js working-directory: pmm-ui-tests - name: Run the Integration tests of pdpgsql - run: npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pdpgsql_integration_test.js --steps --debug working-directory: pmm-ui-tests + run: | + export PMM_UI_URL="http://127.0.0.1:8081/" + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pdpgsql_integration_test.js --steps --debug diff --git a/.github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml deleted file mode 100644 index f2be9a7..0000000 --- a/.github/workflows/PMM_PROXYSQL.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: PMM_PXC_PROXYSQL -on: - workflow_dispatch: - inputs: - pxc_version: - description: "pxc version, example: 8.0.33-25 Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS by default" - required: false - pxc_glibc: - description: "pxc glibc version, example: 2.35" - required: true - pmm_image: - description: "pmm_image, example: perconalab/pmm-server:dev-latest" - required: false - push: - branches: - - main - - pull_request: - branches: - - main - -jobs: - PMM_PXC_TEST: - runs-on: ubuntu-latest - timeout-minutes: 20 - env: - PXC_GLIBC: ${{ github.event.inputs.pxc_glibc || '2.35' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} - steps: - - uses: actions/checkout@v2 - - - name: PXC_VERSION ENV Setup - run: | - if [[ -n "${{ github.event.inputs.pxc_version }}" ]]; then - PXC_VERSION="${{ github.event.inputs.pxc_version }}" - PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION") - PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) - else - wget https://raw.githubusercontent.com/Percona-QA/package-testing/master/VERSIONS - PXC_VERSION="$(grep 'PXC80_VER' VERSIONS | cut -d = -f 2)" - PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION.1") - PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) - fi - echo "PXC_VERSION=${PXC_VERSION}" | sed 's/"//g' >> $GITHUB_ENV - echo "PXC_VERSION_TAR=${PXC_VERSION}.1" | sed 's/"//g' >> $GITHUB_ENV - echo "PXC_VERSION_PATH=${PXC_VERSION_PATH}" >> $GITHUB_ENV - echo "PXC_VERSION_TRIMED=${PXC_VERSION_TRIMED}" >> $GITHUB_ENV - - - name: Concatenate values to environment file - run: | - echo "PXC_TARBALL_PATH=https://downloads.percona.com/downloads/TESTING/${{env.PXC_VERSION_PATH}}/Percona-XtraDB-Cluster_${{env.PXC_VERSION_TAR}}_Linux.x86_64.glibc${{env.PXC_GLIBC}}.tar.gz" >> $GITHUB_ENV - echo "PMM_QA_REPO_URL=https://github.com/percona/pmm-qa/" >> $GITHUB_ENV - echo "PMM_QA_REPO_BRANCH=main" >> $GITHUB_ENV - - - name: Install Ansible and update the node - run: sudo apt-get update -y && sudo apt-get install ansible -y - - - name: Install Percona Release Package - run: wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb && sudo dpkg -i percona-release_latest.generic_all.deb - - - name: Install Enable percona-release experimental for dev-latest installation of pmm2-client - run: sudo percona-release enable-only original experimental - - - name: Install PMM Client - run: sudo apt update -y && sudo apt-get upgrade -y && sudo apt-get install -y pmm2-client - - - name: Delete if the Repo already checked out - run: sudo rm -r pmm-qa || true - - - name: Clone the PMM_QA repo - run: git clone -b ${{ env.PMM_QA_REPO_BRANCH }} ${{ env.PMM_QA_REPO_URL }} - - - name: Give write perimssion to the bash script - run: chmod +x ./pmm-tests/pmm-framework.sh - working-directory: pmm-qa - - - name: Run the pmm container in backgrounds - run: docker run --detach --restart always -p 443:443 -p 80:80 --name pmm-server ${{ env.PMM_IMAGE }} - - - name: Wait before pmm-admin config command - run: sleep 120 - - - name: setup pmm-admin config - run: sudo pmm-admin config --server-insecure-tls --server-url=https://admin:admin@localhost:443 - - - name: Run for PXC tests - run: ./pmm-framework.sh --with-proxysql --pxc-version ${{ env.PXC_VERSION }} --addclient=pxc,1 --pmm2 --pxc-tarball ${{ env.PXC_TARBALL_PATH }} - working-directory: pmm-qa/pmm-tests - - - name: Checkout the repo and install node packages - run: git clone https://github.com/percona/pmm-ui-tests.git && cd pmm-ui-tests && npm ci - - - name: Install npx dependencies - run: cd pmm-ui-tests && sudo npx playwright install-deps && npx codeceptjs def pr.codecept.js - - - name: Run the Integration tests of PXC - run: npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pxc_integration_test.js --steps --debug - working-directory: pmm-ui-tests - diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index 3c7525d..c56fc3b 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -3,14 +3,30 @@ on: workflow_dispatch: inputs: ps_version: - description: "ps version example: 8.0.34-26 , Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS by default" + description: "ps version example: 8.0.34-26 , Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS" + default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS' required: true ps_glibc: description: "ps glibc version, example: 2.28" + default: '2.28' required: false - pmm_image: - description: "pmm_image, example: perconalab/pmm-server:dev-latest" + pmm_qa_branch: + description: "Branch for qa-integration to checkout" required: false + type: string + pmm_ui_tests_branch: + description: "Branch for PMM-UI tests to checkout" + required: false + type: string + pmm_server_image: + description: "pmm_image, example: perconalab/pmm-server:3-dev-latest" + default: 'perconalab/pmm-server:3-dev-latest' + required: false + pmm_client_version: + description: 'PMM Client version (3-dev-latest|pmm3-rc|x.xx.x|https...)' + default: '3-dev-latest' + required: false + type: string push: branches: - main @@ -24,15 +40,21 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 env: + ADMIN_PASSWORD: 'admin' + PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }} PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.28' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} + PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} + PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} + PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_client_version || '3-dev-latest' }} + steps: - uses: actions/checkout@v2 - name: PS_VERSION ENV Setup run: | - if [[ -n "${{ github.event.inputs.ps_version }}" ]]; then + if [[ "${{ github.event.inputs.ps_version }}" != http* ]]; then PS_VERSION="${{ github.event.inputs.ps_version }}" PS_VERSION_PATH=$(echo "ps-$PS_VERSION") PS_VERSION_TRIMED=$(echo ${PS_VERSION%-*}) @@ -56,45 +78,47 @@ jobs: - name: Install Ansible and update the node run: sudo apt-get update -y && sudo apt-get install ansible -y - - name: Install Percona Release Package - run: wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb && sudo dpkg -i percona-release_latest.generic_all.deb - - - name: Install Enable percona-release experimental for dev-latest installation of pmm2-client - run: sudo percona-release enable-only original experimental - - - name: Install PMM Client - run: sudo apt update -y && sudo apt-get upgrade -y && sudo apt-get install -y pmm2-client - - - name: Delete if the Repo already checked out - run: sudo rm -r pmm-qa || true + - name: Run the PMM Server container + run: docker run --detach --restart always -p 443:8443 -p 8081:8080 --name pmm-server ${{ env.PMM_SERVER_IMAGE }} - - name: Clone the PMM_QA repo - run: git clone -b ${{ env.PMM_QA_REPO_BRANCH }} ${{ env.PMM_QA_REPO_URL }} - - - name: Give write perimssion to the bash script - run: chmod +x ./pmm-tests/pmm-framework.sh - working-directory: pmm-qa + - name: Checkout qa-integration repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_QA_BRANCH }} + repository: Percona-Lab/qa-integration + path: ./qa-integration - - name: Run the pmm container in backgrounds - run: docker run --detach --restart always -p 443:443 -p 80:80 --name pmm-server ${{ env.PMM_IMAGE }} + - name: Export path to qa-integration repo + working-directory: qa-integration + run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV - - name: Wait before pmm-admin config command - run: sleep 120 - - - name: setup pmm-admin config - run: sudo pmm-admin config --server-insecure-tls --server-url=https://admin:admin@localhost:443 + - name: Run Setup for E2E Tests + working-directory: qa-integration/pmm_qa + run: | + mkdir -m 777 -p /tmp/backup_data + python3 -m venv virtenv + . virtenv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + if [[ "${{ github.event.inputs.ps_version }}" != http* ]]; then + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS=${{ env.PS_VERSION }} + else + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS,TARBALL=${{ env.PS_TARBALL_PATH }} + fi - - name: Run for PS tests - run: ./pmm-framework.sh --ps-version ${{ env.PS_VERSION }} --setup-pmm-ps-integration --pmm2 --ps-tarball ${{ env.PS_TARBALL_PATH }} --query-source=slowlog - working-directory: pmm-qa/pmm-tests - - - name: Checkout the repo and install node packages - run: git clone https://github.com/percona/pmm-ui-tests.git && cd pmm-ui-tests && npm ci + - name: Checkout PMM UI tests + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_UI_BRANCH }} + repository: percona/pmm-ui-tests + path: ./pmm-ui-tests - name: Install npx dependencies - run: sudo npx playwright install-deps && npx codeceptjs def pr.codecept.js + run: npm ci && npx playwright install --with-deps && npx codeceptjs def pr.codecept.js working-directory: pmm-ui-tests - name: Run the Integration tests of PS - run: npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js --steps --debug + run: | + export PMM_UI_URL="http://127.0.0.1:8081/" + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js --steps --debug working-directory: pmm-ui-tests diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 8c2adf7..94c0305 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -23,7 +23,7 @@ on: push: branches: - main - - -v3 + - v3 pull_request: branches: diff --git a/.github/workflows/PMM_PXC.yaml b/.github/workflows/PMM_PXC.yaml deleted file mode 100644 index da11259..0000000 --- a/.github/workflows/PMM_PXC.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: PMM_PXC -on: - workflow_dispatch: - inputs: - pxc_version: - description: "pxc version, example: 8.0.33-25 Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS by default" - required: false - pxc_glibc: - description: "pxc glibc version, example: 2.35" - required: true - pmm_image: - description: "pmm_image, example: perconalab/pmm-server:dev-latest" - required: false - push: - branches: - - main - - pull_request: - branches: - - main - -jobs: - PMM_PXC_TEST: - runs-on: ubuntu-latest - timeout-minutes: 20 - env: - PXC_GLIBC: ${{ github.event.inputs.pxc_glibc || '2.35' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} - steps: - - uses: actions/checkout@v2 - - - name: PXC_VERSION ENV Setup - run: | - if [[ -n "${{ github.event.inputs.pxc_version }}" ]]; then - PXC_VERSION="${{ github.event.inputs.pxc_version }}" - PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION") - PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) - else - wget https://raw.githubusercontent.com/Percona-QA/package-testing/master/VERSIONS - PXC_VERSION="$(grep 'PXC80_VER' VERSIONS | cut -d = -f 2)" - PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION.1") - PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) - fi - echo "PXC_VERSION=${PXC_VERSION}" | sed 's/"//g' >> $GITHUB_ENV - echo "PXC_VERSION_TAR=${PXC_VERSION}.1" | sed 's/"//g' >> $GITHUB_ENV - echo "PXC_VERSION_PATH=${PXC_VERSION_PATH}" >> $GITHUB_ENV - echo "PXC_VERSION_TRIMED=${PXC_VERSION_TRIMED}" >> $GITHUB_ENV - - - name: Concatenate values to environment file - run: | - echo "PXC_TARBALL_PATH=https://downloads.percona.com/downloads/TESTING/${{env.PXC_VERSION_PATH}}/Percona-XtraDB-Cluster_${{env.PXC_VERSION_TAR}}_Linux.x86_64.glibc${{env.PXC_GLIBC}}.tar.gz" >> $GITHUB_ENV - echo "PMM_QA_REPO_URL=https://github.com/percona/pmm-qa/" >> $GITHUB_ENV - echo "PMM_QA_REPO_BRANCH=main" >> $GITHUB_ENV - - - name: Install Ansible and update the node - run: sudo apt-get update -y && sudo apt-get install ansible -y - - - name: Install Percona Release Package - run: wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb && sudo dpkg -i percona-release_latest.generic_all.deb - - - name: Install Enable percona-release experimental for dev-latest installation of pmm2-client - run: sudo percona-release enable-only original experimental - - - name: Install PMM Client - run: sudo apt update -y && sudo apt-get upgrade -y && sudo apt-get install -y pmm2-client - - - name: Delete if the Repo already checked out - run: sudo rm -r pmm-qa || true - - - name: Clone the PMM_QA repo - run: git clone -b ${{ env.PMM_QA_REPO_BRANCH }} ${{ env.PMM_QA_REPO_URL }} - - - name: Give write perimssion to the bash script - run: chmod +x ./pmm-tests/pmm-framework.sh - working-directory: pmm-qa - - - name: Run the pmm container in backgrounds - run: docker run --detach --restart always -p 443:443 -p 80:80 --name pmm-server ${{ env.PMM_IMAGE }} - - - name: Wait before pmm-admin config command - run: sleep 120 - - - name: setup pmm-admin config - run: sudo pmm-admin config --server-insecure-tls --server-url=https://admin:admin@localhost:443 - - - name: Run for PXC tests - run: ./pmm-framework.sh --pxc-version ${{ env.PXC_VERSION }} --addclient=pxc,1 --pmm2 --pxc-tarball ${{ env.PXC_TARBALL_PATH }} - working-directory: pmm-qa/pmm-tests - - - name: Checkout the repo and install node packages - run: git clone https://github.com/percona/pmm-ui-tests.git && cd pmm-ui-tests && npm ci - - - name: Install npx dependencies - run: cd pmm-ui-tests && sudo npx playwright install-deps && npx codeceptjs def pr.codecept.js - - - name: Run the Integration tests of PXC - run: npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pxc_integration_test.js --steps --debug - working-directory: pmm-ui-tests - diff --git a/.github/workflows/PMM_PXC_PROXYSQL.yaml b/.github/workflows/PMM_PXC_PROXYSQL.yaml new file mode 100644 index 0000000..11bd72d --- /dev/null +++ b/.github/workflows/PMM_PXC_PROXYSQL.yaml @@ -0,0 +1,125 @@ +name: PMM_PXC +on: + workflow_dispatch: + inputs: + pxc_version: + description: "pxc version, example: 8.0.33-25 Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS" + default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS' + required: true + type: string + pxc_glibc: + description: "pxc glibc version, example: 2.35" + default: '2.35' + required: true + pmm_qa_branch: + description: "Branch for qa-integration to checkout" + required: false + type: string + pmm_ui_tests_branch: + description: "Branch for PMM-UI tests to checkout" + required: false + type: string + pmm_server_image: + description: "pmm_image, example: perconalab/pmm-server:3-dev-latest" + default: 'perconalab/pmm-server:3-dev-latest' + required: false + type: string + pmm_client_version: + description: 'PMM Client version (3-dev-latest|pmm3-rc|x.xx.x|https...)' + default: '3-dev-latest' + required: false + type: string + push: + branches: + - v3 + + pull_request: + branches: + - v3 + +jobs: + PMM_PXC_TEST: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ADMIN_PASSWORD: 'admin' + PXC_VERSION: ${{ github.event.inputs.pxc_version || '8.0' }} + PXC_GLIBC: ${{ github.event.inputs.pxc_glibc || '2.35' }} + PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} + PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} + PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} + PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_client_version || '3-dev-latest' }} + + steps: + - uses: actions/checkout@v2 + + - name: PXC_VERSION ENV Setup + run: | + if [[ "${{ github.event.inputs.pxc_version }}" != http* ]]; then + PXC_VERSION="${{ github.event.inputs.pxc_version }}" + PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION") + PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) + else + wget https://raw.githubusercontent.com/Percona-QA/package-testing/master/VERSIONS + PXC_VERSION="$(grep 'PXC80_VER' VERSIONS | cut -d = -f 2)" + PXC_VERSION_PATH=$(echo "pxc-$PXC_VERSION.1") + PXC_VERSION_TRIMED=$(echo ${PXC_VERSION%-*}) + fi + echo "PXC_VERSION=${PXC_VERSION}" | sed 's/"//g' >> $GITHUB_ENV + echo "PXC_VERSION_TAR=${PXC_VERSION}.1" | sed 's/"//g' >> $GITHUB_ENV + echo "PXC_VERSION_PATH=${PXC_VERSION_PATH}" >> $GITHUB_ENV + echo "PXC_VERSION_TRIMED=${PXC_VERSION_TRIMED}" >> $GITHUB_ENV + + - name: Concatenate values to environment file + run: | + echo "PXC_TARBALL_PATH=https://downloads.percona.com/downloads/TESTING/${{env.PXC_VERSION_PATH}}/Percona-XtraDB-Cluster_${{env.PXC_VERSION_TAR}}_Linux.x86_64.glibc${{env.PXC_GLIBC}}.tar.gz" >> $GITHUB_ENV + echo "PMM_QA_REPO_URL=https://github.com/percona/pmm-qa/" >> $GITHUB_ENV + + - name: Install Ansible and update the node + run: sudo apt-get update -y && sudo apt-get install ansible -y + + - name: Run the PMM Server container + run: docker run --detach --restart always -p 443:8443 -p 8081:8080 --name pmm-server ${{ env.PMM_SERVER_IMAGE }} + + - name: Checkout qa-integration repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_QA_BRANCH }} + repository: Percona-Lab/qa-integration + path: ./qa-integration + + - name: Export path to qa-integration repo + working-directory: qa-integration + run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV + + - name: Run Setup for E2E Tests + working-directory: qa-integration/pmm_qa + run: | + mkdir -m 777 -p /tmp/backup_data + python3 -m venv virtenv + . virtenv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + if [[ "${{ github.event.inputs.pxc_version }}" != http* ]]; then + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${{ env.PXC_VERSION }} + else + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC,TARBALL=${{ env.PXC_TARBALL_PATH }} + fi + + - name: Checkout PMM UI tests + uses: actions/checkout@v4 + with: + ref: ${{ env.PMM_UI_BRANCH }} + repository: percona/pmm-ui-tests + path: ./pmm-ui-tests + + - name: Install npx dependencies + run: npm ci && npx playwright install --with-deps && npx codeceptjs def pr.codecept.js + working-directory: pmm-ui-tests + + - name: Run the Integration tests of PXC + run: | + export PMM_UI_URL="http://127.0.0.1:8081/" + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pxc_integration_test.js --steps --debug + working-directory: pmm-ui-tests + From d30f2a4af94e5b50dcd701ee4c68f2ab431b5493 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:29:49 +0530 Subject: [PATCH 56/88] Update PMM_PXC_PROXYSQL.yaml (#75) Fix name on workflow --- .github/workflows/PMM_PXC_PROXYSQL.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PMM_PXC_PROXYSQL.yaml b/.github/workflows/PMM_PXC_PROXYSQL.yaml index 11bd72d..b456521 100644 --- a/.github/workflows/PMM_PXC_PROXYSQL.yaml +++ b/.github/workflows/PMM_PXC_PROXYSQL.yaml @@ -1,4 +1,4 @@ -name: PMM_PXC +name: PMM_PXC_PROXYSQL on: workflow_dispatch: inputs: From 6bde793bc114153084be1606e437c7bd41021c0d Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:29:18 +0530 Subject: [PATCH 57/88] Updates v3 changes for PMM-13331 from pmm2 (#79) --- .github/workflows/PMM_PSMDB_PBM.yml | 6 +-- pmm_psmdb-pbm_setup/configure-agents.sh | 16 +++--- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 12 ++--- pmm_psmdb-pbm_setup/configure-extra-psa.sh | 10 ++-- .../configure-extra-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-psa.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 6 +++ .../docker-compose-sharded.yaml | 10 ++++ pmm_psmdb-pbm_setup/start-rs-only.sh | 6 +-- pmm_psmdb-pbm_setup/start-rs.sh | 16 +++--- pmm_psmdb-pbm_setup/start-sharded.sh | 52 +++++++++---------- .../docker-compose-pmm-psmdb.yml | 1 + pmm_psmdb_diffauth_setup/test-auth.sh | 22 ++++---- 14 files changed, 102 insertions(+), 85 deletions(-) diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 94c0305..9dd6d25 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -32,7 +32,7 @@ on: jobs: test_replica_set: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} @@ -47,7 +47,7 @@ jobs: working-directory: ./pmm_psmdb-pbm_setup test_sharded_cluster: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} @@ -62,7 +62,7 @@ jobs: working-directory: ./pmm_psmdb-pbm_setup test_diff_auth: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 20 env: PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 97e2a56..2498b03 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -13,14 +13,14 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node" - docker-compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent + docker compose -f docker compose-rs.yaml exec -T rs103 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,17 +29,17 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo echo "adding some data" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF use students; db.students.insertMany([ { diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 1839ceb..10649ea 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -13,14 +13,14 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node rs203" - docker-compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent + docker compose -f docker compose-rs.yaml exec -T rs203 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,10 +29,10 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker-compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker-compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh index d667f6b..721029f 100755 --- a/pmm_psmdb-pbm_setup/configure-extra-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring extra PSA replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-extra-replset.sh b/pmm_psmdb-pbm_setup/configure-extra-replset.sh index cff511d..1f7e9c3 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-psa.sh b/pmm_psmdb-pbm_setup/configure-psa.sh index 879adb1..ce006e2 100755 --- a/pmm_psmdb-pbm_setup/configure-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring PSA replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-replset.sh b/pmm_psmdb-pbm_setup/configure-replset.sh index f50bc39..819ceeb 100644 --- a/pmm_psmdb-pbm_setup/configure-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 4b4451d..9da2b1a 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -35,6 +35,7 @@ services: - /tmp/backup_data:/tmp/backup_data:rw - /tmp/mongodb:/tmp privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -61,6 +62,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -87,6 +89,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -115,6 +118,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -141,6 +145,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} @@ -167,6 +172,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw privileged: true + cgroup: host environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 2949bad..b223ad9 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -23,6 +23,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/mongodb:/tmp privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -45,6 +46,7 @@ services: - ./conf/mongod-rs1:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -65,6 +67,7 @@ services: - ./conf/mongod-rs1:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -85,6 +88,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -105,6 +109,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -125,6 +130,7 @@ services: - ./conf/mongod-rs2:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -145,6 +151,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -165,6 +172,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -185,6 +193,7 @@ services: - ./conf/mongod-cfg:/etc/mongod - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true + cgroup: host environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - "PMM_AGENT_SERVER_ADDRESS=${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443}" @@ -206,6 +215,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - ./conf/datagen:/etc/datagen:ro privileged: true + cgroup: host command: "mongos --keyFile=/etc/keyfile --configdb rscfg/rscfg01:27017,rscfg02:27017,rscfg03:27017 --port 27017 --bind_ip 0.0.0.0" networks: - test-network diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index df21e30..23f2442 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -13,9 +13,9 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker-compose -f docker-compose-rs.yaml down -v --remove-orphans -docker-compose -f docker-compose-rs.yaml build --no-cache -docker-compose -f docker-compose-rs.yaml up -d +docker compose -f docker compose-rs.yaml down -v --remove-orphans +docker compose -f docker compose-rs.yaml build --no-cache +docker compose -f docker compose-rs.yaml up -d echo echo "waiting 60 seconds for replica set members to start" sleep 60 diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index cddf62f..c6d3fb4 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -14,16 +14,16 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans -docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml build -docker-compose -f docker-compose-pmm.yaml -f docker-compose-rs.yaml up -d +docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans +docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml build +docker compose -f docker compose-pmm.yaml -f docker compose-rs.yaml up -d echo echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker-compose -f docker-compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass +docker compose -f docker compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass echo "restarting pmm-server" -docker-compose -f docker-compose-pmm.yaml restart pmm-server +docker compose -f docker compose-pmm.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 if [ $mongo_setup_type == "pss" ]; then @@ -36,8 +36,8 @@ tests=${TESTS:-yes} if [ $tests != "no" ]; then echo echo "running tests" - docker-compose -f docker-compose-pmm.yaml run test pytest -s -x --verbose test.py - docker-compose -f docker-compose-pmm.yaml run test chmod -R 777 . + docker compose -f docker compose-pmm.yaml run test pytest -s -x --verbose test.py + docker compose -f docker compose-pmm.yaml run test chmod -R 777 . else echo echo "skipping tests" @@ -46,7 +46,7 @@ cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo echo "cleanup" - docker-compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans + docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans else echo echo "skipping cleanup" diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 91673d3..dd4cbcb 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -5,16 +5,16 @@ pmm_pass=${PMM_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} -docker-compose -f docker-compose-sharded.yaml down -v --remove-orphans -docker-compose -f docker-compose-sharded.yaml build -docker-compose -f docker-compose-sharded.yaml up -d +docker compose -f docker compose-sharded.yaml down -v --remove-orphans +docker compose -f docker compose-sharded.yaml build +docker compose -f docker compose-sharded.yaml up -d echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker-compose -f docker-compose-sharded.yaml exec -T pmm-server change-admin-password password +docker compose -f docker compose-sharded.yaml exec -T pmm-server change-admin-password password echo "restarting pmm-server" -docker-compose -f docker-compose-sharded.yaml restart pmm-server +docker compose -f docker compose-sharded.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 @@ -23,7 +23,7 @@ for node in $nodes do rs=$(echo $node | awk -F "0" '{print $1}') echo "configuring replicaset ${rs} with members priorities" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF config = { "_id" : "${rs}", "members" : [ @@ -49,12 +49,12 @@ EOF sleep 60 echo echo "configuring root user on primary $node replicaset $rs" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles on replicaset $rs" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -84,7 +84,7 @@ EOF EOF echo echo "creating pbm user for replicaset ${rs}" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -99,7 +99,7 @@ EOF EOF echo echo "creating pmm user for replicaset ${rs}" - docker-compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -118,7 +118,7 @@ EOF done echo "configuring configserver replicaset with members priorities" -docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF config = { "_id" : "rscfg", "members" : [ @@ -144,18 +144,18 @@ EOF sleep 60 echo echo "adding shards and creating global mongo user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' echo sleep 20 -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' echo sleep 20 echo echo "configuring pbm and pmm roles" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -185,7 +185,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -200,7 +200,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -224,9 +224,9 @@ nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pbm agent on $node" - docker-compose -f docker-compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker-compose -f docker-compose-sharded.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker compose-sharded.yaml exec -T $node systemctl restart pbm-agent done echo echo "configuring pmm agents" @@ -236,26 +236,26 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker-compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker-compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker compose -f docker compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker-compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" -docker-compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 +docker compose -f docker compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 tests=${TESTS:-yes} if [ $tests != "no" ]; then echo "running tests" - docker-compose -f docker-compose-sharded.yaml run test pytest -s -x --verbose test.py - docker-compose -f docker-compose-sharded.yaml run test chmod -R 777 . + docker compose -f docker compose-sharded.yaml run test pytest -s -x --verbose test.py + docker compose -f docker compose-sharded.yaml run test chmod -R 777 . else echo "skipping tests" fi cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo "cleanup" - docker-compose -f docker-compose-sharded.yaml down -v --remove-orphans + docker compose -f docker compose-sharded.yaml down -v --remove-orphans else echo "skipping cleanup" fi diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index aad57ae..4622980 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -49,6 +49,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' privileged: true + cgroup: host ports: - "43000-43020:43000-43020" healthcheck: diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index a758ef9..58372ca 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -37,25 +37,25 @@ cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; #Start setup -docker-compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans -docker-compose -f docker-compose-pmm-psmdb.yml build -docker-compose -f docker-compose-pmm-psmdb.yml up -d +docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans +docker compose -f docker compose-pmm-psmdb.yml build +docker compose -f docker compose-pmm-psmdb.yml up -d #Add users -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js #Configure PBM -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent #Configure PMM set +e i=1 while [ $i -le 3 ]; do - output=$(docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) exit_code=$? if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then @@ -68,12 +68,12 @@ done #Add Mongo Service random_number=$RANDOM -docker-compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker-compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + docker compose -f docker compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py else echo "skipping tests" fi @@ -81,7 +81,7 @@ fi cleanup=${CLEANUP:-yes} if [ $cleanup = "yes" ]; then echo "cleanup" - docker-compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans + docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf fi From bb97e674f27bfe54817e169ec529113e5ba525b9 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:51:44 +0530 Subject: [PATCH 58/88] Updates fix docker-compose search replace (#80) --- pmm_psmdb-pbm_setup/configure-agents.sh | 16 +++--- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 12 ++--- pmm_psmdb-pbm_setup/configure-extra-psa.sh | 10 ++-- .../configure-extra-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-psa.sh | 10 ++-- pmm_psmdb-pbm_setup/configure-replset.sh | 10 ++-- pmm_psmdb-pbm_setup/start-rs-only.sh | 6 +-- pmm_psmdb-pbm_setup/start-rs.sh | 16 +++--- pmm_psmdb-pbm_setup/start-sharded.sh | 52 +++++++++---------- pmm_psmdb_diffauth_setup/test-auth.sh | 22 ++++---- 10 files changed, 82 insertions(+), 82 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 2498b03..107156f 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -13,14 +13,14 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node" - docker compose -f docker compose-rs.yaml exec -T rs103 systemctl stop pbm-agent + docker compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,17 +29,17 @@ nodes="rs101 rs102 rs103" for node in $nodes do echo "congiguring pmm agent on $node" - docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo echo "adding some data" -docker compose -f docker compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mgodatagen -f /etc/datagen/replicaset.json --uri=mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@127.0.0.1:27017/?replicaSet=rs +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://${pmm_mongo_user}:${pmm_mongo_user_pass}@localhost/?replicaSet=rs" --quiet << EOF use students; db.students.insertMany([ { diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index 10649ea..ce486d8 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -13,14 +13,14 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-rs.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-rs.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent done if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node rs203" - docker compose -f docker compose-rs.yaml exec -T rs203 systemctl stop pbm-agent + docker compose -f docker-compose-rs.yaml exec -T rs203 systemctl stop pbm-agent fi echo echo "configuring pmm agents" @@ -29,10 +29,10 @@ nodes="rs201 rs202 rs203" for node in $nodes do echo "configuring pmm agent on $node" - docker compose -f docker compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup + docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/configure-extra-psa.sh b/pmm_psmdb-pbm_setup/configure-extra-psa.sh index 721029f..00bf29a 100755 --- a/pmm_psmdb-pbm_setup/configure-extra-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring extra PSA replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-extra-replset.sh b/pmm_psmdb-pbm_setup/configure-extra-replset.sh index 1f7e9c3..cade2cf 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs201 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-psa.sh b/pmm_psmdb-pbm_setup/configure-psa.sh index ce006e2..391b6df 100755 --- a/pmm_psmdb-pbm_setup/configure-psa.sh +++ b/pmm_psmdb-pbm_setup/configure-psa.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring PSA replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/configure-replset.sh b/pmm_psmdb-pbm_setup/configure-replset.sh index 819ceeb..d89e571 100644 --- a/pmm_psmdb-pbm_setup/configure-replset.sh +++ b/pmm_psmdb-pbm_setup/configure-replset.sh @@ -8,7 +8,7 @@ pbm_pass=${PBM_PASS:-pbmpass} echo echo "configuring replicaset with members priorities" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF config = { "_id" : "rs", "members" : [ @@ -35,12 +35,12 @@ echo sleep 60 echo echo "configuring root user on primary" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -70,7 +70,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -85,7 +85,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF +docker compose -f docker-compose-rs.yaml exec -T rs101 mongo "mongodb://root:root@localhost/?replicaSet=rs" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_mongo_user}", pwd: "${pmm_mongo_user_pass}", diff --git a/pmm_psmdb-pbm_setup/start-rs-only.sh b/pmm_psmdb-pbm_setup/start-rs-only.sh index 23f2442..d7f95df 100755 --- a/pmm_psmdb-pbm_setup/start-rs-only.sh +++ b/pmm_psmdb-pbm_setup/start-rs-only.sh @@ -13,9 +13,9 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker compose -f docker compose-rs.yaml down -v --remove-orphans -docker compose -f docker compose-rs.yaml build --no-cache -docker compose -f docker compose-rs.yaml up -d +docker compose -f docker-compose-rs.yaml down -v --remove-orphans +docker compose -f docker-compose-rs.yaml build --no-cache +docker compose -f docker-compose-rs.yaml up -d echo echo "waiting 60 seconds for replica set members to start" sleep 60 diff --git a/pmm_psmdb-pbm_setup/start-rs.sh b/pmm_psmdb-pbm_setup/start-rs.sh index c6d3fb4..a4e651b 100755 --- a/pmm_psmdb-pbm_setup/start-rs.sh +++ b/pmm_psmdb-pbm_setup/start-rs.sh @@ -14,16 +14,16 @@ docker network create pmm2-ui-tests_pmm-network || true export COMPOSE_PROFILES=${profile} export MONGO_SETUP_TYPE=${mongo_setup_type} -docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans -docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml build -docker compose -f docker compose-pmm.yaml -f docker compose-rs.yaml up -d +docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans +docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml build +docker compose -f docker-compose-pmm.yaml -f docker-compose-rs.yaml up -d echo echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker compose -f docker compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass +docker compose -f docker-compose-pmm.yaml exec -T pmm-server change-admin-password $pmm_server_admin_pass echo "restarting pmm-server" -docker compose -f docker compose-pmm.yaml restart pmm-server +docker compose -f docker-compose-pmm.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 if [ $mongo_setup_type == "pss" ]; then @@ -36,8 +36,8 @@ tests=${TESTS:-yes} if [ $tests != "no" ]; then echo echo "running tests" - docker compose -f docker compose-pmm.yaml run test pytest -s -x --verbose test.py - docker compose -f docker compose-pmm.yaml run test chmod -R 777 . + docker compose -f docker-compose-pmm.yaml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-pmm.yaml run test chmod -R 777 . else echo echo "skipping tests" @@ -46,7 +46,7 @@ cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo echo "cleanup" - docker compose -f docker compose-rs.yaml -f docker compose-pmm.yaml down -v --remove-orphans + docker compose -f docker-compose-rs.yaml -f docker-compose-pmm.yaml down -v --remove-orphans else echo echo "skipping cleanup" diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index dd4cbcb..4beac37 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -5,16 +5,16 @@ pmm_pass=${PMM_PASS:-pmmpass} pbm_user=${PBM_USER:-pbm} pbm_pass=${PBM_PASS:-pbmpass} -docker compose -f docker compose-sharded.yaml down -v --remove-orphans -docker compose -f docker compose-sharded.yaml build -docker compose -f docker compose-sharded.yaml up -d +docker compose -f docker-compose-sharded.yaml down -v --remove-orphans +docker compose -f docker-compose-sharded.yaml build +docker compose -f docker-compose-sharded.yaml up -d echo "waiting 30 seconds for pmm-server to start" sleep 30 echo "configuring pmm-server" -docker compose -f docker compose-sharded.yaml exec -T pmm-server change-admin-password password +docker compose -f docker-compose-sharded.yaml exec -T pmm-server change-admin-password password echo "restarting pmm-server" -docker compose -f docker compose-sharded.yaml restart pmm-server +docker compose -f docker-compose-sharded.yaml restart pmm-server echo "waiting 30 seconds for pmm-server to start" sleep 30 @@ -23,7 +23,7 @@ for node in $nodes do rs=$(echo $node | awk -F "0" '{print $1}') echo "configuring replicaset ${rs} with members priorities" - docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF config = { "_id" : "${rs}", "members" : [ @@ -49,12 +49,12 @@ EOF sleep 60 echo echo "configuring root user on primary $node replicaset $rs" - docker compose -f docker compose-sharded.yaml exec -T $node mongo --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF echo echo "configuring pbm and pmm roles on replicaset $rs" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -84,7 +84,7 @@ EOF EOF echo echo "creating pbm user for replicaset ${rs}" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -99,7 +99,7 @@ EOF EOF echo echo "creating pmm user for replicaset ${rs}" - docker compose -f docker compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF + docker compose -f docker-compose-sharded.yaml exec -T $node mongo "mongodb://root:root@localhost/?replicaSet=${rs}" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -118,7 +118,7 @@ EOF done echo "configuring configserver replicaset with members priorities" -docker compose -f docker compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 mongo --quiet << EOF config = { "_id" : "rscfg", "members" : [ @@ -144,18 +144,18 @@ EOF sleep 60 echo echo "adding shards and creating global mongo user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs1/rs101:27017,rs102:27017,rs103:27017" )' echo sleep 20 -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet --eval 'sh.addShard( "rs2/rs201:27017,rs202:27017,rs203:27017" )' echo sleep 20 echo echo "configuring pbm and pmm roles" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ @@ -185,7 +185,7 @@ db.getSiblingDB("admin").createRole({ EOF echo echo "creating pbm user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pbm_user}", pwd: "${pbm_pass}", @@ -200,7 +200,7 @@ db.getSiblingDB("admin").createUser({ EOF echo echo "creating pmm user" -docker compose -f docker compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF +docker compose -f docker-compose-sharded.yaml exec -T mongos mongo "mongodb://root:root@localhost" --quiet << EOF db.getSiblingDB("admin").createUser({ user: "${pmm_user}", pwd: "${pmm_pass}", @@ -224,9 +224,9 @@ nodes="rs101 rs102 rs103 rs201 rs202 rs203 rscfg01 rscfg02 rscfg03" for node in $nodes do echo "congiguring pbm agent on $node" - docker compose -f docker compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" + docker compose -f docker-compose-sharded.yaml exec -T $node bash -c "echo \"PBM_MONGODB_URI=mongodb://${pbm_user}:${pbm_pass}@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" echo "restarting pbm agent on $node" - docker compose -f docker compose-sharded.yaml exec -T $node systemctl restart pbm-agent + docker compose -f docker-compose-sharded.yaml exec -T $node systemctl restart pbm-agent done echo echo "configuring pmm agents" @@ -236,26 +236,26 @@ for node in $nodes do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') - docker compose -f docker compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" -docker compose -f docker compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 +docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 tests=${TESTS:-yes} if [ $tests != "no" ]; then echo "running tests" - docker compose -f docker compose-sharded.yaml run test pytest -s -x --verbose test.py - docker compose -f docker compose-sharded.yaml run test chmod -R 777 . + docker compose -f docker-compose-sharded.yaml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-sharded.yaml run test chmod -R 777 . else echo "skipping tests" fi cleanup=${CLEANUP:-yes} if [ $cleanup != "no" ]; then echo "cleanup" - docker compose -f docker compose-sharded.yaml down -v --remove-orphans + docker compose -f docker-compose-sharded.yaml down -v --remove-orphans else echo "skipping cleanup" fi diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 58372ca..41960ae 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -37,25 +37,25 @@ cat pki/private/pmm-test.key pki/issued/pmm-test.crt > certs/client.pem find certs -type f -exec chmod 644 {} \; #Start setup -docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans -docker compose -f docker compose-pmm-psmdb.yml build -docker compose -f docker compose-pmm-psmdb.yml up -d +docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans +docker compose -f docker-compose-pmm-psmdb.yml build +docker compose -f docker-compose-pmm-psmdb.yml up -d #Add users -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet << EOF db.getSiblingDB("admin").createUser({ user: "root", pwd: "root", roles: [ "root", "userAdminAnyDatabase", "clusterAdmin" ] }); EOF -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mongo --quiet "mongodb://root:root@localhost/?replicaSet=rs0" < init/setup_psmdb.js #Configure PBM -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server bash -c "echo \"PBM_MONGODB_URI=mongodb://pbm:pbmpass@127.0.0.1:27017\" > /etc/sysconfig/pbm-agent" +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl restart pbm-agent #Configure PMM set +e i=1 while [ $i -le 3 ]; do - output=$(docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) exit_code=$? if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then @@ -68,12 +68,12 @@ done #Add Mongo Service random_number=$RANDOM -docker compose -f docker compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker compose -f docker compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py else echo "skipping tests" fi @@ -81,7 +81,7 @@ fi cleanup=${CLEANUP:-yes} if [ $cleanup = "yes" ]; then echo "cleanup" - docker compose -f docker compose-pmm-psmdb.yml down -v --remove-orphans + docker compose -f docker-compose-pmm-psmdb.yml down -v --remove-orphans if [[ -n "$PSMDB_VERSION" ]] && [[ "$PSMDB_VERSION" == *"4.2"* ]]; then sed -i 's/MONGODB-X509/MONGODB-X509,MONGODB-AWS/' conf/mongod.conf fi From e44d8fbdf89f0133324a9a4e63871057108bed89 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:21:18 +0530 Subject: [PATCH 59/88] Fix integration test issues (#83) fix cli integration tests --- pmm_qa/pxc_proxysql_setup.yml | 2 +- pmm_qa/setup_docker_client_images.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/pxc_proxysql_setup.yml b/pmm_qa/pxc_proxysql_setup.yml index 967c5d0..0e9b7f1 100644 --- a/pmm_qa/pxc_proxysql_setup.yml +++ b/pmm_qa/pxc_proxysql_setup.yml @@ -33,7 +33,7 @@ - name: Prepare Container for PXC and Proxysql shell: > docker run -d --name={{ pxc_container }} - -p 6032:6032 + -p 6033:6033 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pxc_container diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index f76d873..40cfb46 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -5,9 +5,9 @@ echo echo "Configuring Multiple Docker Images with PMM Server and Client" echo "Please wait...." docker network create docker-client-check || true -docker-compose -f docker-compose-clients.yaml down -v --remove-orphans -docker-compose -f docker-compose-clients.yaml build --no-cache -docker-compose -f docker-compose-clients.yaml up -d +docker compose -f docker-compose-clients.yaml down -v --remove-orphans +docker compose -f docker-compose-clients.yaml build --no-cache +docker compose -f docker-compose-clients.yaml up -d sleep 20 echo "Adding DB Clients to PMM Server" docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true From 8cab71ade362ad75e6a1bab55066118281a41941 Mon Sep 17 00:00:00 2001 From: Puneet Kala Date: Tue, 5 Nov 2024 15:07:55 +0530 Subject: [PATCH 60/88] PMM-7 Fix PMM V3 mysql_tls_setup.yml (#84) * PMM-7 Fix PMM V3 mysql_tls_setup.yml * PMM-7 Fix Update postgresql_tls_setup.yml * PMM-7 mongodb_tls_setup.yml --- pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml | 4 ++-- pmm_qa/tls-ssl-setup/mysql_tls_setup.yml | 4 ++-- pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml b/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml index 6e8ad79..c2cb9f5 100644 --- a/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml @@ -7,8 +7,8 @@ mongodb_version: "{{ lookup('vars', 'extra_mongodb_version', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" mongodb_ssl_container: "{{ lookup('vars', 'extra_mongodb_ssl_container', default=lookup('env','MONGODB_SSL_CONTAINER') | default('mongodb_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: diff --git a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml index 2f19f18..e1e07b2 100644 --- a/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml @@ -11,8 +11,8 @@ mysql_version: "{{ lookup('vars', 'extra_mysql_version', default=lookup('env','MYSQL_VERSION') | default('8.0', true) ) }}" mysql_ssl_container: "{{ lookup('vars', 'extra_mysql_ssl_container', default=lookup('env','MYSQL_SSL_CONTAINER') | default('mysql_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: diff --git a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml index c689f91..ff6bbc3 100644 --- a/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml +++ b/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml @@ -11,8 +11,8 @@ pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('13', true) ) }}" pgsql_ssl_container: "{{ lookup('vars', 'extra_pgsql_ssl_container', default=lookup('env','PGSQL_SSL_CONTAINER') | default('pgsql_ssl', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('pmm3admin!', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" tasks: From 359159f1f3bbd323a8dbe82b8b37e5df2594d650 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:49:01 +0530 Subject: [PATCH 61/88] Update pmm-framework.py (#85) Fix envars override issue. This is a know issue in ansible runner. --- pmm_qa/pmm-framework.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index a9ce849..2e3572c 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -86,7 +86,8 @@ def run_ansible_playbook(playbook_filename, env_vars, args): playbook=playbook_path, inventory='127.0.0.1', cmdline='-l localhost, --connection=local', - envvars=env_vars + envvars=env_vars, + suppress_env_files=True ) print(f'{playbook_filename} playbook execution {r.status}') From 98b0f8e8bacc13bbcc3f2b70b029a75ff1a8acfd Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:34:40 +0530 Subject: [PATCH 62/88] Pmm3 fix pdpgsql and psmdb versions (#86) * Fix versions for pdpgsql and psmdb --- pmm_qa/pdpgsql_pgsm_setup.yml | 4 ++-- pmm_qa/pg_stat_statements_setup.sh | 2 +- pmm_qa/pmm-framework.py | 18 +++++++++++------- pmm_qa/product_version_download_helper | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 6fc4998..3046548 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -6,10 +6,10 @@ become: true become_method: sudo vars: - pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('14', true) ) }}" + pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('15', true) ) }}" pdpgsql_pgsm_container: "{{ lookup('vars', 'extra_pdpgsql_pgsm_container', default=lookup('env','PDPGSQL_PGSM_CONTAINER') | default('pdpgsql_pgsm', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('1.1.1', true) ) }}" diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh index ed05a5f..550d813 100644 --- a/pmm_qa/pg_stat_statements_setup.sh +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -13,7 +13,7 @@ done # If postgres server version is not provided then it will default to version 14. if [ -z "$pgsql_version" ] then - export pgsql_version=14 + export pgsql_version=15 fi # If distribution is not provided then it will default to percona distribution 'PPG' diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2e3572c..5673211 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -9,12 +9,12 @@ # Database configurations database_configs = { "PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, "SSL_PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "latest"], + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, @@ -34,15 +34,15 @@ "TARBALL": ""} }, "PGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "SSL_PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16"], + "versions": ["11", "12", "13", "14", "15", "16", "17"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} }, "PXC": { @@ -492,6 +492,10 @@ def mongo_sharding_setup(script_filename, args): def get_latest_psmdb_version(psmdb_version): if psmdb_version == "latest": return psmdb_version + # workaround till 8.0 is released. + elif psmdb_version in ("8.0", "8.0.1", "8.0.1-1"): + return "8.0.1-1" + # Define the data to be sent in the POST request data = { 'version': f'percona-server-mongodb-{psmdb_version}' @@ -640,7 +644,7 @@ def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), - 'TESTS': 'no', + '`TESTS`': 'no', 'CLEANUP': 'no' } @@ -752,7 +756,7 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): for db_type, options in database_configs.items(): db_parser = subparsers.add_parser(db_type.lower()) for config, value in options['configurations'].items(): - db_parser.add_argument(f'{config}',metavar='', help=f'{config} for {db_type} (default: {value})') + db_parser.add_argument(f'{config}', metavar='', help=f'{config} for {db_type} (default: {value})') # Add arguments parser.add_argument("--database", action='append', nargs=1, diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index d41657f..cb85dff 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -16,6 +16,8 @@ psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mon psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz +psmdb # 8.0.1 # https://downloads.percona.com/downloads/TESTING/psmdb-8.0.1/percona-server-mongodb-8.0.1-1-x86_64.jammy-minimal.tar.gz + ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz From e1c40adda0f2d2ad2f50b9ce835c87d47cb1bf38 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:46:48 +0530 Subject: [PATCH 63/88] Update product_version_download_helper (#87) Removed not required psmdb 8.0 binary from file --- pmm_qa/product_version_download_helper | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index cb85dff..d41657f 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -16,8 +16,6 @@ psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mon psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz -psmdb # 8.0.1 # https://downloads.percona.com/downloads/TESTING/psmdb-8.0.1/percona-server-mongodb-8.0.1-1-x86_64.jammy-minimal.tar.gz - ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz From c8d056a5c520aff3e14196ce9efb24364b2d668d Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:26:10 +0530 Subject: [PATCH 64/88] PMM3 fix ssl qan (#88) * Updates for Mongo SSL QAN --- pmm_psmdb_diffauth_setup/conf/mongod.conf | 4 ++-- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 1 + pmm_psmdb_diffauth_setup/test-auth.sh | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pmm_psmdb_diffauth_setup/conf/mongod.conf b/pmm_psmdb_diffauth_setup/conf/mongod.conf index 4800c3f..0ddce7b 100644 --- a/pmm_psmdb_diffauth_setup/conf/mongod.conf +++ b/pmm_psmdb_diffauth_setup/conf/mongod.conf @@ -40,6 +40,6 @@ replication: replSetName: rs0 operationProfiling: - slowOpThresholdMs: 200 mode: all - rateLimit: 100 + slowOpThresholdMs: 1 + diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 4622980..38f0e5f 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -48,6 +48,7 @@ services: - ./certs:/mongodb_certs - /sys/fs/cgroup:/sys/fs/cgroup:rw - 'psmdb-server-data:/data/db' + - ../pmm_psmdb-pbm_setup/conf/datagen:/etc/datagen:ro privileged: true cgroup: host ports: diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 41960ae..915a101 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -69,6 +69,8 @@ done #Add Mongo Service random_number=$RANDOM docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +#Add some data +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem tests=${TESTS:-yes} if [ $tests = "yes" ]; then From 47bee28b27927ed512693c7f5898efe2313af21a Mon Sep 17 00:00:00 2001 From: Sandra Date: Tue, 16 Jul 2024 22:00:47 +0300 Subject: [PATCH 65/88] PSMDB. Ensure PSMDB related metrics are exported, adjust PSMDB tests for PMM3 and run tests for all major versions --- .github/workflows/PMM_PSMDB_PBM.yml | 10 --- .github/workflows/PMM_PSMDB_PBM_FULL.yml | 75 +++++++++++++++++++ pmm_psmdb-pbm_setup/Dockerfile | 4 +- pmm_psmdb-pbm_setup/test/expected_metrics.txt | 38 ++++++++++ pmm_psmdb-pbm_setup/test/test.py | 48 +++++++++--- pmm_psmdb_diffauth_setup/init/setup_psmdb.js | 2 +- pmm_psmdb_diffauth_setup/test-auth.sh | 7 +- .../test/expected_metrics.txt | 38 ++++++++++ pmm_psmdb_diffauth_setup/test/test.py | 24 +++--- 9 files changed, 212 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/PMM_PSMDB_PBM_FULL.yml create mode 100644 pmm_psmdb-pbm_setup/test/expected_metrics.txt create mode 100644 pmm_psmdb_diffauth_setup/test/expected_metrics.txt diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 9dd6d25..68d1d70 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -20,16 +20,6 @@ on: default: "perconalab/pmm-server:3-dev-latest" required: false - push: - branches: - - main - - v3 - - pull_request: - branches: - - main - - v3 - jobs: test_replica_set: runs-on: ubuntu-22.04 diff --git a/.github/workflows/PMM_PSMDB_PBM_FULL.yml b/.github/workflows/PMM_PSMDB_PBM_FULL.yml new file mode 100644 index 0000000..ef13949 --- /dev/null +++ b/.github/workflows/PMM_PSMDB_PBM_FULL.yml @@ -0,0 +1,75 @@ +name: PMM_PSMDB_PBM_FULL + +on: + workflow_dispatch: + inputs: + pmm_version: + description: "pmm2-client version" + default: "3-dev-latest" + required: false + pmm_image: + description: "pmm-server docker image" + default: "perconalab/pmm-server:3-dev-latest" + required: false + + push: + branches: + - main + - v3 + + pull_request: + branches: + - main + - v3 + +jobs: + test_replica_set: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v2 + - name: Test RS with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./start-rs.sh + working-directory: ./pmm_psmdb-pbm_setup + + test_sharded_cluster: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v2 + - name: Test sharded with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./start-sharded.sh + working-directory: ./pmm_psmdb-pbm_setup + + test_diff_auth: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + psmdb: ["6.0", "7.0", "8.0"] + env: + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} + steps: + - uses: actions/checkout@v3 + - name: Test auth with PSMDB ${{ matrix.psmdb }} + run: | + PSMDB_VERSION=${{ matrix.psmdb }} ./test-auth.sh + working-directory: ./pmm_psmdb_diffauth_setup diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 0e54e18..dc28ff6 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -22,7 +22,7 @@ RUN set -ex; \ export GNUPGHOME="$(mktemp -d)"; \ export PERCONA_TELEMETRY_URL="https://check-dev.percona.com/v1/telemetry/GenericReport"; \ yum install -y gnupg2; \ - if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-60 ; fi ; \ + if [ $PSMDB_VERSION != "latest" ]; then export PSMDB_REPO="$(echo "${PSMDB_VERSION}" | awk -F '.' '{print "psmdb-"$1$2}')" ; else export PSMDB_REPO=psmdb-80 ; fi ; \ gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1; \ gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \ gpg --batch --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-centosofficial; \ @@ -38,7 +38,7 @@ RUN set -ex; \ percona-release enable pmm3-client ${PMM_REPO} && \ yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ yum update --refresh -y && \ - if [ $PSMDB_VERSION == "latest" ]; then \ + if [[ $PSMDB_VERSION == "latest" || "$PSMDB_VERSION" =~ ^[0-9]\.[0-9]$ ]] ; then \ yum -y install percona-server-mongodb \ percona-server-mongodb-tools \ percona-server-mongodb-server \ diff --git a/pmm_psmdb-pbm_setup/test/expected_metrics.txt b/pmm_psmdb-pbm_setup/test/expected_metrics.txt new file mode 100644 index 0000000..4a65db2 --- /dev/null +++ b/pmm_psmdb-pbm_setup/test/expected_metrics.txt @@ -0,0 +1,38 @@ +mongodb_up +mongodb_mongod_global_lock_client +mongodb_mongod_global_lock_current_queue +mongodb_mongod_instance_uptime_seconds +mongodb_mongod_locks_time_acquiring_global_microseconds_total +mongodb_mongod_metrics_cursor_open +mongodb_mongod_metrics_cursor_timed_out_total +mongodb_mongod_metrics_document_total +mongodb_mongod_metrics_get_last_error_wtime_num_total +mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds +mongodb_mongod_metrics_get_last_error_wtimeouts_total +mongodb_mongod_metrics_operation_total +mongodb_mongod_metrics_query_executor_total +mongodb_mongod_metrics_ttl_deleted_documents_total +mongodb_mongod_metrics_ttl_passes_total +mongodb_mongod_op_counters_repl_total +mongodb_mongod_op_latencies_latency_total +mongodb_mongod_op_latencies_ops_total +mongodb_mongod_storage_engine +mongodb_mongod_wiredtiger_blockmanager_bytes_total +mongodb_mongod_wiredtiger_cache_bytes +mongodb_mongod_wiredtiger_cache_bytes_total +mongodb_mongod_wiredtiger_cache_evicted_total +mongodb_mongod_wiredtiger_cache_max_bytes +mongodb_mongod_wiredtiger_cache_overhead_percent +mongodb_mongod_wiredtiger_cache_pages +mongodb_mongod_wiredtiger_cache_pages_total +mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets +mongodb_mongod_wiredtiger_log_bytes_total +mongodb_mongod_wiredtiger_log_operations_total +mongodb_mongod_wiredtiger_log_records_scanned_total +mongodb_mongod_wiredtiger_log_records_total +mongodb_mongod_wiredtiger_session_open_sessions_total +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total +mongodb_mongod_wiredtiger_transactions_total diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index a748fa8..660a5de 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -23,13 +23,13 @@ def test_pmm_services(): - req = requests.post(f"https://{pmm_server_url}/v1/inventory/Services/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/inventory/services", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nGetting all mongodb services:') mongodb = req.json()['mongodb'] print(mongodb) assert mongodb - assert "service_id" in mongodb[0]['service_id'] + assert "service_id" in mongodb[0] for service in mongodb: assert "rs" or "mongos" in service['service_name'] if not "mongos" in service['service_name']: @@ -49,11 +49,11 @@ def test_pmm_add_location(): 'bucket_name': 'bcp' } } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Locations/Add", json=data, + req = requests.post(f"https://{pmm_server_url}/v1/backups/locations", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nAdding new location:') print(req.json()) - assert "location_id" in req.json()['location_id'] + assert "location_id" in req.json() pytest.location_id = req.json()['location_id'] @@ -64,13 +64,15 @@ def test_pmm_logical_backup(): 'name': 'test', 'description': 'test', 'retries': 0, - 'data_model': 'LOGICAL' + 'data_model': 'DATA_MODEL_LOGICAL' } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Start", json=data, + + print(data) + req = requests.post(f"https://{pmm_server_url}/v1/backups:start", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nCreating logical backup:') print(req.json()) - assert "artifact_id" in req.json()['artifact_id'] + assert "artifact_id" in req.json() pytest.artifact_id = req.json()['artifact_id'] @@ -78,7 +80,7 @@ def test_pmm_artifact(): backup_complete = False for i in range(600): done = False - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Artifacts/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/backups/artifacts", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['artifacts'] for artifact in req.json()['artifacts']: @@ -118,11 +120,11 @@ def test_pmm_start_restore(): 'service_id': pytest.service_id, 'artifact_id': pytest.artifact_id } - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Restore", json=data, + req = requests.post(f"https://{pmm_server_url}/v1/backups/restores:start", json=data, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nRestoring logical backup:') print(req.json()) - assert "restore_id" in req.json()['restore_id'] + assert "restore_id" in req.json() pytest.restore_id = req.json()['restore_id'] @@ -132,7 +134,7 @@ def test_pmm_restore(): restore_complete = False for i in range(600): done = False - req = requests.post(f"https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, + req = requests.get(f"https://{pmm_server_url}/v1/backups/restores", json={}, headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['items'] for item in req.json()['items']: @@ -166,3 +168,27 @@ def test_pbm_restore(): restore_complete = True assert restore_complete + +def test_metrics(): + pmm_admin_list = json.loads(docker_rs101.check_output('pmm-admin list --json', timeout=30)) + for agent in pmm_admin_list['agent']: + if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': + agent_id = agent['agent_id'] + agent_port = agent['port'] + break + try: + command = f"curl -s http://pmm:{agent_id}@127.0.0.1:{agent_port}/metrics" + metrics = docker_rs101.run(command, timeout=30) + assert metrics.exit_status == 0, f"Curl command failed with exit status {metrics.exit_status}" + except Exception as e: + pytest.fail(f"Fail to get metrics from exporter") + + try: + with open("expected_metrics.txt", "r") as f: + expected_metrics = {line.strip() for line in f if line.strip()} + except FileNotFoundError: + pytest.fail("Expected metrics file not found") + + for metric in expected_metrics: + if metric not in metrics.stdout: + pytest.fail(f"Metric '{metric}' is missing from the exporter output") diff --git a/pmm_psmdb_diffauth_setup/init/setup_psmdb.js b/pmm_psmdb_diffauth_setup/init/setup_psmdb.js index f1ac4c1..8f1c8fe 100644 --- a/pmm_psmdb_diffauth_setup/init/setup_psmdb.js +++ b/pmm_psmdb_diffauth_setup/init/setup_psmdb.js @@ -1,4 +1,4 @@ -var db = connect("mongodb://dba:secret@localhost:27017/admin"); +var db = connect("mongodb://root:root@localhost:27017/admin"); db.getSiblingDB("admin").createRole({ "role": "pbmAnyAction", "privileges": [{ diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index 915a101..e4f6b3d 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -75,7 +75,7 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen - tests=${TESTS:-yes} if [ $tests = "yes" ]; then echo "running tests" - docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s -x --verbose test.py + output=$(docker compose -f docker-compose-pmm-psmdb.yml run test pytest -s --verbose test.py) else echo "skipping tests" fi @@ -90,3 +90,8 @@ if [ $cleanup = "yes" ]; then else echo "skipping cleanup" fi + +echo "$output" +if echo "$output" | grep -q "\bFAILED\b"; then + exit 1 +fi diff --git a/pmm_psmdb_diffauth_setup/test/expected_metrics.txt b/pmm_psmdb_diffauth_setup/test/expected_metrics.txt new file mode 100644 index 0000000..4a65db2 --- /dev/null +++ b/pmm_psmdb_diffauth_setup/test/expected_metrics.txt @@ -0,0 +1,38 @@ +mongodb_up +mongodb_mongod_global_lock_client +mongodb_mongod_global_lock_current_queue +mongodb_mongod_instance_uptime_seconds +mongodb_mongod_locks_time_acquiring_global_microseconds_total +mongodb_mongod_metrics_cursor_open +mongodb_mongod_metrics_cursor_timed_out_total +mongodb_mongod_metrics_document_total +mongodb_mongod_metrics_get_last_error_wtime_num_total +mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds +mongodb_mongod_metrics_get_last_error_wtimeouts_total +mongodb_mongod_metrics_operation_total +mongodb_mongod_metrics_query_executor_total +mongodb_mongod_metrics_ttl_deleted_documents_total +mongodb_mongod_metrics_ttl_passes_total +mongodb_mongod_op_counters_repl_total +mongodb_mongod_op_latencies_latency_total +mongodb_mongod_op_latencies_ops_total +mongodb_mongod_storage_engine +mongodb_mongod_wiredtiger_blockmanager_bytes_total +mongodb_mongod_wiredtiger_cache_bytes +mongodb_mongod_wiredtiger_cache_bytes_total +mongodb_mongod_wiredtiger_cache_evicted_total +mongodb_mongod_wiredtiger_cache_max_bytes +mongodb_mongod_wiredtiger_cache_overhead_percent +mongodb_mongod_wiredtiger_cache_pages +mongodb_mongod_wiredtiger_cache_pages_total +mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets +mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets +mongodb_mongod_wiredtiger_log_bytes_total +mongodb_mongod_wiredtiger_log_operations_total +mongodb_mongod_wiredtiger_log_records_scanned_total +mongodb_mongod_wiredtiger_log_records_total +mongodb_mongod_wiredtiger_session_open_sessions_total +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds +mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total +mongodb_mongod_wiredtiger_transactions_total diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index 8ac14ba..0cf308b 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -25,20 +25,26 @@ def run_test(add_db_command): pmm_admin_list = json.loads(docker_pmm_client.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: - if agent['agent_type'] == 'MONGODB_EXPORTER': + if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': agent_id = agent['agent_id'] agent_port = agent['port'] break + try: + command = f"curl -s http://pmm:{agent_id}@127.0.0.1:{agent_port}/metrics" + metrics = docker_pmm_client.run(command, timeout=30) + assert metrics.exit_status == 0, f"Curl command failed with exit status {metrics.exit_status}" + except Exception as e: + pytest.fail(f"Fail to get metrics from exporter") - url = f'http://localhost:{agent_port}/metrics' try: - response = docker_pmm_client.check_output(f"curl --request GET --url {url} --header 'Content-Type: " - f"application/json' --user 'pmm:{agent_id}'") - pattern = r'mongodb_up (\d+)' - result = re.search(pattern, response) - assert result is not None, "MongoDB related data isn't exported" - except AssertionError: - pytest.fail(f"Connection to {url} failed") + with open("expected_metrics.txt", "r") as f: + expected_metrics = {line.strip() for line in f if line.strip()} + except FileNotFoundError: + pytest.fail("Expected metrics file not found") + + for metric in expected_metrics: + if metric not in metrics.stdout: + pytest.fail(f"Metric '{metric}' is missing from the exporter output") def test_simple_auth_wo_tls(): From a9a4b83ab0fb41092ac0507c8b4bf3e694fca8eb Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:27:54 +0530 Subject: [PATCH 66/88] Update pmm-framework.py (#90) fix editor automatic changes, to avoid running tests and create more services --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5673211..5315337 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -644,7 +644,7 @@ def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), - '`TESTS`': 'no', + 'TESTS': 'no', 'CLEANUP': 'no' } From 92a2b96abf7645b01e93a074fbd6e73eab8352f7 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:30:09 +0530 Subject: [PATCH 67/88] Update pmm-framework.py (#92) Fix issue, to pick changed password --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5315337..cf5ea16 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -596,7 +596,7 @@ def mongo_ssl_setup(script_filename, args): subprocess.run( ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/password/{admin_password}/g', f'{compose_file_path}']) + subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) From f0c601dee62eb8049990c3a2fd87890a7f134a32 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:26:39 +0530 Subject: [PATCH 68/88] Update to expose minio port number on 9001 (#93) Fix minio port number, as we start pmm-sever on 9000 already in pipelines --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 2 ++ pmm_psmdb-pbm_setup/docker-compose-sharded.yaml | 2 +- pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 9da2b1a..6bdee0f 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -186,6 +186,8 @@ services: image: minio/minio profiles: ["classic", "extra"] container_name: minio + ports: + - "9001:9000" depends_on: - build_member networks: diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index b223ad9..0123ecb 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -229,7 +229,7 @@ services: image: minio/minio container_name: minio ports: - - "9000:9000" + - "9001:9000" networks: - test-network volumes: diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 38f0e5f..5ac5ff6 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -99,6 +99,8 @@ services: minio: image: minio/minio container_name: minio + ports: + - "9001:9000" volumes: - /tmp/minio/backups:/backups environment: From 70755950f67a3421c42297979143b93d5a19e93c Mon Sep 17 00:00:00 2001 From: Sandra Romanchenko <53295797+sandraromanchenko@users.noreply.github.com> Date: Fri, 29 Nov 2024 18:12:46 +0200 Subject: [PATCH 69/88] PSMDB. Remove unnecessary pidFilePath option as forking is already disabled (#97) --- pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf | 4 ---- pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf | 4 ---- pmm_psmdb_diffauth_setup/conf/mongod.conf | 4 ---- 5 files changed, 20 deletions(-) diff --git a/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf index d69f78d..20e2c62 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-cfg/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf index 2dc676f..7ecaf25 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf index b2fa66f..34ad841 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs1/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf b/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf index e0ee0ea..8d7946f 100644 --- a/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf +++ b/pmm_psmdb-pbm_setup/conf/mongod-rs2/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 diff --git a/pmm_psmdb_diffauth_setup/conf/mongod.conf b/pmm_psmdb_diffauth_setup/conf/mongod.conf index 0ddce7b..d83b3d8 100644 --- a/pmm_psmdb_diffauth_setup/conf/mongod.conf +++ b/pmm_psmdb_diffauth_setup/conf/mongod.conf @@ -9,10 +9,6 @@ storage: systemLog: destination: syslog -processManagement: -# fork: true - pidFilePath: /var/run/mongod.pid - net: port: 27017 bindIp: 0.0.0.0 From c063f46d8a938a2f84cda4942f2e50c1a08cae82 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:52:49 +0530 Subject: [PATCH 70/88] Updates fix gh triggers for PS and Proxysql (#101) * Updates fix gh triggers for PS and Proxysql * Updates PDPGSQL version default to 17 --- .github/workflows/PMM_PDPGSQL.yaml | 6 +++--- .../workflows/{PMM_PXC_PROXYSQL.yaml => PMM_PROXYSQL.yaml} | 2 +- .github/workflows/PMM_PS.yaml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{PMM_PXC_PROXYSQL.yaml => PMM_PROXYSQL.yaml} (99%) diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index e457459..097f2e5 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -3,9 +3,9 @@ on: workflow_dispatch: inputs: pdpgsql_version: - description: "PDPGSQL Version available Image versions on github, example: 14, 15, 16" + description: "PDPGSQL Version available Image versions on github, example: 15, 16, 17" required: true - default: '16' + default: '17' type: string pmm_qa_branch: description: "Branch for qa-integration to checkout" @@ -39,7 +39,7 @@ jobs: timeout-minutes: 20 env: ADMIN_PASSWORD: 'admin' - PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '16' }} + PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '17' }} PMM_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} diff --git a/.github/workflows/PMM_PXC_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml similarity index 99% rename from .github/workflows/PMM_PXC_PROXYSQL.yaml rename to .github/workflows/PMM_PROXYSQL.yaml index b456521..2379c9b 100644 --- a/.github/workflows/PMM_PXC_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -1,4 +1,4 @@ -name: PMM_PXC_PROXYSQL +name: PMM_PROXYSQL on: workflow_dispatch: inputs: diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index c56fc3b..05408bc 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -7,8 +7,8 @@ on: default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS' required: true ps_glibc: - description: "ps glibc version, example: 2.28" - default: '2.28' + description: "ps glibc version, example: 2.35" + default: '2.35' required: false pmm_qa_branch: description: "Branch for qa-integration to checkout" @@ -42,7 +42,7 @@ jobs: env: ADMIN_PASSWORD: 'admin' PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }} - PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.28' }} + PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.35' }} PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} PMM_UI_BRANCH: ${{ github.event.inputs.pmm_ui_tests_branch || 'v3' }} From 19ea78648e1d2ce0e1e33f625e00078d029c5cb2 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:33:16 +0530 Subject: [PATCH 71/88] Update PMM_PS.yaml (#102) --- .github/workflows/PMM_PS.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index 05408bc..ae57e8e 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -29,11 +29,11 @@ on: type: string push: branches: - - main + - v3 pull_request: branches: - - main + - v3 jobs: PMM_PS_TEST: From 204660fc4bcb2f8bab2cfbef37c883230e887ccf Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:35:22 +0530 Subject: [PATCH 72/88] Updates to fix PGSQL for PGSM (#103) --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 +++-- pmm_qa/pg_stat_monitor_setup.sh | 2 +- pmm_qa/pmm-framework.py | 9 ++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 3046548..fd3ef2d 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -12,9 +12,10 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" - pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('1.1.1', true) ) }}" + pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('2.1.0', true) ) }}" pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" + distribution: "{{ lookup('vars', 'extra_pdpgsql_distribution', default=lookup('env','DISTRIBUTION') | default('PPG', true) ) }}" tasks: - name: cleanup container for client and DB setup @@ -45,7 +46,7 @@ - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container shell: "{{ item }}" with_items: - - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution PPG --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log + - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log - name: Install pmm2-client on the pdpgsql_pgsm_container shell: "{{ item }}" diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index c3f5d6c..cc6fb54 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -19,7 +19,7 @@ fi # If branch/tag is not provided then it will default to main branch if [ -z "$pgstat_monitor_branch" ] then - export pgstat_monitor_branch="1.1.1" + export pgstat_monitor_branch="2.1.0" fi # If repo is not provided then it will default to percona PGSM repository diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index cf5ea16..129d560 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -284,6 +284,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'DISTRIBUTION': '', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } @@ -335,17 +336,19 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): # Define environment variables for playbook env_vars = { - 'PGSQL_VERSION': pgsql_version, + 'PDPGSQL_VERSION': pgsql_version, + 'PGSTAT_MONITOR_BRANCH': 'main', 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'PDPGSQL_PGSM_CONTAINER': 'pgsql_pgsm_pmm_' + str(pgsql_version), 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'DISTRIBUTION': 'PGDG', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename - playbook_filename = 'pgsql_pgss_setup.yml' + playbook_filename = 'pdpgsql_pgsm_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) From 136d1729ca5fd332630708a40ac22d3175dd3150 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:44:51 +0530 Subject: [PATCH 73/88] Pmm3 pgsql portfix (#106) * Updates to fix PGSQL port --- pmm_qa/pdpgsql_pgsm_setup.yml | 5 +++-- pmm_qa/pmm-framework.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index fd3ef2d..3672241 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -15,6 +15,7 @@ pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('2.1.0', true) ) }}" pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" + port: "{{ lookup('vars', 'extra_pdpgsql_port', default=lookup('env','PORT') | default(5447, true) ) }}" distribution: "{{ lookup('vars', 'extra_pdpgsql_distribution', default=lookup('env','DISTRIBUTION') | default('PPG', true) ) }}" tasks: @@ -32,7 +33,7 @@ - name: Prepare Container for PostgreSQL shell: > sudo docker run -d --name={{ pdpgsql_pgsm_container }} - -p 5447:5432 + -p {{ port }}:5432 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pdpgsql_pgsm_container @@ -46,7 +47,7 @@ - name: Execute Setup script inside the pdpgsql pdpgsql_pgsm_container shell: "{{ item }}" with_items: - - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_pdpgsql_pgsm_{{ pdpgsql_version }}.log + - docker exec {{ pdpgsql_pgsm_container }} bash -xe ./pg_stat_monitor_setup.sh --distribution {{ distribution }} --pgsql_version {{ pdpgsql_version }} --pgstat_monitor_branch {{ pgstat_monitor_branch }} --pgstat_monitor_repo {{ pgstat_monitor_repo }} > setup_{{ pdpgsql_pgsm_container }}.log - name: Install pmm2-client on the pdpgsql_pgsm_container shell: "{{ item }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 129d560..b9129b4 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -284,6 +284,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PORT': 5447, 'DISTRIBUTION': '', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } @@ -343,6 +344,7 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PORT': 5448, 'DISTRIBUTION': 'PGDG', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } From 70715dae7e391b905ee71032a4e086d859edd971 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Tue, 14 Jan 2025 18:17:42 +0530 Subject: [PATCH 74/88] Updates fix pgsm support (#109) --- pmm_qa/pdpgsql_pgsm_setup.yml | 4 ++-- pmm_qa/pgsql_pgss_setup.yml | 2 +- pmm_qa/pmm-framework.py | 12 +++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pmm_qa/pdpgsql_pgsm_setup.yml b/pmm_qa/pdpgsql_pgsm_setup.yml index 3672241..09bb4dc 100644 --- a/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/pmm_qa/pdpgsql_pgsm_setup.yml @@ -15,7 +15,7 @@ pgstat_monitor_branch: "{{ lookup('vars', 'extra_pgstat_monitor_branch', default=lookup('env','PGSTAT_MONITOR_BRANCH') | default('2.1.0', true) ) }}" pgstat_monitor_repo: "{{ lookup('vars', 'extra_pgstat_monitor_repo', default=lookup('env','PGSTAT_MONITOR_REPO') | default('percona/pg_stat_monitor', true) ) }}" use_socket: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','USE_SOCKET') | default('', true) ) }}" - port: "{{ lookup('vars', 'extra_pdpgsql_port', default=lookup('env','PORT') | default(5447, true) ) }}" + pdpgsql_pgsm_port: "{{ lookup('vars', 'extra_pdpgsql_port', default=lookup('env','PDPGSQL_PGSM_PORT') | default(5447, true) ) }}" distribution: "{{ lookup('vars', 'extra_pdpgsql_distribution', default=lookup('env','DISTRIBUTION') | default('PPG', true) ) }}" tasks: @@ -33,7 +33,7 @@ - name: Prepare Container for PostgreSQL shell: > sudo docker run -d --name={{ pdpgsql_pgsm_container }} - -p {{ port }}:5432 + -p {{ pdpgsql_pgsm_port }}:5432 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker pdpgsql_pgsm_container diff --git a/pmm_qa/pgsql_pgss_setup.yml b/pmm_qa/pgsql_pgss_setup.yml index d3a414f..b1674c0 100644 --- a/pmm_qa/pgsql_pgss_setup.yml +++ b/pmm_qa/pgsql_pgss_setup.yml @@ -8,7 +8,7 @@ vars: pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('14', true) ) }}" pgsql_pgss_container: "{{ lookup('vars', 'extra_pgsql_pgss_container', default=lookup('env','PGSQL_PGSS_CONTAINER') | default('pgsql_pgss', true) ) }}" - pgsql_pgss_port: "{{ lookup('vars', 'extra_pgsql_port', default=lookup('env','PGSQL_PGSS_PORT') | default('5438', true) ) }}" + pgsql_pgss_port: "{{ lookup('vars', 'extra_pgsql_port', default=lookup('env','PGSQL_PGSS_PORT') | default('5448', true) ) }}" pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b9129b4..67baaf2 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -284,7 +284,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PORT': 5447, + 'PDPGSQL_PGSM_PORT': 5447, 'DISTRIBUTION': '', 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } @@ -337,20 +337,18 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): # Define environment variables for playbook env_vars = { - 'PDPGSQL_VERSION': pgsql_version, - 'PGSTAT_MONITOR_BRANCH': 'main', + 'PGSQL_VERSION': pgsql_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PDPGSQL_PGSM_CONTAINER': 'pgsql_pgsm_pmm_' + str(pgsql_version), + 'PGSQL_PGSM_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PORT': 5448, - 'DISTRIBUTION': 'PGDG', + 'PGSQL_PGSS_PORT': 5448, 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' } # Ansible playbook filename - playbook_filename = 'pdpgsql_pgsm_setup.yml' + playbook_filename = 'pgsql_pgss_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) From b15a7aafbe65d495f90f71ce0c78f4a4960f2ba2 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:33:08 +0530 Subject: [PATCH 75/88] Update pmm-framework.py (#110) Fixed typo in PGSS container name --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 67baaf2..73c8ed6 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -339,7 +339,7 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): env_vars = { 'PGSQL_VERSION': pgsql_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PGSQL_PGSM_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', From 9f409eec005aaf4f8024627166bd8bfeb5736915 Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 16 Jan 2025 21:30:24 +0530 Subject: [PATCH 76/88] PMM-13668 PMM-13678 Updates add PS 8.4 and MS 8.4 Support (#111) * Updates add PS 8.4 Support, Fix mysql_native_plugin for 8.4. Fix 8.0 and 57 URLS based on new GLIB versions * Updates and also Add support for MS version 8.4 and 8.0.40 * Updates keep defaults as 8.0 only for PS and MySQL for now. --- pmm_qa/client_container_ms_setup.sh | 18 +++++++++++++----- pmm_qa/client_container_ps_setup.sh | 18 +++++++++++++----- pmm_qa/ms_pmm_setup.yml | 2 +- pmm_qa/pmm-framework.py | 6 +++--- pmm_qa/product_version_download_helper | 10 +++++++--- pmm_qa/ps_pmm_setup.yml | 2 +- 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/pmm_qa/client_container_ms_setup.sh b/pmm_qa/client_container_ms_setup.sh index 292377d..c4004c2 100644 --- a/pmm_qa/client_container_ms_setup.sh +++ b/pmm_qa/client_container_ms_setup.sh @@ -17,12 +17,12 @@ fi if [ -z "$ms_version" ] then - export ms_version="8.0.33" + export ms_version="8.0" fi if [ -z "$ms_tarball" ] then - export ms_tarball="https://dev.mysql.com/get/Downloads/MySQL-8/mysql-8.0.33-linux-glibc2.28-x86_64.tar.gz" + export ms_tarball="https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.40-linux-glibc2.17-x86_64-minimal.tar.xz" fi if [ -z "$query_source" ] @@ -49,9 +49,17 @@ dbdeployer unpack ${tar_ball_name} --sandbox-binary=~/ms${ms_version} --overwrit export db_version_sandbox=$(ls ~/ms${ms_version}) export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) +# Initialize my_cnf_options +my_cnf_options="" + +# Check if ps_version is 8.4 or greater to enable the plugin to change the password +if [[ "$ms_version" =~ ^8\.[4-9]([0-9])? || "$ms_version" =~ ^[9-9][0-9]\. ]]; then + my_cnf_options="mysql-native-password=ON" +fi + if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then - dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ms${ms_version} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" @@ -59,7 +67,7 @@ if [[ "$number_of_nodes" == 1 ]];then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --port=$MS_PORT --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --port=$MS_PORT --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" @@ -91,7 +99,7 @@ if [[ "$number_of_nodes" == 1 ]];then pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ms-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port fi else - dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 + dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ms${ms_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index 0b3f84f..3b25f4e 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -17,12 +17,12 @@ fi if [ -z "$ps_version" ] then - export ps_version=8 + export ps_version=8.0 fi if [ -z "$ps_tarball" ] then - export ps_tarball=https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.29-21/binary/tarball/Percona-Server-8.0.29-21-Linux.x86_64.glibc2.17-minimal.tar.gz + export ps_tarball="https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.40-31/binary/tarball/Percona-Server-8.0.40-31-Linux.x86_64.glibc2.35-minimal.tar.gz" fi if [ -z "$query_source" ] @@ -53,15 +53,23 @@ export db_version_sandbox=$(ls ~/ps${ps_version}) export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) +# Initialize my_cnf_options +my_cnf_options="" + +# Check if ps_version is 8.4 or greater to enable the plugin to change the password +if [[ "$ps_version" =~ ^8\.[4-9]([0-9])? || "$ps_version" =~ ^[9-9][0-9]\. ]]; then + my_cnf_options="mysql-native-password=ON" +fi + if [[ "$number_of_nodes" == 1 ]];then if [[ ! -z $group_replication ]]; then - dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" else - dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force + dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" @@ -115,7 +123,7 @@ if [[ "$number_of_nodes" == 1 ]];then pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ps-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port fi else - dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 + dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` for j in `seq 1 $number_of_nodes`; do diff --git a/pmm_qa/ms_pmm_setup.yml b/pmm_qa/ms_pmm_setup.yml index 26951a9..62915e5 100644 --- a/pmm_qa/ms_pmm_setup.yml +++ b/pmm_qa/ms_pmm_setup.yml @@ -44,7 +44,7 @@ - docker cp ./pmm3-client-setup.sh {{ ms_container }}:/ - name: Get Product tarball URL based on the version - shell: cat product_version_download_helper | grep -v pxc | grep mysql | grep "\-{{ ms_version }}" | head -1 | awk -F'# ' '{print $3}' + shell: awk -F'# ' '/-{{ ms_version | regex_escape }}/ && /ms/ {print $3; exit}' product_version_download_helper register: tarball when: lookup('env', 'MS_TARBALL') == '' diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 73c8ed6..aaa6190 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -19,17 +19,17 @@ "TARBALL": ""} }, "MYSQL": { - "versions": ["8.0"], + "versions": ["8.4", "8.0"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, "PS": { - "versions": ["5.7", "8.0"], + "versions": ["5.7", "8.4", "8.0"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, "SSL_MYSQL": { - "versions": ["5.7", "8.0"], + "versions": ["5.7", "8.4", "8.0"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, diff --git a/pmm_qa/product_version_download_helper b/pmm_qa/product_version_download_helper index d41657f..f774f47 100644 --- a/pmm_qa/product_version_download_helper +++ b/pmm_qa/product_version_download_helper @@ -16,11 +16,15 @@ psmdb # 6.0.2 # https://downloads.percona.com/downloads/percona-distribution-mon psmdb # 7.0.2 # https://downloads.percona.com/downloads/percona-server-mongodb-7.0/percona-server-mongodb-7.0.2-1/binary/tarball/percona-server-mongodb-7.0.2-1-x86_64.glibc2.17.tar.gz -ps # 8.0.33 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.33-25/binary/tarball/Percona-Server-8.0.33-25-Linux.x86_64.glibc2.17-minimal.tar.gz +ps # 8.4.3 # https://downloads.percona.com/downloads/Percona-Server-8.4/Percona-Server-8.4.3-3/binary/tarball/Percona-Server-8.4.3-3-Linux.x86_64.glibc2.35-minimal.tar.gz -ps # 5.7.43 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.43-47/binary/tarball/Percona-Server-5.7.43-47-Linux.x86_64.glibc2.17-minimal.tar.gz +ps # 8.0.40 # https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.40-31/binary/tarball/Percona-Server-8.0.40-31-Linux.x86_64.glibc2.35-minimal.tar.gz -ms # 8.0.33 # https://dev.mysql.com/get/Downloads/MySQL-8/mysql-8.0.33-linux-glibc2.28-x86_64.tar.gz +ps # 5.7.44 # https://downloads.percona.com/downloads/Percona-Server-5.7/Percona-Server-5.7.44-48/binary/tarball/Percona-Server-5.7.44-48-Linux.x86_64.glibc2.35-minimal.tar.gz + +ms # 8.4 # https://dev.mysql.com/get/Downloads/MySQL-8.4/mysql-8.4.3-linux-glibc2.17-x86_64-minimal.tar.xz + +ms # 8.0.40 # https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.40-linux-glibc2.17-x86_64-minimal.tar.xz proxysql # 2.6.2 # https://downloads.percona.com/downloads/proxysql2/proxysql2-2.6.2/binary/debian/jammy/x86_64/proxysql2_2.6.2-1.1.jammy_amd64.deb diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index 967a189..8213c33 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -43,7 +43,7 @@ - docker cp ./pmm3-client-setup.sh {{ ps_container }}:/ - name: Get Product tarball URL based on the version - shell: cat product_version_download_helper | grep -v pxc | grep ps | grep "\-{{ ps_version }}" | head -1 | awk -F'# ' '{print $3}' + shell: awk -F'# ' '/-{{ ps_version | regex_escape }}/ && !/pxc/ && /ps/ {print $3; exit}' product_version_download_helper register: tarball when: lookup('env', 'PS_TARBALL') == '' From a32f36785425752ee3dd20682e84aaf99c8facf5 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:06:28 +0200 Subject: [PATCH 77/88] PMM-7 fix workflows (#112) * PMM-7 fix workflows * PMM-7 fix workflows * PMM-7 fix workflows * PMM-7 fix workflows * PMM-7 fix workflows --- .github/workflows/PMM_PDPGSQL.yaml | 7 ++++--- .github/workflows/PMM_PROXYSQL.yaml | 7 ++++--- .github/workflows/PMM_PS.yaml | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index 097f2e5..77aa72c 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -35,8 +35,8 @@ on: jobs: PMM_PDPGSQL_TEST: - runs-on: ubuntu-latest - timeout-minutes: 20 + runs-on: ubuntu-20.04 + timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' PDPGSQL_VERSION: ${{ github.event.inputs.pdpgsql_version || '17' }} @@ -69,6 +69,7 @@ jobs: python3 -m venv virtenv . virtenv/bin/activate pip install --upgrade pip + pip install --force-reinstall -U setuptools pip install -r requirements.txt python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }} @@ -87,4 +88,4 @@ jobs: working-directory: pmm-ui-tests run: | export PMM_UI_URL="http://127.0.0.1:8081/" - npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pdpgsql_integration_test.js --steps --debug + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pdpgsql_integration_test.js diff --git a/.github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml index 2379c9b..d1185f0 100644 --- a/.github/workflows/PMM_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -39,8 +39,8 @@ on: jobs: PMM_PXC_TEST: - runs-on: ubuntu-latest - timeout-minutes: 20 + runs-on: ubuntu-20.04 + timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' PXC_VERSION: ${{ github.event.inputs.pxc_version || '8.0' }} @@ -99,6 +99,7 @@ jobs: python3 -m venv virtenv . virtenv/bin/activate pip install --upgrade pip + pip install --force-reinstall -U setuptools pip install -r requirements.txt if [[ "${{ github.event.inputs.pxc_version }}" != http* ]]; then python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${{ env.PXC_VERSION }} @@ -120,6 +121,6 @@ jobs: - name: Run the Integration tests of PXC run: | export PMM_UI_URL="http://127.0.0.1:8081/" - npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pxc_integration_test.js --steps --debug + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_pxc_integration_test.js working-directory: pmm-ui-tests diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index ae57e8e..ad4750d 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -37,8 +37,8 @@ on: jobs: PMM_PS_TEST: - runs-on: ubuntu-latest - timeout-minutes: 20 + runs-on: ubuntu-20.04 + timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }} @@ -99,6 +99,7 @@ jobs: python3 -m venv virtenv . virtenv/bin/activate pip install --upgrade pip + pip install --force-reinstall -U setuptools pip install -r requirements.txt if [[ "${{ github.event.inputs.ps_version }}" != http* ]]; then python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS=${{ env.PS_VERSION }} @@ -120,5 +121,5 @@ jobs: - name: Run the Integration tests of PS run: | export PMM_UI_URL="http://127.0.0.1:8081/" - npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js --steps --debug + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js working-directory: pmm-ui-tests From 74d03b35566c7da5b8a2ac53b407c8a43a4f989f Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:18:21 +0530 Subject: [PATCH 78/88] Pmm3 fix client for release version (#113) * Updates to fix client version --- pmm_qa/pmm3-client-setup.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pmm_qa/pmm3-client-setup.sh b/pmm_qa/pmm3-client-setup.sh index abc906a..f044a19 100755 --- a/pmm_qa/pmm3-client-setup.sh +++ b/pmm_qa/pmm3-client-setup.sh @@ -72,9 +72,8 @@ if [[ "$client_version" == "pmm3-latest" ]]; then fi ## Only supported for debian based systems for now -if [[ "$client_version" =~ ^"3." ]]; then - ## Note: only experimental packages are present atm. - wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/experimental/p/pmm-client/pmm-client_${client_version}-6.$(lsb_release -sc)_amd64.deb +if [[ "$client_version" =~ ^3\.[0-9]+\.[0-9]+$ ]]; then + wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_${client_version}-7.$(lsb_release -sc)_amd64.deb dpkg -i pmm-client.deb fi From e75ed18bbcc6c85c15c4d1594f70129475a4aefc Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Thu, 6 Mar 2025 13:57:58 +0530 Subject: [PATCH 79/88] Pmm3 add ps replica (#116) * Updates add PS replica test in setup --- .github/workflows/PMM_PS.yaml | 28 +++++++++++++++++++--------- pmm_qa/pmm-framework.py | 3 ++- pmm_qa/ps_pmm_setup.yml | 5 +++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index ad4750d..4d312c2 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -3,11 +3,11 @@ on: workflow_dispatch: inputs: ps_version: - description: "ps version example: 8.0.34-26 , Fetched from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS" - default: 'https://github.com/Percona-QA/package-testing/blob/master/VERSIONS' + description: "PS Version ex: 5.7/8.0/8.4, Minor versions from https://github.com/Percona-QA/package-testing/blob/master/VERSIONS" + default: '8.4' required: true ps_glibc: - description: "ps glibc version, example: 2.35" + description: "PS Glibc version, example: 2.35" default: '2.35' required: false pmm_qa_branch: @@ -19,11 +19,11 @@ on: required: false type: string pmm_server_image: - description: "pmm_image, example: perconalab/pmm-server:3-dev-latest" + description: "PMM Server image, example: perconalab/pmm-server:3-dev-latest" default: 'perconalab/pmm-server:3-dev-latest' required: false pmm_client_version: - description: 'PMM Client version (3-dev-latest|pmm3-rc|x.xx.x|https...)' + description: 'PMM Client version (3-dev-latest|pmm3-rc|3.xx.x|https...)' default: '3-dev-latest' required: false type: string @@ -41,7 +41,7 @@ jobs: timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' - PS_VERSION: ${{ github.event.inputs.ps_version || '8.0' }} + PS_VERSION: ${{ github.event.inputs.ps_version || '8.4' }} PS_GLIBC: ${{ github.event.inputs.ps_glibc || '2.35' }} PMM_SERVER_IMAGE: ${{ github.event.inputs.pmm_server_image || 'perconalab/pmm-server:3-dev-latest' }} PMM_QA_BRANCH: ${{ github.event.inputs.pmm_qa_branch || 'v3' }} @@ -92,7 +92,7 @@ jobs: working-directory: qa-integration run: echo "PATH_TO_PMM_QA=$(pwd)" >> $GITHUB_ENV - - name: Run Setup for E2E Tests + - name: Run Setup for PS E2E Tests working-directory: qa-integration/pmm_qa run: | mkdir -m 777 -p /tmp/backup_data @@ -107,6 +107,16 @@ jobs: python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS,TARBALL=${{ env.PS_TARBALL_PATH }} fi + - name: Run Setup for PS Replica E2E Tests + working-directory: qa-integration/pmm_qa + run: | + . virtenv/bin/activate + if [[ "${{ github.event.inputs.ps_version }}" != http* ]]; then + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS=${{ env.PS_VERSION }},SETUP_TYPE=replica + else + python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PS,TARBALL=${{ env.PS_TARBALL_PATH }},SETUP_TYPE=replica + fi + - name: Checkout PMM UI tests uses: actions/checkout@v4 with: @@ -118,8 +128,8 @@ jobs: run: npm ci && npx playwright install --with-deps && npx codeceptjs def pr.codecept.js working-directory: pmm-ui-tests - - name: Run the Integration tests of PS + - name: Run the Integration tests of PS and PS Replica run: | export PMM_UI_URL="http://127.0.0.1:8081/" - npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js + npx codeceptjs run -c pr.codecept.js tests/qa-integration/pmm_ps_integration_test.js tests/qa-integration/pmm_ps_replica_integration_test.js working-directory: pmm-ui-tests diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index aaa6190..e247420 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -177,7 +177,8 @@ def setup_ps(db_type, db_version=None, db_config=None, args=None): 'PS_NODES': no_of_nodes, 'PS_VERSION': ps_version, 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PS_CONTAINER': 'ps_pmm_' + str(ps_version), + 'PS_CONTAINER': 'ps_pmm_' + str(ps_version) + ('_replica' if setup_type_value in ("replication", "replica") else ''), + 'PS_PORT': 3318 if setup_type_value in ("replication", "replica") else 3317, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), 'PS_TARBALL': get_value('TARBALL', db_type, args, db_config), diff --git a/pmm_qa/ps_pmm_setup.yml b/pmm_qa/ps_pmm_setup.yml index 8213c33..2e28ac7 100644 --- a/pmm_qa/ps_pmm_setup.yml +++ b/pmm_qa/ps_pmm_setup.yml @@ -15,6 +15,7 @@ admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" number_of_nodes: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_NODES') | default('3', true) ) }}" + ps_port: "{{ lookup('vars', 'extra_number_of_nodes', default=lookup('env','PS_PORT') | default('3317', true) ) }}" group_replication: "{{ lookup('vars', 'extra_group_replication', default=lookup('env','GROUP_REPLICATION') | default('', true) ) }}" tasks: @@ -32,7 +33,7 @@ - name: Prepare Container for ps and Proxysql shell: > docker run -d --name={{ ps_container }} - -p 3317:3307 + -p {{ ps_port }}:3307 phusion/baseimage:jammy-1.0.1 - name: Copy all required Artifacts to the docker ps_container @@ -68,5 +69,5 @@ - name: Execute Setup script inside the ps ps_container shell: "{{ item }}" with_items: - - docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ps_{{ ps_version }}.log + - docker exec {{ ps_container }} bash -xe ./client_container_ps_setup.sh --ps_version {{ ps_version }} --ps_tarball {{ ps_tarball }} --number_of_nodes {{ number_of_nodes }} --query_source {{ query_source }} --group_replication {{ group_replication }} > setup_ps_{{ ps_container }}.log From 973356bbccbef686a53271a5ee1fb9aed8fa7bbe Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 6 Mar 2025 11:24:44 +0100 Subject: [PATCH 80/88] PMM-13626: use correct docker container for pmm client --- pmm_qa/setup_docker_client_images.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index 40cfb46..0aa5e43 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -7,7 +7,8 @@ echo "Please wait...." docker network create docker-client-check || true docker compose -f docker-compose-clients.yaml down -v --remove-orphans docker compose -f docker-compose-clients.yaml build --no-cache -docker compose -f docker-compose-clients.yaml up -d +echo "$CLIENT_DOCKER_VERSION is the client docker version" +CLIENT_DOCKER_VERSION=${CLIENT_DOCKER_VERSION} docker compose -f docker-compose-clients.yaml up -d sleep 20 echo "Adding DB Clients to PMM Server" docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true From b5a01f90806d8b518b82a5399f76d16907476064 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 6 Mar 2025 11:33:04 +0100 Subject: [PATCH 81/88] PMM-13626: use correct docker container for pmm client --- pmm_qa/setup_docker_client_images.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index 0aa5e43..4896746 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -15,3 +15,5 @@ docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass docker exec pmm-client-1 pmm-admin add postgresql --query-source=pgstatements --username=pmm --password=pmm-pass --service-name=pdpgsql-1 --host=pdpgsql-1 --port=5432 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true docker exec pmm-client-1 pmm-admin add mongodb --username=pmm --password=pmm-pass --service-name=mongodb-7.0 --host=psmdb-1 --port=27017 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true +docker ps -a + From 2572a0a92a3a674604172c42204bb5d7f9736ff2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 6 Mar 2025 11:33:16 +0100 Subject: [PATCH 82/88] PMM-13626: use correct docker container for pmm client --- pmm_qa/setup_docker_client_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index 4896746..e9cdd99 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -8,7 +8,7 @@ docker network create docker-client-check || true docker compose -f docker-compose-clients.yaml down -v --remove-orphans docker compose -f docker-compose-clients.yaml build --no-cache echo "$CLIENT_DOCKER_VERSION is the client docker version" -CLIENT_DOCKER_VERSION=${CLIENT_DOCKER_VERSION} docker compose -f docker-compose-clients.yaml up -d +CLIENT_DOCKER_VERSION=$CLIENT_DOCKER_VERSION docker compose -f docker-compose-clients.yaml up -d sleep 20 echo "Adding DB Clients to PMM Server" docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true From e04fcd10d3759983747131b166788fe146f31703 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 6 Mar 2025 13:31:20 +0100 Subject: [PATCH 83/88] PMM-13626: use correct docker container for pmm client --- pmm_qa/setup_docker_client_images.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pmm_qa/setup_docker_client_images.sh b/pmm_qa/setup_docker_client_images.sh index e9cdd99..076a4d6 100755 --- a/pmm_qa/setup_docker_client_images.sh +++ b/pmm_qa/setup_docker_client_images.sh @@ -7,13 +7,9 @@ echo "Please wait...." docker network create docker-client-check || true docker compose -f docker-compose-clients.yaml down -v --remove-orphans docker compose -f docker-compose-clients.yaml build --no-cache -echo "$CLIENT_DOCKER_VERSION is the client docker version" CLIENT_DOCKER_VERSION=$CLIENT_DOCKER_VERSION docker compose -f docker-compose-clients.yaml up -d sleep 20 echo "Adding DB Clients to PMM Server" docker exec pmm-client-1 pmm-admin add mysql --username=pmm --password=pmm-pass --service-name=ps-8.0 --query-source=perfschema --host=ps-1 --port=3306 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true docker exec pmm-client-1 pmm-admin add postgresql --query-source=pgstatements --username=pmm --password=pmm-pass --service-name=pdpgsql-1 --host=pdpgsql-1 --port=5432 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true docker exec pmm-client-1 pmm-admin add mongodb --username=pmm --password=pmm-pass --service-name=mongodb-7.0 --host=psmdb-1 --port=27017 --server-url=https://admin:admin@pmm-server-1:8443 --server-insecure-tls=true - -docker ps -a - From 6e876e61373bc91f75b7a89f7fec1468335aae4d Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:41:58 +0530 Subject: [PATCH 84/88] PMM-13871 Updates fix MySQL Replica issue with Auth. (#120) * Updates fix MySQL Replica issue with Auth. * Updates fix PS and GR defaults --- pmm_qa/client_container_ps_setup.sh | 75 ++++++++++++++++++----------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/pmm_qa/client_container_ps_setup.sh b/pmm_qa/client_container_ps_setup.sh index 3b25f4e..3b4ffc3 100644 --- a/pmm_qa/client_container_ps_setup.sh +++ b/pmm_qa/client_container_ps_setup.sh @@ -56,9 +56,14 @@ export SERVICE_RANDOM_NUMBER=$((1 + $RANDOM % 9999)) # Initialize my_cnf_options my_cnf_options="" -# Check if ps_version is 8.4 or greater to enable the plugin to change the password -if [[ "$ps_version" =~ ^8\.[4-9]([0-9])? || "$ps_version" =~ ^[9-9][0-9]\. ]]; then - my_cnf_options="mysql-native-password=ON" +if [[ "$number_of_nodes" -gt 1 ]]; then + # Check if ps_version is greater than 8.0 + if [[ "$ps_version" =~ ^8\.[1-9]([0-9])? || "$ps_version" =~ ^9\.[0-9]+ ]]; then + my_cnf_options="caching_sha2_password_auto_generate_rsa_keys=ON" + else + # MySQL 5.7, create user fails which already exists, to ignore this we do: + my_cnf_options="replicate-ignore-table=mysql.user" + fi fi if [[ "$number_of_nodes" == 1 ]];then @@ -66,14 +71,12 @@ if [[ "$number_of_nodes" == 1 ]];then dbdeployer deploy --topology=group replication ${db_version_sandbox} --single-primary --sandbox-binary=~/ps${ps_version} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'group-single-primary' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';" else dbdeployer deploy single ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --port=${PS_PORT} --remote-access=% --bind-address=0.0.0.0 --force ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'single' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';" if [[ "${query_source}" == "slowlog" ]]; then mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" @@ -123,37 +126,51 @@ if [[ "$number_of_nodes" == 1 ]];then pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=dev --cluster=dev-cluster --replication-set=repl1 ps-single-${SERVICE_RANDOM_NUMBER} 127.0.0.1:$node_port fi else - dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} + dbdeployer deploy multiple ${db_version_sandbox} --sandbox-binary=~/ps${ps_version} --nodes $number_of_nodes --force --remote-access=% --bind-address=0.0.0.0 --my-cnf-options=gtid_mode=ON --my-cnf-options=enforce-gtid-consistency=ON --my-cnf-options=binlog-format=ROW --my-cnf-options=log-slave-updates=ON --my-cnf-options=binlog-checksum=NONE ${my_cnf_options:+--my-cnf-options="$my_cnf_options"} export db_sandbox=$(dbdeployer sandboxes | awk -F' ' '{print $1}') - node_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` - for j in `seq 1 $number_of_nodes`; do - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'msandbox'@'localhost' IDENTIFIED WITH mysql_native_password BY 'msandbox';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'GRgrO9301RuF';" + master_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $1}'` + if echo "$ps_version" | grep '5.7'; then + slave_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $2}'` + else + slave_port=`dbdeployer sandboxes --header | grep ${db_version_sandbox} | grep 'multiple' | awk -F'[' '{print $2}' | awk -F' ' '{print $3}'` + fi + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $master_port -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'GRgrO9301RuF';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $master_port -e "FLUSH PRIVILEGES;" + mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "CREATE USER 'repl'@'%' IDENTIFIED BY 'repl_pass';" + mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'repl'@'%';" + mysql -h 127.0.0.1 -u root -p'GRgrO9301RuF' --port $master_port -e "FLUSH PRIVILEGES;" + if echo "$ps_version" | grep '5.7'; then + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='repl', MASTER_PASSWORD='repl_pass', MASTER_PORT=${master_port}, MASTER_AUTO_POSITION=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "START SLAVE;" + else + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "CHANGE REPLICATION SOURCE TO SOURCE_HOST='127.0.0.1', SOURCE_USER='repl', SOURCE_PASSWORD='repl_pass', SOURCE_PORT=${master_port}, SOURCE_AUTO_POSITION=1, SOURCE_SSL=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $slave_port -e "START REPLICA;" + fi + for port in $master_port $slave_port; do if [[ "${query_source}" == "slowlog" ]]; then - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL slow_query_log='ON';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL long_query_time=0;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_rate_limit=1;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_admin_statements=ON;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL log_slow_slave_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL slow_query_log='ON';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL long_query_time=0;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_rate_limit=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_admin_statements=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL log_slow_slave_statements=ON;" else - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL userstat=1;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL innodb_monitor_enable=all;" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL userstat=1;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL innodb_monitor_enable=all;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME LIKE '%statements%';" if echo "$ps_version" | grep '5.7'; then - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" - mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $node_port -e "SET GLOBAL query_response_time_stats=ON;" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_AUDIT SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_READ SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "INSTALL PLUGIN QUERY_RESPONSE_TIME_WRITE SONAME 'query_response_time.so';" + mysql -h 127.0.0.1 -u msandbox -pmsandbox --port $port -e "SET GLOBAL query_response_time_stats=ON;" fi fi - if [ $(( ${j} % 2 )) -eq 0 ]; then - pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-prod --cluster=ps-prod-cluster --replication-set=ps-repl2 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + if [[ "$port" == "$master_port" ]]; then + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-master-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$master_port else - pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-node-$j-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$node_port + pmm-admin add mysql --query-source=$query_source --username=msandbox --password=msandbox --environment=ps-dev --cluster=ps-dev-cluster --replication-set=ps-repl1 ps-multiple-slave-${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:$slave_port fi #run_workload 127.0.0.1 msandbox msandbox $node_port mysql mysql-multiple-node - node_port=$(($node_port + 1)) sleep 20 done fi From 904c9463a50775246a3073f85a0d3c3ae5325ba6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 24 Mar 2025 13:25:15 +0100 Subject: [PATCH 85/88] PMM-13765: Change runner --- .github/workflows/PMM_PDPGSQL.yaml | 2 +- .github/workflows/PMM_PROXYSQL.yaml | 2 +- .github/workflows/PMM_PS.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index 77aa72c..00fc5ef 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -35,7 +35,7 @@ on: jobs: PMM_PDPGSQL_TEST: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' diff --git a/.github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml index d1185f0..ac0f4ad 100644 --- a/.github/workflows/PMM_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -39,7 +39,7 @@ on: jobs: PMM_PXC_TEST: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index 4d312c2..03e0aff 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -37,7 +37,7 @@ on: jobs: PMM_PS_TEST: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' From 72cb71a5c73d8566ff24564d5ace2690ab367d2c Mon Sep 17 00:00:00 2001 From: saikumar-vs <112166980+saikumar-vs@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:00:37 +0530 Subject: [PATCH 86/88] PMM-13294 MLaunch Support (#76) * MLaunch Support * Updates fix SSL binary removal * Updates to support keyword "shards" as well --- pmm_qa/Dockerfile | 16 +++ pmm_qa/mlaunch_modb_setup.sh | 100 +++++++++++++ pmm_qa/mlaunch_modb_setup.yml | 85 +++++++++++ pmm_qa/mlaunch_psmdb_setup.sh | 136 ++++++++++++++++++ pmm_qa/mlaunch_psmdb_setup.yml | 85 +++++++++++ pmm_qa/mongodb_query.php | 108 ++++++++++++++ pmm_qa/pmm-framework.py | 107 +++++++++++++- pmm_qa/psmdb_setup.sh | 23 ++- pmm_qa/tls-ssl-setup/create_certs.sh | 2 +- pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml | 94 ++++++++++++ .../mongodb/mlaunch_ssl_setup.sh | 70 +++++++++ 11 files changed, 819 insertions(+), 7 deletions(-) create mode 100644 pmm_qa/Dockerfile create mode 100755 pmm_qa/mlaunch_modb_setup.sh create mode 100644 pmm_qa/mlaunch_modb_setup.yml create mode 100755 pmm_qa/mlaunch_psmdb_setup.sh create mode 100644 pmm_qa/mlaunch_psmdb_setup.yml create mode 100644 pmm_qa/mongodb_query.php create mode 100644 pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml create mode 100644 pmm_qa/tls-ssl-setup/mongodb/mlaunch_ssl_setup.sh diff --git a/pmm_qa/Dockerfile b/pmm_qa/Dockerfile new file mode 100644 index 0000000..f78c2c2 --- /dev/null +++ b/pmm_qa/Dockerfile @@ -0,0 +1,16 @@ +FROM php:8.1-alpine + +RUN set -ex \ + && apk --no-cache add postgresql-libs postgresql-dev \ + && docker-php-ext-install pgsql pdo_pgsql \ + && docker-php-ext-install mysqli && docker-php-ext-enable mysqli \ + && apk del postgresql-dev + +RUN apk --update add --virtual build-dependencies build-base openssl-dev autoconf \ + && pecl install mongodb \ + && docker-php-ext-enable mongodb \ + && apk del build-dependencies build-base openssl-dev autoconf \ + && rm -rf /var/cache/apk/* + +RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin/ --filename=composer +RUN composer require mongodb/mongodb diff --git a/pmm_qa/mlaunch_modb_setup.sh b/pmm_qa/mlaunch_modb_setup.sh new file mode 100755 index 0000000..924d458 --- /dev/null +++ b/pmm_qa/mlaunch_modb_setup.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=7.0 +fi + +if [ -z "$mongdb_setup" ]; then + export mongdb_setup=pss +fi + +if [ -z "$metrics_mode" ]; then + export metrics_mode=push +fi + +# Mongo user credtials for the replicat set cluster +export user="dba" +export pwd="test1234" + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl jq git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl +sleep 10 + +#wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh +#chmod +x mongo_startup.sh +export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999))) + +## Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +modb_tarball=$(curl -s "https://www.mongodb.com/try/download/community" | \ + grep -oP "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2004-${mongodb_version//./\\.}\.\d+\.tgz" | \ + sort -V | tail -n 1) + +echo "Downloading ${mongodb_version} from ${modb_tarball}..." +wget -O percona_server_mongodb.tar.gz ${modb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep mongodb-linux) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} modb_${mongodb_version} +rm percona_server_mongodb.tar.gz* + +# For mongodb dependency in Debian +wget http://http.us.debian.org/debian/pool/main/o/openldap/libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb +apt install -y ./libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb + +if [ "$mongodb_setup" == "pss" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 3 + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "psa" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 2 --arbiter + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "sharded" ] || [ "$mongodb_setup" == "shards" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./modb_${mongodb_version}/bin" --replicaset --sharded rs1 rs2 --config 3 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --environment=mongos_shraded_node mongos_shraded_node_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27024 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27025 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27026 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27018 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27019 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27020 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27021 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27022 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27023 + sleep 20 +fi diff --git a/pmm_qa/mlaunch_modb_setup.yml b/pmm_qa/mlaunch_modb_setup.yml new file mode 100644 index 0000000..dfb749c --- /dev/null +++ b/pmm_qa/mlaunch_modb_setup.yml @@ -0,0 +1,85 @@ +--- + +- hosts: all + become: true + become_method: sudo + vars: + modb_version: "{{ lookup('vars', 'extra_modb_version', default=lookup('env','MODB_VERSION') | default('4.4', true) ) }}" + modb_tarball: "{{ lookup('vars', 'extra_modb_tarball', default=lookup('env','MODB_TARBALL') | default('', true) ) }}" + modb_setup: "{{ lookup('vars', 'extra_modb_setup', default=lookup('env','MODB_SETUP') | default('pss', true) ) }}" + modb_container: "{{ lookup('vars', 'extra_modb_container', default=lookup('env','MODB_CONTAINER') | default('MODB', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('v3', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ modb_container }}" | grep -q . && docker stop {{ modb_container }} && docker rm -fv {{ modb_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for modb + shell: > + docker run -d --name={{ modb_container }} + -p 27017:27017 + phusion/baseimage:focal-1.2.0 + + - name: Copy all required Artifacts to the docker modb_container + shell: "{{ item }}" + with_items: + - docker cp ./mlaunch_modb_setup.sh {{ modb_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ modb_container }}:/ + - docker exec {{ modb_container }} apt-get update + - docker exec {{ modb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip + + - name: Install required software's to the docker modb_container + shell: "{{ item }}" + with_items: + - docker exec {{ modb_container }} python3 -m pip install --upgrade pip + - docker exec {{ modb_container }} pip3 install 'mtools[all]' + + - name: Install pmm2-client on the modb_container + shell: "{{ item }}" + with_items: + - docker network connect pmm-qa {{ modb_container }} + - docker exec {{ modb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Setup modb for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ modb_container }} bash -x ./mlaunch_modb_setup.sh --mongodb_version {{ modb_version }} --mongodb_setup {{ modb_setup }} > setup_modb_{{ modb_version }}_{{ modb_setup }}.log + + - name: Setup Load Running Docker Container + shell: "{{ item }}" + with_items: + - rm -rf ~/modb_{{ modb_version }} || true; mkdir -p ~/modb_{{ modb_version }} + - wget -P ~/modb_{{ modb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{pmm_qa_branch}}/pmm_qa/Dockerfile" + - wget -P ~/modb_{{ modb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{pmm_qa_branch}}/pmm_qa/mongodb_query.php" + - docker build --tag php-db ~/modb_{{ modb_version }}/ > ~/docker-build_mongodb_load_{{ modb_version }}_{{ modb_setup }}.log || true + - docker rm mongodb_load_{{ modb_version }}_{{ modb_setup }} || true + - docker run --rm --name mongodb_load_{{ modb_version }}_{{ modb_setup }} --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db composer require mongodb/mongodb || true + + - name: Run load on Replica Set Master(PSS) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "pss" + + - name: Run load on Replica Set Master(PSA) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "psa" + + - name: Run load on Sharded Clusters Master + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log + when: modb_setup == "sharded" \ No newline at end of file diff --git a/pmm_qa/mlaunch_psmdb_setup.sh b/pmm_qa/mlaunch_psmdb_setup.sh new file mode 100755 index 0000000..fc4b274 --- /dev/null +++ b/pmm_qa/mlaunch_psmdb_setup.sh @@ -0,0 +1,136 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=7.0 +fi + +if [ -z "$mongdb_setup" ]; then + export mongdb_setup=pss +fi + +if [ -z "$metrics_mode" ]; then + export metrics_mode=push +fi + +# Mongo user credtials for the replicat set cluster +export user="dba" +export pwd="test1234" + +# Install the dependencies +source ~/.bash_profile || true; +apt-get update +apt-get -y install wget curl jq git gnupg2 lsb-release +apt-get -y install libreadline6-dev systemtap-sdt-dev zlib1g-dev libssl-dev libpam0g-dev python-dev bison make flex libipc-run-perl +sleep 10 + +#wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startup.sh +#chmod +x mongo_startup.sh +export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999))) + +### Detect latest tarball link for specified mongodb_version: 8.0 | 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +psmdb_latest=$( wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) +if [[ "$mongodb_version" == "4.4" ]]; then + psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) +else + psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep jammy-minimal) +fi + +echo "Downloading ${mongodb_version} ..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep percona-server-mongodb) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} psmdb_${mongodb_version} + +## Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +psmdb_tarball=$(curl -s "https://www.mongodb.com/try/download/community" | \ + grep -oP "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2004-${mongodb_version//./\\.}\.\d+\.tgz" | \ + sort -V | tail -n 1) + +echo "Downloading ${mongodb_version} from ${psmdb_tarball}..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz + +export extracted_folder_name=$(ls | grep mongodb-linux) +echo "Extracted folder name ${extracted_folder_name}" +mv ${extracted_folder_name} psmdb_${mongodb_version} + +# TODO: refactor if to match range of versions 6.0+ +if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" || "$mongodb_version" == "8.0" ]]; then +### PSMDB 6+ requires "percona-mongodb-mongosh" additionally + if [[ "$mongodb_version" == "8.0" ]]; then + # Use Mongo 7.0 mongosh itself for 8.0 + psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-7.0" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) + mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh || true) + if [ -z "$mongosh_link" ]; then + psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-6.0" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) + fi + fi + mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh) + echo "Downloading mongosh ${mongosh_link}..." + wget -O mongosh.tar.gz ${mongosh_link} + tar -xvf mongosh.tar.gz + mv percona-mongodb-mongosh* mongosh + cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo + rm mongosh.tar.gz +fi +rm percona_server_mongodb.tar.gz* + +# For mongodb dependency in Debian +wget http://http.us.debian.org/debian/pool/main/o/openldap/libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb +apt install -y ./libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb + +if [ "$mongodb_setup" == "pss" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 3 + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "psa" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 2 --arbiter + sleep 20 + pmm-admin remove mongodb mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27017 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27018 + sleep 2 + pmm-admin remove mongodb mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} || true; pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node --metrics-mode=$metrics_mode mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --debug 127.0.0.1:27019 + sleep 20 +fi + +if [ "$mongodb_setup" == "sharded" ] || [ "$mongodb_setup" == "shards" ]; then + mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --sharded rs1 rs2 --config 3 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --environment=mongos_shraded_node mongos_shraded_node_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27017 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27024 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27025 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=config --environment=mongodb_config_node mongodb_config_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27026 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27018 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27019 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs1 --environment=mongodb_rs_node mongodb_rs1_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27020 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_1_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27021 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_2_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27022 + sleep 2 + pmm-admin add mongodb --enable-all-collectors --cluster mongodb_node_cluster --replication-set=rs2 --environment=mongodb_rs_node mongodb_rs2_3_${SERVICE_RANDOM_NUMBER} --metrics-mode=$metrics_mode --debug 127.0.0.1:27023 + sleep 20 +fi diff --git a/pmm_qa/mlaunch_psmdb_setup.yml b/pmm_qa/mlaunch_psmdb_setup.yml new file mode 100644 index 0000000..7b6fa35 --- /dev/null +++ b/pmm_qa/mlaunch_psmdb_setup.yml @@ -0,0 +1,85 @@ +--- + +- hosts: all + become: true + become_method: sudo + vars: + psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}" + psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}" + psmdb_setup: "{{ lookup('vars', 'extra_psmdb_setup', default=lookup('env','PSMDB_SETUP') | default('pss', true) ) }}" + psmdb_container: "{{ lookup('vars', 'extra_psmdb_container', default=lookup('env','PSMDB_CONTAINER') | default('psmdb', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('v3', true) ) }}" + + tasks: + - name: cleanup container for client and DB setup + shell: > + docker ps -a --filter "name={{ psmdb_container }}" | grep -q . && docker stop {{ psmdb_container }} && docker rm -fv {{ psmdb_container }} + ignore_errors: true + tags: + - cleanup + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for PSMDB + shell: > + docker run -d --name={{ psmdb_container }} + -p 27017:27017 + phusion/baseimage:jammy-1.0.1 + + - name: Copy all required Artifacts to the docker psmdb_container + shell: "{{ item }}" + with_items: + - docker cp ./mlaunch_psmdb_setup.sh {{ psmdb_container }}:/ + - docker cp ./pmm3-client-setup.sh {{ psmdb_container }}:/ + - docker exec {{ psmdb_container }} apt-get update + - docker exec {{ psmdb_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip + + - name: Install required software's to the docker psmdb_container + shell: "{{ item }}" + with_items: + - docker exec {{ psmdb_container }} python3 -m pip install --upgrade pip + - docker exec {{ psmdb_container }} pip3 install 'mtools[all]' + + - name: Install pmm2-client on the psmdb_container + shell: "{{ item }}" + with_items: + - docker network connect pmm-qa {{ psmdb_container }} + - docker exec {{ psmdb_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Setup psmdb for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ psmdb_container }} bash -x ./mlaunch_psmdb_setup.sh --mongodb_version {{ psmdb_version }} --mongodb_setup {{ psmdb_setup }} > setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + + - name: Setup Load Running Docker Container + shell: "{{ item }}" + with_items: + - rm -rf ~/psmdb_{{ psmdb_version }} || true; mkdir -p ~/psmdb_{{ psmdb_version }} + - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{pmm_qa_branch}}/pmm_qa/Dockerfile" + - wget -P ~/psmdb_{{ psmdb_version }}/ "https://raw.githubusercontent.com/Percona-Lab/qa-integration/{{pmm_qa_branch}}/pmm_qa/mongodb_query.php" + - docker build --tag php-db ~/psmdb_{{ psmdb_version }}/ > ~/docker-build_mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }}.log || true + - docker rm mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} || true + - docker run --rm --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db composer require mongodb/mongodb || true + + - name: Run load on Replica Set Master(PSS) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + when: psmdb_setup == "pss" + + - name: Run load on Replica Set Master(PSA) + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + when: psmdb_setup == "psa" + + - name: Run load on Sharded Clusters Master + shell: "{{ item }}" + with_items: + - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log + when: psmdb_setup == "sharded" \ No newline at end of file diff --git a/pmm_qa/mongodb_query.php b/pmm_qa/mongodb_query.php new file mode 100644 index 0000000..494e8a7 --- /dev/null +++ b/pmm_qa/mongodb_query.php @@ -0,0 +1,108 @@ + false]); + +function run_query($db,$collection) +{ + global $client; + $collectionName = "beers" . $collection; + $dbName = "demo" . $db; + $collectionObj = $client->$dbName->$collectionName; + //read + $cursor = $collectionObj->find(); + //update + $collectionObj->updateMany(array("a"=>"a"), + array('$set'=>array("a"=>"a_u"))); + + //count + $collectionObj->count(); + + //distinct + $collectionObj->distinct("a"); + + //aggregate + $collectionObj->aggregate([array('$match' =>array("a"=>"a_u"))]); + + //findAndModify + $collectionObj->findOneAndUpdate(array("a"=>"a_u"), array('$set'=>array("a"=>"a_m"))); + + //delete + $collectionObj->deleteOne(array("a"=>"a_m")); + //create + $result = $collectionObj->insertOne( [ 'a' => 'a', 'b' => 'B', 'c' => $i ] ); +} + +echo("Running Queries...\n"); + +//lets create all db's and data +for($i = 1; $i <= $db; $i++) +{ + $dbName = "demo" . $i; + for ($j = 1; $j <= $collection; $j++) + { + $collectionName = "beers" . $j; + $collectionObj = $client->$dbName->$collectionName; + $result = $collectionObj->insertOne( [ 'a' => 'a', 'b' => 'B', 'c' => $j ] ); + echo "Inserted with Object ID '{$result->getInsertedId()}'"; + } +} + +/* How long we want target to take */ +$target_round_time=1/$target_qps; + +while(1) +{ + $start=microtime(1); + $dbNumber=skewed_rnd(1,$db); + $collectionNumber=skewed_rnd(1,$collection); + run_query($dbNumber,$collectionNumber); + $end=microtime(1); + $round_time=$end-$start; +# echo("Round Took: $round_time\n"); + if($round_time<$target_round_time) /* Went faster than needed */ + { + $sleep=($target_round_time-$round_time)*1000000; +# echo("Sleeping $sleep microseconds\n"); + usleep($sleep); + } +} +?> diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index e247420..6a1d2c2 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -13,6 +13,19 @@ "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", "TARBALL": ""} }, + "MLAUNCH_PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} + }, + "MLAUNCH_MODB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} + }, + "SSL_MLAUNCH": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, "SSL_PSMDB": { "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", @@ -87,7 +100,7 @@ def run_ansible_playbook(playbook_filename, env_vars, args): inventory='127.0.0.1', cmdline='-l localhost, --connection=local', envvars=env_vars, - suppress_env_files=True + suppress_env_files=True, ) print(f'{playbook_filename} playbook execution {r.status}') @@ -408,6 +421,63 @@ def setup_external(db_type, db_version=None, db_config=None, args=None): run_ansible_playbook(playbook_filename, env_vars, args) +def setup_mlaunch_psmdb(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + psmdb_version = os.getenv('PSMDB_VERSION') or db_version or \ + database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'PSMDB_VERSION': psmdb_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PSMDB_CONTAINER': 'psmdb_pmm_' + str(psmdb_version), + 'PSMDB_SETUP': get_value('SETUP_TYPE', db_type, args, db_config), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'mlaunch_psmdb_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + +def setup_mlaunch_modb(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running..Exiting") + exit() + + # Gather Version details + modb_version = os.getenv('MODB_VERSION') or db_version or \ + database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'MODB_VERSION': modb_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MODB_CONTAINER': 'modb_pmm_' + str(modb_version), + 'MODB_SETUP': get_value('SETUP_TYPE', db_type, args, db_config), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'mlaunch_modb_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + def execute_shell_scripts(shell_scripts, project_relative_scripts_dir, env_vars, args): # Get script directory current_directory = os.getcwd() @@ -662,6 +732,34 @@ def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): execute_shell_scripts(shell_scripts, scripts_folder, env_vars, args) +def setup_ssl_mlaunch(db_type, db_version=None, db_config=None, args=None): + # Check if PMM server is running + container_name = get_running_container_name() + if container_name is None and args.pmm_server_ip is None: + print(f"Check if PMM Server is Up and Running...Exiting") + exit(1) + + # Gather Version details + psmdb_version = os.getenv('PSMDB_VERSION') or db_version or \ + database_configs[db_type]["versions"][-1] + + # Define environment variables for playbook + env_vars = { + 'MONGODB_VERSION': psmdb_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MONGODB_SSL_CONTAINER': 'psmdb_ssl_pmm_' + str(psmdb_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'tls-ssl-setup/mlaunch_tls_setup.yml' + + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) + + def setup_pxc_proxysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() @@ -744,6 +842,13 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): setup_ssl_pdpgsql(db_type, db_version, db_config, args) elif db_type == 'SSL_PSMDB': setup_ssl_psmdb(db_type, db_version, db_config, args) + elif db_type == 'MLAUNCH_PSMDB': + setup_mlaunch_psmdb(db_type, db_version, db_config, args) + elif db_type == 'MLAUNCH_MODB': + setup_mlaunch_modb(db_type, db_version, db_config, args) + elif db_type == 'SSL_MLAUNCH': + setup_ssl_mlaunch(db_type, db_version, db_config, args) + else: print(f"Database type {db_type} is not recognised, Exiting...") exit(1) diff --git a/pmm_qa/psmdb_setup.sh b/pmm_qa/psmdb_setup.sh index ca7e62c..dfb243c 100644 --- a/pmm_qa/psmdb_setup.sh +++ b/pmm_qa/psmdb_setup.sh @@ -35,9 +35,13 @@ wget https://raw.githubusercontent.com/Percona-QA/percona-qa/master/mongo_startu chmod +x mongo_startup.sh export SERVICE_RANDOM_NUMBER=$(echo $((1 + $RANDOM % 9999))) -### Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +### Detect latest tarball link for specified mongodb_version: 8.0 | 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) -psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) +if [[ "$mongodb_version" == "4.4" ]]; then + psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) +else + psmdb_tarball=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep focal-minimal) +fi echo "Downloading ${psmdb_latest} ..." wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} @@ -48,16 +52,25 @@ echo "Extracted folder name ${extracted_folder_name}" mv ${extracted_folder_name} psmdb_${mongodb_version} # TODO: refactor if to match range of versions 6.0+ -if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" ]]; then - ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally - echo "Downloading mongosh ..." +if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" || "$mongodb_version" == "8.0" ]]; then +### PSMDB 6+ requires "percona-mongodb-mongosh" additionally + if [[ "$mongodb_version" == "8.0" ]]; then + # Use Mongo 7.0 mongosh itself for 8.0 + psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-7.0" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) + mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh || true) + if [ -z "$mongosh_link" ]; then + psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-6.0" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) + fi + fi mongosh_link=$(wget -q --post-data "version_files=${psmdb_latest}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh) + echo "Downloading mongosh ${mongosh_link}..." wget -O mongosh.tar.gz ${mongosh_link} tar -xvf mongosh.tar.gz mv percona-mongodb-mongosh* mongosh cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo rm mongosh.tar.gz fi +rm percona_server_mongodb.tar.gz* if [ "$mongodb_setup" == "sharded" ]; then bash ./mongo_startup.sh -s -e wiredTiger --mongosExtra="--slowms 1" --mongodExtra="--profile 2 --slowms 1" --configExtra="--profile 2 --slowms 1" --b=./psmdb_${mongodb_version}/bin diff --git a/pmm_qa/tls-ssl-setup/create_certs.sh b/pmm_qa/tls-ssl-setup/create_certs.sh index dcb6d94..809723f 100644 --- a/pmm_qa/tls-ssl-setup/create_certs.sh +++ b/pmm_qa/tls-ssl-setup/create_certs.sh @@ -4,7 +4,7 @@ export PWD=$(pwd) export HOST=localhost mkdir -p certificates pushd certificates -echo -e "\n=== Generating SSL certificates in ${PWD} ===" +echo -e "\n=== Generating SSL certificates in ${PWD}/certificates ===" # Generate self signed root CA cert openssl req -nodes -x509 -newkey rsa:4096 -keyout ca.key -out ca.crt -subj "/C=US/ST=California/L=San Francisco/O=Percona/OU=root/CN=${HOST}/emailAddress=test@percona.com" # Generate server cert to be signed diff --git a/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml b/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml new file mode 100644 index 0000000..f47d885 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml @@ -0,0 +1,94 @@ +--- + +- hosts: all + become: true + become_method: sudo + vars: + mongodb_version: "{{ lookup('vars', 'extra_mongodb_version', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" + mongodb_ssl_container: "{{ lookup('vars', 'extra_mongodb_ssl_container', default=lookup('env','MONGODB_SSL_CONTAINER') | default('mongodb_ssl', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_qa_branch: "{{ lookup('vars', 'extra_pmm_qa_branch', default=lookup('env','PMM_QA_GIT_BRANCH') | default('main', true) ) }}" + + tasks: + - name: Cleanup Docker container for client and DB setup + shell: > + docker ps -a --filter "name={{ mongodb_ssl_container }}" | grep -q . && docker stop {{ mongodb_ssl_container }} && docker rm -fv {{ mongodb_ssl_container }} + ignore_errors: true + tags: + - cleanup + - name: delete network if exist + shell: docker network rm "{{ mongodb_ssl_container }}_network" + ignore_errors: true + tags: + - cleanup + + - name: Create a network + shell: docker network create "{{ mongodb_ssl_container }}_network" + + - name: Create pmm-qa network if not exist + shell: docker network create pmm-qa + ignore_errors: true + + - name: Prepare Container for mongodb ssl container + shell: > + docker run -d --name={{ mongodb_ssl_container }} + --network "{{ mongodb_ssl_container }}_network" + phusion/baseimage:jammy-1.0.1 + + - name: Copy all required Artifacts to the docker mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} mkdir -p artifacts + - docker cp ./mongodb/mlaunch_ssl_setup.sh {{ mongodb_ssl_container }}:/ + - docker cp ./create_certs.sh {{ mongodb_ssl_container }}:/ + - docker cp ../pmm3-client-setup.sh {{ mongodb_ssl_container }}:/ + + - name: Install required software's to the docker psmdb_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} apt-get update + - docker exec {{ mongodb_ssl_container }} apt-get -y install wget curl git gnupg2 lsb-release jq python3 pip + - docker exec {{ mongodb_ssl_container }} python3 -m pip install --upgrade pip + - docker exec {{ mongodb_ssl_container }} pip3 install 'mtools[all]' + + - name: Execute Certs Script inside the mongodb mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -xe ./create_certs.sh > mongodb/setup_mongodb_ssl_{{ mongodb_version }}.log + + - name: Execute Setup script inside the mongodb mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -xe ./mlaunch_ssl_setup.sh --mongodb_version {{ mongodb_version }} >> mongodb/setup_mongodb_ssl_{{ mongodb_version }}.log + + - name: Install pmm2-client on the mongodb_ssl_container + shell: "{{ item }}" + with_items: + - docker network connect pmm-qa {{ mongodb_ssl_container }} + - docker exec {{ mongodb_ssl_container }} bash -x ./pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + + - name: Add pmm-admin binary to path when tar ball installation + shell: docker exec {{ mongodb_ssl_container }} echo "export PATH=$PATH:/pmm2-client/bin" > setup_path.sh + when: '"http" in client_version' + + - name: Remove mongodb service if already added previously + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin remove mongodb {{ mongodb_ssl_container }}_service' + ignore_errors: true + + - name: Add mongodb_ssl for monitoring + shell: "{{ item }}" + with_items: + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin list' + - docker exec {{ mongodb_ssl_container }} bash -c 'source ~/.bash_profile || true; pmm-admin add mongodb --tls --tls-skip-verify --authentication-mechanism=MONGODB-X509 --authentication-database=$external --tls-certificate-key-file=/certificates/client.pem --tls-certificate-key-file-password=/certificates/client.key --tls-ca-file=/certificates/ca.crt {{ mongodb_ssl_container }}_ssl_service' + + - name: Get client cert Files on host + shell: "{{ item }}" + with_items: + - mkdir -p mongodb/{{ mongodb_version }} || true + - docker exec {{ mongodb_ssl_container }} cat /certificates/ca.crt > mongodb/{{ mongodb_version }}/ca.crt + - docker exec {{ mongodb_ssl_container }} cat /certificates/client.key > mongodb/{{ mongodb_version }}/client.key + - docker exec {{ mongodb_ssl_container }} cat /certificates/client.pem > mongodb/{{ mongodb_version }}/client.pem diff --git a/pmm_qa/tls-ssl-setup/mongodb/mlaunch_ssl_setup.sh b/pmm_qa/tls-ssl-setup/mongodb/mlaunch_ssl_setup.sh new file mode 100644 index 0000000..92d4481 --- /dev/null +++ b/pmm_qa/tls-ssl-setup/mongodb/mlaunch_ssl_setup.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +while [ $# -gt 0 ]; do + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + fi + shift +done + +if [ -z "$mongodb_version" ]; then + export mongodb_version=4.4 +fi + +wget https://raw.githubusercontent.com/percona/pmm-qa/main/pmm-tests/mongodb_user_setup.js +### Detect latest tarball link for specified mongodb_version: 7.0 | 6.0 | 5.0 | 4.4 | 4.2 at the moment +#psmdb_latest=$(wget -q --post-data "version=percona-server-mongodb-${mongodb_version}" https://www.percona.com/products-api.php -O - | grep -oP "(?<=value\=\")[^\"]*" | sort -V | tail -1) +psmdb_tarball=$(wget -q --post-data "version_files=percona-server-mongodb-${mongodb_version}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep glibc2\.17-minimal) + +echo "Downloading ${mongodb_version} ..." +wget -O percona_server_mongodb.tar.gz ${psmdb_tarball} +tar -xvf percona_server_mongodb.tar.gz +mv percona-server-mongodb-${mongodb_version}.* psmdb_${mongodb_version} +rm percona_server_mongodb.tar.gz* + +# TODO: refactor if to match range of versions 6.0+ +if [[ "$mongodb_version" == "6.0" || "$mongodb_version" == "7.0" ]]; then + ### PSMDB 6+ requires "percona-mongodb-mongosh" additionally + echo "Downloading mongosh ..." + mongosh_link=$(wget -q --post-data "version_files=percona-server-mongodb-${mongodb_version}&software_files=binary" https://www.percona.com/products-api.php -O - | jq -r '.[] | select(.link | contains("sha") | not) | .link' | grep mongosh) + wget -O mongosh.tar.gz ${mongosh_link} + tar -xvf mongosh.tar.gz + mv percona-mongodb-mongosh* mongosh + cp mongosh/bin/mongosh ./psmdb_${mongodb_version}/bin/mongo + rm mongosh.tar.gz +fi + +# For mongodb dependency in Debian +wget http://http.us.debian.org/debian/pool/main/o/openldap/libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb +apt install -y ./libldap-2.4-2_2.4.47+dfsg-3+deb10u7_amd64.deb + +mlaunch init --bind_ip 0.0.0.0 --binarypath "./psmdb_${mongodb_version}/bin" --replicaset --name rs1 --nodes 3 --sslMode requireSSL --sslPEMKeyFile /certificates/server.pem --sslCAFile /certificates/ca.crt --sslClientCertificate /certificates/client.pem +#bash ./mongo_startup.sh -m --ssl -x -e wiredTiger --mongodExtra="--profile 2 --slowms 1 --bind_ip_all" --b=/psmdb_${mongodb_version}/bin +sleep 20 +./psmdb_${mongodb_version}/bin/mongo --tls --host localhost --port 27017 --tlsCAFile /certificates/ca.crt --tlsCertificateKeyFile /certificates/client.pem mongodb_user_setup.js +cat > add_new_ssl_user.js < Date: Wed, 26 Mar 2025 13:04:01 +0100 Subject: [PATCH 87/88] PMM-13765: Change runner --- .github/workflows/PMM_PDPGSQL.yaml | 2 +- .github/workflows/PMM_PROXYSQL.yaml | 2 +- .github/workflows/PMM_PS.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index 00fc5ef..241333a 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -35,7 +35,7 @@ on: jobs: PMM_PDPGSQL_TEST: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' diff --git a/.github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml index ac0f4ad..79c2e9d 100644 --- a/.github/workflows/PMM_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -39,7 +39,7 @@ on: jobs: PMM_PXC_TEST: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' diff --git a/.github/workflows/PMM_PS.yaml b/.github/workflows/PMM_PS.yaml index 03e0aff..4ffa9c2 100644 --- a/.github/workflows/PMM_PS.yaml +++ b/.github/workflows/PMM_PS.yaml @@ -37,7 +37,7 @@ on: jobs: PMM_PS_TEST: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 40 env: ADMIN_PASSWORD: 'admin' From b7113e0ddda2d6c506d1e68ee803df7a02058a77 Mon Sep 17 00:00:00 2001 From: Vasyl Yurkovych <59879559+yurkovychv@users.noreply.github.com> Date: Thu, 27 Mar 2025 16:29:16 +0200 Subject: [PATCH 88/88] PMM-13890 add agent password argument (#122) * PMM-13890 add agent password argument * PMM-13890 update tests * PMM-13890 update setup * PMM-13890 update setup * PMM-13890 update test * PMM-13890 update test --- pmm_psmdb-pbm_setup/configure-agents.sh | 6 ++++-- pmm_psmdb-pbm_setup/configure-extra-agents.sh | 4 ++-- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 1 - pmm_psmdb-pbm_setup/start-sharded.sh | 4 ++-- pmm_psmdb-pbm_setup/test/test.py | 2 +- pmm_psmdb_diffauth_setup/test-auth.sh | 2 +- pmm_psmdb_diffauth_setup/test/test.py | 16 ++++++++-------- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pmm_psmdb-pbm_setup/configure-agents.sh b/pmm_psmdb-pbm_setup/configure-agents.sh index 107156f..93e9fa9 100644 --- a/pmm_psmdb-pbm_setup/configure-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-agents.sh @@ -18,6 +18,8 @@ do docker compose -f docker-compose-rs.yaml exec -T $node systemctl restart pbm-agent done +docker compose -f docker-compose-rs.yaml exec -T rs101 pbm config --file /etc/pbm/minio.yaml + if [[ $mongo_setup_type == "psa" ]]; then echo "stop pbm agent for arbiter node" docker compose -f docker-compose-rs.yaml exec -T rs103 systemctl stop pbm-agent @@ -31,9 +33,9 @@ do echo "congiguring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs103" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done echo diff --git a/pmm_psmdb-pbm_setup/configure-extra-agents.sh b/pmm_psmdb-pbm_setup/configure-extra-agents.sh index ce486d8..0c96a3d 100644 --- a/pmm_psmdb-pbm_setup/configure-extra-agents.sh +++ b/pmm_psmdb-pbm_setup/configure-extra-agents.sh @@ -31,8 +31,8 @@ do echo "configuring pmm agent on $node" docker compose -f docker-compose-rs.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}._${random_number} $node pmm-agent setup if [[ $mongo_setup_type == "psa" && $node == "rs203" ]]; then - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs1 ${node}_${random_number} 127.0.0.1:27017 else - docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-rs.yaml exec -T $node pmm-admin add mongodb --enable-all-collectors --agent-password=mypass --cluster=replicaset --replication-set=rs1 --username=${pmm_mongo_user} --password=${pmm_mongo_user_pass} ${node}_${random_number} 127.0.0.1:27017 fi done diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 6bdee0f..b140258 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -33,7 +33,6 @@ services: - ./conf/datagen:/etc/datagen:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - - /tmp/mongodb:/tmp privileged: true cgroup: host environment: diff --git a/pmm_psmdb-pbm_setup/start-sharded.sh b/pmm_psmdb-pbm_setup/start-sharded.sh index 4beac37..4fe16b1 100755 --- a/pmm_psmdb-pbm_setup/start-sharded.sh +++ b/pmm_psmdb-pbm_setup/start-sharded.sh @@ -237,10 +237,10 @@ do echo "congiguring pmm agent on $node" rs=$(echo $node | awk -F "0" '{print $1}') docker compose -f docker-compose-sharded.yaml exec -T -e PMM_AGENT_SETUP_NODE_NAME=${node}_${random_number} $node pmm-agent setup - docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 + docker compose -f docker-compose-sharded.yaml exec -T $node pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} ${node}_${random_number} 127.0.0.1:27017 done echo "configuring pmm-agent on primary rscfg01 for mongos instance" -docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 +docker compose -f docker-compose-sharded.yaml exec -T rscfg01 pmm-admin add mongodb --agent-password=mypass --cluster=sharded --username=${pmm_user} --password=${pmm_pass} mongos_${random_number} mongos:27017 echo "adding some data" docker compose -f docker-compose-sharded.yaml exec -T mongos mgodatagen -f /etc/datagen/sharded.json --uri=mongodb://root:root@127.0.0.1:27017 diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index 660a5de..6314f7c 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -173,7 +173,7 @@ def test_metrics(): pmm_admin_list = json.loads(docker_rs101.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': - agent_id = agent['agent_id'] + agent_id = "mypass" agent_port = agent['port'] break try: diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index e4f6b3d..ade6c3f 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -68,7 +68,7 @@ done #Add Mongo Service random_number=$RANDOM -docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster +docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-admin add mongodb psmdb-server_${random_number} --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem --cluster=mycluster #Add some data docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server mgodatagen -f /etc/datagen/replicaset.json --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" --host psmdb-server --port 27017 --tlsCertificateKeyFile=/mongodb_certs/client.pem --tlsCAFile=/mongodb_certs/ca-certs.pem diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index 0cf308b..03d1d88 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -26,7 +26,7 @@ def run_test(add_db_command): pmm_admin_list = json.loads(docker_pmm_client.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: if agent['agent_type'] == 'AGENT_TYPE_MONGODB_EXPORTER': - agent_id = agent['agent_id'] + agent_id = "mypass" agent_port = agent['port'] break try: @@ -48,33 +48,33 @@ def run_test(add_db_command): def test_simple_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ''--host ' + run_test('pmm-admin add mongodb psmdb-server --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ''--host ' 'psmdb-server --port 27017') def test_simple_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ' + run_test('pmm-admin add mongodb psmdb-server --agent-password=mypass --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ' '--host psmdb-server --port 27017 ' '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' '--cluster=mycluster') def test_x509_auth(): - run_test('pmm-admin add mongodb psmdb-server --host=psmdb-server --port 27017 ' + run_test('pmm-admin add mongodb psmdb-server --agent-password=mypass --host=psmdb-server --port 27017 ' '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' '--authentication-mechanism=MONGODB-X509 --authentication-database=\'$external\' ' '--cluster=mycluster') def test_ldap_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + run_test('pmm-admin add mongodb psmdb-server --agent-password=mypass --username="CN=pmm-test" --password=password1 ' '--host=psmdb-server --port 27017 ' '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' '--cluster=mycluster') def test_ldap_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + run_test('pmm-admin add mongodb psmdb-server --agent-password=mypass --username="CN=pmm-test" --password=password1 ' '--host=psmdb-server --port 27017 ' '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' @@ -85,7 +85,7 @@ def test_ldap_auth_tls(): any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_wo_tls(): - run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + run_test(f'pmm-admin add mongodb psmdb-server --agent-password=mypass --username={os.environ.get("AWS_ACCESS_KEY_ID")}' f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' '--host=psmdb-server --port 27017 ' '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' @@ -96,7 +96,7 @@ def test_aws_auth_wo_tls(): any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_tls(): - run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + run_test(f'pmm-admin add mongodb psmdb-server --agent-password=mypass --username={os.environ.get("AWS_ACCESS_KEY_ID")}' f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' '--host=psmdb-server --port 27017 ' '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' '