feat: init project

.commitlintrc

@@ -0,0 +1,9 @@
+{ 
+  "extends": ["@commitlint/config-conventional"],
+  "rules": {
+    "type-enum": [2, "always", [
+        "build", "chore", "ci", "docs", "feat", "feature", "fix", "bugfix", "hotfix",
+         "perf", "refactor", "revert", "style", "test",  "update", "release", "delete"
+      ]]
+  }
+}

.github/workflows/app-cd.yaml

@@ -0,0 +1,123 @@
+name: app-cd
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+env:
+  helmRepoURL: 'https://harbor.cvrgo.com/chartrepo/dds'
+  helmRepoName: 'dds-deploy'
+  k8sNamespace: ''
+  chartPath: 'pipeline'
+  dockerRegistry: 'harbor.cvrgo.com/dds'
+
+jobs:
+  build_test_deploy:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout deepdataspace
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get Git Tag
+        id: get_git_tag
+        run: echo "tag=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
+
+      - name: Set environment variables
+        run: |
+          gitTag="${{ steps.get_git_tag.outputs.tag }}"
+          echo "GIT_TAG=${gitTag}"
+          echo "GIT_TAG=${gitTag}" >> $GITHUB_ENV
+          if [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-dev[0-9]+ ]]; then
+            echo "K8S_NAMESPACE=dds-dev" >> $GITHUB_ENV
+          elif [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+ ]]; then
+            echo "K8S_NAMESPACE=dds-test" >> $GITHUB_ENV
+          elif [[ "${gitTag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "K8S_NAMESPACE=dds-prd" >> $GITHUB_ENV
+          else
+            echo "Invalid git tag. Pipeline terminated."
+            exit 1
+          fi
+
+      - name: Checkout deepdataspace-app-env
+        uses: actions/checkout@v2
+        with:
+          repository: deepdataspace/deepdataspace-app-env
+          ref: 'main'
+          path: deepdataspace-app-env
+          token: ${{ secrets.DDS_TOKEN }}
+
+      - name: Copy frontend build env file 
+        run: | 
+          if [[ "${{ env.K8S_NAMESPACE }}" == 'dds-dev' ]]; then  
+            cp deepdataspace-app-env/.env.dev packages/app/.env.local  
+          elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-test' ]]; then  
+            cp deepdataspace-app-env/.env.test packages/app/.env.local  
+          elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-prd' ]]; then  
+            cp deepdataspace-app-env/.env.prod packages/app/.env.local  
+          fi  
+          echo "TAG_VERSION=${{ env.GIT_TAG }}" >> packages/app/.env.local
+
+      - name: Checkout devops repo
+        uses: actions/checkout@v2
+        with:
+          repository: deepdataspace/deepdataspace-helm
+          ref: 'main'
+          path: deepdataspace-helm
+          token: ${{ secrets.DDS_TOKEN }}
+
+      - name: Copy dockerfiles
+        run: |
+          cp -r deepdataspace-helm/dockerfiles .
+
+      - name: Build Docker image
+        run: |
+          imageName="${{ env.helmRepoName }}"
+          imageTag="${{ env.GIT_TAG }}"
+          docker login ${{ env.dockerRegistry }} -u 'robot$robot-dds' --password ${{ secrets.HARBOR_CREDENTIAL }}
+          docker build -t ${{ env.dockerRegistry }}/${imageName}:${imageTag} -f dockerfiles/deploy.Dockerfile .
+          docker push ${{ env.dockerRegistry }}/${imageName}:${imageTag}
+
+      - name: Copy frontend helm values file
+        working-directory: deepdataspace-helm
+        run: |
+          chartPath="${{ env.chartPath }}"
+          if [[ "${{ env.K8S_NAMESPACE }}" == 'dds-dev' ]]; then 
+            cp ${chartPath}/env/dev-values.yaml ${chartPath}/values.yaml
+          elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-test' ]]; then 
+            cp ${chartPath}/env/test-values.yaml ${chartPath}/values.yaml
+          elif [[ "${{ env.K8S_NAMESPACE }}" == 'dds-prd' ]]; then 
+            cp ${chartPath}/env/prd-values.yaml ${chartPath}/values.yaml
+          fi 
+          
+      - name: Update Helm chart
+        working-directory: deepdataspace-helm
+        run: |
+          imageName="${{ env.helmRepoName }}"
+          imageTag="${{ env.GIT_TAG }}"
+          ver="${{ env.GIT_TAG }}-$(date +%s)"
+          echo "ver=${ver}" >> $GITHUB_ENV
+          chartPath="${{ env.chartPath }}"
+          sed -i "s/^version:.*/version: ${ver}/" ${chartPath}/Chart.yaml
+          sed -i "s/^appVersion:.*/appVersion: ${imageTag}/" ${chartPath}/Chart.yaml
+          sed -i "s/^name:.*/name: ${{ env.K8S_NAMESPACE }}/" ${chartPath}/Chart.yaml
+
+      - name: Push Helm chart to Harbor
+        working-directory: deepdataspace-helm
+        run: |
+          helm repo add --username 'robot$robot-dds' --password ${{ secrets.HARBOR_CREDENTIAL }} ${{ env.helmRepoName }} ${{ env.helmRepoURL }} 
+          helm repo update 
+          helm package ${{ env.chartPath }}
+          helm cm-push ${{ env.chartPath }} ${{ env.helmRepoName }}
+          
+      - name: Deploy to Kubernetes
+        working-directory: deepdataspace-helm
+        run: |
+          chartPath="${{ env.chartPath }}"
+          imageTag="${{ env.GIT_TAG }}"
+          helm repo update
+          helm upgrade --install --wait --namespace ${{ env.K8S_NAMESPACE }} ${{ env.K8S_NAMESPACE }} ${{ env.helmRepoName }}/${{ env.K8S_NAMESPACE }} -f ${chartPath}/values.yaml --version ${{ env.ver }} --set django.imageTag=${imageTag},celery.imageTag=${imageTag}

.github/workflows/tool-ci.yaml

@@ -0,0 +1,57 @@
+name: tool-ci
+
+on:
+  push:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  test_and_scan:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run pnpm audit
+        run: pnpm audit --registry=https://registry.npmjs.org
+
+      - name: Run Test
+        run: |
+          export PYENV_ROOT="$HOME/.pyenv"
+          command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+          eval "$(pyenv init -)"
+          pyenv local dds-cicd
+          pip install -r requirements-dev.txt
+          pytest --cov=deepdataspace -s --cov-report=xml
+
+      - name: Upload Test Report
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+          verbose: true
+
+      - name: Scan Code
+        uses: sonarsource/sonarqube-scan-action@master
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+        with:
+          args: >
+            -Dsonar.projectKey=dds-tool
+            -Dsonar.python.coverage.reportPaths=/data/home/deploy/actions-runner/_work/deepdataspace/deepdataspace/coverage.xml

.gitignore

@@ -0,0 +1,34 @@
+# for local dev
+/op
+.idea
+.vscode
+dev.*
+venv
+/dev/*
+.DS_Store
+/config.py
+.python-version
+
+# for runtime file
+/data
+*.pyc
+*.log
+*.pid
+
+# for tests
+/htmlcov
+/.coverage
+/.coverage.*
+
+# for setup.py
+/dist
+/build
+/MANIFEST
+/deepdataspace.egg-info
+
+# for docs
+/docs
+
+# frontend
+/node_modules
+/packages/deepdataspace-website

.husky/commit-msg

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx --no -- commitlint --edit $1

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+pnpm -r --stream run precommit

.npmrc

@@ -0,0 +1,2 @@
+registry=https://registry.npmmirror.com
+strict-peer-dependencies=false