azure-pipelines.yml

trigger:
  batch: true
  branches:
    include:
      - master
  tags:
    include:
      - "v*"

pr:
  autoCancel: true
  branches:
    include:
      - master
  drafts: true
  paths:
    exclude:
      - AUTHORS
      - CHANGELOG.md
      - CONTRIBUTING.md
      - LICENSE
      - README.md
      - binder/
      - docs/
      - examples/
      - python/perspective/README.md

schedules:
  - cron: "0 0 * * 6"
    displayName: Daily midnight build
    branches:
      include:
        - master
    always: "true"

variables:
  YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn

parameters:
  - name: initSteps
    type: stepList
    default:
      - task: NodeTool@0
        inputs:
          versionSpec: "16.x"

      - bash: npm install -g yarn
        displayName: "Install Yarn"

      - task: Cache@2
        inputs:
          key: 'yarn | "$(Agent.OS)" | yarn.lock'
          restoreKeys: |
            yarn | "$(Agent.OS)"
            yarn
          path: $(YARN_CACHE_FOLDER)
        displayName: Cache Yarn packages
        condition: not(eq(variables['Agent.OS'], 'Windows_NT'))

      - bash: yarn --frozen-lockfile
        displayName: "Install Javascript Dependencies"

  - name: pythonInitSteps
    type: stepList
    default:
      - task: UsePythonVersion@0
        inputs:
          versionSpec: "$(python.version)"
          architecture: "x64"
        displayName: "Use Python $(python.version)"

      - bash: python -m pip install --upgrade pip wheel setuptools "jupyterlab>=3.0.14" numpy "pyarrow>=5" "black==20.8b1" flake8-black
        displayName: "Install Python base dependencies"
        condition: succeeded()

  - name: wasmInitSteps
    type: stepList
    default:
      - bash: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
        displayName: "Install wasm-pack"

      - bash: rustup toolchain install nightly
        displayName: "Install Rust nightly"

      - bash: rustup component add rust-src --toolchain nightly
        displayName: "Install rust-src"

  - name: linuxInitSteps
    type: stepList
    default:
      - bash: sudo node scripts/install_tools.js
        displayName: "Install Linux System dependencies"

  - name: macInitSteps
    type: stepList
    default:
      - bash: brew install boost
        displayName: "Install Mac System dependencies"

  - name: windowsInitSteps
    type: stepList
    default:
      - script: |
          which python > python.txt
          set /p PYTHON=<python.txt
          ln -s %PYTHON% %PYTHON%$(python.version)
          python --version
          which python$(python.version)
        displayName: "Which python"

      # https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/utility/command-line?view=azure-devops&tabs=yaml
      # When you want to run a batch file from another batch file in Windows CMD, you must use the call command, otherwise the first batch file is terminated.
      - script: |
          set VCPKG_DEFAULT_TRIPLET=x64-windows
          set VCPKG_PLATFORM_TOOLSET=v142
          git clone https://github.com/Microsoft/vcpkg
          cd vcpkg
          call bootstrap-vcpkg.bat
          call vcpkg.exe install boost-algorithm boost-filesystem boost-multi-index boost-multiprecision boost-program-options boost-system boost-unordered boost-uuid
          call vcpkg.exe integrate install
        displayName: "Install Windows System dependencies"

  - name: webassemblySteps
    type: stepList
    default:
      - bash: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
        displayName: "Install wasm-pack"

      - bash: rustup toolchain install nightly
        displayName: "Install Rust nightly"

      - bash: rustup component add rust-src --toolchain nightly
        displayName: "Install rust-src"

      - bash: yarn lint_js
        displayName: "Lint Javascript"

      - bash: yarn build_js --ci
        displayName: "Build Javascript"

      - bash: yarn test_js --quiet --ci
        displayName: "Test Javascript"
        env:
          PSP_DOCKER: 1

  - name: pythonBuildSteps
    type: stepList
    default:
      - script: yarn build_python  --ci $(python_flag) $(manylinux_flag)
        displayName: "Build Python"

  - name: pythonBuildVS2019Steps
    type: stepList
    default:
      - script: yarn build_python  --ci $(python_flag) $(manylinux_flag)
        displayName: "Build Python"
        env:
          # azure pipelines are bad, so force windows-2019 to use VS2019
          PSP_GENERATOR: Visual Studio 16 2019
      
  - name: pythonBuildDockerSteps
    type: stepList
    default:
      - bash: yarn lint_python
        displayName: "Lint Python"

      - bash: yarn build_python  --ci $(python_flag) $(manylinux_flag)
        displayName: "Build Python"
        env:
          PSP_DOCKER: 1

  - name: pythonCoverageSteps
    type: stepList
    default:
      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFiles: "python/perspective/python_junit.xml"
          testRunTitle: "Publish test results for Python $(python.version) $(manylinux_flag)"

      - task: PublishCodeCoverageResults@1
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: "$(System.DefaultWorkingDirectory)/**/*coverage.xml"

  - name: pythonSDistSteps
    type: stepList
    default:
      # Build a python sdist
      - bash: export SDIST_PUBLISH_NAME=perspective-python-$(Build.SourceBranchName)-$(Build.BuildId).tar.gz && cd $(System.DefaultWorkingDirectory)/python/perspective && ./scripts/build_sdist.sh
        displayName: "Build Python SDist"

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/python/perspective/dist/"
          artifactName: "perspective-python-$(Build.SourceBranchName)-$(Build.BuildId)"

  - name: pythonMacWheelSteps
    type: stepList
    default:
      # Build a python wheel for Mac 10.14
      - bash: yarn _wheel_python $(python_flag) --macos
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')))
        displayName: "Build Mac wheel"

      # Test the wheel
      - bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) --macos
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')))
        displayName: "Test Mac Wheel"

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')))
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/python/perspective/dist/"
          artifactName: "$(artifact_name)"

  - name: pythonLinuxWheelSteps
    type: stepList
    default:
      # Build a python wheel for Manylinux
      - bash: yarn _wheel_python --ci $(python_flag) $(manylinux_flag)
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
        displayName: "Build Manylinux Wheel"
        env:
          PSP_DOCKER: 1

      # Test the wheel
      - bash: cd $(System.DefaultWorkingDirectory)/python/perspective/scripts && ./test_wheels.sh $(python_flag) $(manylinux_flag)
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
        displayName: "Test Manylinux Wheel"

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), ne(variables['manylinux_flag'], ''))
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/python/perspective/wheelhouse/"
          artifactName: "$(artifact_name)"

  - name: pythonWindowsWheelVS2019Steps
    type: stepList
    default:
      # Build a python wheel for Windows
      - bash: yarn _wheel_python $(python_flag)
        displayName: "Build Windows Wheel"
        env:
          # azure pipelines are bad, so force windows-2019 to use VS2019
          PSP_GENERATOR: Visual Studio 16 2019

      # Save the artifact to Azure storage
      - task: PublishPipelineArtifact@1
        condition: and(succeeded(), or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')))
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/python/perspective/dist/"
          artifactName: "$(artifact_name)"

jobs:
  - job: "WebAssembly"
    pool:
      vmImage: "ubuntu-latest"

    strategy:
      matrix:
        Base:
          # Do we care to test elsewhere?
          python.version: "3.9"

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.wasmInitSteps }}
      - ${{ parameters.linuxInitSteps }}
      - ${{ parameters.webassemblySteps }}

      - task: PublishTestResults@2
        condition: succeeded()
        inputs:
          testRunner: JUnit
          testResultsFiles: "junit.xml"

      - task: PublishPipelineArtifact@1
        condition: failed()
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/rust/perspective-viewer/test/screenshots/"
          artifactName: "perspective-viewer-$(Build.BuildNumber)"

      - task: PublishPipelineArtifact@1
        condition: failed()
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-viewer-datagrid/test/screenshots/"
          artifactName: "perspective-viewer-datagrid-$(Build.BuildNumber)"

      - task: PublishPipelineArtifact@1
        condition: failed()
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-viewer-d3fc/test/screenshots/"
          artifactName: "perspective-viewer-d3fc-$(Build.BuildNumber)"

      - task: PublishPipelineArtifact@1
        condition: failed()
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-workspace/test/screenshots/"
          artifactName: "perspective-workspace-$(Build.BuildNumber)"

      # `yarn pack` the public packages so the Jupyterlab tests can use the
      # already-built `dist` folder for perspective, perspective-viewer,
      # perspective-viewer-datagrid and perspective-viewer-d3fc.

      - bash: cd $(System.DefaultWorkingDirectory)/packages/perspective && yarn pack --filename=perspective-dist.tgz
        displayName: "yarn pack perspective"

      - bash: cd $(System.DefaultWorkingDirectory)/rust/perspective-viewer && yarn pack --filename=perspective-viewer-dist.tgz
        displayName: "yarn pack perspective-viewer"

      - bash: cd $(System.DefaultWorkingDirectory)/packages/perspective-viewer-datagrid && yarn pack --filename=perspective-viewer-datagrid-dist.tgz
        displayName: "yarn pack perspective-viewer-datagrid"

      - bash: cd $(System.DefaultWorkingDirectory)/packages/perspective-viewer-d3fc && yarn pack --filename=perspective-viewer-d3fc-dist.tgz
        displayName: "yarn pack perspective-viewer-d3fc"

      - bash: cd $(System.DefaultWorkingDirectory)/packages/perspective-jupyterlab && yarn pack --filename=perspective-jupyterlab-dist.tgz
        displayName: "yarn pack perspective-jupyterlab"

      # Upload the packed versions
      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective/perspective-dist.tgz"
          artifactName: "perspective-dist"

      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/rust/perspective-viewer/perspective-viewer-dist.tgz"
          artifactName: "perspective-viewer-dist"

      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-viewer-datagrid/perspective-viewer-datagrid-dist.tgz"
          artifactName: "perspective-viewer-datagrid-dist"

      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-viewer-d3fc/perspective-viewer-d3fc-dist.tgz"
          artifactName: "perspective-viewer-d3fc-dist"

      - task: PublishPipelineArtifact@1
        inputs:
          targetPath: "$(System.DefaultWorkingDirectory)/packages/perspective-jupyterlab/perspective-jupyterlab-dist.tgz"
          artifactName: "perspective-jupyterlab-dist"

  - job: "Linux"
    pool:
      vmImage: "ubuntu-latest"

    strategy:
      matrix:
        Python37ManyLinux2010:
          python.version: "3.7"
          python_flag: ""
          manylinux_flag: "--manylinux2010"
          artifact_name: "cp37-cp37m-manylinux2010_x86_64"

        ? ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')) }}
        : Python36ManyLinux2010:
            python.version: "3.6"
            python_flag: "--python36"
            manylinux_flag: "--manylinux2010"
            artifact_name: "cp36-cp36m-manylinux2010_x86_64"
          Python36ManyLinux2014:
            python.version: "3.6"
            python_flag: "--python36"
            manylinux_flag: "--manylinux2014"
            artifact_name: "cp36-cp36m-manylinux2014_x86_64"
          # Python37ManyLinux2010 is always built
          Python37ManyLinux2014:
            python.version: "3.7"
            python_flag: ""
            manylinux_flag: "--manylinux2014"
            artifact_name: "cp37-cp37m-manylinux2014_x86_64"
          Python38ManyLinux2010:
            python.version: "3.8"
            python_flag: "--python38"
            manylinux_flag: "--manylinux2010"
            artifact_name: "cp38-cp38-manylinux2010_x86_64"
          Python38ManyLinux2014:
            python.version: "3.8"
            python_flag: "--python38"
            manylinux_flag: "--manylinux2014"
            artifact_name: "cp38-cp38-manylinux2014_x86_64"
          Python39ManyLinux2010:
            python.version: "3.9"
            python_flag: "--python39"
            manylinux_flag: "--manylinux2010"
            artifact_name: "cp39-cp39-manylinux2010_x86_64"
          Python39ManyLinux2014:
            python.version: "3.9"
            python_flag: "--python39"
            manylinux_flag: "--manylinux2014"
            artifact_name: "cp39-cp39-manylinux2014_x86_64"

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.pythonBuildDockerSteps }}
      - ${{ parameters.pythonCoverageSteps }}
      - ${{ parameters.pythonLinuxWheelSteps }}

  # - job: 'Linux_Python_SDist'
  #   pool:
  #     vmImage: 'ubuntu-18.04'

  #   strategy:
  #     matrix:
  #       Python37:
  #         python.version: '3.7'
  #         python_flag: ''

  #   steps:
  #     - ${{ parameters.initSteps }}
  #     - ${{ parameters.linuxInitSteps }}
  #     - ${{ parameters.pythonBuildSteps }}
  #     - ${{ parameters.pythonSDistSteps }}

  - job: "Windows"
    pool:
      vmImage: "windows-2019"

    strategy:
      matrix:
        Python37:
          python.version: "3.7"
          python_flag: ""
          artifact_name: "cp37-cp37m-win64_amd"
        ? ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual')) }}
        : Python38:
            python.version: "3.8"
            python_flag: "--python38"
            artifact_name: "cp38-cp38m-win64_amd"
          Python39:
            python.version: "3.9"
            python_flag: "--python39"
            artifact_name: "cp39-cp39m-win64_amd"

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.windowsInitSteps }}
      - ${{ parameters.pythonBuildVS2019Steps }}

  # Separate out windows wheel builds as the windows
  # build takes 37 years to finish
  - job: "Windows_Wheels"
    pool:
      vmImage: "windows-2019"
    condition: or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual'))

    strategy:
      matrix:
        Python37:
          python.version: "3.7"
          python_flag: ""
          artifact_name: "cp37-cp37m-win64_amd"
        Python38:
          python.version: "3.8"
          python_flag: "--python38"
          artifact_name: "cp38-cp38m-win64_amd"
        Python39:
          python.version: "3.9"
          python_flag: "--python39"
          artifact_name: "cp39-cp39m-win64_amd"

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.windowsInitSteps }}
      - ${{ parameters.pythonWindowsWheelVS2019Steps }}

  - job: "MacOS_Catalina"
    pool:
      vmImage: "macos-10.15"
    condition: or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule'), eq(variables['Build.Reason'], 'Manual'))

    strategy:
      matrix:
          Python36:
            python.version: "3.6"
            python_flag: "--python36"
            artifact_name: "cp36-cp36m-macosx_10_15_x86_64"

          Python37:
            python.version: "3.7"
            python_flag: ""
            artifact_name: "cp37-cp37m-macosx_10_15_x86_64"

          Python38:
            python.version: "3.8"
            python_flag: "--python38"
            artifact_name: "cp38-cp38-macosx_10_15_x86_64"
            
          Python39:
            python.version: "3.9"
            python_flag: "--python39"
            artifact_name: "cp39-cp39-macosx_10_15_x86_64"
          

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.macInitSteps }}
      - ${{ parameters.pythonBuildSteps }}
      - ${{ parameters.pythonCoverageSteps }}
      - ${{ parameters.pythonMacWheelSteps }}

  - job: "MacOS_BigSur"
    pool:
      vmImage: "macos-11"

    strategy:
      matrix:
        Python39:
          python.version: "3.9"
          python_flag: "--python39"
          artifact_name: "cp39-cp39-macosx_11_0_x86_64"

    steps:
      - ${{ parameters.initSteps }}
      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.macInitSteps }}
      - ${{ parameters.pythonBuildSteps }}
      - ${{ parameters.pythonCoverageSteps }}
      - ${{ parameters.pythonMacWheelSteps }}

  - job: "Jupyterlab"
    dependsOn: ["WebAssembly", "Linux"]
    pool:
      vmImage: "ubuntu-latest"
    variables:
      python.version: "3.7"
      python_flag: ""
      manylinux_flag: "--manylinux2010"
      artifact_name: "cp37-cp37m-manylinux2010_x86_64"

    steps:
      # don't use InitSteps because we don't want to cache local puppeteer
      - task: NodeTool@0
        inputs:
          versionSpec: "16.x"

      - bash: npm install -g yarn
        displayName: "Install Yarn"

      - bash: yarn --frozen-lockfile
        displayName: "Install Javascript Dependencies"

      - ${{ parameters.pythonInitSteps }}
      - ${{ parameters.wasmInitSteps }}

      - task: DownloadPipelineArtifact@2
        displayName: "Download Python 3.7 Perspective Wheel"
        inputs:
          artifact: $(artifact_name)
          path: $(System.DefaultWorkingDirectory)/wheel

      - bash: python -m pip install $(System.DefaultWorkingDirectory)/wheel/*.whl
        displayName: "Install perspective-python from wheel"


      - bash: yarn add -W --dev puppeteer@9.0.0
        displayName: "Install Puppeteer"

      - task: DownloadPipelineArtifact@2
        displayName: "Download perspective dist"
        inputs:
          artifact: "perspective-dist"
          path: $(System.DefaultWorkingDirectory)

      - task: DownloadPipelineArtifact@2
        displayName: "Download perspective-viewer dist"
        inputs:
          artifact: "perspective-viewer-dist"
          path: $(System.DefaultWorkingDirectory)

      - task: DownloadPipelineArtifact@2
        displayName: "Download perspective-viewer-datagrid dist"
        inputs:
          artifact: "perspective-viewer-datagrid-dist"
          path: $(System.DefaultWorkingDirectory)

      - task: DownloadPipelineArtifact@2
        displayName: "Download perspective-viewer-d3fc dist"
        inputs:
          artifact: "perspective-viewer-d3fc-dist"
          path: $(System.DefaultWorkingDirectory)

      - task: DownloadPipelineArtifact@2
        displayName: "Download perspective-jupyterlab dist"
        inputs:
          artifact: "perspective-jupyterlab-dist"
          path: $(System.DefaultWorkingDirectory)

      - bash: tar --strip-components=1 -xvf $(System.DefaultWorkingDirectory)/perspective-dist.tgz -C $(System.DefaultWorkingDirectory)/packages/perspective package
        displayName: "Install perspective dist"

      - bash: tar --strip-components=1 -xvf $(System.DefaultWorkingDirectory)/perspective-viewer-dist.tgz -C $(System.DefaultWorkingDirectory)/rust/perspective-viewer package
        displayName: "Install perspective-viewer dist"

      - bash: tar --strip-components=1 -xvf $(System.DefaultWorkingDirectory)/perspective-viewer-datagrid-dist.tgz -C $(System.DefaultWorkingDirectory)/packages/perspective-viewer-datagrid package
        displayName: "Install perspective-viewer-datagrid dist"

      - bash: tar --strip-components=1 -xvf $(System.DefaultWorkingDirectory)/perspective-viewer-d3fc-dist.tgz -C $(System.DefaultWorkingDirectory)/packages/perspective-viewer-d3fc package
        displayName: "Install perspective-viewer-d3fc dist"

      - bash: tar --strip-components=1 -xvf $(System.DefaultWorkingDirectory)/perspective-jupyterlab-dist.tgz -C $(System.DefaultWorkingDirectory)/packages/perspective-jupyterlab package
        displayName: "Install perspective-jupyterlab dist"

      - bash: jupyter labextension install $(System.DefaultWorkingDirectory)/packages/perspective-jupyterlab
        displayName: "Install perspective-jupyterlab labextension"

      - bash: yarn test_js --jupyter --debug
        displayName: "Run Jupyterlab tests"
        env:
          PACKAGE: perspective-jupyterlab