Updated CI setup to assist issue #100 (#104)

* Updated CI setup to assist issue #100 * Adds odoc Strictly speaking, not necessary for this PR. However, it's needed in future and keeping the CI setup close to hello-reason helps in maintenance * Build docs only for root
esy · Jan 27, 2020 · aeb5233 · aeb5233
1 parent 6463aed
commit aeb5233
Show file tree

Hide file tree

Showing 16 changed files with 510 additions and 138 deletions.
diff --git a/.ci/build-platform.yml b/.ci/build-platform.yml
@@ -1,48 +1,68 @@
 parameters:
   platform: "macOS"
   vmImage: "macOS-10.13"
-  STAGING_DIRECTORY: $(Agent.HomeDirectory)/STAGING
-  STAGING_DIRECTORY_UNIX: $(Agent.HomeDirectory)/STAGING
-  ESY__CACHE_SOURCE_TARBALL_PATH: $(Agent.HomeDirectory)/.esy/source/i
 
 jobs:
   - job: ${{ parameters.platform }}
     pool:
       vmImage: ${{ parameters.vmImage }}
       demands: node.js
     timeoutInMinutes: 120 # This is mostly for Windows
-    variables:
-      STAGING_DIRECTORY: ${{ parameters.STAGING_DIRECTORY }}
-      STAGING_DIRECTORY_UNIX: ${{ parameters.STAGING_DIRECTORY_UNIX }}
-      ESY__CACHE_INSTALL_PATH: ${{ parameters.ESY__CACHE_INSTALL_PATH }}
-      ESY__CACHE_SOURCE_TARBALL_PATH: ${{ parameters.ESY__CACHE_SOURCE_TARBALL_PATH }}
-
-    steps:
-      - template: use-calc-esy-install-path.yml
-      - template: utils/use-node.yml
-      - template: utils/use-esy.yml
-      - template: utils/restore-build-cache.yml
-      - script: "esy install"
-        displayName: "esy install"
-      - script: "esy build"
-        displayName: "esy build"
-      - script: "esy npm-release"
-        displayName: "esy npm-release"
-      - script: "esy test"
-        displayName: "Running unit tests"
-      - ${{ if eq(parameters.platform, 'Windows') }}:
-          - script: mkdir $(System.DefaultWorkingDirectory)\..\tests-tmp-dir
-            displayName: 'Creating temporary workspace for tests'
-          - script: .\_build\install\default\bin\Runner.exe
-            displayName: 'Running e2e tests'
-            env:
-              TEMP: $(System.DefaultWorkingDirectory)\..\tests-tmp-dir
-      - ${{ if ne(parameters.platform, 'Windows') }}:
-          - script: ./_build/install/default/bin/Runner.exe
-            displayName: 'Running e2e tests'
-            env:
-              OCAMLRUNPARAM: 'b'
-      - template: utils/publish-build-cache.yml
+    steps: 
+      - powershell: $Env:Path
+        continueOnError: true
+        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        displayName: "Print env in powershell"
+      # Needed so that the mingw tar doesn't shadow the system tar. See
+      # pipelines.yaml. We need windows bsdtar from system32, not the mingw
+      # one. Note powershell doesn't need escaping of backslashes.
+      - powershell: Write-Host "##vso[task.setvariable variable=PATH;]C:\Windows\system32;${env:PATH}"
+        continueOnError: true
+        condition: eq(variables['AGENT.OS'], 'Windows_NT')
+        displayName: "Make sure windows/system32 is at front of path if windows"
+      - powershell: $Env:Path
+        continueOnError: true
+        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        displayName: "Print env in powershell"
+      - powershell: get-command tar
+        continueOnError: true
+        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        displayName: "Print where tar is located"
+      - powershell: tar --help
+        continueOnError: true
+        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        displayName: "Print tar help"
+      - bash: |
+          # COMPUTE THE ESY INSTALL CACHE LOCATION AHEAD OF TIME
+          DESIRED_LEN="86"
+          # Note: This will need to change when upgrading esy version
+          # that reenables long paths on windows.
+          if [ "$AGENT_OS" == "Windows_NT" ]; then
+            DESIRED_LEN="33"
+          fi
+          HOME_ESY3="$HOME/.esy/3"
+          HOME_ESY3_LEN=${#HOME_ESY3}
+          NUM_UNDERS=$(echo "$(($DESIRED_LEN-$HOME_ESY3_LEN))")
+          UNDERS=$(printf "%-${NUM_UNDERS}s" "_")
+          UNDERS="${UNDERS// /_}"
+          THE_ESY__CACHE_INSTALL_PATH=${HOME_ESY3}${UNDERS}/i
+          if [ "$AGENT_OS" == "Windows_NT" ]; then
+            THE_ESY__CACHE_INSTALL_PATH=$( cygpath --mixed --absolute "$THE_ESY__CACHE_INSTALL_PATH")
+          fi
+          echo "THE_ESY__CACHE_INSTALL_PATH: $THE_ESY__CACHE_INSTALL_PATH"
+          # This will be exposed as an env var ESY__CACHE_INSTALL_PATH, or an
+          # Azure var esy__cache_install_path
+          echo "##vso[task.setvariable variable=esy__cache_install_path]$THE_ESY__CACHE_INSTALL_PATH"
+      # - bash: |
+      #     which esy
+      #     echo "$( which esy )"
+      #     echo "##vso[task.setvariable variable=esy_bin_location]$(which esy)"
+      #   displayName: "Find esy binary"
+      # - bash: echo ${ESY_BIN_LOCATION}
+      # displayName: "Print esy bin location"
+      - bash: env
+        displayName: "Print environment"
+      - template: esy-build-steps.yml
       - task: PublishBuildArtifacts@1
         displayName: "Publish Artifact: ${{ parameters.platform }}"
         inputs:

diff --git a/.ci/esy-build-steps.yml b/.ci/esy-build-steps.yml
@@ -0,0 +1,19 @@
+# Cross-platform set of build steps for building esy projects
+
+steps:
+  - template: utils/use-node.yml
+  - template: utils/use-esy.yml
+  - script: "esy install"
+    displayName: "esy install"
+  - template: utils/restore-build-cache.yml
+  - script: "esy build"
+    displayName: "esy build"
+  - template: utils/create-docs.yml
+  - script: "esy test"
+    displayName: "Test command"
+  - script: "esy release"
+    displayName: "esy release"
+  - template: utils/publish-build-cache.yml
+
+  # Run tests or any additional steps here
+  # - script: esy b dune runtest
diff --git a/.ci/opam-build-steps.yml b/.ci/opam-build-steps.yml
@@ -0,0 +1,38 @@
+# Shared steps for building OPAM projects
+
+steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '8.9'
+  - script: brew install gpatch
+    displayName: 'brew install gpatch'
+  - script: brew install opam
+    displayName: 'brew install opam'
+  - script: opam --version
+    displayName: 'Check opam version'
+  - script: opam init --auto-setup --dot-profile=~/.bash_profile
+    displayName: 'opam init'
+  - script: opam remote add ocamlorg https://opam.ocaml.org || true
+    displayName: 'opam remote add ocamlorg'
+  - script: opam remove default || true
+    displayName: 'opam remove default'
+  - script: opam update
+    displayName: 'opam update'
+  - script: opam switch create $(OCAML_VERSION)
+    displayName: 'Use OCaml version: $(OCAML_VERSION)'
+  - script: opam install -y dune
+    displayName: 'opam install -y dune'
+  - script: opam install -y menhir
+    displayName: 'opam install -y menhir'
+  - script: opam install -y utop
+    displayName: 'opam install -y utop'
+  - script: make clean-for-ci
+    displayName: 'make clean-for-ci'
+  - script: opam pin add -y reason .
+    displayName: "opam pin add -y reason ."
+  - script: opam pin add -y rtop .
+    displayName: "opam pin add -y rtop ."
+  - script: eval $(opam env); make test-ci
+    displayName: "make test-ci"
+  - script: git diff --exit-code
+    displayName: "Check git is clean"
diff --git a/.ci/pipelines-release.js b/.ci/pipelines-release.js
@@ -2,7 +2,35 @@ const fs = require("fs");
 const path = require("path");
 
 console.log("Creating package.json");
-const mainPackageJson = require("../package.json");
+
+// From the project root pwd
+const mainPackageJsonPath =
+  fs.existsSync('esy.json') ?
+  'esy.json' : 'package.json';
+
+const exists = fs.existsSync(mainPackageJsonPath);
+if (!exists) {
+  console.error("No package.json or esy.json at " + mainPackageJsonPath);
+  process.exit(1);
+}
+// Now require from this script's location.
+const mainPackageJson = require(path.join('..', mainPackageJsonPath));
+const bins =
+  Array.isArray(mainPackageJson.esy.release.bin) ?
+  mainPackageJson.esy.release.bin.reduce(
+    (acc, curr) => Object.assign({ [curr]: "bin/" + curr }, acc),
+    {}
+  ) :
+  Object.keys(mainPackageJson.esy.release.bin).reduce(
+    (acc, currKey) => Object.assign({ [currKey]: "bin/" + mainPackageJson.esy.release.bin[currKey] }, acc),
+    {}
+  );
+
+const rewritePrefix =
+  mainPackageJson.esy &&
+  mainPackageJson.esy.release &&
+  mainPackageJson.esy.release.rewritePrefix;
+
 const packageJson = JSON.stringify(
   {
     name: mainPackageJson.name,
@@ -11,11 +39,12 @@ const packageJson = JSON.stringify(
     description: mainPackageJson.description,
     repository: mainPackageJson.repository,
     scripts: {
-      postinstall: "node ./postinstall.js"
+      postinstall:
+        rewritePrefix ?
+        "ESY_RELEASE_REWRITE_PREFIX=true node ./postinstall.js" :
+        "node ./postinstall.js"
     },
-    bin: mainPackageJson.esy.release.bin.reduce((acc, curr) => {
-      return Object.assign({ [curr]: "bin/" + curr }, acc);
-    }, {}),
+    bin: bins,
     files: [
       "_export/",
       "bin/",
@@ -65,13 +94,21 @@ const placeholderFile = `:; echo "You need to have postinstall enabled"; exit $?
 @ECHO OFF
 ECHO You need to have postinstall enabled`;
 fs.mkdirSync(path.join(__dirname, "..", "_release", "bin"));
-const binPath = path.join(
-  __dirname,
-  "..",
-  "_release",
-  "bin",
-  mainPackageJson.esy.release.bin[0]
-);
 
-fs.writeFileSync(binPath, placeholderFile);
-fs.chmodSync(binPath, 0777);
+Object.keys(bins).forEach(
+  name => {
+    if(bins[name]) {
+      const binPath = path.join(
+        __dirname,
+        "..",
+        "_release",
+        bins[name]
+      );
+      fs.writeFileSync(binPath, placeholderFile);
+      fs.chmodSync(binPath, 0777);
+    } else {
+      console.log("bins[name] name=" + name + " was empty. Weird.");
+      console.log(bins);
+    }
+  }
+);
diff --git a/.ci/use-calc-esy-install-path.yml b/.ci/use-calc-esy-install-path.yml
diff --git a/.ci/utils/create-docs.yml b/.ci/utils/create-docs.yml
@@ -4,7 +4,7 @@ steps:
     displayName: "Build docs"
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
-  - script: echo '##vso[task.setvariable variable=docsPath]'$(esy echo '#{self.target_dir}/default/_doc/_html')
+  - script: echo '##vso[task.setvariable variable=docsPath]'$(esy echo '_build/default/_doc/_html')
     displayName: "Save docsPath in variable"
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 

diff --git a/.ci/utils/publish-build-cache.yml b/.ci/utils/publish-build-cache.yml
@@ -1,24 +1,37 @@
 # Steps for publishing project cache
 
 steps:
-  - bash: 'mkdir -p $(STAGING_DIRECTORY_UNIX)'
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+  - bash: 'mkdir -p $(Build.StagingDirectory)'
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
     displayName: '[Cache][Publish] Create cache directory'
 
-  - bash: 'cd $(ESY__CACHE_INSTALL_PATH) && tar -czf $(STAGING_DIRECTORY_UNIX)/esy-cache.tar .'
-    workingDirectory: ''
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-    displayName: '[Cache][Publish] Tar esy cache directory'
+  # continueOnError because on windows it has a permission denied error but the
+  # export succeeds.
+  - script: "esy export-dependencies"
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
+    displayName: "esy export-dependencies"
 
-  # - bash: 'cd $(ESY__NPM_ROOT) && tar -czf $(STAGING_DIRECTORY_UNIX)/npm-cache.tar .'
-  #   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+  - bash: pwd && ls _export/* && mv _export '$(Build.StagingDirectory)' && ls '$(Build.StagingDirectory)/_export/'
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
+    displayName: '[Cache][Publish] move export to staging dir'
+
+  # - bash: cd $ESY__CACHE_INSTALL_PATH && tar -czf $(Build.StagingDirectory)/esy-cache.tar .
+  #   workingDirectory: ''
+  #   condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
+  #   displayName: '[Cache][Publish] Tar esy cache directory'
+
+  # - bash: 'cd $(ESY__NPM_ROOT) && tar -czf $(Build.StagingDirectory)/npm-cache.tar .'
+  #   condition: and(succeeded(), eq(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))
   #   displayName: '[Cache][Publish] Tar npm cache directory'
 
   - task: PublishBuildArtifacts@1
     displayName: '[Cache][Publish] Upload tarball'
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
+    # TODO: The CI Build caches are pulled down by the last successful buildID
+    # for the target branch.
     inputs:
-        pathToPublish: '$(STAGING_DIRECTORY)'
-        artifactName: 'cache-$(Agent.OS)-install'
+        pathToPublish: '$(Build.StagingDirectory)'
+        artifactName: 'cache-$(Agent.OS)-install-$(esy__ci_cache_version)'
         parallel: true
         parallelCount: 8