ARROW-1577: [JS] add ASF release scripts

@wesm does [this](apache@b270dba#diff-fc8acbd4f42fb5e6b0cad14928b68115R59) look good? Author: Paul Taylor <[email protected]> Author: Wes McKinney <[email protected]> Closes apache#1346 from trxcllnt/js-asf-release-scripts and squashes the following commits: ac246cd [Wes McKinney] Update npm-release.sh while integration testing from release tarball is hard 469beea [Wes McKinney] Add to READMEs about how to release and verify JS release 160f5c3 [Paul Taylor] remove tag-version-prefix from npmrc 854020d [Wes McKinney] Run unit tests without integration tests for now 9485922 [Wes McKinney] Add JavaScript release verification script dd06c46 [Wes McKinney] Create tag in source release script 2c6ce86 [Paul Taylor] number of npm-release.sh script arguments is 0 e1c3a96 [Paul Taylor] npm install before release, use npm version --git-tag-version with tag-version-prefix 5870043 [Wes McKinney] Number of arguments is 2 1133fb8 [Wes McKinney] Remove upstream Arrow version tag. Bump JS version to 0.2.0 30b602f [Paul Taylor] add dev/release/js-source-release.sh script be82c08 [Paul Taylor] add npm-release.sh script to publish npm modules f72fc83 [Paul Taylor] update test_js commands in verify-release-candidate.sh 5194951 [Paul Taylor] npm pack the js sources into the release tarball d73c47a [Paul Taylor] prepare js lib for release 7d9c84a [Paul Taylor] remove tslib patches as it doesn't work on windows
giantwhale · Nov 23, 2017 · ea0fb37 · ea0fb37
1 parent 1524ed7
commit ea0fb37
Show file tree

Hide file tree

Showing 14 changed files with 292 additions and 35 deletions.
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
@@ -51,7 +51,7 @@ pushd $ARROW_JS_DIR
 npm run lint
 npm run build
 # create initial test data
-npm run test:createTestData
+npm run create:testdata
 # run once to write the snapshots
 npm test -- -t ts -u --integration
 # run again to test all builds against the snapshots

diff --git a/dev/README.md b/dev/README.md
@@ -113,3 +113,11 @@ Studio 2015):
 ```
 dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
 ```
+
+### Verifying the JavaScript release
+
+For JavaScript-specific releases, use a different verification script:
+
+```shell
+bash dev/release/js-verify-release-candidate.sh 0.7.0 0
+```
diff --git a/dev/release/README b/dev/release/README
@@ -49,3 +49,16 @@ Logon to the apache repository: https://repository.apache.org/#stagingRepositori
 Select the arrow staging repository you just just created: orgapachearrow-100x
 Click the "close" button
 Once validation has passed, click the "release" button
+
+# Releasing JavaScript
+
+* Make release branch then tag the release
+
+git checkout -b release-js-X.Y.Z
+git tag -a apache-arrow-js-X.Y.Z
+
+* Build the source release (requires NodeJS)
+
+dev/release/js-source-release.sh X.Y.Z $RC_NUM
+
+* After release vote push tag, and rebase master on release branch
diff --git a/dev/release/js-source-release.sh b/dev/release/js-source-release.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+set -e
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <js-version> <rc-num>"
+  exit
+fi
+
+js_version=$1
+rc=$2
+
+tag=apache-arrow-js-${js_version}
+tagrc=${tag}-rc${rc}
+
+echo "Preparing source for tag ${tag}"
+
+tarball=${tag}.tar.gz
+
+# cd to $ARROW_HOME/js
+cd $SOURCE_DIR/../../js
+JS_SRC_DIR="$PWD"
+# npm pack the js source files
+npm install
+
+npm version --no-git-tag-version $js_version
+git add package.json
+git commit -m "[Release] Apache Arrow JavaScript $js_version"
+git tag -a ${tag}
+
+release_hash=`git rev-list $tag 2> /dev/null | head -n 1 `
+
+if [ -z "$release_hash" ]; then
+  echo "Cannot continue: unknown git tag: $tag"
+  exit
+fi
+
+echo "Using commit $release_hash"
+
+cd $SOURCE_DIR
+
+rm -rf js-tmp
+# `npm pack` writes the .tgz file to the current dir, so cd into js-tmp
+mkdir -p js-tmp
+cd js-tmp
+# run npm pack on `arrow/js`
+npm pack ${JS_SRC_DIR}
+# unzip and remove the npm pack tarball
+tar -xzf *.tgz && rm *.tgz
+# `npm pack` puts files in a dir called "package"
+cp $JS_SRC_DIR/../NOTICE.txt package
+cp $JS_SRC_DIR/../LICENSE.txt package
+# rename "package" to $tag
+mv package ${tag}
+tar czf ${tarball} ${tag}
+rm -rf ${tag}
+
+${SOURCE_DIR}/run-rat.sh ${tarball}
+
+# sign the archive
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+gpg --print-md MD5 ${tarball} > ${tarball}.md5
+sha1sum $tarball > ${tarball}.sha1
+sha256sum $tarball > ${tarball}.sha256
+sha512sum $tarball > ${tarball}.sha512
+
+# check out the arrow RC folder
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow js-rc-tmp
+
+# add the release candidate for the tag
+mkdir -p js-rc-tmp/${tagrc}
+cp ${tarball}* js-rc-tmp/${tagrc}
+svn add js-rc-tmp/${tagrc}
+svn ci -m 'Apache Arrow JavaScript ${version} RC${rc}' js-rc-tmp/${tagrc}
+
+cd -
+
+# clean up
+rm -rf js-tmp
+
+echo "Success! The release candidate is available here:"
+echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
+echo ""
+echo "Commit SHA1: ${release_hash}"
diff --git a/dev/release/js-verify-release-candidate.sh b/dev/release/js-verify-release-candidate.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - nodejs >= 6.0.0 (best way is to use nvm)
+
+case $# in
+  2) VERSION="$1"
+     RC_NUMBER="$2"
+     ;;
+
+  *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+     exit 1
+     ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+  curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+  download_dist_file apache-arrow-js-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+  download_dist_file KEYS
+  gpg --import KEYS
+}
+
+fetch_archive() {
+  local dist_name=$1
+  download_rc_file ${dist_name}.tar.gz
+  download_rc_file ${dist_name}.tar.gz.asc
+  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha512
+  gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+  if [ "$(uname)" == "Darwin" ]; then
+    shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  else
+    sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+  fi
+}
+
+setup_tempdir() {
+  cleanup() {
+    rm -fr "$TMPDIR"
+  }
+  trap cleanup EXIT
+  TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+setup_tempdir "arrow-js-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-js-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-js-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+echo 'Release candidate looks good!'
+exit 0
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -173,7 +173,16 @@ test_glib() {
 test_js() {
   pushd js
   npm install
-  npm run validate
+  # clean, lint, and build JS source
+  npm run clean:all
+  npm run lint
+  npm run build
+  # create initial integration test data
+  npm run create:testdata
+  # run once to write the snapshots
+  npm test -- -t ts -u --integration
+  # run again to test all builds against the snapshots
+  npm test -- --integration
   popd
 }
 

diff --git a/js/.npmrc b/js/.npmrc
@@ -1,2 +1,2 @@
 save-prefix=
-package-lock=false
+package-lock=false
diff --git a/js/README.md b/js/README.md
@@ -183,22 +183,24 @@ The base `apache-arrow` package includes all the compilation targets for conveni
 The targets are also published under the `@apache-arrow` namespace:
 
 ```sh
-npm install @apache-arrow/es5-cjs # ES5 CommonJS target
-npm install @apache-arrow/es5-esm # ES5 ESModules target
-npm install @apache-arrow/es5-umd # ES5 UMD target
-npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target
-npm install @apache-arrow/es2015-esm # ES2015 ESModules target
-npm install @apache-arrow/es2015-umd # ES2015 UMD target
-npm install @apache-arrow/esnext-esm # ESNext CommonJS target
-npm install @apache-arrow/esnext-esm # ESNext ESModules target
-npm install @apache-arrow/esnext-umd # ESNext UMD target
+npm install apache-arrow # <-- combined es5/CommonJS + UMD, es2015/ESModules + UMD, and TypeScript package
+npm install @apache-arrow/ts # standalone TypeScript package
+npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package
+npm install @apache-arrow/es5-esm # standalone es5/ESModules package
+npm install @apache-arrow/es5-umd # standalone es5/UMD package
+npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package
+npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package
+npm install @apache-arrow/es2015-umd # standalone es2015/UMD package
+npm install @apache-arrow/esnext-esm # standalone esNext/CommonJS package
+npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package
+npm install @apache-arrow/esnext-umd # standalone esNext/UMD package
 ```
 
 ### Why we package like this
 
 The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes.
 
-If you think we missed a compilation target and it's a blocker for adoption, please open an issue. We're here for you ❤️.
+If you think we missed a compilation target and it's a blocker for adoption, please open an issue.
 
 # License
 

diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
@@ -48,13 +48,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
     module: `${mainExport}.mjs`,
     browser: `${mainExport}.es5.min.js`,
     [`browser:es2015`]: `${mainExport}.es2015.min.js`,
-    [`@std/esm`]: { esm: `mjs` },
-    // Temporary workaround until https://github.com/Microsoft/tslib/pull/44 is merged
-    scripts: {
-        postinstall: `npm i shx && npm run tslib_mjs && npm run tslib_pkg && npm r shx`,
-        tslib_mjs: `shx cp $(node -e \"console.log(require.resolve('tslib/tslib.es6.js'))\") $(node -e \"var r=require,p=r('path');console.log(p.join(p.dirname(r.resolve('tslib')),'tslib.mjs'))\")`,
-        tslib_pkg: `node -e \"var r=require,p=r('path'),f=r('fs'),k=p.join(p.dirname(r.resolve('tslib')),'package.json'),x=JSON.parse(f.readFileSync(k));x.main='tslib';f.writeFileSync(k,JSON.stringify(x))\"`
-    }
+    [`@std/esm`]: { esm: `mjs` }
 });
 
 const createTypeScriptPackageJson = (target, format) => (orig) => ({

diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js
@@ -73,12 +73,13 @@ const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
 const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
 
 const testFilesDir = path.join(ARROW_HOME, 'js/test/data');
+const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__');
 const cppFilesDir = path.join(testFilesDir, 'cpp');
 const javaFilesDir = path.join(testFilesDir, 'java');
 const jsonFilesDir = path.join(testFilesDir, 'json');
 
 async function cleanTestData() {
-    return await del(`${testFilesDir}/**`);
+    return await del([`${testFilesDir}/**`, `${snapshotsDir}/**`]);
 }
 
 async function createTestJSON() {

diff --git a/js/gulp/util.js b/js/gulp/util.js
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+const fs = require('fs');
 const path = require(`path`);
 const pump = require(`pump`);
 const { Observable, ReplaySubject } = require('rxjs');
@@ -27,13 +28,26 @@ const releasesRootDir = `targets`;
 const knownTargets = [`es5`, `es2015`, `esnext`];
 const knownModules = [`cjs`, `esm`, `cls`, `umd`];
 const moduleFormatsToSkipCombosOf = { cls: true };
-const metadataFiles = [`../LICENSE.txt`, `../NOTICE.txt`, `README.md`];
 const packageJSONFields = [
   `version`, `license`, `description`,
   `author`, `homepage`, `repository`,
   `bugs`, `keywords`,  `dependencies`
 ];
 
+const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => {
+    let err = false, prefixes = [`./`, `../`];
+    let p = prefixes.find((prefix) => {
+        try {
+            fs.statSync(path.resolve(path.join(prefix, filename)));
+        } catch (e) { return false; }
+        return true;
+    });
+    if (!p) {
+        throw new Error(`Couldn't find ${filename} in ./ or ../`);
+    }
+    return path.join(p, filename);
+});
+
 // see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988
 const gCCLanguageNames = {
     es5: `ECMASCRIPT5`,

diff --git a/js/gulpfile.js b/js/gulpfile.js
@@ -86,8 +86,8 @@ const buildConcurrent = (tasks) => () =>
             .merge(...knownTargets.map((target) =>
                 del(`${targetDir(target, `cls`)}/**`)))));
 
-gulp.task(`test:cleanTestData`, cleanTestData);
-gulp.task(`test:createTestData`, createTestData);
+gulp.task(`clean:testdata`, cleanTestData);
+gulp.task(`create:testdata`, createTestData);
 gulp.task( `test`, gulp.series(getTasks(`test`)));
 gulp.task(`debug`, gulp.series(getTasks(`debug`)));
 gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));

diff --git a/js/npm-release.sh b/js/npm-release.sh
@@ -16,5 +16,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
 
-lerna publish --yes --skip-git --force-publish=*
+# validate the targets pass all tests before publishing
+npm install
+# npx run-s clean:all lint create:testdata build
+# npm run test -- -t ts -u --integration
+# npm run test -- --integration
+npx run-s clean:all lint build
+npm run test
+
+# publish the JS target modules to npm
+npx lerna publish --yes --skip-git --force-publish=*