ARROW-7675: [R][CI] Move Windows CI from Appveyor to GHA

This also includes https://issues.apache.org/jira/browse/ARROW-7202 Closes apache#6581 from nealrichardson/r-windows-gha Lead-authored-by: Neal Richardson <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
davidhao3300 · Mar 12, 2020 · 52fb3c6 · 52fb3c6
1 parent 7a0467f
commit 52fb3c6
Show file tree

Hide file tree

Showing 9 changed files with 166 additions and 227 deletions.
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -21,13 +21,21 @@ on:
   push:
     paths:
       - '.github/workflows/r.yml'
-      - 'ci/**'
+      - 'ci/scripts/r_*.sh'
+      - 'ci/scripts/cpp_*.sh'
+      - 'ci/scripts/PKGBUILD'
+      - 'ci/etc/rprofile'
+      - 'ci/docker/**'
       - 'cpp/**'
       - 'r/**'
   pull_request:
     paths:
       - '.github/workflows/r.yml'
-      - 'ci/**'
+      - 'ci/scripts/r_*.sh'
+      - 'ci/scripts/cpp_*.sh'
+      - 'ci/scripts/PKGBUILD'
+      - 'ci/etc/rprofile'
+      - 'ci/docker/**'
       - 'cpp/**'
       - 'r/**'
 
@@ -110,3 +118,46 @@ jobs:
           docker login -u ${{ secrets.DOCKERHUB_USER }} \
                        -p ${{ secrets.DOCKERHUB_TOKEN }}
           docker-compose push r
+
+  windows:
+    name: AMD64 Windows RTools ${{ matrix.rtools }}
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: test on rtools40
+        rtools: [35]
+    env:
+      TEST_R_WITH_ARROW: "TRUE"
+      ARROW_R_CXXFLAGS: '-Werror'
+      _R_CHECK_TESTS_NLINES_: 0
+    steps:
+      - run: git config --global core.autocrlf false
+      - uses: numworks/setup-msys2@v1
+        with:
+          update: true
+      - uses: actions/checkout@v2
+      - uses: r-lib/actions/setup-r@master
+        with:
+          rtools-version: ${{ matrix.rtools }}
+      - name: Build Arrow C++
+        run: msys2do ci/scripts/r_windows_build.sh
+      - name: Install R package dependencies
+        shell: Rscript {0}
+        run: |
+          options(pkgType="win.binary")
+          install.packages(c("remotes", "rcmdcheck"))
+          remotes::install_deps("r", dependencies = TRUE)
+      - name: Check
+        shell: Rscript {0}
+        run: |
+          Sys.setenv(
+            RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "libarrow.zip"),
+            MAKEFLAGS = paste0("-j", parallel::detectCores())
+          )
+          rcmdcheck::rcmdcheck("r",
+            build_args = '--no-build-vignettes',
+            args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'),
+            error_on = 'warning',
+            check_dir = 'check'
+          )
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -1765,7 +1765,7 @@ preceding version 3 has the following license:
 
 This project includes code from the rtools-backports project.
 
-* ci/PKGBUILD and ci/appveyor-build-r.sh are based on code
+* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
   from the rtools-backports project.
 
 Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.

diff --git a/appveyor.yml b/appveyor.yml
@@ -19,14 +19,15 @@
 os: Visual Studio 2015
 
 only_commits:
-  # Skip commits not related to Python, C++, R, or Ruby
+  # Skip commits not related to Python or C++
   files:
     - appveyor.yml
-    - ci/
+    - ci/appveyor*
+    - ci/*.bat
+    - ci/conda*
     - cpp/
     - format/
     - python/
-    - r/
 
 cache:
   - C:\Users\Appveyor\clcache1
@@ -73,13 +74,6 @@ environment:
     # NOTE: Since ARROW-5403 we have disabled the static CRT build
     # - JOB: "Static_Crt_Build"
     #   GENERATOR: Ninja
-    - JOB: "R with libarrow"
-      USE_CLCACHE: false
-      TEST_R_WITH_ARROW: "TRUE"
-      RWINLIB_LOCAL: '%APPVEYOR_BUILD_FOLDER%\libarrow.zip'
-      ARROW_R_CXXFLAGS: '-Werror'
-      _R_CHECK_TESTS_NLINES_: 0
-      WARNINGS_ARE_ERRORS: 1
 
   MSVC_DEFAULT_OPTIONS: ON
   APPVEYOR_SAVE_CACHE_ON_ERROR: true
@@ -97,63 +91,6 @@ before_build:
 build_script:
   - call ci\appveyor-build.bat
 
-for:
--
-  matrix:
-    only:
-      - JOB: "R with libarrow"
-  init:
-    ps: |
-          $ErrorActionPreference = "Stop"
-          Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1"
-          Import-Module '..\appveyor-tool.ps1'
-
-  before_build:
-    - cd r
-    - ps: Bootstrap
-    - cd %APPVEYOR_BUILD_FOLDER%
-
-  build_script:
-    - rmdir /s /Q C:\OpenSSL-Win32 C:\OpenSSL-Win64
-    - C:\msys64\usr\bin\pacman --noconfirm --ask 20 --sync --refresh --refresh --sysupgrade --sysupgrade
-    - C:\msys64\usr\bin\bash --login -c "$(cygpath ${APPVEYOR_BUILD_FOLDER})/ci/appveyor-build-r.sh"
-    - pushd r
-    - travis-tool.sh install_deps
-
-  test_script:
-    - travis-tool.sh run_tests
-
-  on_failure:
-    - travis-tool.sh dump_logs
-
-  environment:
-    NOT_CRAN: true
-
-  artifacts:
-    - path: '*.pkg.tar.xz'
-      name: binaries
-
-    - path: 'build\*.zip'
-      name: binaries
-
-    - path: 'r\*.Rcheck\**\*.log'
-      name: Logs
-
-    - path: 'r\*.Rcheck\**\*.out'
-      name: Logs
-
-    - path: 'r\*.Rcheck\**\*.fail'
-      name: Logs
-
-    - path: 'r\*.Rcheck\**\*.Rout'
-      name: Logs
-
-    - path: 'r\*_*.tar.gz'
-      name: Bits
-
-    - path: 'r\*_*.zip'
-      name: Bits
-
 # Disable test discovery
 test: off
 

diff --git a/ci/appveyor-build-r.sh b/ci/appveyor-build-r.sh
diff --git a/ci/PKGBUILD → ci/scripts/PKGBUILD b/ci/PKGBUILD → ci/scripts/PKGBUILD
@@ -36,7 +36,7 @@ options=("staticlibs" "strip" "!buildflags")
 
 # For installing from a local checkout, set source_dir to . and don't include
 # a "source" param below
-source_dir="$APPVEYOR_BUILD_FOLDER"
+source_dir="$GITHUB_WORKSPACE"
 # else
 # source_dir=apache-${_realname}-${pkgver}
 
@@ -49,7 +49,6 @@ source_dir="$APPVEYOR_BUILD_FOLDER"
 # sha256sums=("SKIP")
 # source_dir="${APPVEYOR_BUILD_FOLDER}/${source_dir}"
 
-cmake_build_type=release
 cpp_build_dir=build-${CARCH}-cpp
 
 pkgver() {
@@ -76,28 +75,28 @@ build() {
     ${MINGW_PREFIX}/bin/cmake.exe \
     ${ARROW_CPP_DIR} \
     -G "MSYS Makefiles" \
-    -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \
-    -DCMAKE_BUILD_TYPE=${cmake_build_type} \
-    -DARROW_BUILD_STATIC=ON \
+    -DARROW_BOOST_USE_SHARED=OFF \
     -DARROW_BUILD_SHARED=OFF \
+    -DARROW_BUILD_STATIC=ON \
+    -DARROW_BUILD_UTILITIES=OFF \
     -DARROW_COMPUTE=ON \
     -DARROW_CSV=ON \
     -DARROW_DATASET=ON \
     -DARROW_FILESYSTEM=ON \
-    -DARROW_JSON=ON \
-    -DARROW_PARQUET=ON \
     -DARROW_HDFS=OFF \
-    -DARROW_BOOST_USE_SHARED=OFF \
     -DARROW_JEMALLOC=OFF \
-    -DARROW_WITH_ZLIB=ON \
-    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_JSON=ON \
+    -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \
+    -DARROW_PARQUET=ON \
+    -DARROW_USE_GLOG=OFF \
     -DARROW_WITH_LZ4=ON \
+    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_WITH_ZLIB=ON \
     -DARROW_WITH_ZSTD=ON \
-    -DARROW_USE_GLOG=OFF \
-    -DARROW_BUILD_UTILITIES=OFF \
-    -DThrift_ROOT="${MINGW_PREFIX}"
+    -DCMAKE_BUILD_TYPE="release" \
+    -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX}
 
-  make
+  make -j2
   popd
 }
 

diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -x
+
+pacman --sync --noconfirm ccache
+
+wget https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf
+cp -f pacman.conf /etc/pacman.conf
+
+pacman --noconfirm -Scc
+pacman --noconfirm -Syyu
+pacman --noconfirm --needed -S git base-devel binutils zip
+
+# Install core build stuff
+pacman --noconfirm --needed -S mingw-w64-{i686,x86_64}-{toolchain,crt,winpthreads,gcc,libtre,pkg-config,xz}
+
+# Force static linking
+rm -f /mingw32/lib/*.dll.a
+rm -f /mingw64/lib/*.dll.a
+export PKG_CONFIG="/${MINGW_PREFIX}/bin/pkg-config --static"
+
+cp ci/scripts/PKGBUILD .
+export PKGEXT='.pkg.tar.xz' # pacman default changed to .zst in 2020, but keep the old ext for compat
+unset BOOST_ROOT
+printenv
+makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild
+
+# Collect the build artifacts and make the shape of zip file that rwinlib expects
+ls
+mkdir build
+cp mingw* build
+cd build
+
+# This may vary by system/CI provider
+MSYS_LIB_DIR="D:/a/_temp/msys/msys64"
+
+ls $MSYS_LIB_DIR/mingw64/lib/
+ls $MSYS_LIB_DIR/mingw32/lib/
+
+VERSION=$(grep Version ../r/DESCRIPTION | cut -d " " -f 2)
+DST_DIR="arrow-$VERSION"
+
+# Untar the two builds we made
+ls | xargs -n 1 tar -xJf
+mkdir $DST_DIR
+# Grab the headers from one, either one is fine
+mv mingw64/include $DST_DIR
+
+# Make the rest of the directory structure
+# lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
+mkdir -p $DST_DIR/lib-4.9.3/x64
+mkdir -p $DST_DIR/lib-4.9.3/i386
+# lib is for the new gcc 8 toolchain (Rtools 4.0)
+mkdir -p $DST_DIR/lib/x64
+mkdir -p $DST_DIR/lib/i386
+
+# Move the 64-bit versions of libarrow into the expected location
+mv mingw64/lib/*.a $DST_DIR/lib-4.9.3/x64
+# Same for the 32-bit versions
+mv mingw32/lib/*.a $DST_DIR/lib-4.9.3/i386
+
+# These are from https://dl.bintray.com/rtools/backports/
+cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,boost*}.a $DST_DIR/lib-4.9.3/x64
+cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,boost*}.a $DST_DIR/lib-4.9.3/i386
+
+# These are from https://dl.bintray.com/rtools/mingw{32,64}/
+cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64
+cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386
+
+# Create build artifact
+zip -r ${DST_DIR}.zip $DST_DIR
+
+# Copy that to a file name/path that does not vary by version number so we
+# can easily find it in the R package tests on CI
+cp ${DST_DIR}.zip ../libarrow.zip