Simplify Appveyor build script (JuliaLang#31917)

.appveyor.yml

@@ -1,9 +1,19 @@
 environment:
-#  USEMSVC: "1"
+  global:
+    CCACHE_DIR: C:\ccache
+    CYG_MIRROR: http://cygwin.mirror.constant.com
+    CYG_CACHE: '%CYG_ROOT%\var\cache\setup'
+    CYG_BASH: '%CYG_ROOT%\bin\bash'
+
   matrix:
-  - ARCH: "i686"
+  - MINGW_ARCH: "i686"
+    CYG_ROOT: C:\cygwin
+    CYG_SETUP: setup-x86.exe
     JULIA_TEST_MAXRSS_MB: 500
-  - ARCH: "x86_64"
+
+  - MINGW_ARCH: "x86_64"
+    CYG_ROOT: C:\cygwin64
+    CYG_SETUP: setup-x86_64.exe
     JULIA_TEST_MAXRSS_MB: 450
 
 # Only build on master and PR's for now, not personal branches
@@ -30,35 +40,23 @@ notifications:
     on_build_failure: false
     on_build_status_changed: false
 
-clone_depth: 50
+cache:
+  - '%CYG_CACHE%'
+  - '%CCACHE_DIR%'
 
 init:
-# Carriage returns are bad
   - git config --global core.autocrlf input
 
-cache:
-# Cache large downloads to avoid network unreliability
-  - C:\cygdownloads
-  - llvm-3.9.1-i686-w64-mingw32-juliadeps-r07.7z
-  - llvm-3.9.1-x86_64-w64-mingw32-juliadeps-r07.7z
-  - C:\ccache
+install:
+    - '%CYG_ROOT%\%CYG_SETUP% -gnq  -R "%CYG_ROOT%" -s "%CYG_MIRROR%" -l "%CYG_CACHE%" -P make,python2,libiconv,curl,time,p7zip,ccache,mingw64-%MINGW_ARCH%-gcc-g++,mingw64-%MINGW_ARCH%-gcc-fortran > NULL 2>&1'
+    - '%CYG_ROOT%\bin\cygcheck -dc cygwin'
 
 build_script:
-# If there's a newer build queued for the same PR, cancel this one
-  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
-        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
-        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
-        throw "There are newer queued builds for this pull request, failing early." }
-# Remove C:\MinGW\bin from the path, the version of MinGW installed on
-# AppVeyor is not compatible with the cross-compiled Julia Windows binaries
-  - set PATH=%PATH:C:\MinGW\bin;=%
-# Remove git's msys2 from path, since it conflicts with cygwin1.dll
-  - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
-  - ps: contrib/windows/install-cygwin.ps1
-#  - '"%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64'
-  - C:\cygwin-%ARCH%\bin\sh.exe --login /cygdrive/c/projects/julia/contrib/windows/appveyor_build.sh
+  - 'echo Building Julia'
+  - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER && ./contrib/windows/appveyor_build.sh"'
 
 test_script:
+  - 'echo Testing Julia'
   - usr\bin\julia -e "Base.require(Main, :InteractiveUtils).versioninfo()"
   - usr\bin\julia --sysimage-native-code=no -e "true"
   - cd julia-* && .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl all &&

contrib/windows/appveyor_build.sh

@@ -1,216 +1,37 @@
 #!/bin/sh
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# Script to compile Windows Julia, using binary dependencies from nightlies.
-# Should work in MSYS assuming 7zip is installed and on the path,
-# or Cygwin or Linux assuming make, curl, p7zip, and mingw64-$ARCH-gcc-g++
-# are installed
+# Script to compile Windows Julia on Appveyor using a Cygwin host
+# make sure you cd to the main julia directory beforing running
+# uses environment variables defined in .appveyor.yml: MINGW_ARCH
 
-# Run in top-level Julia directory
-cd `dirname "$0"`/../..
 # Stop on error
 set -e
-# Make sure stdin exists (not true on appveyor msys2)
-exec < /dev/null
 
-curlflags="curl --retry 10 -k -L -y 5"
-checksum_download() {
-  # checksum_download filename url
-  f=$1
-  url=$2
-  if [ -e "$f" ]; then
-    deps/tools/jlchecksum "$f" 2> /dev/null && return
-    echo "Checksum for '$f' changed, download again." >&2
-  fi
-  echo "Downloading '$f'"
-  $curlflags -O "$url"
-  deps/tools/jlchecksum "$f"
-}
+# Make sure stdin exists (apparently sometimes needed for mysys2?)
+exec < /dev/null
 
-# If ARCH environment variable not set, choose based on uname -m
-if [ -z "$ARCH" -a -z "$XC_HOST" ]; then
-  export ARCH=`uname -m`
-elif [ -z "$ARCH" ]; then
-  ARCH=`echo $XC_HOST | sed 's/-w64-mingw32//'`
-fi
+export TERM=ansi  # make sure escape sequences print out properly on appveyor?
 
-echo "" > Make.user
-echo "" > get-deps.log
 # set MARCH for consistency with how binaries get built
-if [ "$ARCH" = x86_64 ]; then
-  bits=64
-  archsuffix=64
-  exc=seh
+if [ "$MINGW_ARCH" = x86_64 ]; then
   echo "override MARCH = x86-64" >> Make.user
-  echo 'USE_BLAS64 = 1' >> Make.user
-  echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas64_' >> Make.user
-  echo 'LIBBLASNAME = libopenblas64_' >> Make.user
-else
-  bits=32
-  archsuffix=86
-  exc=sjlj
+elif [ "$MINGW_ARCH" = i686 ]; then
   echo "override MARCH = pentium4" >> Make.user
-  echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user
-  echo 'LIBBLASNAME = libopenblas' >> Make.user
-fi
-echo "override JULIA_CPU_TARGET=generic;native" >> Make.user
-
-# Set XC_HOST if in Cygwin or Linux
-case $(uname) in
-  CYGWIN*)
-    if [ -z "$XC_HOST" ]; then
-      XC_HOST="$ARCH-w64-mingw32"
-      echo "XC_HOST = $XC_HOST" >> Make.user
-    fi
-    CROSS_COMPILE="$XC_HOST-"
-    # Set BUILD_MACHINE and HOSTCC in case we don't have Cygwin gcc installed
-    echo "override BUILD_MACHINE = $ARCH-pc-cygwin" >> Make.user
-    if [ -z "`which gcc 2>/dev/null`" ]; then
-      echo 'override HOSTCC = $(CROSS_COMPILE)gcc' >> Make.user
-    fi
-    SEVENZIP="7z"
-    ;;
-  Linux)
-    if [ -z "$XC_HOST" ]; then
-      XC_HOST="$ARCH-w64-mingw32"
-      echo "XC_HOST = $XC_HOST" >> Make.user
-    fi
-    CROSS_COMPILE="$XC_HOST-"
-    make win-extras >> get-deps.log
-    SEVENZIP="wine dist-extras/7z.exe"
-    ;;
-  *)
-    CROSS_COMPILE=""
-    SEVENZIP="7z"
-    ;;
-esac
-
-# Download most recent Julia binary for dependencies
-if ! [ -e julia-installer.exe ]; then
-  f=julia-latest-win$bits.exe
-  echo "Downloading $f"
-  $curlflags -O https://julialangnightlies-s3.julialang.org/bin/winnt/x$archsuffix/$f
-  echo "Extracting $f"
-  $SEVENZIP x -y $f >> get-deps.log
-fi
-mkdir -p usr
-for i in bin/*.dll; do
-  $SEVENZIP e -y julia-installer.exe "$i" \
-    -ousr\\`dirname $i | sed -e 's|/julia||' -e 's|/|\\\\|g'` >> get-deps.log
-done
-for i in share/julia/base/pcre_h.jl; do
-  $SEVENZIP e -y julia-installer.exe "$i" -obase >> get-deps.log
-  # Touch the file to adjust the modification time, thereby (hopefully) avoiding
-  # issues with clock skew during the build
-  touch "base/$(basename $i)"
-done
-echo "override PCRE_INCL_PATH =" >> Make.user
-# Remove libjulia.dll if it was copied from downloaded binary
-rm -f usr/bin/libjulia.dll
-rm -f usr/bin/libjulia-debug.dll
-rm -f usr/bin/libgcc_s_s*-1.dll
-rm -f usr/bin/libgfortran-3.dll
-rm -f usr/bin/libquadmath-0.dll
-rm -f usr/bin/libssp-0.dll
-rm -f usr/bin/libstdc++-6.dll
-rm -f usr/bin/libccalltest.dll
-rm -f usr/bin/libpthread.dll
-
-if [ -z "$USEMSVC" ]; then
-  if [ -z "`which ${CROSS_COMPILE}gcc 2>/dev/null`" ]; then
-    f=$ARCH-4.9.2-release-win32-$exc-rt_v4-rev3.7z
-    checksum_download \
-        "$f" "https://bintray.com/artifact/download/tkelman/generic/$f"
-    echo "Extracting $f"
-    $SEVENZIP x -y $f >> get-deps.log
-    export PATH=$PWD/mingw$bits/bin:$PATH
-    # If there is a version of make.exe here, it is mingw32-make which won't work
-    rm -f mingw$bits/bin/make.exe
-  fi
-  export AR=${CROSS_COMPILE}ar
-  mkdir -p usr/tools
-else
-  echo "override USEMSVC = 1" >> Make.user
-  echo "override ARCH = $ARCH" >> Make.user
-  echo "override XC_HOST = " >> Make.user
-  export CC="$PWD/deps/srccache/libuv/compile cl -nologo -MD -Z7"
-  export AR="$PWD/deps/srccache/libuv/ar-lib lib"
-  export LD="$PWD/linkld link"
-  echo "override CC = $CC" >> Make.user
-  echo 'override CXX = $(CC) -EHsc' >> Make.user
-  echo "override AR = $AR" >> Make.user
-  echo "override LD = $LD -DEBUG" >> Make.user
-
-  f=llvm-3.3-$ARCH-msvc12-juliadeps.7z
-  checksum_download \
-      "$f" "https://bintray.com/artifact/download/tkelman/generic/$f"
-  echo "Extracting $f"
-  $SEVENZIP x -y $f >> get-deps.log
 fi
+echo "override XC_HOST = $MINGW_ARCH-w64-mingw32" >> Make.user
+echo "override JULIA_CPU_TARGET = generic;native" >> Make.user
 
-if [ -z "`which make 2>/dev/null`" ]; then
-  if [ -n "`uname | grep CYGWIN`" ]; then
-    echo "Install the Cygwin package for 'make' and try again."
-    exit 1
-  fi
-  f="/make/make-3.81-2/make-3.81-2-msys-1.0.11-bin.tar"
-  if ! [ -e `basename $f.lzma` ]; then
-    echo "Downloading `basename $f`"
-    $curlflags -O http://sourceforge.net/projects/mingw/files/MSYS/Base$f.lzma
-  fi
-  $SEVENZIP x -y `basename $f.lzma` >> get-deps.log
-  tar -xf `basename $f`
-  export PATH=$PWD/bin:$PATH
-fi
-
-
-for lib in SUITESPARSE ARPACK BLAS LAPACK \
-    GMP MPFR PCRE LIBUNWIND; do
-  echo "USE_SYSTEM_$lib = 1" >> Make.user
-done
-echo 'override LIBLAPACK = $(LIBBLAS)' >> Make.user
-echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user
-
-# Remaining dependencies:
-# libuv since its static lib is no longer included in the binaries
-# openlibm since we need it as a static library to work properly
-# utf8proc since its headers are not in the binary download
-echo 'override DEP_LIBS = libuv utf8proc' >> Make.user
-
-if [ -n "$USEMSVC" ]; then
-  # Openlibm doesn't build well with MSVC right now
-  echo 'USE_SYSTEM_OPENLIBM = 1' >> Make.user
-  # Since we don't have a static library for openlibm
-  echo 'override UNTRUSTED_SYSTEM_LIBM = 0' >> Make.user
-
-  # Compile libuv and utf8proc without -TP first, then add -TP
-  make -C deps install-libuv install-utf8proc
-  cp usr/lib/uv.lib usr/lib/libuv.a
-  echo 'override CC += -TP' >> Make.user
-  echo 'override DEP_LIBS += dsfmt' >> Make.user
-
-  # Create a modified version of compile for wrapping link
-  sed -e 's/-link//' -e 's/cl/link/g' -e 's/ -Fe/ -OUT:/' \
-    -e 's|$dir/$lib|$dir/lib$lib|g' deps/srccache/libuv/compile > linkld
-  chmod +x linkld
-else
-  # Use BinaryBuilder
-  echo 'USE_BINARYBUILDER_LLVM = 1' >> Make.user
-  echo 'USE_BINARYBUILDER_OPENBLAS = 1' >> Make.user
-  echo 'USE_BINARYBUILDER_SUITESPARSE = 1' >> Make.user
-  echo 'BINARYBUILDER_LLVM_ASSERTS = 1' >> Make.user
-  echo 'override DEP_LIBS += llvm openlibm openblas suitesparse' >> Make.user
-  export CCACHE_DIR=/cygdrive/c/ccache
-  echo 'USECCACHE=1' >> Make.user
-  make check-whitespace
-  make VERBOSE=1 -C base version_git.jl.phony
-  echo 'NO_GIT = 1' >> Make.user
-fi
+echo 'USE_BINARYBUILDER = 1' >> Make.user
+echo 'BINARYBUILDER_LLVM_ASSERTS = 1' >> Make.user
 echo 'FORCE_ASSERTIONS = 1' >> Make.user
+echo 'USECCACHE = 1' >> Make.user
+echo 'VERBOSE = 1' >> Make.user
 
 cat Make.user
-make -j3 VERBOSE=1 release
-make -j3 VERBOSE=1 install
-make VERBOSE=1 JULIA=../../usr/bin/julia.exe BIN=. "$(make print-CC)" -C test/embedding release
+make check-whitespace
+make -j3 release
+make -j3 install
+make JULIA=../../usr/bin/julia.exe BIN=. "$(make print-CC)" -C test/embedding release
 make build-stats
 ccache -s