Pass native build args from top-level scripts (dotnet#2071)

* Pass native build args from top-level scripts

* Remove obsolete message; switch platform/arch order

* Move export to a separate line

* Use URL-encoding to escape msbuild CMakeArgs property

eng/build.sh

@@ -24,6 +24,7 @@ usage()
   echo "  --configuration <value>    Build configuration: Debug or Release (short: -c)"
   echo "  --verbosity <value>        MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
   echo "  --binaryLog                Output binary log (short: -bl)"
+  echo "  --cross                    Optional argument to signify cross compilation"
   echo "  --help                     Print help and exit (short: -h)"
   echo ""
 
@@ -45,11 +46,20 @@ usage()
   echo "  --testscope                Test scope, allowed values: innerloop, outerloop, all"
   echo "  --allconfigurations        Build packages for all build configurations"
   echo ""
+
+  echo "Native build settings:"
+  echo "  --clang                    Optional argument to build using clang in PATH (default)"
+  echo "  --clangx.y                 Optional argument to build using clang version x.y"
+  echo "  --cmakeargs                User-settable additional arguments passed to CMake."
+  echo "  --gcc                      Optional argument to build using gcc in PATH (default)"
+  echo "  --gccx.y                   Optional argument to build using gcc version x.y"
+
   echo "Command line arguments starting with '/p:' are passed through to MSBuild."
   echo "Arguments can also be passed in with a single hyphen."
 }
 
 arguments=''
+cmakeargs=''
 extraargs=''
 build=false
 buildtests=false
@@ -128,6 +138,22 @@ while [[ $# > 0 ]]; do
       arguments="$arguments /p:LibrariesConfiguration=$2"
       shift 2
       ;;
+     -cross)
+      arguments="$arguments /p:CrossBuild=True"
+      shift 1
+      ;;
+     -clang*)
+      arguments="$arguments /p:Compiler=$opt"
+      shift 1
+      ;;
+     -cmakeargs)
+      cmakeargs="${cmakeargs} ${opt} $2"
+      shift 2
+      ;;
+     -gcc*)
+      arguments="$arguments /p:Compiler=$opt"
+      shift 1
+      ;;
       *)
       ea=$1
 
@@ -159,5 +185,8 @@ if [ ${#actInt[@]} -eq 0 ]; then
     arguments="-restore -build $arguments"
 fi
 
-arguments="$arguments $extraargs"
+# URL-encode space (%20) to avoid quoting issues until the msbuild call in /eng/common/tools.sh.
+# In *proj files (XML docs), URL-encoded string are rendered in their decoded form.
+cmakeargs="${cmakeargs// /%20}"
+arguments="$arguments /p:CMakeArgs=\"$cmakeargs\" $extraargs"
 "$scriptroot/common/build.sh" $arguments

eng/native/build-commons.sh

@@ -29,7 +29,7 @@ isMSBuildOnNETCoreSupported()
 
     if [[ ( "$__HostOS" == "Linux" )  && ( "$__HostArch" == "x64" || "$__HostArch" == "arm" || "$__HostArch" == "arm64" ) ]]; then
         __IsMSBuildOnNETCoreSupported=1
-    elif [[ "$__HostArch" == "x64" && ( "$__HostOS" == "OSX" || "$__HostOS" == "FreeBSD" ) ]]; then
+    elif [[ ( "$__HostOS" == "OSX" || "$__HostOS" == "FreeBSD" ) && "$__HostArch" == "x64" ]]; then
         __IsMSBuildOnNETCoreSupported=1
     fi
 }
@@ -213,7 +213,6 @@ case "$CPUName" in
         ;;
 
     armv7l)
-        echo "Unsupported CPU $CPUName detected, build might not succeed!"
         __BuildArch=arm
         __HostArch=arm
         ;;
@@ -456,7 +455,8 @@ __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:
 
 # Configure environment if we are doing a verbose build
 if [[ "$__VerboseBuild" == 1 ]]; then
-    export VERBOSE=1
+    VERBOSE=1
+    export VERBOSE
     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
 fi
 
@@ -466,9 +466,11 @@ fi
 
 # Configure environment if we are doing a cross compile.
 if [[ "$__CrossBuild" == 1 ]]; then
-    export CROSSCOMPILE=1
+    CROSSCOMPILE=1
+    export CROSSCOMPILE
     if [[ ! -n "$ROOTFS_DIR" ]]; then
-        export ROOTFS_DIR="$__RepoRootDir/.tools/rootfs/$__BuildArch"
+        ROOTFS_DIR="$__RepoRootDir/.tools/rootfs/$__BuildArch"
+        export ROOTFS_DIR
     fi
 fi
 

eng/native/init-distro-rid.sh

@@ -74,10 +74,12 @@ initNonPortableDistroRid()
     fi
 
     if [ -n "${nonPortableBuildID}" ]; then
-        export __DistroRid="${nonPortableBuildID}"
+        __DistroRid="${nonPortableBuildID}"
 
         # We are using a non-portable build rid. Force __PortableBuild to false.
-        export __PortableBuild=0
+        __PortableBuild=0
+
+        export __DistroRid __PortableBuild
     fi
 }
 
@@ -125,7 +127,8 @@ initDistroRidGlobal()
 
     if [ "$buildArch" = "armel" ]; then
         # Armel cross build is Tizen specific and does not support Portable RID build
-        export __PortableBuild=0
+        __PortableBuild=0
+        export __PortableBuild
         isPortable=0
     fi
 
@@ -134,7 +137,8 @@ initDistroRidGlobal()
     if [ -z "${__DistroRid}" ]; then
         # The non-portable build rid was not set. Set the portable rid.
 
-        export __PortableBuild=1
+        __PortableBuild=1
+        export __PortableBuild
         local distroRid=""
 
         # Check for musl-based distros (e.g Alpine Linux, Void Linux).
@@ -153,7 +157,8 @@ initDistroRidGlobal()
             fi
         fi
 
-        export __DistroRid="${distroRid}"
+        __DistroRid="${distroRid}"
+        export __DistroRid
     fi
 
     if [ -z "$__DistroRid" ]; then
@@ -164,6 +169,7 @@ initDistroRidGlobal()
         echo "__DistroRid: ${__DistroRid}"
         echo "__RuntimeId: ${__DistroRid}"
 
-        export __RuntimeId="${__DistroRid}"
+        __RuntimeId="${__DistroRid}"
+        export __RuntimeId
     fi
 }

src/coreclr/build-test.sh

@@ -5,8 +5,10 @@ build_test_wrappers()
     if [[ "$__BuildTestWrappers" -ne -0 ]]; then
         echo "${__MsgPrefix}Creating test wrappers..."
 
-        export __Exclude="${__ProjectDir}/tests/issues.targets"
-        export __BuildLogRootName="Tests_XunitWrapper"
+        __Exclude="${__ProjectDir}/tests/issues.targets"
+        __BuildLogRootName="Tests_XunitWrapper"
+
+        export __Exclude __BuildLogRootName
 
         buildVerbosity="Summary"
 
@@ -79,7 +81,8 @@ generate_layout()
     fi
 
     # Set up the directory for MSBuild debug logs.
-    export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+    MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+    export MSBUILDDEBUGPATH
 
     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 
@@ -101,7 +104,8 @@ generate_layout()
         xUnitTestBinBase="$__TestWorkingDir"
     fi
 
-    export CORE_ROOT="$xUnitTestBinBase"/Tests/Core_Root
+    CORE_ROOT="$xUnitTestBinBase"/Tests/Core_Root
+    export CORE_ROOT
 
     if [[ -d "${CORE_ROOT}" ]]; then
         rm -rf "$CORE_ROOT"
@@ -255,7 +259,9 @@ build_Tests()
         fi
     fi
 
-    export __CMakeBinDir="$__TestBinDir"
+    __CMakeBinDir="$__TestBinDir"
+    export __CMakeBinDir
+
     if [[ ! -d "$__TestIntermediatesDir" ]]; then
         mkdir -p "$__TestIntermediatesDir"
     fi
@@ -291,7 +297,8 @@ build_Tests()
     fi
 
     # Set up the directory for MSBuild debug logs.
-    export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+    MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+    export MSBUILDDEBUGPATH
 
     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 
@@ -384,23 +391,26 @@ build_MSBuild_projects()
         # See https://github.com/Microsoft/msbuild/issues/2993
 
         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
-        export __SkipPackageRestore=false
-        export __SkipTargetingPackBuild=false
-        export __NumberOfTestGroups=3
+        __SkipPackageRestore=false
+        __SkipTargetingPackBuild=false
+        __NumberOfTestGroups=3
 
         __AppendToLog=false
 
         if [[ -n "$__priority1" ]]; then
-            export __NumberOfTestGroups=10
+            __NumberOfTestGroups=10
         fi
 
+        export __SkipPackageRestore __SkipTargetingPackBuild __NumberOfTestGroups
+
         for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
         do
             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 
-            export __TestGroupToBuild="$testGroupToBuild"
+            __TestGroupToBuild="$testGroupToBuild"
+            export __TestGroupToBuild
 
             # Generate build command
             buildArgs=("$projectName")
@@ -427,8 +437,11 @@ build_MSBuild_projects()
                 echo "    $__BuildErr"
                 exit 1
             fi
-            export __SkipPackageRestore=true
-            export __SkipTargetingPackBuild=true
+
+            __SkipPackageRestore=true
+            __SkipTargetingPackBuild=true
+            export __SkipPackageRestore __SkipTargetingPackBuild
+
             __AppendToLog=true
         done
     else
@@ -538,7 +551,9 @@ __CodeCoverage=
 __IncludeTests=INCLUDE_TESTS
 
 # Set the various build properties here so that CMake and MSBuild can pick them up
-export __ProjectDir="$__ProjectRoot"
+__ProjectDir="$__ProjectRoot"
+export __ProjectDir
+
 __BuildTestWrappers=1
 __BuildTestWrappersOnly=
 __Compiler=clang

src/coreclr/build.sh

@@ -120,13 +120,15 @@ build_cross_architecture_components()
         return
     fi
 
-    export __CMakeBinDir="$crossArchBinDir"
-    export CROSSCOMPILE=0
+    __CMakeBinDir="$crossArchBinDir"
+    CROSSCOMPILE=0
+    export __CMakeBinDir CROSSCOMPILE
 
     __CMakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CROSS_COMPONENTS_BUILD=1 $__CMakeArgs"
     build_native "$__CrossArch" "$__ProjectRoot" "$__ProjectRoot" "$intermediatesForBuild" "cross-architecture components"
 
-    export CROSSCOMPILE=1
+    CROSSCOMPILE=1
+    export CROSSCOMPILE
 }
 
 build_CoreLib_ni()
@@ -135,7 +137,8 @@ build_CoreLib_ni()
     local __CoreLibILDir=$2
 
     if [[ "$__PartialNgen" == 1 ]]; then
-        export COMPlus_PartialNGen=1
+        COMPlus_PartialNGen=1
+        export COMPlus_PartialNGen
     fi
 
     if [[ -e "$__CrossGenCoreLibLog" ]]; then
@@ -454,8 +457,10 @@ __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 # Set the remaining variables based upon the determined build configuration
 __BinDir="$__RootBinDir/bin/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
 __PackagesBinDir="$__BinDir/.nuget"
-export __IntermediatesDir="$__RootBinDir/obj/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
-export __ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj/coreclr"
+__IntermediatesDir="$__RootBinDir/obj/coreclr/$__BuildOS.$__BuildArch.$__BuildType"
+__ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj/coreclr"
+export __IntermediatesDir __ArtifactsIntermediatesDir
+
 __CrossComponentBinDir="$__BinDir"
 
 __CrossArch="$__HostArch"
@@ -470,19 +475,22 @@ if [[ -z "$HOME" ]]; then
     if [[ ! -d "$__ProjectDir/temp_home" ]]; then
         mkdir temp_home
     fi
-    export HOME="$__ProjectDir"/temp_home
+    HOME="$__ProjectDir"/temp_home
+    export HOME
     echo "HOME not defined; setting it to $HOME"
 fi
 
 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 # This is where all built CoreClr libraries will copied to.
-export __CMakeBinDir="$__BinDir"
+__CMakeBinDir="$__BinDir"
+export __CMakeBinDir
 
 # Make the directories necessary for build if they don't exist
 setup_dirs_local
 
 # Set up the directory for MSBuild debug logs.
-export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+export MSBUILDDEBUGPATH
 
 # Check prereqs.
 check_prereqs

src/coreclr/coreclr.proj

@@ -6,8 +6,11 @@
     <ItemGroup>
       <_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows))" Include="-skiptests" />
       <_CoreClrBuildArg Condition="'$(TargetArchitecture)' != ''" Include="-$(TargetArchitecture)" />
+      <_CoreClrBuildArg Include="$(CMakeArgs)" />
       <_CoreClrBuildArg Include="-$(Configuration.ToLower())" />
+      <_CoreClrBuildArg Include="$(Compiler)" />
       <_CoreClrBuildArg Condition="'$(ContinuousIntegrationBuild)' == 'true'" Include="-ci" />
+      <_CoreClrBuildArg Condition="'$(CrossBuild)' == 'true'" Include="-cross" />
       <_CoreClrBuildArg Condition="$([MSBuild]::IsOsPlatform(Windows)) and ('$(TargetArchitecture)' == 'x86' or '$(TargetArchitecture)' == 'x64') and '$(Configuration)' == 'Release'" Include="-enforcepgo" />
       <_CoreClrBuildArg Condition="!$([MSBuild]::IsOsPlatform(Windows)) and '$(Configuration)' == 'Release'" Include="-stripsymbols" />
       <_CoreClrBuildArg Condition="'$(OfficialBuildId)' != ''" Include="-officialbuildid=$(OfficialBuildId)" />

src/installer/corehost/build.proj

@@ -30,6 +30,8 @@
       <BuildArgs>$(ConfigurationGroup) $(TargetArchitecture) -apphostver "$(AppHostVersion)" -hostver "$(HostVersion)" -fxrver "$(HostResolverVersion)" -policyver "$(HostPolicyVersion)" -commithash "$(LatestCommit)"</BuildArgs>
       <BuildArgs Condition="'$(PortableBuild)' != 'true'">$(BuildArgs) -portablebuild=false</BuildArgs>
       <BuildArgs Condition="'$(CrossBuild)' == 'true'">$(BuildArgs) -cross</BuildArgs>
+      <BuildArgs Condition="'$(Compiler)' != ''">$(BuildArgs) $(Compiler)</BuildArgs>
+      <BuildArgs Condition="'$(CMakeArgs)' != ''">$(BuildArgs) $(CMakeArgs)</BuildArgs>
       <BuildArgs Condition="'$(StripSymbols)' == 'true'">$(BuildArgs) -stripsymbols</BuildArgs>
     </PropertyGroup>
 

src/installer/corehost/build.sh

@@ -79,7 +79,7 @@ __IntermediatesDir="$__RootBinDir/obj/$__DistroRidLower.$__BuildType"
 
 export __BinDir __IntermediatesDir
 
-__CMakeArgs="-DCLI_CMAKE_HOST_VER=\"$__host_ver\" -DCLI_CMAKE_COMMON_HOST_VER=\"$__apphost_ver\" -DCLI_CMAKE_HOST_FXR_VER=\"$__fxr_ver\""
+__CMakeArgs="-DCLI_CMAKE_HOST_VER=\"$__host_ver\" -DCLI_CMAKE_COMMON_HOST_VER=\"$__apphost_ver\" -DCLI_CMAKE_HOST_FXR_VER=\"$__fxr_ver\" $__CMakeArgs"
 __CMakeArgs="-DCLI_CMAKE_HOST_POLICY_VER=\"$__policy_ver\" -DCLI_CMAKE_PKG_RID=\"$__DistroRid\" -DCLI_CMAKE_COMMIT_HASH=\"$__commit_hash\" $__CMakeArgs"
 
 if [[ "$__PortableBuild" == 1 ]]; then

src/libraries/Native/build-native.proj

@@ -20,13 +20,14 @@
       <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> -stripsymbols</_StripSymbolsArg>
       <_PortableBuildArg Condition="'$(PortableBuild)' != 'true'"> -portablebuild=false</_PortableBuildArg>
       <_CrossBuildArg Condition="'$(CrossBuild)' == 'true'"> -cross</_CrossBuildArg>
+      <_CMakeArgs Condition="'$(CMakeArgs)' != ''"> $(CMakeArgs)</_CMakeArgs>
 
       <!--
         BuildNativeCompiler is a pass-through argument, to pass an argument to build-native.sh. It is intended to be
         used to force a specific compiler toolset.
       -->
       <_BuildNativeCompilerArg Condition="'$(BuildNativeCompiler)' != ''"> $(BuildNativeCompiler)</_BuildNativeCompilerArg>
-      <_BuildNativeUnixArgs>$(_BuildNativeArgs)$(_ProcessCountArg)$(_StripSymbolsArg)$(_PortableBuildArg)$(_CrossBuildArg)$(_BuildNativeCompilerArg)</_BuildNativeUnixArgs>
+      <_BuildNativeUnixArgs>$(_BuildNativeArgs)$(_ProcessCountArg)$(_StripSymbolsArg)$(_PortableBuildArg)$(_CrossBuildArg)$(_BuildNativeCompilerArg)$(_CMakeArgs) $(Compiler)</_BuildNativeUnixArgs>
     </PropertyGroup>
 
     <Message Text="$(MSBuildProjectDirectory)/build-native.sh $(_BuildNativeUnixArgs)" Importance="High"/>