Skip to content

Commit

Permalink
Cleanup functional tests / Structure them like libraries tests (dotne…
Browse files Browse the repository at this point in the history
…t#46810)

You should now be able to run the functional tests locally the same way as running the libraries tests.

Example:
`./dotnet.sh build /t:Test src/tests/FunctionalTests/wasm/Interpreter/console/ /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:Configuration=Release`

Co-authored-by: Steve Pfister <[email protected]>
Co-authored-by: Maxim Lipnin <[email protected]>
  • Loading branch information
3 people authored Jan 15, 2021
1 parent 7b24471 commit c260375
Show file tree
Hide file tree
Showing 32 changed files with 169 additions and 521 deletions.
7 changes: 6 additions & 1 deletion eng/testing/AndroidRunnerTemplate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ TARGET_OS=$3
TEST_NAME=$4
XHARNESS_OUT="$EXECUTION_DIR/xharness-output"

if [ -n "$5" ]; then
EXPECTED_EXIT_CODE="--expected-exit-code $5"
fi

cd $EXECUTION_DIR

# it doesn't support parallel execution yet, so, here is a hand-made semaphore:
Expand All @@ -35,7 +39,8 @@ $HARNESS_RUNNER android test \
--instrumentation="net.dot.MonoRunner" \
--package-name="net.dot.$ASSEMBLY_NAME" \
--app="$EXECUTION_DIR/bin/$TEST_NAME.apk" \
--output-directory="$XHARNESS_OUT"
--output-directory="$XHARNESS_OUT" \
$EXPECTED_EXIT_CODE

_exitCode=$?

Expand Down
11 changes: 9 additions & 2 deletions eng/testing/AppleRunnerTemplate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,15 @@ ASSEMBLY_NAME=$1
TARGET_ARCH=$2
TARGET_OS=$3
TEST_NAME=$4
XHARNESS_CMD="test"
XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
XCODE_PATH=$(xcode-select -p)/../..

if [ -n "$5" ]; then
XHARNESS_CMD="run"
EXPECTED_EXIT_CODE="--expected-exit-code $5"
fi

if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "x86" ]]; then TARGET=ios-simulator-32; fi
if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "x64" ]]; then TARGET=ios-simulator-64; fi
if [[ "$TARGET_OS" == "iOS" && "$TARGET_ARCH" == "arm" ]]; then TARGET=ios-device; fi
Expand Down Expand Up @@ -46,11 +52,12 @@ else
HARNESS_RUNNER="dotnet xharness"
fi

$HARNESS_RUNNER ios test \
$HARNESS_RUNNER ios $XHARNESS_CMD \
--app="$EXECUTION_DIR/$TEST_NAME/$SCHEME_SDK/$TEST_NAME.app" \
--targets="$TARGET" \
--xcode="$XCODE_PATH" \
--output-directory="$XHARNESS_OUT"
--output-directory="$XHARNESS_OUT" \
$EXPECTED_EXIT_CODE

_exitCode=$?

Expand Down
5 changes: 4 additions & 1 deletion eng/testing/WasmRunnerTemplate.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

EXECUTION_DIR=$(dirname $0)
SCENARIO=$3

cd $EXECUTION_DIR

Expand All @@ -18,7 +19,9 @@ else
HARNESS_RUNNER="dotnet xharness"
fi

if [ -z "$XHARNESS_COMMAND" ]; then
if [ "$SCENARIO" == "WasmTestOnBrowser" ]; then
XHARNESS_COMMAND="test-browser"
elif [ -z "$XHARNESS_COMMAND" ]; then
XHARNESS_COMMAND="test"
fi

Expand Down
15 changes: 10 additions & 5 deletions eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

<PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
<!-- We need to set this in order to get extensibility on xunit category traits and other arguments we pass down to xunit via MSBuild properties -->
<RunScriptCommand>$HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT -- $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
<RunScriptCommand Condition="'$(IsFunctionalTest)' != 'true'">$HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT -- $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
<RunScriptCommand Condition="'$(IsFunctionalTest)' == 'true'">$HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT --expected-exit-code=$(ExpectedExitCode) -- $(RunTestsJSArguments) --run $(AssemblyName).dll --testing</RunScriptCommand>
<EventSourceSupport>false</EventSourceSupport>
<UseSystemResourceKeys>true</UseSystemResourceKeys>
<EnableUnsafeUTF7Encoding>false</EnableUnsafeUTF7Encoding>
Expand All @@ -38,6 +39,8 @@
<AndroidAbi Condition="'$(TargetArchitecture)' == 'arm'">armeabi-v7a</AndroidAbi>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'x64'">x86_64</AndroidAbi>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'x86'">x86</AndroidAbi>

<MainLibraryFileName Condition="'$(MainLibraryFileName)' == ''">AndroidTestRunner.dll</MainLibraryFileName>
</PropertyGroup>

<Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
Expand All @@ -54,7 +57,7 @@
RuntimeIdentifier="$(RuntimeIdentifier)"
ProjectName="$(AssemblyName)"
MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackNativeDir)include\mono-2.0"
MainLibraryFileName="AndroidTestRunner.dll"
MainLibraryFileName="$(MainLibraryFileName)"
ForceInterpreter="$(MonoForceInterpreter)"
StripDebugSymbols="False"
OutputDir="$(BundleDir)"
Expand Down Expand Up @@ -92,6 +95,7 @@

<PropertyGroup>
<Optimized>true</Optimized>
<MainLibraryFileName Condition="'$(MainLibraryFileName)' == ''">AppleTestRunner.dll</MainLibraryFileName>
</PropertyGroup>
<ItemGroup>
<AotInputAssemblies Condition="'$(RunAOTCompilation)' == 'true'" Include="$(PublishDir)*.dll" Exclude="$(PublishDir)System.Runtime.WindowsRuntime.dll">
Expand Down Expand Up @@ -122,7 +126,8 @@
ProjectName="$(AssemblyName)"
MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackNativeDir)include\mono-2.0"
Assemblies="@(BundleAssemblies)"
MainLibraryFileName="AppleTestRunner.dll"
MainLibraryFileName="$(MainLibraryFileName)"
ForceAOT="$(RunAOTCompilation)"
ForceInterpreter="$(MonoForceInterpreter)"
UseConsoleUITemplate="True"
GenerateXcodeProject="True"
Expand Down Expand Up @@ -186,8 +191,8 @@
<PropertyGroup>
<WasmAppDir>$(BundleDir)</WasmAppDir>
<WasmBuildDir>$(PublishDir)</WasmBuildDir>
<WasmMainAssemblyPath>$(PublishDir)WasmTestRunner.dll</WasmMainAssemblyPath>
<WasmMainJSPath>$(MonoProjectRoot)\wasm\runtime-test.js</WasmMainJSPath>
<WasmMainAssemblyPath Condition="'$(WasmMainAssemblyPath)' == ''">$(PublishDir)WasmTestRunner.dll</WasmMainAssemblyPath>
<WasmMainJSPath Condition="'$(WasmMainJSPath)' == ''">$(MonoProjectRoot)\wasm\runtime-test.js</WasmMainJSPath>
<WasmInvariantGlobalization>$(InvariantGlobalization)</WasmInvariantGlobalization>
<WasmGenerateRunV8Script>true</WasmGenerateRunV8Script>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions eng/testing/tests.targets
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@
<RunTestsCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and
$([MSBuild]::VersionGreaterThanOrEquals($(TargetFrameworkVersion), '$(NETCoreAppCurrentVersion)'))">$(RunTestsCommand) --runtime-path "$(TestHostRootPath.TrimEnd('\/'))"</RunTestsCommand>
<RunTestsCommand Condition="'$(TestRspFile)' != '' and '$(RuntimeFlavor)' != 'Mono'">$(RunTestsCommand) --rsp-file "$(TestRspFile)"</RunTestsCommand>
<RunTestsCommand Condition="'$(TargetsMobile)' == 'true'">"$(RunScriptOutputPath)" $(AssemblyName) $(TargetArchitecture) $(TargetOS) $(TestProjectName)</RunTestsCommand>
<RunTestsCommand Condition="'$(TargetOS)' == 'Browser'">"$(RunScriptOutputPath)" $(JSEngine) $(AssemblyName).dll $(_withoutCategories.Replace(';', ' -notrait category='))</RunTestsCommand>
<RunTestsCommand Condition="'$(TargetsMobile)' == 'true'">"$(RunScriptOutputPath)" $(AssemblyName) $(TargetArchitecture) $(TargetOS) $(TestProjectName) $(ExpectedExitCode)</RunTestsCommand>
<RunTestsCommand Condition="'$(TargetOS)' == 'Browser'">"$(RunScriptOutputPath)" $(JSEngine) $(AssemblyName).dll $(Scenario)</RunTestsCommand>
</PropertyGroup>

<!-- Invoke the run script with the test host as the runtime path. -->
Expand Down
5 changes: 3 additions & 2 deletions src/libraries/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@

<!-- Define test projects and companions -->
<PropertyGroup>
<IsTestProject>false</IsTestProject>
<IsTestProject Condition="'$(IsTestProject)' == ''">false</IsTestProject>
<IsTestProject Condition="$(MSBuildProjectName.EndsWith('.UnitTests')) or $(MSBuildProjectName.EndsWith('.Tests'))">true</IsTestProject>

<IsTestSupportProject>false</IsTestSupportProject>
Expand Down Expand Up @@ -132,7 +132,8 @@
<AnyOSPlatformConfig>AnyOS.AnyCPU.$(Configuration)</AnyOSPlatformConfig>
<UnixPlatformConfig>Unix.$(Platform).$(Configuration)</UnixPlatformConfig>
<TestArchiveRoot>$(ArtifactsDir)helix/</TestArchiveRoot>
<TestArchiveTestsRoot>$(TestArchiveRoot)tests/</TestArchiveTestsRoot>
<TestArchiveTestsRoot Condition="$(IsFunctionalTest) != true">$(TestArchiveRoot)tests/</TestArchiveTestsRoot>
<TestArchiveTestsRoot Condition="$(IsFunctionalTest) == true">$(TestArchiveRoot)runonly/</TestArchiveTestsRoot>
<TestArchiveTestsDir>$(TestArchiveTestsRoot)$(OSPlatformConfig)/</TestArchiveTestsDir>
<TestArchiveRuntimeRoot>$(TestArchiveRoot)runtime/</TestArchiveRuntimeRoot>

Expand Down
6 changes: 4 additions & 2 deletions src/libraries/sendtohelixhelp.proj
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,8 @@
<HelixCorrelationPayload Include="chromedriver" Uri="$(SeleniumUrl)" Condition="'$(TargetOS)' == 'Browser'" />

<_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" />
<_WorkItem Include="$(TestArchiveRoot)runonly/**/*.Console.Test.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' != 'WasmTestOnBrowser'" />
<_WorkItem Include="$(TestArchiveRoot)runonly/**/*.Browser.Test.zip" Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'" />

<HelixWorkItem Include="@(_WorkItem -> '%(FileName)')">
<PayloadArchive>%(Identity)</PayloadArchive>
Expand All @@ -258,7 +260,7 @@

<ItemGroup Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' != 'WasmTestOnBrowser'">
<!-- Create a work item for run-only WASM console app -->
<_RunOnlyWorkItem Include="$(TestArchiveRoot)runonly/**/console/*.zip" />
<_RunOnlyWorkItem Include="$(TestArchiveRoot)runonly/**/*.Console.Sample.zip" />
<HelixWorkItem Include="@(_RunOnlyWorkItem -> '%(FileName)')" >
<PayloadArchive>%(Identity)</PayloadArchive>
<!-- No RunTests script generated for the sample project so we just use the direct command -->
Expand All @@ -268,7 +270,7 @@

<ItemGroup Condition="'$(TargetOS)' == 'Browser' and '$(Scenario)' == 'WasmTestOnBrowser'">
<!-- Create a work item for run-only WASM browser app -->
<_RunOnlyWorkItem Include="$(TestArchiveRoot)runonly/**/browser/*.zip" />
<_RunOnlyWorkItem Include="$(TestArchiveRoot)runonly/**/*.Browser.Sample.zip" />
<HelixWorkItem Include="@(_RunOnlyWorkItem -> '%(FileName)')" >
<PayloadArchive>%(Identity)</PayloadArchive>
<!-- No RunTests script generated for the sample project so we just use the direct command -->
Expand Down
4 changes: 3 additions & 1 deletion src/libraries/tests.proj
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,10 @@
<ItemGroup Condition="'$(ArchiveTests)' == 'true' and ('$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS')">
<ProjectReference Include="$(MonoProjectRoot)netcore\sample\iOS\Program.csproj"
BuildInParallel="false" />
<!-- TODO: remove Exclude after "tests.mobile.targets(107,5): error : Unable to compile method" fixed -->
<ProjectReference Include="$(RepoRoot)\src\tests\FunctionalTests\iOS\**\*.Test.csproj"
BuildInParallel="false" />
BuildInParallel="false"
Exclude="$(RepoRoot)\src\tests\FunctionalTests\iOS\Simulator\AOT\iOS.Simulator.Aot.Test.csproj"/>
</ItemGroup>

<ItemGroup Condition="'$(ArchiveTests)' == 'true' and '$(TargetOS)' == 'Android'">
Expand Down
Original file line number Diff line number Diff line change
@@ -1,81 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">

<Import Project="..\..\common.props" />

<!-- Redirect 'dotnet publish' to in-tree runtime pack -->
<Target Name="TrickRuntimePackLocation" AfterTargets="ProcessFrameworkReferences">
<ItemGroup>
<RuntimePack>
<PackageDirectory>$(ArtifactsBinDir)microsoft.netcore.app.runtime.$(RuntimeIdentifier)\$(Configuration)</PackageDirectory>
</RuntimePack>
</ItemGroup>
<Message Text="Packaged ID: %(RuntimePack.PackageDirectory)" Importance="high" />
</Target>

<UsingTask TaskName="AndroidAppBuilderTask" AssemblyFile="$(AndroidAppBuilderTasksAssemblyPath)"/>

<Target Name="BuildApp" AfterTargets="CopyFilesToPublishDirectory">
<PropertyGroup>
<StripDebugSymbols>False</StripDebugSymbols>
<StripDebugSymbols Condition="'$(Configuration)' == 'Release'">True</StripDebugSymbols>
<AdbTool>$(ANDROID_SDK_ROOT)\platform-tools\adb</AdbTool>
<ApkDir>$(PublishDir)apk\</ApkDir>
</PropertyGroup>

<RemoveDir Directories="$(ApkDir)" />

<PropertyGroup>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'arm64'">arm64-v8a</AndroidAbi>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'arm'">armeabi-v7a</AndroidAbi>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'x64'">x86_64</AndroidAbi>
<AndroidAbi Condition="'$(TargetArchitecture)' == 'x86'">x86</AndroidAbi>
</PropertyGroup>

<!-- TODO: delete once we switch to Android Crypto -->
<Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
SourceFiles="$(ANDROID_OPENSSL_AAR)\prefab\modules\crypto\libs\android.$(AndroidAbi)\libcrypto.so"
DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true"/>
<Copy Condition="'$(ANDROID_OPENSSL_AAR)' != ''"
SourceFiles="$(ANDROID_OPENSSL_AAR)\prefab\modules\ssl\libs\android.$(AndroidAbi)\libssl.so"
DestinationFolder="$(PublishDir)" SkipUnchangedFiles="true"/>

<AndroidAppBuilderTask
RuntimeIdentifier="$(RuntimeIdentifier)"
SourceDir="$(PublishDir)"
ProjectName="Android.Emulator.Interpreter.Test"
ForceInterpreter="True"
MonoRuntimeHeaders="$(MicrosoftNetCoreAppRuntimePackDir)\native\include\mono-2.0"
MainLibraryFileName="Android.Emulator.Interpreter.Test.dll"
StripDebugSymbols="$(StripDebugSymbols)"
OutputDir="$(ApkDir)">
<Output TaskParameter="ApkBundlePath" PropertyName="ApkBundlePath" />
<Output TaskParameter="ApkPackageId" PropertyName="ApkPackageId" />
</AndroidAppBuilderTask>

<Message Importance="High" Text="Apk: $(ApkBundlePath)"/>
<Message Importance="High" Text="PackageId: $(ApkPackageId)"/>
</Target>
<PropertyGroup>
<OutputType>Exe</OutputType>
<MonoForceInterpreter>true</MonoForceInterpreter>
<RunAOTCompilation>false</RunAOTCompilation>
<TestRuntime>true</TestRuntime>
<TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
<MainLibraryFileName>Android.Emulator.Interpreter.Test.dll</MainLibraryFileName>
<ExpectedExitCode>42</ExpectedExitCode>
</PropertyGroup>

<ItemGroup>
<Compile Include="Program.cs" />
</ItemGroup>

<Target Name="CopySampleAppToHelixTestDir"
AfterTargets="Build"
DependsOnTargets="Publish;BuildApp" >
<PropertyGroup>
<OSPlatformConfig>$(TargetOS).AnyCPU.$(Configuration)</OSPlatformConfig>
<HelixArchiveRoot>$(ArtifactsDir)helix/</HelixArchiveRoot>
<HelixArchiveRunOnlyRoot>$(HelixArchiveRoot)runonly/</HelixArchiveRunOnlyRoot>
<HelixArchiveRunOnlyAppsDir>$(HelixArchiveRunOnlyRoot)$(OSPlatformConfig)/</HelixArchiveRunOnlyAppsDir>
</PropertyGroup>
<ItemGroup>
<_apkFiles Include="$(ApkBundlePath)" />
</ItemGroup>
<Copy SourceFiles="@(_apkFiles)"
DestinationFolder="$(HelixArchiveRunOnlyAppsDir)/%(RecursiveDir)" />

<Message Importance="High" Text="ApkBundlePath: $(ApkBundlePath)"/>
<Message Importance="High" Text="TestArchiveTestsDir: $(TestArchiveTestsDir)"/>
</Target>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<RunAOTCompilation>false</RunAOTCompilation>
<TestRuntime>true</TestRuntime>
<TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
<MainLibraryFileName>Android.Emulator.JIT.Test.dll</MainLibraryFileName>
<ExpectedExitCode>42</ExpectedExitCode>
</PropertyGroup>

<ItemGroup>
<Compile Include="Program.cs" />
</ItemGroup>
</Project>
13 changes: 13 additions & 0 deletions src/tests/FunctionalTests/Android/Emulator/JIT/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;

public class Program
{
public static int Main(string[] args)
{
Console.WriteLine("Hello, Android!"); // logcat
return 42;
}
}
13 changes: 0 additions & 13 deletions src/tests/FunctionalTests/Android/common.props

This file was deleted.

4 changes: 3 additions & 1 deletion src/tests/FunctionalTests/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
<RunAnalyzers>false</RunAnalyzers>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<IsTestProject>true</IsTestProject>
<IsFunctionalTest>true</IsFunctionalTest>
</PropertyGroup>

<Import Project="..\..\mono\Directory.Build.props" />
<Import Project="..\..\libraries\Directory.Build.props" />
</Project>
2 changes: 1 addition & 1 deletion src/tests/FunctionalTests/Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
<Project>
<Import Project="..\..\mono\Directory.Build.targets" />
<Import Project="..\..\libraries\Directory.Build.targets" />
</Project>
Loading

0 comments on commit c260375

Please sign in to comment.