Run helix tests locally (#19031)

docs/Helix.md

@@ -12,17 +12,10 @@ For more info about helix see: [SDK](https://github.com/dotnet/arcade/blob/maste
 To run Helix tests for one particular test project:
 
 ```
-cd src/MyCode/test
-dotnet msbuild /t:Helix
+.\eng\scripts\RunHelix.ps1 -Project path\mytestproject.csproj
 ```
 
-To run tests for the entire repo, run:
-
-```
-.\eng\scripts\TestHelix.ps1
-```
-
-This will restore, and then publish all of the test projects including some bootstrapping scripts that will install the correct dotnet runtime/sdk before running the test assemblies on the helix machine, and upload the job to helix.
+This will restore, and then publish all the test project including some bootstrapping scripts that will install the correct dotnet runtime/sdk before running the test assembly on the helix machine(s), and upload the job to helix.
 
 
 ## How do I look at the results of a helix run on Azure Pipelines?

eng/helix/content/runtests.cmd

@@ -23,11 +23,6 @@ powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePo
 
 set HELIX=%helixQueue%
 
-if (%targetFrameworkIdentifier%==.NETFramework) (
-    xunit.console.exe %target% -xml testResults.xml
-    exit /b %ERRORLEVEL%
-)
-
 %DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt
 find /c "Exception thrown" discovered.txt
 REM "ERRORLEVEL is not %ERRORLEVEL%" https://blogs.msdn.microsoft.com/oldnewthing/20080926-00/?p=20743/
@@ -41,8 +36,8 @@ set exit_code=0
 
 set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
 set QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
-if (%quarantined%==true) (
-    echo Running all tests.
+if %quarantined%==true (
+    echo Running quarantined tests.
     %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%QUARANTINE_FILTER%
     if errorlevel 1 (
         echo Failure in flaky test 1>&2

eng/helix/helix.proj

@@ -42,12 +42,12 @@
   <!-- Items with the type "HelixProperties" will become arbitrary JSON associated with the job -->
   <!-- NOTE: These are global to the whole Job, so don't include project-specific content here. -->
   <ItemGroup>
-    <HelixProperties Include="buildNumber" Value="$(BUILD_BUILDNUMBER)" />
-    <HelixProperties Include="buildId" Value="$(BUILD_BUILDID)" />
-    <HelixProperties Include="azdoProject" Value="$(SYSTEM_TEAMPROJECT)" />
-    <HelixProperties Include="buildDefinition" Value="$(BUILD_DEFINITIONNAME)" />
-    <HelixProperties Include="commitSha" Value="$(BUILD_SOURCEVERSION)" />
-    <HelixProperties Include="branch" Value="$(BUILD_SOURCEBRANCH)" />
+    <HelixProperties Include="buildNumber" Value="$(BUILD_BUILDNUMBER)" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
+    <HelixProperties Include="buildId" Value="$(BUILD_BUILDID)" Condition="'$(BUILD_BUILDID)' != ''" />
+    <HelixProperties Include="azdoProject" Value="$(SYSTEM_TEAMPROJECT)" Condition="'$(SYSTEM_TEAMPROJECT)' != ''" />
+    <HelixProperties Include="buildDefinition" Value="$(BUILD_DEFINITIONNAME)" Condition="'$(BUILD_DEFINITIONNAME)' != ''" />
+    <HelixProperties Include="commitSha" Value="$(BUILD_SOURCEVERSION)" Condition="'$(BUILD_SOURCEVERSION)' != ''" />
+    <HelixProperties Include="branch" Value="$(BUILD_SOURCEBRANCH)" Condition="'$(BUILD_SOURCEBRANCH)' != ''" />
     <HelixProperties Condition="'$(RunQuarantinedTests)' == 'true'" Include="runType" Value="quarantined" />
     <HelixProperties Condition="'$(RunQuarantinedTests)' != 'true'" Include="runType" Value="unquarantined" />
   </ItemGroup>

eng/scripts/RunHelix.ps1

@@ -0,0 +1,41 @@
+<#
+.SYNOPSIS
+    Runs the specified test project on a Helix machine.
+.DESCRIPTION
+    This script runs the Helix msbuild task on the given project and publishes then uploads the output and runs tests on the Helix machine(s) passed in.
+.PARAMETER Project
+    The test project to publish and send to Helix.
+.PARAMETER HelixQueues
+    Set the Helix queues to use, the list is ';' separated.
+    Some supported queues:
+    Ubuntu.1604.Amd64.Open
+    Ubuntu.1804.Amd64.Open
+    Windows.10.Amd64.Open
+    Windows.81.Amd64.Open
+    Windows.7.Amd64.Open
+    OSX.1014.Amd64.Open
+    Centos.7.Amd64.Open
+    Debian.8.Amd64.Open
+    Debian.9.Amd64.Open
+    Redhat.7.Amd64.Open
+.PARAMETER RunQuarantinedTests
+    By default quarantined tests are not run. Set this to $true to run only the quarantined tests.
+#>
+param(
+    [Parameter(Mandatory=$true)]
+    [string]$Project,
+    [string]$HelixQueues = "Windows.10.Amd64.Open",
+    [bool]$RunQuarantinedTests = $false
+)
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+$env:BUILD_REASON="PullRequest"
+$env:BUILD_SOURCEBRANCH="local"
+$env:BUILD_REPOSITORY_NAME="aspnetcore"
+$env:SYSTEM_TEAMPROJECT="aspnetcore"
+
+$HelixQueues = $HelixQueues -replace ";", "%3B"
+dotnet msbuild $Project /t:Helix /p:IsRequiredCheck=true /p:IsHelixDaily=true /p:HelixTargetQueues=$HelixQueues /p:RunQuarantinedTests=$RunQuarantinedTests