Support installing .NET SDKs from a protected feed (#27405)

* Support installing .NET SDKs from a protected feed
- align variable names with the `tools.*` scripts
  - the `tools.*` scripts perform the actual SDK installation
- always pass feed-related variables to `InstallDotNet` in Helix `runtests.*` scripts

.azure/pipelines/ci.yml

@@ -95,13 +95,15 @@ variables:
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-MSRC-Storage
   - name: _InternalRuntimeDownloadArgs
-    value: -DotNetRuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet -DotNetRuntimeSourceFeedKey
-           $(dotnetclimsrc-read-sas-token-base64) /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
-  # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have
-  # to be passed directly. This is awkward, since we pass the same info above, but we have
-  # to have it in two different forms
+    value: -RuntimeSourceFeed https://dotnetclimsrc.blob.core.windows.net/dotnet
+           -RuntimeSourceFeedKey $(dotnetclimsrc-read-sas-token-base64)
+           /p:DotNetAssetRootAccessTokenSuffix='$(dotnetclimsrc-read-sas-token-base64)'
+  # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This
+  # is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically.
   - name: _InternalRuntimeDownloadCodeSignArgs
-    value: /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
+    value: $(_InternalRuntimeDownloadArgs)
+           /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet
+           /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
   - group: DotNet-HelixApi-Access
   - name: _UseHelixOpenQueues
     value: 'false'

build.ps1

@@ -81,11 +81,11 @@ MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]
 .PARAMETER MSBuildArguments
 Additional MSBuild arguments to be passed through.
 
-.PARAMETER DotNetRuntimeSourceFeed
-Additional feed that can be used when downloading .NET runtimes
+.PARAMETER RuntimeSourceFeed
+Additional feed that can be used when downloading .NET runtimes and SDKs
 
-.PARAMETER DotNetRuntimeSourceFeedKey
-Key for feed that can be used when downloading .NET runtimes
+.PARAMETER RuntimeSourceFeedKey
+Key for feed that can be used when downloading .NET runtimes and SDKs
 
 .EXAMPLE
 Building both native and managed projects.
@@ -174,8 +174,10 @@ param(
 
     # Optional arguments that enable downloading an internal
     # runtime or runtime from a non-default location
-    [string]$DotNetRuntimeSourceFeed,
-    [string]$DotNetRuntimeSourceFeedKey,
+    [Alias('DotNetRuntimeSourceFeed')]
+    [string]$RuntimeSourceFeed,
+    [Alias('DotNetRuntimeSourceFeedKey')]
+    [string]$RuntimeSourceFeedKey,
 
     # Capture the rest
     [Parameter(ValueFromRemainingArguments = $true)]
@@ -257,9 +259,9 @@ if (-not $Configuration) {
 $MSBuildArguments += "/p:Configuration=$Configuration"
 
 [string[]]$ToolsetBuildArguments = @()
-if ($DotNetRuntimeSourceFeed -or $DotNetRuntimeSourceFeedKey) {
-    $runtimeFeedArg = "/p:DotNetRuntimeSourceFeed=$DotNetRuntimeSourceFeed"
-    $runtimeFeedKeyArg = "/p:DotNetRuntimeSourceFeedKey=$DotNetRuntimeSourceFeedKey"
+if ($RuntimeSourceFeed -or $RuntimeSourceFeedKey) {
+    $runtimeFeedArg = "/p:DotNetRuntimeSourceFeed=$RuntimeSourceFeed"
+    $runtimeFeedKeyArg = "/p:DotNetRuntimeSourceFeedKey=$RuntimeSourceFeedKey"
     $MSBuildArguments += $runtimeFeedArg
     $MSBuildArguments += $runtimeFeedKeyArg
     $ToolsetBuildArguments += $runtimeFeedArg

build.sh

@@ -30,8 +30,8 @@ build_installers=''
 build_projects=''
 target_arch='x64'
 configuration=''
-dotnet_runtime_source_feed=''
-dotnet_runtime_source_feed_key=''
+runtime_source_feed=''
+runtime_source_feed_key=''
 
 if [ "$(uname)" = "Darwin" ]; then
     target_os_name='osx'
@@ -77,8 +77,8 @@ Options:
     --excludeCIBinarylog              Don't output binary log by default in CI builds (short: -nobl).
     --verbosity|-v                    MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]
 
-    --dotnet-runtime-source-feed      Additional feed that can be used when downloading .NET runtimes
-    --dotnet-runtime-source-feed-key  Key for feed that can be used when downloading .NET runtimes
+    --runtime-source-feed             Additional feed that can be used when downloading .NET runtimes and SDKs
+    --runtime-source-feed-key         Key for feed that can be used when downloading .NET runtimes and SDKs
 
 Description:
     This build script installs required tools and runs an MSBuild command on this repository
@@ -209,15 +209,15 @@ while [[ $# -gt 0 ]]; do
         -excludeCIBinarylog|-nobl)
             exclude_ci_binary_log=true
             ;;
-        -dotnet-runtime-source-feed|-dotnetruntimesourcefeed)
+        -dotnet-runtime-source-feed|-dotnetruntimesourcefeed|-runtime-source-feed|-runtimesourcefeed)
             shift
-            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed" && __usage
-            dotnet_runtime_source_feed="${1:-}"
+            [ -z "${1:-}" ] && __error "Missing value for parameter --runtime-source-feed" && __usage
+            runtime_source_feed="${1:-}"
             ;;
-        -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey)
+        -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey|-runtime-source-feed-key|-runtimesourcefeedkey)
             shift
-            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed-key" && __usage
-            dotnet_runtime_source_feed_key="${1:-}"
+            [ -z "${1:-}" ] && __error "Missing value for parameter --runtime-source-feed-key" && __usage
+            runtime_source_feed_key="${1:-}"
             ;;
         *)
             msbuild_args[${#msbuild_args[*]}]="$1"
@@ -290,9 +290,9 @@ msbuild_args[${#msbuild_args[*]}]="-p:Configuration=$configuration"
 
 # Set up additional runtime args
 toolset_build_args=()
-if [ ! -z "$dotnet_runtime_source_feed$dotnet_runtime_source_feed_key" ]; then
-    runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$dotnet_runtime_source_feed"
-    runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$dotnet_runtime_source_feed_key"
+if [ ! -z "$runtime_source_feed$runtime_source_feed_key" ]; then
+    runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$runtime_source_feed"
+    runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$runtime_source_feed_key"
     msbuild_args[${#msbuild_args[*]}]=$runtimeFeedArg
     msbuild_args[${#msbuild_args[*]}]=$runtimeFeedKeyArg
     toolset_build_args[${#toolset_build_args[*]}]=$runtimeFeedArg

eng/helix/content/runtests.cmd

@@ -25,14 +25,18 @@ set "PATH=%DOTNET_ROOT%;%PATH%;%HELIX_CORRELATION_PAYLOAD%\node\bin"
 echo Set path to: "%PATH%"
 echo.
 
-echo "InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true"
-powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true"
-echo.
-
 IF [%$feedCred%] == [] (
+    echo "InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true"
+    powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true '' '' $true"
+    echo.
+
     echo "InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true '' '' $true"
     powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true '' '' $true"
 ) else (
+    echo "InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true https://dotnetclimsrc.blob.core.windows.net/dotnet ... $true"
+    powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$sdkVersion% %$arch% '' $true https://dotnetclimsrc.blob.core.windows.net/dotnet %$feedCred% $true"
+    echo.
+
     echo "InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true https://dotnetclimsrc.blob.core.windows.net/dotnet ... $true"
     powershell.exe -noLogo -NoProfile -ExecutionPolicy unrestricted -command ". eng\common\tools.ps1; InstallDotNet %DOTNET_ROOT% %$runtimeVersion% %$arch% dotnet $true https://dotnetclimsrc.blob.core.windows.net/dotnet %$feedCred% $true"
 )

eng/helix/content/runtests.sh

@@ -31,15 +31,16 @@ YELLOW="\033[0;33m"
 MAGENTA="\033[0;95m"
 
 . eng/common/tools.sh
-echo "InstallDotNet $DOTNET_ROOT $dotnet_sdk_version '' '' true"
-InstallDotNet $DOTNET_ROOT $dotnet_sdk_version "" "" true || {
-  exit_code=$?
-  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
-  ExitWithExitCode $exit_code
-}
-echo
 
 if [[ -z "${10:-}" ]]; then
+    echo "InstallDotNet $DOTNET_ROOT $dotnet_sdk_version '' '' true"
+    InstallDotNet $DOTNET_ROOT $dotnet_sdk_version "" "" true || {
+      exit_code=$?
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+      ExitWithExitCode $exit_code
+    }
+    echo
+
     echo "InstallDotNet $DOTNET_ROOT $dotnet_runtime_version '' dotnet true"
     InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true || {
       exit_code=$?
@@ -47,6 +48,14 @@ if [[ -z "${10:-}" ]]; then
       ExitWithExitCode $exit_code
     }
 else
+    echo "InstallDotNet $DOTNET_ROOT $dotnet_sdk_version '' '' true https://dotnetclimsrc.blob.core.windows.net/dotnet ..."
+    InstallDotNet $DOTNET_ROOT $dotnet_sdk_version "" "" true https://dotnetclimsrc.blob.core.windows.net/dotnet ${10} || {
+      exit_code=$?
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
+      ExitWithExitCode $exit_code
+    }
+    echo
+
     echo "InstallDotNet $DOTNET_ROOT $dotnet_runtime_version '' dotnet true https://dotnetclimsrc.blob.core.windows.net/dotnet ..."
     InstallDotNet $DOTNET_ROOT $dotnet_runtime_version "" dotnet true https://dotnetclimsrc.blob.core.windows.net/dotnet ${10} || {
       exit_code=$?

eng/scripts/CodeCheck.ps1

@@ -7,8 +7,10 @@ param(
     [switch]$ci,
     # Optional arguments that enable downloading an internal
     # runtime or runtime from a non-default location
-    [string]$DotNetRuntimeSourceFeed,
-    [string]$DotNetRuntimeSourceFeedKey
+    [Alias('DotNetRuntimeSourceFeed')]
+    [string]$RuntimeSourceFeed,
+    [Alias('DotNetRuntimeSourceFeedKey')]
+    [string]$RuntimeSourceFeedKey
 )
 
 $ErrorActionPreference = 'Stop'
@@ -47,8 +49,8 @@ function LogError {
 try {
     if ($ci) {
         # Install dotnet.exe
-        if ($DotNetRuntimeSourceFeed -or $DotNetRuntimeSourceFeedKey) {
-            & $repoRoot/restore.cmd -ci -nobl -noBuildNodeJS -DotNetRuntimeSourceFeed $DotNetRuntimeSourceFeed -DotNetRuntimeSourceFeedKey $DotNetRuntimeSourceFeedKey
+        if ($RuntimeSourceFeed -or $RuntimeSourceFeedKey) {
+            & $repoRoot/restore.cmd -ci -nobl -noBuildNodeJS -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
         }
         else{
             & $repoRoot/restore.cmd -ci -nobl -noBuildNodeJS

eng/scripts/ci-source-build.sh

@@ -33,8 +33,8 @@ reporoot="$(dirname "$(dirname "$scriptroot")")"
 #    mv "$reporoot/global.bak.json" "$reporoot/global.json"
 #}" EXIT
 
-dotnet_runtime_source_feed=''
-dotnet_runtime_source_feed_key=''
+runtime_source_feed=''
+runtime_source_feed_key=''
 other_args=()
 
 #
@@ -46,8 +46,8 @@ __usage() {
 Arguments:
     <Arguments>...                    Arguments passed to the command. Variable number of arguments allowed.
 
-    --dotnet-runtime-source-feed      Additional feed that can be used when downloading .NET runtimes
-    --dotnet-runtime-source-feed-key  Key for feed that can be used when downloading .NET runtimes
+    --runtime-source-feed             Additional feed that can be used when downloading .NET runtimes and SDKs
+    --runtime-source-feed-key         Key for feed that can be used when downloading .NET runtimes and SDKs
 
 Description:
    This script is meant for testing source build by imitating some of the input parameters and conditions.
@@ -73,15 +73,15 @@ while [[ $# -gt 0 ]]; do
             __usage --no-exit
             exit 0
             ;;
-        -dotnet-runtime-source-feed|-dotnetruntimesourcefeed)
+        -dotnet-runtime-source-feed|-dotnetruntimesourcefeed|-runtime_source_feed|-runtimesourcefeed)
             shift
-            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed" && __usage
-            dotnet_runtime_source_feed="${1:-}"
+            [ -z "${1:-}" ] && __error "Missing value for parameter --runtime-source-feed" && __usage
+            runtime_source_feed="${1:-}"
             ;;
-        -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey)
+        -dotnet-runtime-source-feed-key|-dotnetruntimesourcefeedkey|-runtime_source_feed_key|-runtimesourcefeedkey)
             shift
-            [ -z "${1:-}" ] && __error "Missing value for parameter --dotnet-runtime-source-feed-key" && __usage
-            dotnet_runtime_source_feed_key="${1:-}"
+            [ -z "${1:-}" ] && __error "Missing value for parameter --runtime-source-feed-key" && __usage
+            runtime_source_feed_key="${1:-}"
             ;;
         *)
             other_args[${#other_args[*]}]="$1"
@@ -92,9 +92,14 @@ done
 
 # Set up additional runtime args
 runtime_feed_args=()
-if [ ! -z "$dotnet_runtime_source_feed$dotnet_runtime_source_feed_key" ]; then
-    runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$dotnet_runtime_source_feed"
-    runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$dotnet_runtime_source_feed_key"
+if [ ! -z "$runtime_source_feed$runtime_source_feed_key" ]; then
+    runtime_feed_args[${#runtime_feed_args[*]}]="-runtimesourcefeed"
+    runtime_feed_args[${#runtime_feed_args[*]}]="$runtime_source_feed"
+    runtime_feed_args[${#runtime_feed_args[*]}]="-runtimesourcefeedKey"
+    runtime_feed_args[${#runtime_feed_args[*]}]="$runtime_source_feed_key"
+
+    runtimeFeedArg="/p:DotNetRuntimeSourceFeed=$runtime_source_feed"
+    runtimeFeedKeyArg="/p:DotNetRuntimeSourceFeedKey=$runtime_source_feed_key"
     runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedArg
     runtime_feed_args[${#runtime_feed_args[*]}]=$runtimeFeedKeyArg
 fi