Backport cleanups to infrastructure

* Fix README links to use https
* Add a bigger timeout to PushToBlobFeed
* Remove hard-coded restore source for the 2.1.4 build
* Fail the build if korebuild.json cannot be parsed
* Fix output path for sharedfx .tar.gz files to avoid max path issues

README.md

@@ -16,10 +16,10 @@ All published ASP.NET Core packages can be found on <https://www.nuget.org/profi
 Commonly referenced packages:
 
 [app-metapackage-nuget]:  https://nuget.org/packages/Microsoft.AspNetCore.App
-[app-metapackage-nuget-badge]: http://img.shields.io/nuget/v/Microsoft.AspNetCore.App.svg?style=flat-square&label=nuget
+[app-metapackage-nuget-badge]: https://img.shields.io/nuget/v/Microsoft.AspNetCore.App.svg?style=flat-square&label=nuget
 
 [metapackage-nuget]:  https://nuget.org/packages/Microsoft.AspNetCore
-[metapackage-nuget-badge]: http://img.shields.io/nuget/v/Microsoft.AspNetCore.svg?style=flat-square&label=nuget
+[metapackage-nuget-badge]: https://img.shields.io/nuget/v/Microsoft.AspNetCore.svg?style=flat-square&label=nuget
 
 Package                           | NuGet.org
 :---------------------------------|:---------------------------------------------------------
@@ -37,10 +37,10 @@ packages that will not be supported in a officially released build.
 Commonly referenced packages:
 
 [app-metapackage-myget]:  https://dotnet.myget.org/feed/dotnet-core/package/nuget/Microsoft.AspNetCore.App
-[app-metapackage-myget-badge]: http://img.shields.io/dotnet.myget/dotnet-core/v/Microsoft.AspNetCore.App.svg?style=flat-square&label=myget
+[app-metapackage-myget-badge]: https://img.shields.io/dotnet.myget/dotnet-core/v/Microsoft.AspNetCore.App.svg?style=flat-square&label=myget
 
 [metapackage-myget]:  https://dotnet.myget.org/feed/dotnet-core/package/nuget/Microsoft.AspNetCore
-[metapackage-myget-badge]: http://img.shields.io/dotnet.myget/dotnet-core/v/Microsoft.AspNetCore.svg?style=flat-square&label=myget
+[metapackage-myget-badge]: https://img.shields.io/dotnet.myget/dotnet-core/v/Microsoft.AspNetCore.svg?style=flat-square&label=myget
 
 Package                           | MyGet
 :---------------------------------|:---------------------------------------------------------

build/Publish.targets

@@ -20,6 +20,7 @@
     </PublishDependsOn>
 
     <!-- Settings for pushing to the transport feed -->
+    <PushToBlobFeed_UploadTimeoutMinutes>10</PushToBlobFeed_UploadTimeoutMinutes>
     <PushToBlobFeed_Overwrite Condition="'$(PushToBlobFeed_Overwrite)' == ''">false</PushToBlobFeed_Overwrite>
     <PushToBlobFeed_MaxClients Condition="'$(PushToBlobFeed_MaxClients)' == ''">8</PushToBlobFeed_MaxClients>
     <BlobFileRelativePathBase Condition="'$(BlobFileRelativePathBase)' == ''">assets</BlobFileRelativePathBase>
@@ -287,6 +288,7 @@
                     AccountKey="$(PublishBlobFeedKey)"
                     ItemsToPush="@(PackageToPublishToTransport)"
                     Overwrite="$(PushToBlobFeed_Overwrite)"
+                    UploadTimeoutInMinutes="$(PushToBlobFeed_UploadTimeoutMinutes)"
                     ManifestBranch="$(BuildBranch)"
                     ManifestBuildId="$(Version)"
                     ManifestBuildData="ProductVersion=$(PackageVersion);UniverseCommitHash=$(CommitHash)"
@@ -300,6 +302,7 @@
                     ItemsToPush="@(FilesToPublishToTransport)"
                     PublishFlatContainer="true"
                     Overwrite="$(PushToBlobFeed_Overwrite)"
+                    UploadTimeoutInMinutes="$(PushToBlobFeed_UploadTimeoutMinutes)"
                     ManifestBranch="$(BuildBranch)"
                     ManifestBuildId="$(Version)"
                     ManifestBuildData="ProductVersion=$(PackageVersion);UniverseCommitHash=$(CommitHash)"

build/SharedFx.props

@@ -1,7 +1,7 @@
 <Project>
   <PropertyGroup>
     <!-- directories -->
-    <_WorkRoot>$(RepositoryRoot).w\</_WorkRoot>
+    <_WorkRoot>$(RepositoryRoot).w\$(SharedFxRID)\</_WorkRoot>
     <_WorkLayoutDir>$(_WorkRoot).l\</_WorkLayoutDir>
     <_WorkOutputDir>$(_WorkRoot).o\</_WorkOutputDir>
     <_MetapackageSrcRoot>$(RepositoryRoot)src\Packages\</_MetapackageSrcRoot>
@@ -24,6 +24,7 @@
     <LibExtension Condition="$([MSBuild]::IsOSPlatform('Windows'))">.dll</LibExtension>
     <LibExtension Condition="$([MSBuild]::IsOSPlatform('OSX'))">.dylib</LibExtension>
     <ExeExtension Condition="$([MSBuild]::IsOSPlatform('Windows'))">.exe</ExeExtension>
+    <SharedFrameworkTargetFramework>netcoreapp2.1</SharedFrameworkTargetFramework>
 
     <!-- installers -->
     <SharedFxInstallerName>aspnetcore-runtime</SharedFxInstallerName>

build/SharedFx.targets

@@ -2,7 +2,7 @@
   <Import Project="SharedFx.props" />
 
   <PropertyGroup>
-    <SharedFxOutputPath>$([MSBuild]::NormalizeDirectory($(ArtifactsDir)))assets\Runtime\$(PackageVersion)\</SharedFxOutputPath>
+    <SharedFxOutputPath>$([MSBuild]::NormalizeDirectory($(ArtifactsDir)))runtime\</SharedFxOutputPath>
   </PropertyGroup>
 
   <Target Name="GetMetapackageArtifactInfo">
@@ -450,41 +450,41 @@
       Condition="Exists('$(_SymbolsSourceDir)%(AllPortablePDBsToPublish.SymbolsPackageFilename)')" />
     <Copy
       SourceFiles="$(_WorkRoot)SymbolsPackages\%(AppPortablePDBsToPublish.SymbolsPackageFilename)%(AppPortablePDBsToPublish.SymbolsRecursivePath)"
-      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True"
       Condition="Exists('$(_WorkRoot)SymbolsPackages\%(AppPortablePDBsToPublish.SymbolsPackageFilename)\%(AppPortablePDBsToPublish.SymbolsRecursivePath)')" />
     <Copy
       SourceFiles="$(_WorkRoot)SymbolsPackages\%(AllPortablePDBsToPublish.SymbolsPackageFilename)%(AllPortablePDBsToPublish.SymbolsRecursivePath)"
-      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True"
       Condition="Exists('$(_WorkRoot)SymbolsPackages\%(AllPortablePDBsToPublish.SymbolsPackageFilename)\%(AllPortablePDBsToPublish.SymbolsRecursivePath)')" />
 
     <!-- Copy over DLLs and PDBs -->
     <Copy
       SourceFiles="%(AppPortablePDBsToPublish.PortablePDB)"
-      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True"
       Condition="Exists('%(AppPortablePDBsToPublish.PortablePDB)')" />
     <Copy
       SourceFiles="%(AllPortablePDBsToPublish.PortablePDB)"
-      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True"
       Condition="Exists('%(AllPortablePDBsToPublish.PortablePDB)')" />
     <Copy
       SourceFiles="@(AppCrossGenSymbols)"
-      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True" />
     <Copy
       SourceFiles="@(AllCrossGenSymbols)"
-      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True" />
     <Copy
       SourceFiles="$(AppSharedFxCrossgenDirectory)%(AppCrossGenOutput.RecursiveDir)%(AppCrossGenOutput.FileName)%(AppCrossGenOutput.Extension)"
-      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AppSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True" />
     <Copy
       SourceFiles="$(AllSharedFxCrossgenDirectory)%(AllCrossGenOutput.RecursiveDir)%(AllCrossGenOutput.FileName)%(AllCrossGenOutput.Extension)"
-      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\netcoreapp2.1"
+      DestinationFolder="$(AllSharedFxSymbolsDirectory)runtimes\$(SharedFxRID)\lib\$(SharedFrameworkTargetFramework)"
       OverwriteReadOnlyFiles="True" />
 
     <!-- Create symbols nupkg -->
@@ -542,6 +542,7 @@
         DotNetPackageVersionPropsPath=$(GeneratedPackageVersionPropsPath);
         SharedFxOutputPath=$(SharedFxIntermediateOutputPath);
         RepositoryCommit=$(RepositoryCommit);
+        VSTestLogger=$([MSBuild]::Escape('trx;LogFileName=$(UnitTestFxTrxLogFile)'));
         SharedFxRuntimeIdentifier=$(SharedFXRid)
       </UnitTestFxTestProps>
     </PropertyGroup>

build/SharedFxInstaller.targets

@@ -217,7 +217,7 @@
   </Target>
 
   <Target Name="GenerateDebs" DependsOnTargets="_EnsureInstallerPrerequisites">
-  <PropertyGroup>
+    <PropertyGroup>
       <Deb_DotnetRuntimeDependencyId>dotnet-runtime-$(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion)</Deb_DotnetRuntimeDependencyId>
       <Deb_DotnetRuntimeDependencyVersion>$(MicrosoftNETCoreAppPackageVersion)</Deb_DotnetRuntimeDependencyVersion>
       <!-- Needed some creativity to convert the PackageVersion M.N.P-Build to the installer version M.N.P~Build, The conditional handles stabilized builds -->

build/repo.props

@@ -55,7 +55,7 @@
       Arch="x86"
       Feed="$(DotNetAssetRootUrl)"
       FeedCredential="$(DotNetAssetRootAccessTokenSuffix)" />
-    
+
       <!--
         The build doesn't support compiling the shared runtime on one machine along with running tests,
         so this is enables installing the shared runtime from a previous build.

build/sources.props

@@ -11,7 +11,6 @@
       $(RestoreSources);
       https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
       https://api.nuget.org/v3/index.json;
-      https://dotnetfeed.blob.core.windows.net/orchestrated-release-2-1/20180725-02/final/index.json;
     </RestoreSources>
     <RestoreSources Condition=" '$(DotNetBuildOffline)' != 'true' AND '$(DisableMyGetRestoreSources)' != 'true' ">
       $(RestoreSources);

build/tools/NuspecBaselineGenerator/NuspecBaselineGenerator.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-   <Import Project="..\..\sources.props" />
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="NuGet.Packaging" Version="4.7.0" />
-    <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="2.2.3" />
-  </ItemGroup>
-</Project>

build/tools/NuspecBaselineGenerator/NuspecBaselineGenerator.sln

@@ -1,34 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.26124.0
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuspecBaselineGenerator", "NuspecBaselineGenerator.csproj", "{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|Any CPU = Release|Any CPU
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|x64.ActiveCfg = Debug|x64
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|x64.Build.0 = Debug|x64
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|x86.ActiveCfg = Debug|x86
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Debug|x86.Build.0 = Debug|x86
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|x64.ActiveCfg = Release|x64
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|x64.Build.0 = Release|x64
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|x86.ActiveCfg = Release|x86
-		{1A7FFC6E-7343-4AAD-A047-4D7097FBD7BF}.Release|x86.Build.0 = Release|x86
-	EndGlobalSection
-EndGlobal

build/tools/NuspecBaselineGenerator/Program.cs

@@ -1,87 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations;
-using System.IO;
-using System.Linq;
-using System.Xml.Linq;
-using McMaster.Extensions.CommandLineUtils;
-using NuGet.Packaging;
-
-namespace NuspecBaselineGenerator
-{
-    class Program
-    {
-        static void Main(string[] args) => CommandLineApplication.Execute<Program>(args);
-
-        [Required]
-        [DirectoryExists]
-        [Argument(0, Description = "Path(s) to directories containing .nupkg files from previous releases.")]
-        public string[] Directories { get; }
-
-        [Required]
-        [Option(Description = "The path to the artifacts.props file")]
-        [FileExists]
-        public string Artifacts { get; }
-
-        [Option(Description = "Show verbose output")]
-        public bool Verbose { get; }
-
-        private void OnExecute()
-        {
-            var doc = XDocument.Load(Artifacts);
-            var versions = new List<(string, string)>();
-            foreach (var dir in Directories)
-            {
-                foreach (var nupkg in Directory.EnumerateFiles(dir, "*.nupkg"))
-                {
-                    using (var reader = new PackageArchiveReader(nupkg))
-                    {
-                        var identity = reader.GetIdentity();
-                        versions.Add((identity.Id, identity.Version.ToNormalizedString()));
-                        LogVerbose($"Found package {identity.Id}/{identity.Version} ({nupkg})");
-                    }
-                }
-            }
-
-            LogVerbose($"Found {versions.Count} package(s)");
-
-            void WriteAttribute(XElement element, string attr)
-            {
-                var attribute = element.Attribute(attr);
-                if (attribute != null)
-                {
-                    Console.Write($" {attr}=\"{attribute.Value}\"");
-                }
-            }
-
-            foreach (var item in versions.OrderBy(i => i.Item1))
-            {
-                var element = doc
-                    .Descendants("PackageArtifact")
-                    .SingleOrDefault(p => p.Attribute("Include")?.Value == item.Item1);
-
-
-                Console.Write($"<ExternalDependency Include=\"{item.Item1}\" Version=\"{item.Item2}\"");
-                if (element != null)
-                {
-                    WriteAttribute(element, "Analyzer");
-                    WriteAttribute(element, "AllMetapackage");
-                    WriteAttribute(element, "AppMetapackage");
-                    WriteAttribute(element, "LZMA");
-                    WriteAttribute(element, "PackageType");
-                    WriteAttribute(element, "Category");
-                }
-                Console.WriteLine(" />");
-            }
-        }
-
-        private void LogVerbose(string message)
-        {
-            if (!Verbose)
-            {
-                return;
-            }
-            Console.WriteLine(message);
-        }
-    }
-}

run.sh

@@ -250,17 +250,20 @@ if [ -f "$config_file" ]; then
             config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")"
             config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")"
         else
-            __warn "$config_file is invalid JSON. Its settings will be ignored."
+            __error "$config_file is invalid JSON. Its settings will be ignored."
+            exit 1
         fi
     elif __machine_has python ; then
         if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then
             config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")"
             config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")"
         else
-            __warn "$config_file is invalid JSON. Its settings will be ignored."
+            __error "$config_file is invalid JSON. Its settings will be ignored."
+            exit 1
         fi
     else
-        __warn 'Missing required command: jq or pyton. Could not parse the JSON file. Its settings will be ignored.'
+        __error 'Missing required command: jq or python. Could not parse the JSON file. Its settings will be ignored.'
+        exit 1
     fi
 
     [ ! -z "${config_channel:-}" ] && channel="$config_channel"

scripts/common.psm1

@@ -177,14 +177,13 @@ function Set-GithubInfo(
 function CommitUpdatedVersions(
     [hashtable]$updatedVars,
     [xml]$dependencies,
-    [string]$depsPath)
+    [string]$depsPath,
+    [string]$subject = 'Updating external dependencies')
 {
     $count = $updatedVars.Count
     if ($count -gt 0) {
         & git add build\dependencies.props
 
-        $subject = "Updating external dependencies"
-
         $gitConfigArgs = @()
         if ($env:GITHUB_USER) {
             $gitConfigArgs += '-c',"user.name=$env:GITHUB_USER"