Merge branch 'release/2.2' => 'master' (#10023)

build/sources.props

@@ -27,11 +27,6 @@
       $(RestoreSources);
       https://dotnet.myget.org/F/aspnetcore-dev/api/v3/index.json;
     </RestoreSources>
-    <!-- Only used to fetch Microsoft.NETFramework.ReferenceAssemblies for x-plat netfx builds. -->
-    <RestoreSources Condition=" '$(OS)' != 'Windows_NT' ">
-      $(RestoreSources);
-      https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json;
-    </RestoreSources>
 
     <!-- In an orchestrated build, this may be overriden to other Azure feeds. -->
     <DotNetAssetRootUrl Condition="'$(DotNetAssetRootUrl)'==''">https://dotnetcli.blob.core.windows.net/dotnet/</DotNetAssetRootUrl>

eng/Versions.props

@@ -132,7 +132,7 @@
     <!-- Build tool dependencies -->
     <InternalAspNetCoreSdkPackageVersion>$(KoreBuildVersion)</InternalAspNetCoreSdkPackageVersion>
     <InternalAspNetCoreSdkPackageVersion Condition=" '$(KoreBuildVersion)' == '' ">3.0.0-build-20190430.1</InternalAspNetCoreSdkPackageVersion>
-    <MicrosoftNETFrameworkReferenceAssembliesPackageVersion>1.0.0-alpha-004</MicrosoftNETFrameworkReferenceAssembliesPackageVersion>
+    <MicrosoftNETFrameworkReferenceAssembliesPackageVersion>1.0.0-preview.1</MicrosoftNETFrameworkReferenceAssembliesPackageVersion>
     <MicrosoftNETTestSdkPackageVersion>15.9.0</MicrosoftNETTestSdkPackageVersion>
     <MicrosoftSourceLinkGitHubPackageVersion>1.0.0-beta2-18618-05</MicrosoftSourceLinkGitHubPackageVersion>
     <MicrosoftSourceLinkVstsGitPackageVersion>1.0.0-beta2-18618-05</MicrosoftSourceLinkVstsGitPackageVersion>

eng/tools/BaselineGenerator/BaselineGenerator.csproj

@@ -3,12 +3,12 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.1</TargetFramework>
-    <StartArguments>-o "$(MSBuildThisFileDirectory)../../Baseline.Designer.props" --v3 -s https://dotnetfeed.blob.core.windows.net/dotnet-core/flatcontainer</StartArguments>
+    <StartArguments>-s https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</StartArguments>
     <StartWorkingDirectory>$(MSBuildThisFileDirectory)../../</StartWorkingDirectory>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NuGet.Packaging" Version="4.8.0" />
+    <PackageReference Include="NuGet.Protocol" Version="4.8.2" />
     <PackageReference Include="Microsoft.Extensions.CommandLineUtils" Version="1.1.0" />
   </ItemGroup>
 

eng/tools/BaselineGenerator/Program.cs

@@ -3,14 +3,20 @@
 
 using System;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
 using System.Xml.Linq;
 using Microsoft.Extensions.CommandLineUtils;
+using NuGet.Common;
+using NuGet.Configuration;
 using NuGet.Packaging;
-using NuGet.Packaging.Core;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using NuGet.Versioning;
 
 namespace PackageBaselineGenerator
 {
@@ -26,13 +32,13 @@ namespace PackageBaselineGenerator
 
         private readonly CommandOption _source;
         private readonly CommandOption _output;
-        private readonly CommandOption _feedv3;
+        private readonly CommandOption _update;
 
         public Program()
         {
-            _source = Option("-s|--source <SOURCE>", "The NuGet v2 source of the package to fetch", CommandOptionType.SingleValue);
+            _source = Option("-s|--package-source <SOURCE>", "The NuGet source of packages to fetch", CommandOptionType.SingleValue);
             _output = Option("-o|--output <OUT>", "The generated file output path", CommandOptionType.SingleValue);
-            _feedv3 = Option("--v3", "Sources is nuget v3", CommandOptionType.NoValue);
+            _update = Option("-u|--update", "Regenerate the input (Baseline.xml) file.", CommandOptionType.NoValue);
 
             Invoke = () => Run().GetAwaiter().GetResult();
         }
@@ -40,22 +46,43 @@ namespace PackageBaselineGenerator
         private async Task<int> Run()
         {
             var source = _source.HasValue()
-                ? _source.Value()
-                : "https://www.nuget.org/api/v2/package";
-
-            var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") != null
-                ? Environment.GetEnvironmentVariable("NUGET_PACKAGES")
-                : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages");
+                ? _source.Value().TrimEnd('/')
+                : "https://api.nuget.org/v3/index.json";
+            if (_output.HasValue() && _update.HasValue())
+            {
+                await Error.WriteLineAsync("'--output' and '--update' options must not be used together.");
+                return 1;
+            }
 
-            var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp");
-            Directory.CreateDirectory(tempDir);
+            var inputPath = Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml");
+            var input = XDocument.Load(inputPath);
+            var packageSource = new PackageSource(source);
+            var providers = Repository.Provider.GetCoreV3(); // Get v2 and v3 API support
+            var sourceRepository = new SourceRepository(packageSource, providers);
+            if (_update.HasValue())
+            {
+                return await RunUpdateAsync(inputPath, input, sourceRepository);
+            }
 
-            var input = XDocument.Load(Path.Combine(Directory.GetCurrentDirectory(), "Baseline.xml"));
+            var feedType = await sourceRepository.GetFeedType(CancellationToken.None);
+            var feedV3 = feedType == FeedType.HttpV3;
+            var packageBase = source + "/package";
+            if (feedV3)
+            {
+                var resources = await sourceRepository.GetResourceAsync<ServiceIndexResourceV3>();
+                packageBase = resources.GetServiceEntryUri(ServiceTypes.PackageBaseAddress).ToString().TrimEnd('/');
+            }
 
             var output = _output.HasValue()
                 ? _output.Value()
                 : Path.Combine(Directory.GetCurrentDirectory(), "Baseline.Designer.props");
 
+            var packageCache = Environment.GetEnvironmentVariable("NUGET_PACKAGES") ??
+                Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages");
+
+            var tempDir = Path.Combine(Directory.GetCurrentDirectory(), "obj", "tmp");
+            Directory.CreateDirectory(tempDir);
+
             var baselineVersion = input.Root.Attribute("Version").Value;
 
             var doc = new XDocument(
@@ -66,7 +93,6 @@ namespace PackageBaselineGenerator
                         new XElement("AspNetCoreBaselineVersion", baselineVersion))));
 
             var client = new HttpClient();
-
             foreach (var pkg in input.Root.Descendants("Package"))
             {
                 var id = pkg.Attribute("Id").Value;
@@ -80,25 +106,26 @@ namespace PackageBaselineGenerator
 
                 if (!File.Exists(nupkgPath))
                 {
-                    var url = _feedv3.HasValue()
-                        ? $"{source}/{id.ToLowerInvariant()}/{version}/{id.ToLowerInvariant()}.{version}.nupkg"
-                        : $"{source}/{id}/{version}";
-                    Console.WriteLine($"Downloading {url}");
-
-                    var response = await client.GetStreamAsync(url);
+                    var url = feedV3 ?
+                        $"{packageBase}/{id.ToLowerInvariant()}/{version}/{id.ToLowerInvariant()}.{version}.nupkg" :
+                        $"{packageBase}/{id}/{version}";
 
-                    using (var file = File.Create(nupkgPath))
+                    Console.WriteLine($"Downloading {url}");
+                    using (var response = await client.GetStreamAsync(url))
                     {
-                        await response.CopyToAsync(file);
+                        using (var file = File.Create(nupkgPath))
+                        {
+                            await response.CopyToAsync(file);
+                        }
                     }
                 }
 
-
                 using (var reader = new PackageArchiveReader(nupkgPath))
                 {
                     doc.Root.Add(new XComment($" Package: {id}"));
 
-                    var propertyGroup = new XElement("PropertyGroup",
+                    var propertyGroup = new XElement(
+                        "PropertyGroup",
                         new XAttribute("Condition", $" '$(PackageId)' == '{id}' "),
                         new XElement("BaselinePackageVersion", version));
                     doc.Root.Add(propertyGroup);
@@ -122,12 +149,181 @@ namespace PackageBaselineGenerator
                 Encoding = Encoding.UTF8,
                 Indent = true,
             };
+
             using (var writer = XmlWriter.Create(output, settings))
             {
                 doc.Save(writer);
             }
+
             Console.WriteLine($"Generated file in {output}");
+
             return 0;
         }
+
+        private async Task<int> RunUpdateAsync(
+            string documentPath,
+            XDocument document,
+            SourceRepository sourceRepository)
+        {
+            var packageMetadataResource = await sourceRepository.GetResourceAsync<PackageMetadataResource>();
+            var logger = new Logger(Error, Out);
+            var hasChanged = false;
+            using (var cacheContext = new SourceCacheContext { NoCache = true })
+            {
+                var versionAttribute = document.Root.Attribute("Version");
+                hasChanged = await TryUpdateVersionAsync(
+                    versionAttribute,
+                    "Microsoft.AspNetCore.App",
+                    packageMetadataResource,
+                    logger,
+                    cacheContext);
+
+                foreach (var package in document.Root.Descendants("Package"))
+                {
+                    var id = package.Attribute("Id").Value;
+                    versionAttribute = package.Attribute("Version");
+                    var attributeChanged = await TryUpdateVersionAsync(
+                        versionAttribute,
+                        id,
+                        packageMetadataResource,
+                        logger,
+                        cacheContext);
+
+                    hasChanged |= attributeChanged;
+                }
+            }
+
+            if (hasChanged)
+            {
+                await Out.WriteLineAsync($"Updating {documentPath}.");
+
+                var settings = new XmlWriterSettings
+                {
+                    Async = true,
+                    CheckCharacters = true,
+                    CloseOutput = false,
+                    Encoding = Encoding.UTF8,
+                    Indent = true,
+                    IndentChars = "  ",
+                    NewLineOnAttributes = false,
+                    OmitXmlDeclaration = true,
+                    WriteEndDocumentOnClose = true,
+                };
+
+                using (var stream = File.OpenWrite(documentPath))
+                {
+                    using (var writer = XmlWriter.Create(stream, settings))
+                    {
+                        await document.SaveAsync(writer, CancellationToken.None);
+                    }
+                }
+            }
+            else
+            {
+                await Out.WriteLineAsync("No new versions found");
+            }
+
+            return 0;
+        }
+
+        private static async Task<bool> TryUpdateVersionAsync(
+            XAttribute versionAttribute,
+            string packageId,
+            PackageMetadataResource packageMetadataResource,
+            ILogger logger,
+            SourceCacheContext cacheContext)
+        {
+            var searchMetadata = await packageMetadataResource.GetMetadataAsync(
+                packageId,
+                includePrerelease: false,
+                includeUnlisted: true, // Microsoft.AspNetCore.DataOrotection.Redis package is not listed.
+                sourceCacheContext: cacheContext,
+                log: logger,
+                token: CancellationToken.None);
+
+            var currentVersion = NuGetVersion.Parse(versionAttribute.Value);
+            var versionRange = new VersionRange(
+                currentVersion,
+                new FloatRange(NuGetVersionFloatBehavior.Patch, currentVersion));
+
+            var latestVersion = versionRange.FindBestMatch(
+                searchMetadata.Select(metadata => metadata.Identity.Version));
+
+            if (latestVersion == null)
+            {
+                logger.LogWarning($"Unable to find latest version of '{packageId}'.");
+                return false;
+            }
+
+            var hasChanged = false;
+            if (latestVersion != currentVersion)
+            {
+                hasChanged = true;
+                versionAttribute.Value = latestVersion.ToNormalizedString();
+            }
+
+            return hasChanged;
+        }
+
+        private class Logger : ILogger
+        {
+            private readonly TextWriter _error;
+            private readonly TextWriter _out;
+
+            public Logger(TextWriter error, TextWriter @out)
+            {
+                _error = error;
+                _out = @out;
+            }
+
+            public void Log(LogLevel level, string data)
+            {
+                switch (level)
+                {
+                    case LogLevel.Debug:
+                        LogDebug(data);
+                        break;
+                    case LogLevel.Error:
+                        LogError(data);
+                        break;
+                    case LogLevel.Information:
+                        LogInformation(data);
+                        break;
+                    case LogLevel.Minimal:
+                        LogMinimal(data);
+                        break;
+                    case LogLevel.Verbose:
+                        LogVerbose(data);
+                        break;
+                    case LogLevel.Warning:
+                        LogWarning(data);
+                        break;
+                }
+            }
+
+            public void Log(ILogMessage message) => Log(message.Level, message.Message);
+
+            public Task LogAsync(LogLevel level, string data)
+            {
+                Log(level, data);
+                return Task.CompletedTask;
+            }
+
+            public Task LogAsync(ILogMessage message) => LogAsync(message.Level, message.Message);
+
+            public void LogDebug(string data) => _out.WriteLine($"Debug: {data}");
+
+            public void LogError(string data) => _error.WriteLine($"Error: {data}");
+
+            public void LogInformation(string data) => _out.WriteLine($"Information: {data}");
+
+            public void LogInformationSummary(string data) => _out.WriteLine($"Summary: {data}");
+
+            public void LogMinimal(string data) => _out.WriteLine($"Minimal: {data}");
+
+            public void LogVerbose(string data) => _out.WriteLine($"Verbose: {data}");
+
+            public void LogWarning(string data) => _out.WriteLine($"Warning: {data}");
+        }
     }
 }

eng/tools/BaselineGenerator/README.md

@@ -1,10 +1,20 @@
-BaselineGenerator
-=================
+# BaselineGenerator
 
 This tool is used to generate an MSBuild file which sets the "baseline" against which servicing updates are built.
 
 ## Usage
 
-1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of this information can typically
-  be found in the build.xml file generated during ProdCon builds. See https://github.com/dotnet/versions/blob/master/build-info/dotnet/product/cli/release/2.1.6/build.xml for example.
+Add `--package-source {source}` to the commands below if the packages of interest are not all hosted on NuGet.org.
+
+### Auto-update
+
+1. Run `dotnet run --update` in this project folder.
+2. Run `dotnet run` in this project.
+
+### Manual update
+
+1. Add to the [Baseline.xml](/eng/Baseline.xml) a list of package ID's and their latest released versions. The source of
+this information can typically be found in the build.xml file generated during ProdCon builds. See
+<file://vsufile/patches/sign/NET/CORE_BUILDS/3.0.X/3.0.0/preview5/3.0.100-preview5-011568/manifest.txt> for example.
+Update the version at the top of baseline.xml to match prior release (even if no packages changed in the prior release).
 2. Run `dotnet run` on this project.