aspnetcore/eng/tools/RepoTasks/DownloadFile.cs

// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace RepoTasks;

public class DownloadFile : Microsoft.Build.Utilities.Task
{
    [Required]
    public string Uri { get; set; }

    /// <summary>
    /// If this field is set and the task fail to download the file from `Uri`, with a NotFound
    /// status, it will try to download the file from `PrivateUri`.
    /// </summary>
    public string PrivateUri { get; set; }

    /// <summary>
    /// Suffix for the private URI in base64 form (for SAS compatibility)
    /// </summary>
    public string PrivateUriSuffix { get; set; }

    public int MaxRetries { get; set; } = 5;

    [Required]
    public string DestinationPath { get; set; }

    public bool Overwrite { get; set; }

    public override bool Execute()
    {
        return ExecuteAsync().GetAwaiter().GetResult();
    }

    private async System.Threading.Tasks.Task<bool> ExecuteAsync()
    {
        string destinationDir = Path.GetDirectoryName(DestinationPath);
        if (!Directory.Exists(destinationDir))
        {
            Directory.CreateDirectory(destinationDir);
        }

        if (File.Exists(DestinationPath) && !Overwrite)
        {
            return true;
        }

        const string FileUriProtocol = "file://";

        if (Uri.StartsWith(FileUriProtocol, StringComparison.Ordinal))
        {
            var filePath = Uri.Substring(FileUriProtocol.Length);
            Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'");
            File.Copy(filePath, DestinationPath);
            return true;
        }

        List<string> errorMessages = new List<string>();
        bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages);

        if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri))
        {
            string uriSuffix = "";
            if (!string.IsNullOrEmpty(PrivateUriSuffix))
            {
                var uriSuffixBytes = System.Convert.FromBase64String(PrivateUriSuffix);
                uriSuffix = System.Text.Encoding.UTF8.GetString(uriSuffixBytes);
            }
            downloadStatus = await DownloadWithRetriesAsync($"{PrivateUri}{uriSuffix}", DestinationPath, errorMessages);
        }

        if (downloadStatus != true)
        {
            foreach (var error in errorMessages)
            {
                Log.LogError(error);
            }
        }

        return downloadStatus == true;
    }

    /// <summary>
    /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success.
    /// </summary>
    /// <param name="source">URL to the file to be downloaded.</param>
    /// <param name="target">Local path where to put the downloaded file.</param>
    /// <returns>true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable.</returns>
    private async Task<bool?> DownloadWithRetriesAsync(string source, string target, List<string> errorMessages)
    {
        Random rng = new Random();

        Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'");

        using (var httpClient = new HttpClient { Timeout = TimeSpan.FromMinutes(7.5) })
        {
            for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++)
            {
                try
                {
                    var httpResponse = await httpClient.GetAsync(source);

                    Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}");

                    // The Azure Storage REST API returns '400 - Bad Request' in some cases
                    // where the resource is not found on the storage.
                    // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
                    if (httpResponse.StatusCode == HttpStatusCode.NotFound ||
                        httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0)
                    {
                        errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}");
                        return false;
                    }

                    httpResponse.EnsureSuccessStatusCode();

                    using (var outStream = File.Create(target))
                    {
                        await httpResponse.Content.CopyToAsync(outStream);
                    }

                    Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}");
                    return true;
                }
                catch (Exception e)
                {
                    Log.LogMessage(MessageImportance.High, $"returning error in {source} ");
                    errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}");
                    File.Delete(target);
                }

                await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000));
            }
        }

        Log.LogMessage(MessageImportance.High, $"giving up {source} ");
        errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries.");
        return null;
    }
}