ソースを参照

1.多线程下载支持配置请求,可调用MultiThreadDownloader的Configure方法进行配置请求头;
2.一些时间日期API的优化(感谢@yzpopulation)

懒得勤快 5 年 前
コミット
fb02f79041

+ 1 - 1
Masuit.Tools.Core/Masuit.Tools.Core.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
-    <Version>2.2.8.3</Version>
+    <Version>2.2.8.4</Version>
     <Authors>懒得勤快</Authors>
     <Company>masuit.com</Company>
     <Description>包含一些常用的操作类,大都是静态类,加密解密,反射操作,硬件信息,字符串扩展方法,日期时间扩展操作,大文件拷贝,图像裁剪,html处理,验证码、NoSql等常用封装。

+ 43 - 13
Masuit.Tools/Net/MultiThreadDownloader.cs

@@ -25,7 +25,8 @@ namespace Masuit.Tools.Net
 
         private string _url;
         private bool _rangeAllowed;
-
+        private readonly HttpWebRequest _request;
+        private Action<HttpWebRequest> _requestConfigure = req => req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36";
         #endregion
 
         #region 公共属性
@@ -61,7 +62,20 @@ namespace Masuit.Tools.Net
         /// <summary>
         /// 已接收字节数
         /// </summary>
-        public long TotalBytesReceived => PartialDownloaderList.Where(t => t != null).Sum(t => t.TotalBytesRead);
+        public long TotalBytesReceived
+        {
+            get
+            {
+                try
+                {
+                    return PartialDownloaderList.Where(t => t != null).Sum(t => t.TotalBytesRead);
+                }
+                catch (Exception e)
+                {
+                    return 0;
+                }
+            }
+        }
 
         /// <summary>
         /// 总进度
@@ -123,6 +137,7 @@ namespace Masuit.Tools.Net
             PartialDownloaderList = new List<PartialDownloader>();
             _aop = AsyncOperationManager.CreateOperation(null);
             FilePath = savePath;
+            _request = WebRequest.Create(sourceUrl) as HttpWebRequest;
         }
 
         /// <summary>
@@ -182,7 +197,7 @@ namespace Masuit.Tools.Net
             temp.DownloadPartCompleted += temp_DownloadPartCompleted;
             temp.DownloadPartProgressChanged += temp_DownloadPartProgressChanged;
             PartialDownloaderList.Add(temp);
-            temp.Start();
+            temp.Start(_requestConfigure);
         }
 
         void temp_DownloadPartProgressChanged(object sender, EventArgs e)
@@ -209,7 +224,7 @@ namespace Masuit.Tools.Net
 
         void CreateFirstPartitions()
         {
-            Size = GetContentLength(_url, ref _rangeAllowed, ref _url);
+            Size = GetContentLength(ref _rangeAllowed, ref _url);
             int maximumPart = (int)(Size / (25 * 1024));
             maximumPart = maximumPart == 0 ? 1 : maximumPart;
             if (!_rangeAllowed)
@@ -227,13 +242,19 @@ namespace Masuit.Tools.Net
                 temp.DownloadPartProgressChanged += temp_DownloadPartProgressChanged;
                 temp.DownloadPartCompleted += temp_DownloadPartCompleted;
                 PartialDownloaderList.Add(temp);
-                temp.Start();
+                temp.Start(_requestConfigure);
             }
         }
 
         void MergeParts()
         {
-            List<PartialDownloader> mergeOrderedList = SortPDsByFrom(PartialDownloaderList);
+            var mergeOrderedList = SortPDsByFrom(PartialDownloaderList);
+            var dir = new FileInfo(FilePath).DirectoryName;
+            if (!Directory.Exists(dir))
+            {
+                Directory.CreateDirectory(dir);
+            }
+
             using var fs = new FileStream(FilePath, FileMode.Create, FileAccess.ReadWrite);
             long totalBytesWritten = 0;
             int mergeProgress = 0;
@@ -337,6 +358,15 @@ namespace Masuit.Tools.Net
             BubbleSort(list);
         }
 
+        /// <summary>
+        /// 配置请求头
+        /// </summary>
+        /// <param name="config"></param>
+        public void Configure(Action<HttpWebRequest> config)
+        {
+            _requestConfigure = config;
+        }
+
         /// <summary>
         /// 获取内容长度
         /// </summary>
@@ -344,12 +374,12 @@ namespace Masuit.Tools.Net
         /// <param name="rangeAllowed"></param>
         /// <param name="redirectedUrl"></param>
         /// <returns></returns>
-        public static long GetContentLength(string url, ref bool rangeAllowed, ref string redirectedUrl)
+        public long GetContentLength(ref bool rangeAllowed, ref string redirectedUrl)
         {
-            var req = WebRequest.Create(url) as HttpWebRequest;
-            req.UserAgent = "Mozilla/4.0 (compatible; MSIE 11.0; Windows NT 6.2; .NET CLR 1.0.3705;)";
-            req.ServicePoint.ConnectionLimit = 4;
-            using var resp = req.GetResponse() as HttpWebResponse;
+            _request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36";
+            _request.ServicePoint.ConnectionLimit = 4;
+            _requestConfigure(_request);
+            using var resp = _request.GetResponse() as HttpWebResponse;
             redirectedUrl = resp.ResponseUri.OriginalString;
             var ctl = resp.ContentLength;
             rangeAllowed = resp.Headers.AllKeys.Select((v, i) => new
@@ -357,7 +387,7 @@ namespace Masuit.Tools.Net
                 HeaderName = v,
                 HeaderValue = resp.Headers[i]
             }).Any(k => k.HeaderName.ToLower().Contains("range") && k.HeaderValue.ToLower().Contains("byte"));
-            req.Abort();
+            _request.Abort();
             return ctl;
         }
 
@@ -409,7 +439,7 @@ namespace Masuit.Tools.Net
                     temp.DownloadPartCompleted += temp_DownloadPartCompleted;
                     PartialDownloaderList.Add(temp);
                     PartialDownloaderList[i].To = PartialDownloaderList[i].CurrentPosition;
-                    temp.Start();
+                    temp.Start(_requestConfigure);
                 }
             }
         }

+ 6 - 5
Masuit.Tools/Net/PartialDownloader.cs

@@ -150,18 +150,19 @@ namespace Masuit.Tools.Net
             _lastSpeeds = new int[10];
         }
 
-        void DownloadProcedure()
+        void DownloadProcedure(Action<HttpWebRequest> config)
         {
             using var file = new FileStream(FullPath, FileMode.Create, FileAccess.ReadWrite);
             var sw = new Stopwatch();
             if (WebRequest.Create(Url) is HttpWebRequest req)
             {
-                req.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
+                req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36";
                 req.AllowAutoRedirect = true;
                 req.MaximumAutomaticRedirections = 5;
                 req.ServicePoint.ConnectionLimit += 1;
                 req.ServicePoint.Expect100Continue = true;
-                req.ProtocolVersion = HttpVersion.Version10;
+                req.ProtocolVersion = HttpVersion.Version11;
+                config(req);
                 ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12 | SecurityProtocolType.Ssl3;
                 ServicePointManager.Expect100Continue = true;
                 if (RangeAllowed)
@@ -231,10 +232,10 @@ namespace Masuit.Tools.Net
         /// <summary>
         /// 启动下载
         /// </summary>
-        public void Start()
+        public void Start(Action<HttpWebRequest> config)
         {
             Stopped = false;
-            var procThread = new Thread(DownloadProcedure);
+            var procThread = new Thread(_ => DownloadProcedure(config));
             procThread.Start();
         }
 

BIN
Masuit.Tools/Properties/AssemblyInfo.cs


+ 1 - 1
Masuit.Tools/package.nuspec

@@ -4,7 +4,7 @@
     <!--*-->
     <id>Masuit.Tools</id>
     <!--*-->
-    <version>2.2.8.2</version>
+    <version>2.2.8.4</version>
     <title>Masuit.Tools</title>
     <!--*-->
     <authors>masuit.com</authors>

+ 18 - 19
Test/Program.cs

@@ -1,8 +1,5 @@
-using Masuit.Tools;
-using Masuit.Tools.RandomSelector;
+using Masuit.Tools.Net;
 using System;
-using System.Collections.Generic;
-using System.Linq;
 
 namespace Test
 {
@@ -10,22 +7,24 @@ namespace Test
     {
         static void Main(string[] args)
         {
-            var result = new List<string>();
-            for (int i = 0; i < 1000; i++)
+            var mtd = new MultiThreadDownloader("https://git.imweb.io/ldqk0/imgbed/raw/master/2020/01/05/sdcgssa1wlxc.jpg", Environment.GetEnvironmentVariable("temp"), "Y:\\1.jpg", 8);
+            mtd.Configure(req =>
             {
-                result.Add(new List<WeightedItem<string>>()
-                {
-                    new WeightedItem<string>("A", 1),
-                    new WeightedItem<string>("B", 3),
-                    new WeightedItem<string>("C", 4),
-                    new WeightedItem<string>("D", 4),
-                }.WeightedItem());
-            }
-
-            foreach (var g in result.GroupBy(s => s).OrderByDescending(g => g.Count()))
-            {
-                Console.WriteLine(g.Key + ":" + g.Count());
-            }
+                req.Referer = "https://masuit.com";
+                req.Headers.Add("Origin", "https://baidu.com");
+            });
+            mtd.TotalProgressChanged += (sender, e) =>
+              {
+                  var downloader = sender as MultiThreadDownloader;
+                  Console.WriteLine("下载进度:" + downloader.TotalProgress + "%");
+                  Console.WriteLine("下载速度:" + downloader.TotalSpeedInBytes / 1024 / 1024 + "MBps");
+              };
+            mtd.FileMergeProgressChanged += (sender, e) =>
+              {
+                  Console.WriteLine("下载完成");
+              };
+            mtd.Start();//开始下载
+            Console.ReadKey();
         }
     }
 }