请求日志优化

src/Masuit.MyBlogs.Core/Common/TrackData.cs

@@ -1,32 +1,57 @@
-using System;
+using Masuit.Tools;
+using Newtonsoft.Json;
+using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
 
 namespace Masuit.MyBlogs.Core.Common
 {
-    public class TrackData
+    public static class TrackData
     {
         /// <summary>
         /// 请求日志
         /// </summary>
-        public static ConcurrentDictionary<string, int> RequestLogs { get; } = new ConcurrentDictionary<string, int>();
+        public static ConcurrentDictionary<string, RequestLog> RequestLogs { get; } = new ConcurrentDictionary<string, RequestLog>();
 
         /// <summary>
         /// 刷写日志
         /// </summary>
         public static void DumpLog()
         {
-            var logPath = Path.Combine(AppContext.BaseDirectory + "logs", "req" + DateTime.Now.ToString("yyyyMMdd") + ".txt");
-            if (!File.Exists(logPath))
+            var logPath = Path.Combine(AppContext.BaseDirectory + "logs", DateTime.Now.ToString("yyyyMMdd"), "req.txt").CreateFileIfNotExist();
+            File.WriteAllLines(logPath, RequestLogs.Values.SelectMany(g => g.RequestUrls).GroupBy(s => s).ToDictionary(x => x.Key, x => x.Count()).OrderBy(x => x.Key).ThenByDescending(x => x.Value).Select(g => g.Value + "\t" + g.Key), Encoding.UTF8);
+            File.AppendAllLines(logPath, new[] { "", $"累计处理请求数：{RequestLogs.Sum(kv => kv.Value.Count)}" });
+
+            logPath = Path.Combine(AppContext.BaseDirectory + "logs", DateTime.Now.ToString("yyyyMMdd"), "ua.txt").CreateFileIfNotExist();
+            File.WriteAllLines(logPath, RequestLogs.Values.SelectMany(g => g.UserAgents).GroupBy(s => s).ToDictionary(x => x.Key, x => x.Count()).OrderBy(x => x.Key).ThenByDescending(x => x.Value).Select(g => g.Value + "\t" + g.Key), Encoding.UTF8);
+
+            logPath = Path.Combine(AppContext.BaseDirectory + "logs", DateTime.Now.ToString("yyyyMMdd"), "ip.txt").CreateFileIfNotExist();
+            File.WriteAllLines(logPath, RequestLogs.Keys.Select(s => new { s, loc = s.GetIPLocation() }).OrderBy(x => x.loc).Select(g => g.s + "\t" + g.loc), Encoding.UTF8);
+
+            logPath = Path.Combine(AppContext.BaseDirectory + "logs", DateTime.Now.ToString("yyyyMMdd"), "raw.json").CreateFileIfNotExist();
+            File.WriteAllText(logPath, RequestLogs.ToJsonString(new JsonSerializerSettings() { Formatting = Formatting.Indented }), Encoding.UTF8);
+            RequestLogs.Clear();
+        }
+
+        private static string CreateFileIfNotExist(this string filepath)
+        {
+            var fileInfo = new FileInfo(filepath);
+            if (!fileInfo.Exists)
             {
-                File.Create(logPath).Dispose();
+                fileInfo.Directory.Create();
             }
 
-            File.WriteAllLines(logPath, RequestLogs.OrderBy(x => x.Key).ThenByDescending(x => x.Value).Select(x => x.Value + "\t" + x.Key), Encoding.UTF8);
-            File.AppendAllLines(logPath, new[] { "", $"累计处理请求数：{RequestLogs.Sum(kv => kv.Value)}" });
-            RequestLogs.Clear();
+            return filepath;
         }
     }
+
+    public class RequestLog
+    {
+        public HashSet<string> UserAgents { get; } = new HashSet<string>();
+        public HashSet<string> RequestUrls { get; } = new HashSet<string>();
+        public int Count { get; set; }
+    }
 }

src/Masuit.MyBlogs.Core/Common/UA.cs

@@ -198,9 +198,7 @@ namespace Masuit.MyBlogs.Core.Common
             {"Uptimebot", "Uptimebot"},
             {"Sogou web spider", "Sogou Web Spider"},
             {"TelegramBot", "Telegram Bot"},
-            {"DNSPod", "DNSPod"},
-            {"spider", "other spider"},
-            {"bot", "other bot"}
+            {"DNSPod", "DNSPod"}
         };
 
         protected string agent;

src/Masuit.MyBlogs.Core/Extensions/RequestInterceptMiddleware.cs

@@ -44,6 +44,7 @@ namespace Masuit.MyBlogs.Core.Extensions
                 return;
             }
 
+            var ip = context.Connection.RemoteIpAddress.ToString();
             var path = HttpUtility.UrlDecode(request.Path + request.QueryString, Encoding.UTF8);
             var requestUrl = HttpUtility.UrlDecode(request.Scheme + "://" + request.Host + path);
             var match = Regex.Match(path ?? "", CommonHelper.BanRegex);
@@ -51,7 +52,7 @@ namespace Masuit.MyBlogs.Core.Extensions
             {
                 BackgroundJob.Enqueue(() => HangfireBackJob.InterceptLog(new IpIntercepter()
                 {
-                    IP = context.Connection.RemoteIpAddress.ToString(),
+                    IP = ip,
                     RequestUrl = requestUrl,
                     Time = DateTime.Now,
                     UserAgent = request.Headers[HeaderNames.UserAgent],
@@ -92,7 +93,18 @@ namespace Masuit.MyBlogs.Core.Extensions
                     var q = request.QueryString.Value.Trim('?');
                     requestUrl = requestUrl.Replace(q, q.Split('&').Where(s => !s.StartsWith("cid") && !s.StartsWith("uid")).Join("&"));
                 }
-                TrackData.RequestLogs.AddOrUpdate(requestUrl, 1, (s, i) => i + 1);
+                TrackData.RequestLogs.AddOrUpdate(ip, new RequestLog()
+                {
+                    Count = 1,
+                    RequestUrls = { requestUrl },
+                    UserAgents = { request.Headers[HeaderNames.UserAgent] }
+                }, (s, i) =>
+                {
+                    i.UserAgents.Add(request.Headers[HeaderNames.UserAgent]);
+                    i.RequestUrls.Add(requestUrl);
+                    i.Count++;
+                    return i;
+                });
             }
 
             if (string.IsNullOrEmpty(context.Session.Get<string>(SessionKey.TimeZone)))

src/Masuit.MyBlogs.Core/Infrastructure/Repository/BaseRepository.cs

@@ -744,9 +744,7 @@ namespace Masuit.MyBlogs.Core.Infrastructure.Repository
         /// <returns>删除成功</returns>
         public virtual int DeleteEntity(Expression<Func<T, bool>> where)
         {
-            var query = DataContext.Set<T>().Where(where);
-            DataContext.RemoveRange(query);
-            return DataContext.SaveChanges();
+            return DataContext.Set<T>().Where(where).Delete();
         }
 
         /// <summary>
@@ -756,9 +754,7 @@ namespace Masuit.MyBlogs.Core.Infrastructure.Repository
         /// <returns>删除成功</returns>
         public virtual async Task<int> DeleteEntityAsync(Expression<Func<T, bool>> where)
         {
-            var query = DataContext.Set<T>().Where(where);
-            DataContext.RemoveRange(query);
-            return await DataContext.SaveChangesAsync();
+            return await DataContext.Set<T>().Where(where).DeleteAsync().ConfigureAwait(false);
         }
 
         /// <summary>

src/Masuit.MyBlogs.Core/Masuit.MyBlogs.Core.csproj

@@ -37,13 +37,13 @@
         <PackageReference Include="CacheManager.Microsoft.Extensions.Caching.Memory" Version="2.0.0-beta-1629" />
         <PackageReference Include="CacheManager.Serialization.Json" Version="2.0.0-beta-1629" />
         <PackageReference Include="CSRedisCore" Version="3.6.5" />
-        <PackageReference Include="Hangfire" Version="1.7.13" />
+        <PackageReference Include="Hangfire" Version="1.7.14" />
         <PackageReference Include="Hangfire.Autofac" Version="2.3.1" />
         <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
         <PackageReference Include="htmldiff.net-core" Version="1.3.6" />
         <PackageReference Include="IP2Region" Version="1.2.0" />
         <PackageReference Include="Karambolo.AspNetCore.Bundling.NUglify" Version="3.4.1" />
-        <PackageReference Include="MaxMind.GeoIP2" Version="3.2.0" />
+        <PackageReference Include="MaxMind.GeoIP2" Version="3.3.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.8" />
         <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="3.1.8" />
         <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="3.1.8" />
@@ -51,7 +51,7 @@
         <PackageReference Include="OpenXmlPowerTools-NetStandard" Version="4.4.21" />
         <PackageReference Include="MiniProfiler.EntityFrameworkCore" Version="4.2.1" />
         <PackageReference Include="PanGu.HighLight" Version="1.0.0" />
-        <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.2.0" />
+        <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.2.1" />
         <PackageReference Include="Svg" Version="3.1.1" />
         <PackageReference Include="System.Linq.Dynamic.Core" Version="1.2.2" />
         <PackageReference Include="TimeZoneConverter" Version="3.3.0" />