using Masuit.Tools.Mime;
using System.Net;
using System.Text.RegularExpressions;
using Masuit.Tools.Files;
using Polly;
namespace Masuit.MyBlogs.Core.Extensions.UEditor;
///
/// Crawler 的摘要说明
///
public class CrawlerHandler(HttpContext context) : Handler(context)
{
private readonly HttpClient _httpClient = context.RequestServices.GetRequiredService().CreateClient();
private readonly IConfiguration _configuration = context.RequestServices.GetRequiredService();
public override async Task Process()
{
var form = await Request.ReadFormAsync();
string[] sources = form["source[]"];
if (sources is { Length: > 0 })
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(sources.Length * 2));
return WriteJson(new
{
state = "SUCCESS",
list = await sources.SelectAsync(async s =>
{
var crawler = new Crawler(s, _httpClient, _configuration, Context);
var fetch = await Policy.Handle().RetryAsync(3).WrapAsync(Policy.Handle().FallbackAsync(crawler)).ExecuteAsync(() => crawler.Fetch(cts.Token));
return new
{
state = fetch.State,
source = fetch.SourceUrl,
url = fetch.ServerUrl
};
})
});
}
return WriteJson(new
{
state = "参数错误:没有指定抓取源"
});
}
}
public class Crawler(string sourceUrl, HttpClient httpClient, IConfiguration configuration, HttpContext httpContext)
{
public string SourceUrl { get; set; } = sourceUrl;
public string ServerUrl { get; set; }
public string State { get; set; }
public async Task Fetch(CancellationToken token)
{
if (!SourceUrl.IsExternalAddress())
{
State = "INVALID_URL";
return this;
}
httpClient.DefaultRequestHeaders.Referrer = new Uri(SourceUrl);
var stream = await httpClient.GetAsync(configuration["HttpClientProxy:UriPrefix"] + SourceUrl, token).ContinueWith(task =>
{
if (task.IsCompletedSuccessfully)
{
var response = task.Result;
if (response.StatusCode != HttpStatusCode.OK)
{
State = "远程地址返回了错误的状态吗:" + response.StatusCode;
return new PooledMemoryStream();
}
var fileName = Path.GetFileNameWithoutExtension(SourceUrl).Next(s => Regex.Matches(s, @"\w+").LastOrDefault()?.Value);
ServerUrl = PathFormatter.Format(fileName, CommonHelper.SystemSettings.GetOrAdd("UploadPath", "upload") + UeditorConfig.GetString("catcherPathFormat")) + MimeMapper.ExtTypes[response.Content.Headers.ContentType?.MediaType ?? "image/jpeg"];
return response.Content.ReadAsStreamAsync().Result;
}
State = "远程请求失败";
return new PooledMemoryStream();
});
if (stream.Length == 0)
{
return this;
}
var format = await Image.DetectFormatAsync(stream).ContinueWith(t => t.IsCompletedSuccessfully ? t.Result : null);
stream.Position = 0;
if (format != null)
{
ServerUrl = ServerUrl.Replace(Path.GetExtension(ServerUrl), "." + format.Name.ToLower());
if (!Regex.IsMatch(format.Name, "JPEG|PNG|Webp|GIF", RegexOptions.IgnoreCase))
{
using var image = await Image.LoadAsync(stream, token);
await image.SaveAsJpegAsync(stream, token);
ServerUrl = ServerUrl.Replace(Path.GetExtension(ServerUrl), ".jpg");
}
}
var savePath = AppContext.BaseDirectory + "wwwroot" + ServerUrl;
var (url, success) = await httpContext.RequestServices.GetRequiredService().UploadImage(stream, savePath, token);
if (success)
{
ServerUrl = url;
}
else
{
Directory.CreateDirectory(Path.GetDirectoryName(savePath));
await stream.SaveFileAsync(savePath);
}
State = "SUCCESS";
return this;
}
}