增加 图片识别 缓存，缓存日期有等级
This commit is contained in:
chfei committed
1 parent
150e8644b7
commit
9bb17a67d2
5 files changed
+176
-19
No files matched your search
@@ -1,4 +1,5 @@
|
||||
using FreeSql.DataAnnotations;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Store.Modules.CloudProduct.Entities;
|
||||
|
||||
@@ -14,4 +15,31 @@ public sealed class ImageAnalysisCache
|
||||
public string RecognitionResult { get; set; } = "";
|
||||
/// <summary>缓存最近更新时间，UTC。</summary>
|
||||
public DateTime UpdatedAt { get; set; }
|
||||
|
||||
/// <summary>从 UpdatedAt 起算的缓存有效小时数，命中时不续期。</summary>
|
||||
public int ExpiresAt { get; set; }
|
||||
|
||||
/// <summary>每轮成功识别后更新有效期：空结果逐级延长，有内容则重置级别。</summary>
|
||||
public void RecordExpiration(bool isEmpty, DateTime utcNow)
|
||||
{
|
||||
// 新记录和上次有内容的记录，首次为空时均从 24 小时起算。
|
||||
ExpiresAt = isEmpty ? ExpiresAt switch
|
||||
{
|
||||
24 => 48,
|
||||
48 => 96,
|
||||
96 => 192,
|
||||
192 or 384 => 384,
|
||||
_ => 24
|
||||
} : 720;
|
||||
UpdatedAt = utcNow;
|
||||
}
|
||||
|
||||
/// <summary>仅复用有效期内的数组，旧对象格式需要重新识别。</summary>
|
||||
public string[]? ReadFreshResult(DateTime utcNow)
|
||||
{
|
||||
if (utcNow >= UpdatedAt.AddHours(ExpiresAt) || !RecognitionResult.TrimStart().StartsWith('['))
|
||||
return null;
|
||||
|
||||
return JsonSerializer.Deserialize<string[]>(RecognitionResult);
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ CloudProduct 提供以下 POST 接口，成功时 ApiResult 的 code 为 200。
|
||||
|
||||
## 配置与部署
|
||||
|
||||
图片识别沿用专用数据库。将旧插件配置中的 `ProductInspection` 节迁入 CloudProduct 对应环境的配置文件，环境变量前缀由 `Plugins__ProductInspection__` 改为 `Plugins__CloudProduct__`。例如，密钥使用 `Plugins__CloudProduct__ProductInspection__ApiKey`，为空时读取 `DASHSCOPE_API_KEY`。部署时移出旧的 `Plugins/Store.ProductInspection` 目录，避免宿主重复加载质检功能。
|
||||
图片识别缓存使用插件的 `CloudProductConnection` 数据库连接。将旧插件配置中的 `ProductInspection` 节迁入 CloudProduct 对应环境的配置文件，环境变量前缀由 `Plugins__ProductInspection__` 改为 `Plugins__CloudProduct__`。例如，密钥使用 `Plugins__CloudProduct__ProductInspection__ApiKey`，为空时读取 `DASHSCOPE_API_KEY`。部署时移出旧的 `Plugins/Store.ProductInspection` 目录，避免宿主重复加载质检功能。
|
||||
|
||||
服务端将客户端提交的 MD5 转为小写，命中缓存后直接返回结果。未命中时调用配置的千问接口，最大并发数由 `ProductInspection:MaxConcurrentAnalysis` 控制。
|
||||
|
||||
@@ -19,4 +19,6 @@ CloudProduct 提供以下 POST 接口，成功时 ApiResult 的 code 为 200。
|
||||
|
||||
从仓库根目录运行 `dotnet run --project Tests/ProductInspection.Checks`，检查专用库结构并通过本地模拟模型验证图片识别和缓存。数据库连接可用 `Plugins__CloudProduct__ProductInspection__ConnectionString` 覆盖。
|
||||
|
||||
已有可用缓存表的部署只需迁移插件配置。旧库升级命令 `dotnet run --project Tests/ProductInspection.Checks -- --initialize` 会更名旧结果列，移除 DetectionText、LogoBrands 列及四张旧任务相关表，并同步缓存表结构。执行前请备份数据库，先归档或清理仍使用 SHA-256 键的缓存。
|
||||
已有缓存表升级时，先暂停旧服务写入，从仓库根目录运行 `dotnet run --project Tests/ProductInspection.Checks -- --migrate-cache-expiration`，再启动新版本。命令读取 `CloudProductConnection`，可通过 `Plugins__CloudProduct__CloudProductConnection` 指定目标库。数据库的 `ExpiresAt` 保存整数小时数，连续空结果的下一档由上一次的小时数决定。迁移保留原更新时间，将旧记录回填为对应小时数。服务器 UTC 时间达到 `UpdatedAt.AddHours(ExpiresAt)` 时缓存过期。空结果从 1 天开始，连续为空时逐级延长至 2、4、8、16 天并封顶。有内容的结果缓存 30 天，之后首次为空时重新从 1 天开始。
|
||||
|
||||
旧库升级命令 `dotnet run --project Tests/ProductInspection.Checks -- --initialize` 会更名旧结果列，移除 DetectionText、LogoBrands 列及四张旧任务相关表，并同步缓存表结构。执行前请备份数据库，先归档或清理仍使用 SHA-256 键的缓存。
|
||||
@@ -14,15 +14,18 @@ public sealed class ImageAnalysisQuotaService : IDisposable
|
||||
{
|
||||
private readonly HttpClient client;
|
||||
private readonly PluginContext<CloudProductPlugin> context;
|
||||
|
||||
private readonly string ApiKey;
|
||||
/// <summary>从站点根地址创建客户端，定期更新连接以跟随 DNS 变化。</summary>
|
||||
public ImageAnalysisQuotaService(PluginContext<CloudProductPlugin> context)
|
||||
{
|
||||
this.context = context;
|
||||
var url = context.RequireConfiguration("ImageAnalysisQuota:Url");
|
||||
ApiKey = context.RequireConfiguration("ImageAnalysisQuota:APIKey");
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var endpoint) ||
|
||||
(endpoint.Scheme != Uri.UriSchemeHttp && endpoint.Scheme != Uri.UriSchemeHttps))
|
||||
throw new InvalidOperationException("ImageAnalysisQuota:Url 必须是 HTTP 或 HTTPS 站点地址。");
|
||||
if (string.IsNullOrEmpty(ApiKey))
|
||||
throw new InvalidOperationException("ApiKey 不能为空");
|
||||
|
||||
client = new HttpClient(new SocketsHttpHandler
|
||||
{
|
||||
@@ -46,7 +49,7 @@ public sealed class ImageAnalysisQuotaService : IDisposable
|
||||
// 外部协议要求 token 和 username 放在查询字符串中，统一编码特殊字符。
|
||||
var requestUrl = QueryHelpers.AddQueryString("/wxxd_user/UseImageAnalysisQuota", new Dictionary<string, string?>
|
||||
{
|
||||
["token"] = context.RequireConfiguration("ImageAnalysisQuota:APIKey"),
|
||||
["token"] = ApiKey,
|
||||
["username"] = request.UserName
|
||||
});
|
||||
using var response = await client.PostAsync(requestUrl, null, cancellationToken);
|
||||
@@ -60,6 +63,36 @@ public sealed class ImageAnalysisQuotaService : IDisposable
|
||||
return true;
|
||||
}
|
||||
throw new AppException(result.Msg);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 使用指定用户的识别额度，返回外部接口原始响应；HTTP 失败直接抛出异常。
|
||||
/// 额度扣减不自动重试，避免请求已执行但响应丢失时重复扣减。
|
||||
/// </summary>
|
||||
public async Task<int> GetUserImageAnalysisQuota(UseImageAnalysisQuotaRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
RequestValidator.Validate(request);
|
||||
// 外部协议要求 token 和 username 放在查询字符串中，统一编码特殊字符。
|
||||
var requestUrl = QueryHelpers.AddQueryString("/wxxd_user/GetUserImageAnalysisQuota", new Dictionary<string, string?>
|
||||
{
|
||||
["token"] = ApiKey,
|
||||
["username"] = request.UserName
|
||||
});
|
||||
using var response = await client.PostAsync(requestUrl, null, cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
// shortcut: 尚未提供响应协议，明确业务成功字段后再添加响应 DTO 和业务失败判断。
|
||||
string html = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
var options = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
var result = JsonSerializer.Deserialize<ApiResult<int>>(html, options);
|
||||
if (result != null && result.Success)
|
||||
{
|
||||
return result.Data;
|
||||
}
|
||||
throw new AppException(result.Msg);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using MAX.Core.DependencyInjection;
|
||||
using MAX.Framework.Commons;
|
||||
using MAX.Framework.Security;
|
||||
using MAX.Framework.Validation;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -52,24 +53,43 @@ public sealed class ProductInspectionService(IInspectionIntegration integration,
|
||||
else if (image.AsSpan(0, 4).SequenceEqual("RIFF"u8) && image.AsSpan(8, 4).SequenceEqual("WEBP"u8)) mediaType = "image/webp";
|
||||
else throw new ValidationException("请上传 JPEG、PNG 或 WebP 图片。");
|
||||
var cached = await database.Select<ImageAnalysisCache>().Where(x => x.CacheKey == md5).FirstAsync();
|
||||
string[] result;
|
||||
// 旧对象格式缺少逐项阅读顺序，重新识别成功后再覆盖。
|
||||
if (cached?.RecognitionResult.TrimStart().StartsWith('[') == true)
|
||||
result = JsonSerializer.Deserialize<string[]>(cached.RecognitionResult)!;
|
||||
else
|
||||
var result = cached?.ReadFreshResult(DateTime.UtcNow);
|
||||
if (result is null)
|
||||
{
|
||||
await gate.WaitAsync(cancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
var useImageQuota = await imageAnalysisQuotaService.GetUserImageAnalysisQuota(new UseImageAnalysisQuotaRequest
|
||||
{
|
||||
UserName = request.UserName
|
||||
});
|
||||
if (useImageQuota <= 0)
|
||||
{
|
||||
throw new AppException("AI检测可用数量为0");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "调用图片识别扣费程序失败");
|
||||
throw new AppException("AI检测可用数量为0");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new MemoryStream(image, writable: false);
|
||||
var recognized = await integration.AnalyzeAsync(stream, mediaType, cancellationToken);
|
||||
result = recognized.Distinct(StringComparer.Ordinal).ToArray();
|
||||
await database.InsertOrUpdate<ImageAnalysisCache>().SetSource(new ImageAnalysisCache
|
||||
{
|
||||
CacheKey = md5,
|
||||
RecognitionResult = JsonSerializer.Serialize(result, CacheJsonOptions),
|
||||
UpdatedAt = DateTime.UtcNow
|
||||
}).ExecuteAffrowsAsync();
|
||||
var entry = cached ?? new ImageAnalysisCache { CacheKey = md5 };
|
||||
var previousUpdate = entry.UpdatedAt;
|
||||
entry.RecognitionResult = JsonSerializer.Serialize(result, CacheJsonOptions);
|
||||
entry.RecordExpiration(result.Length == 0, DateTime.UtcNow);
|
||||
// 同一缓存失效后可能同时触发多次识别，只有首个写入者推进级别。
|
||||
if (cached is null)
|
||||
await database.InsertOrUpdate<ImageAnalysisCache>().SetSource(entry).IfExistsDoNothing().ExecuteAffrowsAsync();
|
||||
else
|
||||
await database.Update<ImageAnalysisCache>().SetSource(entry)
|
||||
.Where(x => x.UpdatedAt == previousUpdate).ExecuteAffrowsAsync();
|
||||
}
|
||||
finally { gate.Release(); }
|
||||
}
|
||||
|
||||
@@ -1,12 +1,86 @@
|
||||
using System.Text.Json;
|
||||
using Store.Modules.CloudProduct.Entities;
|
||||
|
||||
using var config = JsonDocument.Parse(File.ReadAllText("Plugins/CloudProduct/appsettings.json"));
|
||||
// 使用固定时间验证过期边界，无需等待或连接数据库。
|
||||
var now = DateTime.UtcNow;
|
||||
var cache = new ImageAnalysisCache { UpdatedAt = now, RecognitionResult = "[]" };
|
||||
foreach (var days in new[] { 1, 2, 4, 8, 16, 16 })
|
||||
{
|
||||
cache.RecordExpiration(true, now);
|
||||
if (cache.ExpiresAt != days * 24
|
||||
|| cache.ReadFreshResult(now.AddDays(days).AddTicks(-1)) is not { Length: 0 }
|
||||
|| cache.ReadFreshResult(now.AddDays(days)) != null)
|
||||
throw new Exception("连续空识别结果的缓存期限错误。");
|
||||
now = now.AddDays(days);
|
||||
}
|
||||
cache.RecognitionResult = """["文字"]""";
|
||||
cache.RecordExpiration(false, now);
|
||||
if (cache.ExpiresAt != 720
|
||||
|| cache.ReadFreshResult(now.AddDays(30).AddTicks(-1)) is not ["文字"]
|
||||
|| cache.ReadFreshResult(now.AddDays(30)) != null)
|
||||
throw new Exception("非空识别结果应重置级别并缓存三十天。");
|
||||
cache.RecognitionResult = "{}";
|
||||
if (cache.ReadFreshResult(now) != null)
|
||||
throw new Exception("旧对象格式应立即重新识别。");
|
||||
cache.RecordExpiration(true, now.AddDays(30));
|
||||
if (cache.ExpiresAt != 24)
|
||||
throw new Exception("非空结果后的首次空结果应重新从一天开始。");
|
||||
if (args.Contains("--cache-policy"))
|
||||
{
|
||||
Console.WriteLine("PASS cache expiration checks");
|
||||
return;
|
||||
}
|
||||
|
||||
using var config = JsonDocument.Parse(File.ReadAllText("Plugins/CloudProduct/appsettings.json"), new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
|
||||
const string tableQuery = "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() ORDER BY TABLE_NAME";
|
||||
const string columnsQuery = "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'image_analysis_cache' ORDER BY ORDINAL_POSITION";
|
||||
if (args.Contains("--migrate-cache-expiration"))
|
||||
{
|
||||
// 运行时缓存使用 CloudProductConnection；历史初始化命令仍针对原专用库。
|
||||
var cacheConnection = Environment.GetEnvironmentVariable("Plugins__CloudProduct__CloudProductConnection")
|
||||
?? config.RootElement.GetProperty("CloudProductConnection").GetString();
|
||||
using var cacheDb = new FreeSql.FreeSqlBuilder().UseConnectionString(FreeSql.DataType.MySql, cacheConnection).Build();
|
||||
const string expirationTypeQuery = """
|
||||
SELECT DATA_TYPE FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'image_analysis_cache' AND COLUMN_NAME = 'ExpiresAt'
|
||||
""";
|
||||
var columns = await cacheDb.Ado.QueryAsync<string>(columnsQuery);
|
||||
var expirationType = (await cacheDb.Ado.QueryAsync<string>(expirationTypeQuery)).SingleOrDefault();
|
||||
Console.WriteLine($"Cache columns before migration: {string.Join(", ", columns)}; ExpiresAt: {expirationType ?? "absent"}");
|
||||
if (!columns.Contains("RecognitionResult") || !columns.Contains("UpdatedAt"))
|
||||
throw new InvalidOperationException("当前运行库缺少缓存表或基础列。");
|
||||
if (expirationType is not (null or "int" or "datetime" or "timestamp"))
|
||||
throw new InvalidOperationException("不支持的 ExpiresAt 字段类型。");
|
||||
// 用临时列完成日期到时长的转换，回填完成后才一次性替换原列。
|
||||
var hasDateColumn = expirationType is "datetime" or "timestamp";
|
||||
var hoursColumn = hasDateColumn ? "ExpiresAtHours" : "ExpiresAt";
|
||||
if (!columns.Contains(hoursColumn))
|
||||
await cacheDb.Ado.ExecuteNonQueryAsync($"ALTER TABLE image_analysis_cache ADD {hoursColumn} INT NULL");
|
||||
// 已有整数时长原样保留；旧日期列按原到期时间与更新时间的差值换算。
|
||||
var initialHours = "IF(TRIM(RecognitionResult) = '[]', 24, 720)";
|
||||
if (hasDateColumn)
|
||||
initialHours = $"COALESCE(TIMESTAMPDIFF(HOUR, UpdatedAt, ExpiresAt), {initialHours})";
|
||||
await cacheDb.Ado.ExecuteNonQueryAsync($"UPDATE image_analysis_cache SET {hoursColumn} = {initialHours} WHERE {hoursColumn} IS NULL");
|
||||
if (hasDateColumn)
|
||||
await cacheDb.Ado.ExecuteNonQueryAsync("ALTER TABLE image_analysis_cache DROP COLUMN ExpiresAt, CHANGE ExpiresAtHours ExpiresAt INT NOT NULL");
|
||||
else if ((await cacheDb.Ado.QueryAsync<string>("SELECT IS_NULLABLE FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'image_analysis_cache' AND COLUMN_NAME = 'ExpiresAt'")).Single() == "YES")
|
||||
await cacheDb.Ado.ExecuteNonQueryAsync("ALTER TABLE image_analysis_cache MODIFY ExpiresAt INT NOT NULL");
|
||||
|
||||
var finalType = (await cacheDb.Ado.QueryAsync<string>(expirationTypeQuery)).Single();
|
||||
var invalidRows = Convert.ToInt64(await cacheDb.Ado.ExecuteScalarAsync("""
|
||||
SELECT COUNT(*) FROM image_analysis_cache
|
||||
WHERE ExpiresAt IS NULL OR ExpiresAt NOT IN (24, 48, 96, 192, 384, 720)
|
||||
"""));
|
||||
if (finalType != "int" || invalidRows != 0)
|
||||
throw new InvalidOperationException($"迁移校验失败，ExpiresAt 类型 {finalType}，异常行数 {invalidRows}。");
|
||||
if (columns.Contains("EmptyResultStreak"))
|
||||
await cacheDb.Ado.ExecuteNonQueryAsync("ALTER TABLE image_analysis_cache DROP COLUMN EmptyResultStreak");
|
||||
Console.WriteLine("PASS database migration: cache lifetime stored in ExpiresAt hours");
|
||||
return;
|
||||
}
|
||||
var connection = Environment.GetEnvironmentVariable("Plugins__CloudProduct__ProductInspection__ConnectionString")
|
||||
?? config.RootElement.GetProperty("ProductInspection").GetProperty("ConnectionString").GetString();
|
||||
using var db = new FreeSql.FreeSqlBuilder().UseConnectionString(FreeSql.DataType.MySql, connection).Build();
|
||||
const string tableQuery = "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() ORDER BY TABLE_NAME";
|
||||
const string columnsQuery = "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'image_analysis_cache' ORDER BY ORDINAL_POSITION";
|
||||
if (args.Contains("--initialize"))
|
||||
{
|
||||
// 旧表只移除两个派生字段，保留缓存键、响应原文及更新时间。
|
||||
@@ -30,7 +104,7 @@ var cacheColumns = await db.Ado.QueryAsync<string>(columnsQuery);
|
||||
Console.WriteLine("Cache columns: " + string.Join(", ", cacheColumns));
|
||||
Console.WriteLine($"Cache rows: {await db.Select<ImageAnalysisCache>().CountAsync()}");
|
||||
if (args.Contains("--schema")) return;
|
||||
if (!cacheColumns.ToHashSet().SetEquals(new[] { "CacheKey", "RecognitionResult", "UpdatedAt" }))
|
||||
if (!cacheColumns.ToHashSet().SetEquals(new[] { "CacheKey", "RecognitionResult", "UpdatedAt", "ExpiresAt" }))
|
||||
throw new Exception("图片识别缓存结构不匹配。");
|
||||
if (!tables.Contains("image_analysis_cache"))
|
||||
throw new Exception("商品检测专用库缺少图片缓存表。");
|
||||
|
||||
Reference in new issue
Block a user