合并云商品分支到 master 并适配插件结构

This commit is contained in:
chfei committed 2026-11-02 10:01:33 +08:00
commit 1fa91a9cfb
15 files changed
+968 -48

No files matched your search

@@ -39,6 +39,9 @@ namespace Store.Modules.CloudProduct
.UseStorage(new MySqlStorage(provider.GetRequiredKeyedService<IFreeSql>("CloudProductConnection"), new MySqlStorageOptions() { TablesPrefix = "Hangfire_" })));
services.AddHangfireServer();
services.AddTransient<GoodsIndexJob>();
services.AddTransient<GoodsViolationIndexJob>();
services.AddAuthorization(options => options.AddPolicy("CloudGoodsViolationWrite",
policy => policy.RequireAuthenticatedUser().RequireClaim("permission", "cloudgoods:violation:write")));
services.AddKeyedSingleton<IFreeSql>("CloudProductConnection", (_, _) =>
new FreeSql.FreeSqlBuilder().UseConnectionString(FreeSql.DataType.MySql, connectionStringBuilder.ConnectionString)
.UseAdoConnectionPool(true).Build());
@@ -75,6 +78,8 @@ namespace Store.Modules.CloudProduct
"goods-index-pending",
job => job.ProcessPendingAsync(),
cron);
app.ApplicationServices.GetRequiredService<IRecurringJobManager>().AddOrUpdate<GoodsViolationIndexJob>(
"goods-violation-delete-pending", job => job.ProcessPendingAsync(), "* * * * *");
}
catch (Exception ex)
{
@@ -2,6 +2,7 @@ using Asp.Versioning;
using MAX.Framework.Commons;
using MAX.Framework.Dtos;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Store.Framework.Base;
using Store.Modules.CloudProduct.Dtos;
@@ -12,6 +13,8 @@ using System.Text.Json.Nodes;
namespace Store.Modules.CloudProduct.Controllers
{
/// <summary>云商品管理:商品查询、导入、违规上报及搜索索引维护。</summary>
[Tags("云商品")]
[ApiVersionNeutral]
[Route("api/CloudGoods/[action]")]
public class GoodsController : AppControllerBase
@@ -20,31 +23,62 @@ namespace Store.Modules.CloudProduct.Controllers
private readonly IOpenSearchService _searchService;
private readonly IRawDataStorage _rawDataStorage;
private readonly GoodsIndexJob _goodsIndexJob;
private readonly GoodsViolationService _violations;
public GoodsController(IGoodsService goodsService, IOpenSearchService searchService, IRawDataStorage rawDataStorage, GoodsIndexJob goodsIndexJob)
public GoodsController(IGoodsService goodsService, IOpenSearchService searchService, IRawDataStorage rawDataStorage, GoodsIndexJob goodsIndexJob, GoodsViolationService violations)
{
_violations = violations;
_goodsService = goodsService;
_searchService = searchService;
_rawDataStorage = rawDataStorage;
_goodsIndexJob = goodsIndexJob;
}
/// <summary>获取全部违规原因字典,包含未启用违规判定的原因,按ID升序返回。</summary>
[HttpGet]
[EndpointSummary("获取违规原因字典")]
public Task<List<ViolationReasonDto>> GetViolationReasons()
=> _violations.GetReasonsAsync();
/// <summary>批量上报违规商品;重复上报合并原因。</summary>
[HttpPost]
[EndpointSummary("批量上报违规商品")]
public Task<List<ReportGoodsViolationsResult>> ReportViolations([FromBody] ReportGoodsViolationItem[] items)
=> _violations.ReportAsync(items);
/// <summary>批量解除商品违规,恢复正常商品索引;失败后由后台索引任务重试。</summary>
[HttpPost]
[EndpointSummary("解除商品违规并恢复索引")]
public Task<RemoveGoodsViolationsResult> RemoveViolations([FromBody] long[] ids)
=> _violations.RemoveAsync(ids);
/// <summary>按商品编号批量查询是否违规。</summary>
[HttpPost]
[EndpointSummary("批量查询商品违规状态")]
public Task<List<GoodsViolationStatus>> GetViolationStatus([FromBody] long[] ids)
=> _violations.GetStatusAsync(ids);
/// <summary>获取服务器当前日期(长日期格式)。</summary>
[HttpGet]
[EndpointSummary("获取服务器当前日期")]
public async Task<string> Time()
{
return DateTime.Now.ToLongDateString();
}
/// <summary>手动执行待处理商品的索引任务。</summary>
[HttpGet]
[EndpointSummary("处理待建立索引的商品")]
public async Task Job()
{
await _goodsIndexJob.ProcessPendingAsync();
}
/// <summary>
/// 搜索
/// 分页搜索商品,可按标题、价格、类目、销量和平台筛选。
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
/// <param name="request">商品筛选条件及分页参数。</param>
/// <returns>符合条件的商品列表及总数。</returns>
[HttpPost]
[EndpointSummary("分页搜索商品")]
public async Task<PageData<GoodsEntity>> SearchGoods(GoodsSearchRequest request)
{
// 1. 参数基础校验
@@ -145,6 +179,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// </summary>
[HttpGet]
[ActionName("SearchGoodsToIds")]
[EndpointSummary("按关键词搜索商品摘要")]
public async Task<PageData<GoodsSearchDocument>> SearchGoodsSummary(string keyword, int pageindex = 1, int pagesize = 20)
{
if (string.IsNullOrWhiteSpace(keyword))
@@ -172,6 +207,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 导入测试数据:读取 ImportConverters/data.txt 并走完整解析 + 入库 + 分词流程。
/// </summary>
[HttpGet]
[EndpointSummary("导入测试商品数据")]
public async Task<GoodsImportResult> ImportTestData()
{
var filePath = GetDataFilePath();
@@ -198,6 +234,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 导入接口:接收客户端批量上传的 JSON 文件,解析后入库并写入分词索引。
/// </summary>
[HttpPost]
[EndpointSummary("批量导入商品 JSON 文件")]
public async Task<GoodsImportResult> ImportFiles([FromForm] List<IFormFile> files)
{
if (files == null || files.Count == 0)
@@ -247,6 +284,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 查询 OpenSearch 状态。
/// </summary>
[HttpGet]
[EndpointSummary("查询 OpenSearch 服务状态")]
public async Task<OpenSearchStatusDto> GetOpenSearchStatus()
{
return await _searchService.GetStatusAsync();
@@ -256,6 +294,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 检查搜索索引是否存在。
/// </summary>
[HttpGet]
[EndpointSummary("检查商品搜索索引是否存在")]
public async Task<bool> IndexExists()
{
return await _searchService.IndexExistsAsync();
@@ -265,6 +304,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 检查索引映射(分析器)是否符合当前服务要求。
/// </summary>
[HttpGet]
[EndpointSummary("检查商品索引映射是否正常")]
public async Task<bool> IndexMappingHealthy()
{
return await _searchService.IsIndexMappingHealthyAsync();
@@ -274,6 +314,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 创建搜索索引(含 IK 分词映射,已存在且映射正常时跳过)。
/// </summary>
[HttpPost]
[EndpointSummary("创建商品搜索索引")]
public async Task<bool> CreateIndex()
{
return await _searchService.CreateIndexAsync();
@@ -283,6 +324,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 删除搜索索引。
/// </summary>
[HttpPost]
[EndpointSummary("删除商品搜索索引")]
public async Task DeleteIndex()
{
await _searchService.DeleteIndexAsync();
@@ -292,6 +334,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 手动刷新搜索索引,使最新写入立即可搜索。
/// </summary>
[HttpPost]
[EndpointSummary("刷新商品搜索索引")]
public async Task RefreshIndex()
{
await _searchService.RefreshIndexAsync();
@@ -301,6 +344,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 手动新增或覆盖一条搜索文档。
/// </summary>
[HttpPost]
[EndpointSummary("新增或覆盖商品搜索文档")]
public async Task AddSearchDocument([FromBody] GoodsSearchDocument document)
{
if (document == null || document.Id <= 0)
@@ -315,20 +359,36 @@ namespace Store.Modules.CloudProduct.Controllers
/// 读取指定商品详情:解压缩原始数据并返回 JSON。
/// </summary>
[HttpPost]
[EndpointSummary("查询商品详情")]
public async Task<JsonNode> GetGoodsDetail([FromQuery] long goodsId)
{
var raw = await GetRawDataOrThrowAsync(goodsId);
var node = JsonNode.Parse(raw);
return node ?? throw new AppException("商品详情解析失败", ResultCode.BusinessError);
try
{
var raw = await GetRawDataOrThrowAsync(goodsId);
var node = JsonNode.Parse(raw);
return node ?? throw new AppException("商品详情解析失败", ResultCode.BusinessError);
}
catch(Exception ex)
{
throw new AppException(ex.Message);
}
}
/// <summary>
/// 读取指定商品的原始数据(解压缩后原样返回)。
/// </summary>
[HttpPost]
[EndpointSummary("读取商品原始数据")]
public async Task<string> GetGoodsRawData([FromQuery] long goodsId)
{
return await GetRawDataOrThrowAsync(goodsId);
try
{
return await GetRawDataOrThrowAsync(goodsId);
}
catch (Exception ex)
{
throw new AppException(ex.Message);
}
}
/// <summary>
@@ -336,6 +396,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// ids 为空时重建全部商品。
/// </summary>
[HttpPost]
[EndpointSummary("重建商品搜索文档")]
public async Task<long> RebuildIndex([FromBody] long[]? ids)
{
return await _goodsService.RebuildSearchIndexAsync(ids);
@@ -346,6 +407,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// 软删除将 Status 置 1,硬删除物理删除记录;两种方式都会删除 OpenSearch 中对应文档。
/// </summary>
[HttpPost]
[EndpointSummary("批量删除商品")]
public async Task<long> DeleteGoods([FromBody] long[]? ids)
{
if (ids == null || ids.Length == 0)
@@ -361,6 +423,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// </summary>
[HttpGet]
[ActionName("ClearSearch")]
[EndpointSummary("清空并重建商品搜索索引")]
public async Task<bool> ClearSearchIndex()
{
return await _searchService.ClearIndexAsync();
@@ -371,6 +434,7 @@ namespace Store.Modules.CloudProduct.Controllers
/// </summary>
[HttpGet]
[ActionName("GetSearchCount")]
[EndpointSummary("统计商品搜索文档数量")]
public async Task<long> GetSearchDocumentCount()
{
return await _searchService.CountDocumentsAsync();
@@ -383,7 +447,10 @@ namespace Store.Modules.CloudProduct.Controllers
var raw = await _rawDataStorage.ReadAsync(goodsId);
if (raw == null)
{
System.IO.File.WriteAllText("./NoGoods.txt", goodsId.ToString());
throw new AppException($"商品原始数据不存在: {goodsId}", ResultCode.NotFound);
}
return raw;
}
@@ -0,0 +1,36 @@
namespace Store.Modules.CloudProduct.Dtos;
public class ViolationReasonDto
{
public string Name { get; set; } = string.Empty;
public bool IsViolation { get; set; }
}
public class ReportGoodsViolationItem
{
public long GoodsId { get; set; }
/// <summary>原因名称,例如“出售假冒商品”;新名称默认仅记录字典,不判定违规。</summary>
public string Reasons { get; set; } = string.Empty;
}
public class ReportGoodsViolationsResult
{
public long GoodsId { get; set; }
/// <summary>存在违规记录且已完成索引下架时为 true,否则为 false。</summary>
public bool IsOffShelf { get; set; }
}
public class GoodsViolationStatus
{
public long GoodsId { get; set; }
public bool IsViolation { get; set; }
}
public class RemoveGoodsViolationsResult
{
public int RemovedCount { get; set; }
public long[] RestoredGoodsIds { get; set; } = [];
public long[] PendingRestoreGoodsIds { get; set; } = [];
/// <summary>商品不存在、已删除或恢复期间再次被上报违规的编号。</summary>
public long[] SkippedGoodsIds { get; set; } = [];
}
@@ -0,0 +1,22 @@
using FreeSql.DataAnnotations;
namespace Store.Modules.CloudProduct.Entities;
[Table(Name = "goods_violation")]
public class GoodsViolationEntity
{
[Column(Name = "goods_id", IsPrimary = true)]
public long GoodsId { get; set; }
[Column(Name = "reason_id", IsPrimary = true, IsNullable = false)]
public uint ReasonId { get; set; }
[Column(Name = "is_index_deleted")]
public bool IsIndexDeleted { get; set; }
[Column(Name = "created_time")]
public DateTime CreatedTime { get; set; }
[Column(Name = "updated_time")]
public DateTime UpdatedTime { get; set; }
}
@@ -0,0 +1,16 @@
using FreeSql.DataAnnotations;
namespace Store.Modules.CloudProduct.Entities;
[Table(Name = "violation_reason")]
public class ViolationReasonEntity
{
[Column(Name = "id", IsPrimary = true, IsIdentity = true)]
public uint Id { get; set; }
[Column(Name = "name", StringLength = 100, IsNullable = false)]
public string Name { get; set; } = string.Empty;
[Column(Name = "is_violation", IsNullable = false)]
public bool IsViolation { get; set; }
}
@@ -0,0 +1,118 @@
using FreeSql;
using MAX.Core.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using Store.Modules.CloudProduct.Entities;
using Store.Modules.CloudProduct.Dtos;
namespace Store.Modules.CloudProduct.Repositorys;
public class GoodsViolationRepository(
[FromKeyedServices("CloudProductConnection")] IFreeSql fsql)
: BaseRepository<GoodsViolationEntity>(fsql), IScopedDependency
{
public Task<List<ViolationReasonDto>> GetReasonsAsync() => Orm.Select<ViolationReasonEntity>()
.OrderBy(x => x.Id).ToListAsync(x => new ViolationReasonDto
{
Name = x.Name,
IsViolation = x.IsViolation
});
public Task<List<long>> GetViolationIdsAsync(long[] ids) => Select
.Where(x => ids.Contains(x.GoodsId)).Distinct().ToListAsync(x => x.GoodsId);
public async Task<List<long>> GetOffShelfGoodsIdsAsync(long[] ids)
{
var records = await Select.Where(x => ids.Contains(x.GoodsId)).ToListAsync();
return records.GroupBy(x => x.GoodsId).Where(x => x.All(row => row.IsIndexDeleted))
.Select(x => x.Key).ToList();
}
// 解除违规和设置待同步必须一起提交,进程中断后后台索引任务仍可恢复。
public async Task<int> RemoveAndScheduleRestoreAsync(long[] ids)
{
using var unit = Orm.CreateUnitOfWork();
var violations = unit.GetRepository<GoodsViolationEntity>();
var goods = unit.GetRepository<GoodsEntity>();
var removedIds = await violations.Select.Where(x => ids.Contains(x.GoodsId)).Distinct().ToListAsync(x => x.GoodsId);
await violations.DeleteAsync(x => ids.Contains(x.GoodsId));
await goods.UpdateDiy.Set(x => x.IsSegmented, (byte)0)
.Where(x => ids.Contains(x.GoodsId) && x.Status == 0).ExecuteAffrowsAsync();
unit.Commit();
return removedIds.Count;
}
public Task<List<GoodsEntity>> GetRestorableGoodsAsync(long[] ids) => Orm.Select<GoodsEntity>()
.Where(x => ids.Contains(x.GoodsId) && x.Status == 0).ToListAsync();
public Task<int> MarkRestoredAsync(long[] ids) => Orm.Update<GoodsEntity>()
.Set(x => x.IsSegmented, (byte)1)
.Where(x => ids.Contains(x.GoodsId) && x.Status == 0).ExecuteAffrowsAsync();
// 调用方持有 GoodsIndexLock,事务内合并原因,避免并发覆盖。
public async Task<long[]> MergeAsync(Dictionary<long, string[]> items)
{
using var unit = Orm.CreateUnitOfWork();
var repository = unit.GetRepository<GoodsViolationEntity>();
var reasonRepository = unit.GetRepository<ViolationReasonEntity>();
var reasonIds = new Dictionary<string, uint>(StringComparer.Ordinal);
foreach (var name in items.Values.SelectMany(x => x).Distinct(StringComparer.Ordinal))
{
// 让数据库按字典表的排序规则匹配名称,与唯一索引保持一致。
var reason = await reasonRepository.Select.Where(x => x.Name == name).FirstAsync();
// 新类型只收录到字典,绝不因首次上报自动启用商品封禁。
reason ??= await reasonRepository.InsertAsync(new ViolationReasonEntity { Name = name, IsViolation = false });
// 数据库排序规则可能忽略部分字符差异;判定时仍要求名称完整相等。
if (reason.IsViolation && string.Equals(reason.Name, name, StringComparison.Ordinal))
reasonIds.Add(name, reason.Id);
}
var ids = items.Keys.ToArray();
var existing = (await repository.Select.Where(x => ids.Contains(x.GoodsId))
.ToListAsync()).ToLookup(x => x.GoodsId);
var now = DateTime.UtcNow;
var recordedIds = new List<long>();
foreach (var (goodsId, reasons) in items)
{
var accepted = reasons.Where(reasonIds.ContainsKey).Select(x => reasonIds[x]).Distinct().ToArray();
// 忽略原因仅记录字典,不新增或改变该商品已有的违规状态。
if (accepted.Length == 0) continue;
var rows = existing[goodsId].ToArray();
var previous = rows.Select(x => x.ReasonId).ToHashSet();
var added = accepted.Where(x => !previous.Contains(x)).ToArray();
if (added.Length == 0 && rows.All(x => x.IsIndexDeleted)) continue;
// 下架是商品级操作:新增原因或存在失败记录时,整件商品重新标记待下架。
foreach (var row in rows)
{
row.IsIndexDeleted = false;
row.UpdatedTime = now;
await repository.UpdateAsync(row);
}
foreach (var reasonId in added)
{
await repository.InsertAsync(new GoodsViolationEntity
{
GoodsId = goodsId, ReasonId = reasonId, IsIndexDeleted = false,
CreatedTime = now, UpdatedTime = now
});
}
recordedIds.Add(goodsId);
}
unit.Commit();
return recordedIds.ToArray();
}
public Task<int> MarkDeletedAsync(long[] ids) => Orm.Update<GoodsViolationEntity>()
.Set(x => x.IsIndexDeleted, true)
.Set(x => x.UpdatedTime, DateTime.UtcNow)
.Where(x => ids.Contains(x.GoodsId)).ExecuteAffrowsAsync();
public Task<List<long>> GetBatchAsync(long afterId, int size, bool pendingOnly) => Select
.Where(x => x.GoodsId > afterId)
.WhereIf(pendingOnly, x => !x.IsIndexDeleted)
.Distinct().OrderBy(x => x.GoodsId).Limit(size).ToListAsync(x => x.GoodsId);
public Task<List<long>> GetByReasonAsync(uint reasonId, long afterId, int size) => Select
.Where(x => x.ReasonId == reasonId)
.Where(x => x.GoodsId > afterId).OrderBy(x => x.GoodsId)
.Limit(size).ToListAsync(x => x.GoodsId);
}
@@ -0,0 +1,42 @@
using MAX.Core.DependencyInjection;
using MAX.Framework.Commons;
using MAX.Plugins;
using Microsoft.Extensions.DependencyInjection;
using MySql.Data.MySqlClient;
using System.Security.Cryptography;
using System.Text;
namespace Store.Modules.CloudProduct.Services;
/// <summary>跨实例串行化违规上报和商品索引写入,防止下架后被旧批次写回。</summary>
public class GoodsIndexLock([FromKeyedServices("CloudProduct")] PluginContext context) : ISingletonDependency
{
public async Task<IAsyncDisposable> AcquireAsync()
{
// 禁用连接池:即使释放锁失败,关闭物理连接也会释放会话锁。
var settings = new MySqlConnectionStringBuilder(context.Configuration["CloudProductConnection"])
{
Pooling = false
};
var connection = new MySqlConnection(settings.ConnectionString);
var name = "goods-index:" + Convert.ToHexString(SHA256.HashData(
Encoding.UTF8.GetBytes(settings.Database)))[..40];
try
{
await connection.OpenAsync();
using var command = connection.CreateCommand();
command.CommandText = "SELECT GET_LOCK(@name, 30)";
command.CommandTimeout = 35;
command.Parameters.AddWithValue("@name", name);
var result = await command.ExecuteScalarAsync();
if (result == null || result == DBNull.Value || Convert.ToInt32(result) != 1)
throw new AppException("商品索引正在处理,请稍后重试");
return connection;
}
catch
{
await connection.DisposeAsync();
throw;
}
}
}
+23 -23
View File
@@ -25,9 +25,11 @@ namespace Store.Modules.CloudProduct.Services
private readonly IOpenSearchService _searchService;
private readonly IRawDataStorage _rawDataStorage;
private readonly IConfiguration _configuration;
private readonly GoodsViolationService _violations;
public GoodsService(GoodsRepository repository, IOpenSearchService searchService, IRawDataStorage rawDataStorage, [FromKeyedServices("CloudProduct")] PluginContext context)
public GoodsService(GoodsRepository repository, IOpenSearchService searchService, IRawDataStorage rawDataStorage, [FromKeyedServices("CloudProduct")] PluginContext context, GoodsViolationService violations)
{
_violations = violations;
_repository = repository;
_searchService = searchService;
_rawDataStorage = rawDataStorage;
@@ -148,7 +150,6 @@ namespace Store.Modules.CloudProduct.Services
var indexImmediately = IsImmediateIndexing();
var documents = new List<GoodsSearchDocument>();
var indexedIds = new List<long>();
foreach (var item in items)
{
@@ -196,7 +197,6 @@ namespace Store.Modules.CloudProduct.Services
MinPrice = saved.MinPrice,
SoldQuantity = saved.SoldQuantity,
});
indexedIds.Add(saved.GoodsId);
}
result.SuccessCount++;
@@ -212,7 +212,7 @@ namespace Store.Modules.CloudProduct.Services
{
try
{
await _searchService.BulkIndexGoodsAsync(documents);
var indexedIds = await _searchService.BulkIndexGoodsAsync(documents);
await MarkIndexedAsync(indexedIds);
}
catch (Exception ex)
@@ -235,8 +235,9 @@ namespace Store.Modules.CloudProduct.Services
if (batchSize <= 0)
batchSize = 200;
var pending = await _repository.Select
var pending = await _repository.Select.As("g")
.Where(x => x.IsSegmented == 0 && x.Status == 0)
.Where("NOT EXISTS (SELECT 1 FROM goods_violation v WHERE v.goods_id = g.goods_id)")
.OrderBy(x => x.GoodsId)
.Limit(batchSize)
.ToListAsync();
@@ -260,10 +261,9 @@ namespace Store.Modules.CloudProduct.Services
})
.ToList();
await _searchService.BulkIndexGoodsAsync(documents);
await MarkIndexedAsync(pending.Select(x => x.GoodsId));
return pending.Count;
var indexedIds = await _searchService.BulkIndexGoodsAsync(documents);
await MarkIndexedAsync(indexedIds);
return indexedIds.Length;
}
/// <summary>
@@ -310,22 +310,22 @@ namespace Store.Modules.CloudProduct.Services
/// </summary>
public async Task<long> RebuildSearchIndexAsync(IEnumerable<long>? ids = null)
{
List<GoodsEntity> goodsList;
var idList = ids?.ToList();
if (idList == null || idList.Count == 0)
await _violations.DeleteViolationIndexesAsync(pendingOnly: false);
var idList = ids?.Distinct().ToArray() ?? [];
long lastId = 0, count = 0;
while (true)
{
goodsList = await _repository.Select.ToListAsync();
var goodsList = await _repository.Select.As("g")
.Where(x => x.Status == 0 && x.GoodsId > 0 && x.Id > lastId)
.WhereIf(idList.Length > 0, x => idList.Contains(x.GoodsId))
.Where("NOT EXISTS (SELECT 1 FROM goods_violation v WHERE v.goods_id = g.goods_id)")
.OrderBy(x => x.Id).Limit(1000).ToListAsync();
if (goodsList.Count == 0) break;
var indexedIds = await _searchService.RebuildIndexAsync(goodsList);
await MarkIndexedAsync(indexedIds);
count += indexedIds.Length;
lastId = goodsList[^1].Id;
}
else
{
goodsList = await _repository.Select
.Where(x => idList.Contains(x.GoodsId))
.ToListAsync();
}
var count = await _searchService.RebuildIndexAsync(goodsList);
await MarkIndexedAsync(goodsList.Where(x => x.GoodsId > 0).Select(x => x.GoodsId));
return count;
}
@@ -0,0 +1,13 @@
using Microsoft.Extensions.DependencyInjection;
namespace Store.Modules.CloudProduct.Services;
public class GoodsViolationIndexJob(IServiceScopeFactory scopeFactory)
{
public async Task ProcessPendingAsync()
{
using var scope = scopeFactory.CreateScope();
await scope.ServiceProvider.GetRequiredService<GoodsViolationService>()
.DeleteViolationIndexesAsync();
}
}
@@ -0,0 +1,124 @@
using MAX.Core.DependencyInjection;
using MAX.Framework.Commons;
using MAX.Framework.Dtos;
using Microsoft.Extensions.Logging;
using Store.Modules.CloudProduct.Dtos;
using Store.Modules.CloudProduct.Repositorys;
namespace Store.Modules.CloudProduct.Services;
public class GoodsViolationService(
GoodsViolationRepository repository,
GoodsIndexLock indexLock,
IOpenSearchService search,
ILogger<GoodsViolationService> logger) : IScopedDependency
{
public Task<List<ViolationReasonDto>> GetReasonsAsync() => repository.GetReasonsAsync();
public static Dictionary<long, string[]> ValidateAndGroup(ReportGoodsViolationItem[]? items)
{
if (items == null || items.Length is < 1 or > 1000)
throw new AppException("每次请提供1-1000条违规记录", ResultCode.ValidationError);
for (var i = 0; i < items.Length; i++)
{
var item = items[i];
if (item == null || item.GoodsId <= 0 || string.IsNullOrWhiteSpace(item.Reasons)
|| item.Reasons.Trim().Length > 100)
throw new AppException($"第{i + 1}条记录须提供有效商品编号和一个违规原因,原因去除首尾空白后须为1-100个字符", ResultCode.ValidationError);
}
return items.GroupBy(x => x.GoodsId).ToDictionary(x => x.Key,
x => x.Select(y => y.Reasons.Trim()).Distinct(StringComparer.Ordinal).ToArray());
}
public async Task<List<ReportGoodsViolationsResult>> ReportAsync(ReportGoodsViolationItem[]? items)
{
var grouped = ValidateAndGroup(items);
await using var lease = await indexLock.AcquireAsync();
var pendingIds = await repository.MergeAsync(grouped);
if (pendingIds.Length > 0)
{
try
{
await search.DeleteDocumentsAsync(pendingIds);
await repository.MarkDeletedAsync(pendingIds);
}
catch (Exception ex)
{
logger.LogError(ex, "违规记录已保存,{Count}个商品等待重试下架", pendingIds.Length);
}
}
// 返回商品当前持久化状态,包括本次原因被忽略但历史已下架的商品。
var offShelfIds = (await repository.GetOffShelfGoodsIdsAsync(grouped.Keys.ToArray())).ToHashSet();
return grouped.Keys.Select(id => new ReportGoodsViolationsResult
{
GoodsId = id,
IsOffShelf = offShelfIds.Contains(id)
}).ToList();
}
public async Task<List<GoodsViolationStatus>> GetStatusAsync(long[]? ids)
{
if (ids == null || ids.Length is < 1 or > 1000 || ids.Any(x => x <= 0))
throw new AppException("请提供1-1000个正数商品编号", ResultCode.ValidationError);
var distinct = ids.Distinct().ToArray();
var violations = (await repository.GetViolationIdsAsync(distinct)).ToHashSet();
return distinct.Select(x => new GoodsViolationStatus { GoodsId = x, IsViolation = violations.Contains(x) }).ToList();
}
public async Task<RemoveGoodsViolationsResult> RemoveAsync(long[]? ids)
{
if (ids == null || ids.Length is < 1 or > 1000 || ids.Any(x => x <= 0))
throw new AppException("请提供1-1000个正数商品编号", ResultCode.ValidationError);
var distinct = ids.Distinct().ToArray();
var result = new RemoveGoodsViolationsResult();
await using (var lease = await indexLock.AcquireAsync())
result.RemovedCount = await repository.RemoveAndScheduleRestoreAsync(distinct);
// 索引服务自行持锁并再次检查违规,避免与新的上报交错时重新上架违规商品。
var goods = await repository.GetRestorableGoodsAsync(distinct);
var candidates = goods.Select(x => x.GoodsId).Distinct().ToArray();
result.SkippedGoodsIds = distinct.Except(candidates).ToArray();
if (candidates.Length == 0) return result;
try
{
var restored = await search.RebuildIndexAsync(goods);
if (restored.Length > 0)
{
await using var lease = await indexLock.AcquireAsync();
var blocked = await repository.GetViolationIdsAsync(restored);
restored = restored.Except(blocked).ToArray();
if (restored.Length > 0) await repository.MarkRestoredAsync(restored);
}
result.RestoredGoodsIds = restored;
result.SkippedGoodsIds = distinct.Except(restored).ToArray();
}
catch (Exception ex)
{
logger.LogError(ex, "已解除违规,{Count}个商品等待后台任务恢复索引", candidates.Length);
result.PendingRestoreGoodsIds = candidates;
}
return result;
}
// 暂不注册 HTTP 路由。
public Task<List<long>> GetGoodsIdsByReasonAsync(uint reasonId, long afterGoodsId = 0, int pageSize = 100)
{
if (reasonId == 0 || afterGoodsId < 0 || pageSize is < 1 or > 1000)
throw new AppException("原因ID或分页参数无效", ResultCode.ValidationError);
return repository.GetByReasonAsync(reasonId, afterGoodsId, pageSize);
}
public async Task DeleteViolationIndexesAsync(bool pendingOnly = true)
{
long afterId = 0;
while (true)
{
await using var lease = await indexLock.AcquireAsync();
var ids = await repository.GetBatchAsync(afterId, 1000, pendingOnly);
if (ids.Count == 0) return;
await search.DeleteDocumentsAsync(ids);
await repository.MarkDeletedAsync(ids.ToArray());
afterId = ids[^1];
}
}
}
@@ -16,7 +16,7 @@ namespace Store.Modules.CloudProduct.Services
public interface IOpenSearchService : ISingletonDependency
{
Task AddDocumentAsync(GoodsSearchDocument document);
Task BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents);
Task<long[]> BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents);
Task<bool> ClearIndexAsync();
Task<long> CountDocumentsAsync();
Task<bool> CreateIndexAsync();
@@ -26,7 +26,7 @@ namespace Store.Modules.CloudProduct.Services
Task<bool> IndexExistsAsync();
Task<bool> IsIndexMappingHealthyAsync();
Task RefreshIndexAsync();
Task<long> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList);
Task<long[]> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList);
Task<(long Total, List<GoodsSearchDocument> Documents)> SearchByTitleAsync(GoodsSearchRequest request, GoodsSearchSortBy sortBy = GoodsSearchSortBy.Relevance);
}
@@ -41,10 +41,15 @@ namespace Store.Modules.CloudProduct.Services
// 批量写入/建索引/删除索引等管理类操作超时(数据量大时需要更长,可挪到配置文件)
private static readonly TimeSpan AdminTimeout = TimeSpan.FromSeconds(30);
private readonly ILogger<OpenSearchService> _logger;
private readonly IFreeSql _database;
private readonly GoodsIndexLock _indexLock;
public OpenSearchService([FromKeyedServices("CloudProduct")] PluginContext context,ILogger<OpenSearchService> logger)
{
var config = context.Configuration.GetSection("Search");
public OpenSearchService([FromKeyedServices("CloudProduct")] PluginContext context, ILogger<OpenSearchService> logger,
[FromKeyedServices("CloudProductConnection")] IFreeSql database, GoodsIndexLock indexLock)
{
_database = database;
_indexLock = indexLock;
var config = context.Configuration.GetSection("Search");
_nodeUrl = config.GetValue<string>("Url") ?? "";
_indexName = config.GetValue<string>("IndexName") ?? "";
_client = CreateClient();
@@ -206,12 +211,18 @@ namespace Store.Modules.CloudProduct.Services
/// <summary>
/// ============ 3.1 批量写入商品搜索文档(存在则覆盖) ============
/// </summary>
public async Task BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents)
public async Task<long[]> BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents)
{
if (documents == null) throw new ArgumentNullException(nameof(documents));
var list = documents.ToList();
if (list.Count == 0) return;
if (list.Count == 0) return [];
await using var lease = await _indexLock.AcquireAsync();
var ids = list.Select(x => x.Id).Distinct().ToArray();
var blocked = (await _database.Select<GoodsViolationEntity>()
.Where(x => ids.Contains(x.GoodsId)).ToListAsync(x => x.GoodsId)).ToHashSet();
list = list.Where(x => !blocked.Contains(x.Id)).ToList();
if (list.Count == 0) return [];
// 确保索引已按 IK 分词映射创建,否则中文分词无效
await CreateIndexAsync();
@@ -222,19 +233,20 @@ namespace Store.Modules.CloudProduct.Services
.RequestConfiguration(r => r.RequestTimeout(AdminTimeout))
);
if (!bulkResponse.IsValid)
if (!bulkResponse.IsValid || bulkResponse.Errors)
{
throw new AppException("写入 OpenSearch 失败: " + bulkResponse.DebugInformation);
}
// 刷新索引,保证数据立即可搜索(仅用于演示,生产环境不建议频繁 Refresh)
await RefreshIndexAsync();
return list.Select(x => x.Id).Distinct().ToArray();
}
/// <summary>
/// ============ 3.2 根据数据库商品重建搜索索引 ============
/// </summary>
public async Task<long> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList)
public async Task<long[]> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList)
{
if (goodsList == null) throw new ArgumentNullException(nameof(goodsList));
@@ -242,6 +254,8 @@ namespace Store.Modules.CloudProduct.Services
.Where(x => x != null && x.GoodsId > 0)
.Select(x => new GoodsSearchDocument
{
Platform = x.Platform,
ThumbUrl = x.ThumbUrl,
Id = x.GoodsId,
Title = x.Title ?? string.Empty,
CatId1 = x.CatId1,
@@ -253,10 +267,7 @@ namespace Store.Modules.CloudProduct.Services
})
.ToList();
if (documents.Count == 0) return 0;
await BulkIndexGoodsAsync(documents);
return documents.Count;
return await BulkIndexGoodsAsync(documents);
}
/// <summary>
@@ -271,7 +282,8 @@ namespace Store.Modules.CloudProduct.Services
.RequestConfiguration(r => r.RequestTimeout(QueryTimeout)));
if (!exists.Exists)
{
return;
if (exists.ApiCall?.HttpStatusCode == 404) return;
throw new AppException("检查 OpenSearch 索引失败: " + exists.DebugInformation);
}
var response = await _client.BulkAsync(b =>
@@ -284,7 +296,9 @@ namespace Store.Modules.CloudProduct.Services
return b;
});
if (!response.IsValid)
if (response.ApiCall?.HttpStatusCode is not (>= 200 and < 300)
|| response.Items.Count != list.Count
|| response.Items.Any(x => x.Status != 404 && (x.Error != null || x.Status < 200 || x.Status >= 300)))
{
throw new AppException("删除 OpenSearch 文档失败: " + response.DebugInformation);
}
@@ -298,6 +312,9 @@ namespace Store.Modules.CloudProduct.Services
public async Task AddDocumentAsync(GoodsSearchDocument document)
{
if (document == null) throw new ArgumentNullException(nameof(document));
await using var lease = await _indexLock.AcquireAsync();
if (await _database.Select<GoodsViolationEntity>().AnyAsync(x => x.GoodsId == document.Id))
throw new AppException("违规商品不能写入搜索索引");
var response = await _client.IndexAsync(document, i => i
.Index(_indexName)
@@ -524,6 +541,7 @@ namespace Store.Modules.CloudProduct.Services
/// </summary>
public async Task<bool> ClearIndexAsync()
{
await using var lease = await _indexLock.AcquireAsync();
var existsResponse = await _client.Indices.ExistsAsync(_indexName, d => d
.RequestConfiguration(r => r.RequestTimeout(QueryTimeout)));
if (!existsResponse.Exists)
@@ -541,7 +559,10 @@ namespace Store.Modules.CloudProduct.Services
}
Console.WriteLine($"索引 {_indexName} 已删除");
return await CreateIndexAsync();
var created = await CreateIndexAsync();
await _database.Update<GoodsEntity>().Set(x => x.IsSegmented, (byte)0)
.Where(x => x.Status == 0).ExecuteAffrowsAsync();
return created;
}
}
}