合并云商品分支到 master 并适配插件结构
This commit is contained in:
commit
1fa91a9cfb
15 files changed
+968
-48
No files matched your search
@@ -0,0 +1,42 @@
|
||||
using MAX.Core.DependencyInjection;
|
||||
using MAX.Framework.Commons;
|
||||
using MAX.Plugins;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using MySql.Data.MySqlClient;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Store.Modules.CloudProduct.Services;
|
||||
|
||||
/// <summary>跨实例串行化违规上报和商品索引写入,防止下架后被旧批次写回。</summary>
|
||||
public class GoodsIndexLock([FromKeyedServices("CloudProduct")] PluginContext context) : ISingletonDependency
|
||||
{
|
||||
public async Task<IAsyncDisposable> AcquireAsync()
|
||||
{
|
||||
// 禁用连接池:即使释放锁失败,关闭物理连接也会释放会话锁。
|
||||
var settings = new MySqlConnectionStringBuilder(context.Configuration["CloudProductConnection"])
|
||||
{
|
||||
Pooling = false
|
||||
};
|
||||
var connection = new MySqlConnection(settings.ConnectionString);
|
||||
var name = "goods-index:" + Convert.ToHexString(SHA256.HashData(
|
||||
Encoding.UTF8.GetBytes(settings.Database)))[..40];
|
||||
try
|
||||
{
|
||||
await connection.OpenAsync();
|
||||
using var command = connection.CreateCommand();
|
||||
command.CommandText = "SELECT GET_LOCK(@name, 30)";
|
||||
command.CommandTimeout = 35;
|
||||
command.Parameters.AddWithValue("@name", name);
|
||||
var result = await command.ExecuteScalarAsync();
|
||||
if (result == null || result == DBNull.Value || Convert.ToInt32(result) != 1)
|
||||
throw new AppException("商品索引正在处理,请稍后重试");
|
||||
return connection;
|
||||
}
|
||||
catch
|
||||
{
|
||||
await connection.DisposeAsync();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,9 +25,11 @@ namespace Store.Modules.CloudProduct.Services
|
||||
private readonly IOpenSearchService _searchService;
|
||||
private readonly IRawDataStorage _rawDataStorage;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly GoodsViolationService _violations;
|
||||
|
||||
public GoodsService(GoodsRepository repository, IOpenSearchService searchService, IRawDataStorage rawDataStorage, [FromKeyedServices("CloudProduct")] PluginContext context)
|
||||
public GoodsService(GoodsRepository repository, IOpenSearchService searchService, IRawDataStorage rawDataStorage, [FromKeyedServices("CloudProduct")] PluginContext context, GoodsViolationService violations)
|
||||
{
|
||||
_violations = violations;
|
||||
_repository = repository;
|
||||
_searchService = searchService;
|
||||
_rawDataStorage = rawDataStorage;
|
||||
@@ -148,7 +150,6 @@ namespace Store.Modules.CloudProduct.Services
|
||||
|
||||
var indexImmediately = IsImmediateIndexing();
|
||||
var documents = new List<GoodsSearchDocument>();
|
||||
var indexedIds = new List<long>();
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
@@ -196,7 +197,6 @@ namespace Store.Modules.CloudProduct.Services
|
||||
MinPrice = saved.MinPrice,
|
||||
SoldQuantity = saved.SoldQuantity,
|
||||
});
|
||||
indexedIds.Add(saved.GoodsId);
|
||||
}
|
||||
|
||||
result.SuccessCount++;
|
||||
@@ -212,7 +212,7 @@ namespace Store.Modules.CloudProduct.Services
|
||||
{
|
||||
try
|
||||
{
|
||||
await _searchService.BulkIndexGoodsAsync(documents);
|
||||
var indexedIds = await _searchService.BulkIndexGoodsAsync(documents);
|
||||
await MarkIndexedAsync(indexedIds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -235,8 +235,9 @@ namespace Store.Modules.CloudProduct.Services
|
||||
if (batchSize <= 0)
|
||||
batchSize = 200;
|
||||
|
||||
var pending = await _repository.Select
|
||||
var pending = await _repository.Select.As("g")
|
||||
.Where(x => x.IsSegmented == 0 && x.Status == 0)
|
||||
.Where("NOT EXISTS (SELECT 1 FROM goods_violation v WHERE v.goods_id = g.goods_id)")
|
||||
.OrderBy(x => x.GoodsId)
|
||||
.Limit(batchSize)
|
||||
.ToListAsync();
|
||||
@@ -260,10 +261,9 @@ namespace Store.Modules.CloudProduct.Services
|
||||
})
|
||||
.ToList();
|
||||
|
||||
await _searchService.BulkIndexGoodsAsync(documents);
|
||||
|
||||
await MarkIndexedAsync(pending.Select(x => x.GoodsId));
|
||||
return pending.Count;
|
||||
var indexedIds = await _searchService.BulkIndexGoodsAsync(documents);
|
||||
await MarkIndexedAsync(indexedIds);
|
||||
return indexedIds.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -310,22 +310,22 @@ namespace Store.Modules.CloudProduct.Services
|
||||
/// </summary>
|
||||
public async Task<long> RebuildSearchIndexAsync(IEnumerable<long>? ids = null)
|
||||
{
|
||||
List<GoodsEntity> goodsList;
|
||||
|
||||
var idList = ids?.ToList();
|
||||
if (idList == null || idList.Count == 0)
|
||||
await _violations.DeleteViolationIndexesAsync(pendingOnly: false);
|
||||
var idList = ids?.Distinct().ToArray() ?? [];
|
||||
long lastId = 0, count = 0;
|
||||
while (true)
|
||||
{
|
||||
goodsList = await _repository.Select.ToListAsync();
|
||||
var goodsList = await _repository.Select.As("g")
|
||||
.Where(x => x.Status == 0 && x.GoodsId > 0 && x.Id > lastId)
|
||||
.WhereIf(idList.Length > 0, x => idList.Contains(x.GoodsId))
|
||||
.Where("NOT EXISTS (SELECT 1 FROM goods_violation v WHERE v.goods_id = g.goods_id)")
|
||||
.OrderBy(x => x.Id).Limit(1000).ToListAsync();
|
||||
if (goodsList.Count == 0) break;
|
||||
var indexedIds = await _searchService.RebuildIndexAsync(goodsList);
|
||||
await MarkIndexedAsync(indexedIds);
|
||||
count += indexedIds.Length;
|
||||
lastId = goodsList[^1].Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
goodsList = await _repository.Select
|
||||
.Where(x => idList.Contains(x.GoodsId))
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
var count = await _searchService.RebuildIndexAsync(goodsList);
|
||||
await MarkIndexedAsync(goodsList.Where(x => x.GoodsId > 0).Select(x => x.GoodsId));
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace Store.Modules.CloudProduct.Services;
|
||||
|
||||
public class GoodsViolationIndexJob(IServiceScopeFactory scopeFactory)
|
||||
{
|
||||
public async Task ProcessPendingAsync()
|
||||
{
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
await scope.ServiceProvider.GetRequiredService<GoodsViolationService>()
|
||||
.DeleteViolationIndexesAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using MAX.Core.DependencyInjection;
|
||||
using MAX.Framework.Commons;
|
||||
using MAX.Framework.Dtos;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Store.Modules.CloudProduct.Dtos;
|
||||
using Store.Modules.CloudProduct.Repositorys;
|
||||
|
||||
namespace Store.Modules.CloudProduct.Services;
|
||||
|
||||
public class GoodsViolationService(
|
||||
GoodsViolationRepository repository,
|
||||
GoodsIndexLock indexLock,
|
||||
IOpenSearchService search,
|
||||
ILogger<GoodsViolationService> logger) : IScopedDependency
|
||||
{
|
||||
public Task<List<ViolationReasonDto>> GetReasonsAsync() => repository.GetReasonsAsync();
|
||||
|
||||
public static Dictionary<long, string[]> ValidateAndGroup(ReportGoodsViolationItem[]? items)
|
||||
{
|
||||
if (items == null || items.Length is < 1 or > 1000)
|
||||
throw new AppException("每次请提供1-1000条违规记录", ResultCode.ValidationError);
|
||||
for (var i = 0; i < items.Length; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (item == null || item.GoodsId <= 0 || string.IsNullOrWhiteSpace(item.Reasons)
|
||||
|| item.Reasons.Trim().Length > 100)
|
||||
throw new AppException($"第{i + 1}条记录须提供有效商品编号和一个违规原因,原因去除首尾空白后须为1-100个字符", ResultCode.ValidationError);
|
||||
}
|
||||
return items.GroupBy(x => x.GoodsId).ToDictionary(x => x.Key,
|
||||
x => x.Select(y => y.Reasons.Trim()).Distinct(StringComparer.Ordinal).ToArray());
|
||||
}
|
||||
|
||||
public async Task<List<ReportGoodsViolationsResult>> ReportAsync(ReportGoodsViolationItem[]? items)
|
||||
{
|
||||
var grouped = ValidateAndGroup(items);
|
||||
await using var lease = await indexLock.AcquireAsync();
|
||||
var pendingIds = await repository.MergeAsync(grouped);
|
||||
if (pendingIds.Length > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await search.DeleteDocumentsAsync(pendingIds);
|
||||
await repository.MarkDeletedAsync(pendingIds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "违规记录已保存,{Count}个商品等待重试下架", pendingIds.Length);
|
||||
}
|
||||
}
|
||||
// 返回商品当前持久化状态,包括本次原因被忽略但历史已下架的商品。
|
||||
var offShelfIds = (await repository.GetOffShelfGoodsIdsAsync(grouped.Keys.ToArray())).ToHashSet();
|
||||
return grouped.Keys.Select(id => new ReportGoodsViolationsResult
|
||||
{
|
||||
GoodsId = id,
|
||||
IsOffShelf = offShelfIds.Contains(id)
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public async Task<List<GoodsViolationStatus>> GetStatusAsync(long[]? ids)
|
||||
{
|
||||
if (ids == null || ids.Length is < 1 or > 1000 || ids.Any(x => x <= 0))
|
||||
throw new AppException("请提供1-1000个正数商品编号", ResultCode.ValidationError);
|
||||
var distinct = ids.Distinct().ToArray();
|
||||
var violations = (await repository.GetViolationIdsAsync(distinct)).ToHashSet();
|
||||
return distinct.Select(x => new GoodsViolationStatus { GoodsId = x, IsViolation = violations.Contains(x) }).ToList();
|
||||
}
|
||||
|
||||
public async Task<RemoveGoodsViolationsResult> RemoveAsync(long[]? ids)
|
||||
{
|
||||
if (ids == null || ids.Length is < 1 or > 1000 || ids.Any(x => x <= 0))
|
||||
throw new AppException("请提供1-1000个正数商品编号", ResultCode.ValidationError);
|
||||
var distinct = ids.Distinct().ToArray();
|
||||
var result = new RemoveGoodsViolationsResult();
|
||||
await using (var lease = await indexLock.AcquireAsync())
|
||||
result.RemovedCount = await repository.RemoveAndScheduleRestoreAsync(distinct);
|
||||
|
||||
// 索引服务自行持锁并再次检查违规,避免与新的上报交错时重新上架违规商品。
|
||||
var goods = await repository.GetRestorableGoodsAsync(distinct);
|
||||
var candidates = goods.Select(x => x.GoodsId).Distinct().ToArray();
|
||||
result.SkippedGoodsIds = distinct.Except(candidates).ToArray();
|
||||
if (candidates.Length == 0) return result;
|
||||
try
|
||||
{
|
||||
var restored = await search.RebuildIndexAsync(goods);
|
||||
if (restored.Length > 0)
|
||||
{
|
||||
await using var lease = await indexLock.AcquireAsync();
|
||||
var blocked = await repository.GetViolationIdsAsync(restored);
|
||||
restored = restored.Except(blocked).ToArray();
|
||||
if (restored.Length > 0) await repository.MarkRestoredAsync(restored);
|
||||
}
|
||||
result.RestoredGoodsIds = restored;
|
||||
result.SkippedGoodsIds = distinct.Except(restored).ToArray();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "已解除违规,{Count}个商品等待后台任务恢复索引", candidates.Length);
|
||||
result.PendingRestoreGoodsIds = candidates;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// 暂不注册 HTTP 路由。
|
||||
public Task<List<long>> GetGoodsIdsByReasonAsync(uint reasonId, long afterGoodsId = 0, int pageSize = 100)
|
||||
{
|
||||
if (reasonId == 0 || afterGoodsId < 0 || pageSize is < 1 or > 1000)
|
||||
throw new AppException("原因ID或分页参数无效", ResultCode.ValidationError);
|
||||
return repository.GetByReasonAsync(reasonId, afterGoodsId, pageSize);
|
||||
}
|
||||
|
||||
public async Task DeleteViolationIndexesAsync(bool pendingOnly = true)
|
||||
{
|
||||
long afterId = 0;
|
||||
while (true)
|
||||
{
|
||||
await using var lease = await indexLock.AcquireAsync();
|
||||
var ids = await repository.GetBatchAsync(afterId, 1000, pendingOnly);
|
||||
if (ids.Count == 0) return;
|
||||
await search.DeleteDocumentsAsync(ids);
|
||||
await repository.MarkDeletedAsync(ids.ToArray());
|
||||
afterId = ids[^1];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ namespace Store.Modules.CloudProduct.Services
|
||||
public interface IOpenSearchService : ISingletonDependency
|
||||
{
|
||||
Task AddDocumentAsync(GoodsSearchDocument document);
|
||||
Task BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents);
|
||||
Task<long[]> BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents);
|
||||
Task<bool> ClearIndexAsync();
|
||||
Task<long> CountDocumentsAsync();
|
||||
Task<bool> CreateIndexAsync();
|
||||
@@ -26,7 +26,7 @@ namespace Store.Modules.CloudProduct.Services
|
||||
Task<bool> IndexExistsAsync();
|
||||
Task<bool> IsIndexMappingHealthyAsync();
|
||||
Task RefreshIndexAsync();
|
||||
Task<long> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList);
|
||||
Task<long[]> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList);
|
||||
Task<(long Total, List<GoodsSearchDocument> Documents)> SearchByTitleAsync(GoodsSearchRequest request, GoodsSearchSortBy sortBy = GoodsSearchSortBy.Relevance);
|
||||
}
|
||||
|
||||
@@ -41,10 +41,15 @@ namespace Store.Modules.CloudProduct.Services
|
||||
// 批量写入/建索引/删除索引等管理类操作超时(数据量大时需要更长,可挪到配置文件)
|
||||
private static readonly TimeSpan AdminTimeout = TimeSpan.FromSeconds(30);
|
||||
private readonly ILogger<OpenSearchService> _logger;
|
||||
private readonly IFreeSql _database;
|
||||
private readonly GoodsIndexLock _indexLock;
|
||||
|
||||
public OpenSearchService([FromKeyedServices("CloudProduct")] PluginContext context,ILogger<OpenSearchService> logger)
|
||||
{
|
||||
var config = context.Configuration.GetSection("Search");
|
||||
public OpenSearchService([FromKeyedServices("CloudProduct")] PluginContext context, ILogger<OpenSearchService> logger,
|
||||
[FromKeyedServices("CloudProductConnection")] IFreeSql database, GoodsIndexLock indexLock)
|
||||
{
|
||||
_database = database;
|
||||
_indexLock = indexLock;
|
||||
var config = context.Configuration.GetSection("Search");
|
||||
_nodeUrl = config.GetValue<string>("Url") ?? "";
|
||||
_indexName = config.GetValue<string>("IndexName") ?? "";
|
||||
_client = CreateClient();
|
||||
@@ -206,12 +211,18 @@ namespace Store.Modules.CloudProduct.Services
|
||||
/// <summary>
|
||||
/// ============ 3.1 批量写入商品搜索文档(存在则覆盖) ============
|
||||
/// </summary>
|
||||
public async Task BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents)
|
||||
public async Task<long[]> BulkIndexGoodsAsync(IEnumerable<GoodsSearchDocument> documents)
|
||||
{
|
||||
if (documents == null) throw new ArgumentNullException(nameof(documents));
|
||||
|
||||
var list = documents.ToList();
|
||||
if (list.Count == 0) return;
|
||||
if (list.Count == 0) return [];
|
||||
await using var lease = await _indexLock.AcquireAsync();
|
||||
var ids = list.Select(x => x.Id).Distinct().ToArray();
|
||||
var blocked = (await _database.Select<GoodsViolationEntity>()
|
||||
.Where(x => ids.Contains(x.GoodsId)).ToListAsync(x => x.GoodsId)).ToHashSet();
|
||||
list = list.Where(x => !blocked.Contains(x.Id)).ToList();
|
||||
if (list.Count == 0) return [];
|
||||
|
||||
// 确保索引已按 IK 分词映射创建,否则中文分词无效
|
||||
await CreateIndexAsync();
|
||||
@@ -222,19 +233,20 @@ namespace Store.Modules.CloudProduct.Services
|
||||
.RequestConfiguration(r => r.RequestTimeout(AdminTimeout))
|
||||
);
|
||||
|
||||
if (!bulkResponse.IsValid)
|
||||
if (!bulkResponse.IsValid || bulkResponse.Errors)
|
||||
{
|
||||
throw new AppException("写入 OpenSearch 失败: " + bulkResponse.DebugInformation);
|
||||
}
|
||||
|
||||
// 刷新索引,保证数据立即可搜索(仅用于演示,生产环境不建议频繁 Refresh)
|
||||
await RefreshIndexAsync();
|
||||
return list.Select(x => x.Id).Distinct().ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ============ 3.2 根据数据库商品重建搜索索引 ============
|
||||
/// </summary>
|
||||
public async Task<long> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList)
|
||||
public async Task<long[]> RebuildIndexAsync(IEnumerable<GoodsEntity> goodsList)
|
||||
{
|
||||
if (goodsList == null) throw new ArgumentNullException(nameof(goodsList));
|
||||
|
||||
@@ -242,6 +254,8 @@ namespace Store.Modules.CloudProduct.Services
|
||||
.Where(x => x != null && x.GoodsId > 0)
|
||||
.Select(x => new GoodsSearchDocument
|
||||
{
|
||||
Platform = x.Platform,
|
||||
ThumbUrl = x.ThumbUrl,
|
||||
Id = x.GoodsId,
|
||||
Title = x.Title ?? string.Empty,
|
||||
CatId1 = x.CatId1,
|
||||
@@ -253,10 +267,7 @@ namespace Store.Modules.CloudProduct.Services
|
||||
})
|
||||
.ToList();
|
||||
|
||||
if (documents.Count == 0) return 0;
|
||||
|
||||
await BulkIndexGoodsAsync(documents);
|
||||
return documents.Count;
|
||||
return await BulkIndexGoodsAsync(documents);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -271,7 +282,8 @@ namespace Store.Modules.CloudProduct.Services
|
||||
.RequestConfiguration(r => r.RequestTimeout(QueryTimeout)));
|
||||
if (!exists.Exists)
|
||||
{
|
||||
return;
|
||||
if (exists.ApiCall?.HttpStatusCode == 404) return;
|
||||
throw new AppException("检查 OpenSearch 索引失败: " + exists.DebugInformation);
|
||||
}
|
||||
|
||||
var response = await _client.BulkAsync(b =>
|
||||
@@ -284,7 +296,9 @@ namespace Store.Modules.CloudProduct.Services
|
||||
return b;
|
||||
});
|
||||
|
||||
if (!response.IsValid)
|
||||
if (response.ApiCall?.HttpStatusCode is not (>= 200 and < 300)
|
||||
|| response.Items.Count != list.Count
|
||||
|| response.Items.Any(x => x.Status != 404 && (x.Error != null || x.Status < 200 || x.Status >= 300)))
|
||||
{
|
||||
throw new AppException("删除 OpenSearch 文档失败: " + response.DebugInformation);
|
||||
}
|
||||
@@ -298,6 +312,9 @@ namespace Store.Modules.CloudProduct.Services
|
||||
public async Task AddDocumentAsync(GoodsSearchDocument document)
|
||||
{
|
||||
if (document == null) throw new ArgumentNullException(nameof(document));
|
||||
await using var lease = await _indexLock.AcquireAsync();
|
||||
if (await _database.Select<GoodsViolationEntity>().AnyAsync(x => x.GoodsId == document.Id))
|
||||
throw new AppException("违规商品不能写入搜索索引");
|
||||
|
||||
var response = await _client.IndexAsync(document, i => i
|
||||
.Index(_indexName)
|
||||
@@ -524,6 +541,7 @@ namespace Store.Modules.CloudProduct.Services
|
||||
/// </summary>
|
||||
public async Task<bool> ClearIndexAsync()
|
||||
{
|
||||
await using var lease = await _indexLock.AcquireAsync();
|
||||
var existsResponse = await _client.Indices.ExistsAsync(_indexName, d => d
|
||||
.RequestConfiguration(r => r.RequestTimeout(QueryTimeout)));
|
||||
if (!existsResponse.Exists)
|
||||
@@ -541,7 +559,10 @@ namespace Store.Modules.CloudProduct.Services
|
||||
}
|
||||
Console.WriteLine($"索引 {_indexName} 已删除");
|
||||
|
||||
return await CreateIndexAsync();
|
||||
var created = await CreateIndexAsync();
|
||||
await _database.Update<GoodsEntity>().Set(x => x.IsSegmented, (byte)0)
|
||||
.Where(x => x.Status == 0).ExecuteAffrowsAsync();
|
||||
return created;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user