From fe3943796827a9ff448aa244bd56ba29a57dc2d0 Mon Sep 17 00:00:00 2001 From: chfei Date: Mon, 10 Aug 2026 11:29:06 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E8=AE=A1=20sql=E8=AF=AD=E5=8F=A5=20?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BLL/WXXD/WXShopRevenueStatisticsManage.cs | 2 +- .../KJD_WebService/Utils/ApiRequestLogger.cs | 206 ++++++++++++++---- 2 files changed, 168 insertions(+), 40 deletions(-) diff --git a/KJDService/KJD_WebService/BLL/WXXD/WXShopRevenueStatisticsManage.cs b/KJDService/KJD_WebService/BLL/WXXD/WXShopRevenueStatisticsManage.cs index 9ec8adc..6a32003 100644 --- a/KJDService/KJD_WebService/BLL/WXXD/WXShopRevenueStatisticsManage.cs +++ b/KJDService/KJD_WebService/BLL/WXXD/WXShopRevenueStatisticsManage.cs @@ -92,7 +92,7 @@ namespace KJD_Web.BLL.WXXD sb.Append("LEFT JOIN ( SELECT order_id,`status`,refund_amount,id FROM erp.`oms_wei_refund` WHERE id IN ( SELECT MAX(id) FROM erp.`oms_wei_refund` GROUP BY order_id)) AS C ON A.order_id = C.order_id "); sb.Append("WHERE A.create_time BETWEEN " + startTimestamp + " AND " + endTimestamp + " "); sb.Append("AND A.`status` > 10 "); - sb.Append("AND B.shop_id IN (" + shop_ids + ") "); + sb.Append("AND A.shop_id IN (" + shop_ids + ") "); sb.Append("ORDER BY A.create_time asc"); string conn = System.Configuration.ConfigurationManager.ConnectionStrings["ConnKJD"].ConnectionString; diff --git a/KJDService/KJD_WebService/Utils/ApiRequestLogger.cs b/KJDService/KJD_WebService/Utils/ApiRequestLogger.cs index 118e499..9e00220 100644 --- a/KJDService/KJD_WebService/Utils/ApiRequestLogger.cs +++ b/KJDService/KJD_WebService/Utils/ApiRequestLogger.cs @@ -1,9 +1,11 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Configuration; using System.IO; using System.Text; using System.Text.RegularExpressions; +using System.Threading; using System.Web; using Newtonsoft.Json; @@ -11,11 +13,14 @@ namespace KJD_Web.Utils { /// /// API 请求录制工具：挂接在 Global.asax 的 BeginRequest / EndRequest， - /// 捕获请求链接(action/service)、请求内容(body/query)与响应，按天落盘为 JSON Lines， + /// 捕获请求链接(action/service)、请求内容(body/query)与响应， + /// 按请求路径生成文件夹，每一条请求落盘为一个 JSON 文件， /// 用于 .NET Core 迁移后的回放校验。 /// 配置（Web.config appSettings）： /// ApiRequestLog = 1 开启 / 0 关闭（默认开启） /// ApiRequestLogDir = 日志目录（默认 ~/App_Data/ApiLogs） + /// 存储上限：达到 MaxStoredRequests（常量）后跳过不再存储。 + /// 写盘为后台异步：请求线程只入队，由后台线程序列化并落盘，不阻塞接口返回。 /// public static class ApiRequestLogger { @@ -23,6 +28,11 @@ namespace KJD_Web.Utils private const string FilterItemKey = "__ApiResponseCapture__"; private const int MaxBodyLogLength = 100000; + /// 最大存储条数（所有请求路径合计）。达到该数量后，跳过不再存储新请求。 + private const int MaxStoredRequests = 10000; + /// 后台队列最大积压条数；队列满时丢弃新日志（不影响接口返回）。 + private const int MaxQueuedEntries = 5000; + private static readonly object SyncRoot = new object(); private static readonly Regex SensitiveKeyValueRegex = new Regex( @"(?i)(\b(?:password|passwd|pwd|token|access_token|secret|app_secret|client_secret|sign|signature|api_key|authorization|key)=)([^&\s""]*)", @@ -43,6 +53,14 @@ namespace KJD_Web.Utils private static bool _configLoaded; private static bool _enabled = true; private static string _logDir; + /// 已存储条数；-1 表示尚未统计（由后台线程统计）。 + private static long _storedCount = -1; + /// 后台写盘队列：请求线程只入队，不落盘。 + private static readonly BlockingCollection _queue = + new BlockingCollection(new ConcurrentQueue(), MaxQueuedEntries); + private static Thread _writerThread; + /// 已创建过的文件夹缓存，避免每次写盘重复 CreateDirectory。 + private static readonly HashSet _createdDirs = new HashSet(StringComparer.OrdinalIgnoreCase); private sealed class RequestRecord { @@ -59,6 +77,14 @@ namespace KJD_Web.Utils public string ua; } + /// 待后台写盘的一条日志。 + private sealed class LogEntry + { + public RequestRecord Record; + public byte[] ResponseBytes; + public int StatusCode; + } + /// 请求入口：解析 action/service、读取并缓存请求体、安装响应捕获。 public static void BeginRequest(HttpContext context) { @@ -66,6 +92,7 @@ namespace KJD_Web.Utils { EnsureConfig(context); if (!_enabled) return; + if (Interlocked.Read(ref _storedCount) >= MaxStoredRequests) return; // 达到上限，跳过录制 string path = context.Request.Path ?? ""; if (!IsApiPath(path)) return; @@ -104,7 +131,7 @@ namespace KJD_Web.Utils catch { } } - /// 请求出口：合并响应，写一条 JSON Lines 记录。 + /// 请求出口：只把日志放入后台队列，由后台线程序列化并写盘，不阻塞请求。 public static void EndRequest(HttpContext context) { try @@ -113,36 +140,16 @@ namespace KJD_Web.Utils var record = context.Items[ItemKey] as RequestRecord; if (record == null) return; - string responseText = null; + byte[] responseBytes = null; try { var filter = context.Items[FilterItemKey] as ApiResponseCaptureStream; - if (filter != null) responseText = Truncate(Desensitize(filter.GetCapturedText())); + if (filter != null) responseBytes = filter.GetCapturedBytes(); } catch { } - var line = JsonConvert.SerializeObject(new - { - id = record.id, - time = record.time.ToString("yyyy-MM-dd HH:mm:ss.fff"), - service = record.service, - action = record.action, - method = record.method, - url = record.url, - path = record.path, - query = record.query, - body = record.body, - response = responseText, - ip = record.ip, - ua = record.ua, - status = SafeGet(() => context.Response.StatusCode, 0) - }); - - string fileName = Path.Combine(_logDir, "api_" + DateTime.Now.ToString("yyyyMMdd") + ".jsonl"); - lock (SyncRoot) - { - File.AppendAllText(fileName, line + Environment.NewLine, Encoding.UTF8); - } + int status = SafeGet(() => context.Response.StatusCode, 0); + _queue.TryAdd(new LogEntry { Record = record, ResponseBytes = responseBytes, StatusCode = status }); } catch { } } @@ -165,10 +172,139 @@ namespace KJD_Web.Utils if (!Directory.Exists(_logDir)) Directory.CreateDirectory(_logDir); } catch { } + StartWriterThread(); // 后台线程负责统计条数与落盘 _configLoaded = true; } } + /// 统计日志目录下已存储的 JSON 请求文件数量。 + private static long CountStoredFiles() + { + try + { + if (!Directory.Exists(_logDir)) return 0; + return Directory.GetFiles(_logDir, "*.json", SearchOption.AllDirectories).LongLength; + } + catch { return 0; } + } + + /// 按请求路径生成存储文件夹（每段路径一层子目录，清洗非法字符、防目录穿越）。 + private static string GetRecordDirectory(string path) + { + string p = (path ?? "").Trim().TrimStart('/').TrimStart('\\'); + if (string.IsNullOrEmpty(p)) p = "_root"; + + var sb = new StringBuilder(_logDir); + string[] segs = p.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + foreach (string seg in segs) + { + if (seg == "." || seg == "..") continue; // 防目录穿越 + string clean = SanitizeSegment(seg); + if (string.IsNullOrEmpty(clean)) continue; + sb.Append(Path.DirectorySeparatorChar).Append(clean); + } + return sb.ToString(); + } + + /// 清洗路径段为合法的文件夹名。 + private static string SanitizeSegment(string segment) + { + if (string.IsNullOrEmpty(segment)) return null; + char[] invalid = Path.GetInvalidFileNameChars(); + var sb = new StringBuilder(segment.Length); + foreach (char c in segment) + { + sb.Append(Array.IndexOf(invalid, c) >= 0 ? '_' : c); + } + string s = sb.ToString().Trim().TrimEnd('.'); + if (s.Length == 0) return "_"; + if (s.Length > 80) s = s.Substring(0, 80); + return s; + } + + /// 启动后台写盘线程（仅一次）。 + private static void StartWriterThread() + { + if (_writerThread != null) return; + lock (SyncRoot) + { + if (_writerThread != null) return; + var thread = new Thread(WriterLoop) { IsBackground = true, Name = "ApiRequestLogWriter" }; + _writerThread = thread; + thread.Start(); + } + } + + /// 后台写盘主循环：先统计已有条数，再持续消费队列。 + private static void WriterLoop() + { + try + { + if (Interlocked.Read(ref _storedCount) < 0) + Interlocked.Exchange(ref _storedCount, CountStoredFiles()); + } + catch { } + + foreach (LogEntry entry in _queue.GetConsumingEnumerable()) + { + try { WriteEntry(entry); } + catch { } + } + } + + /// 在后台线程中把一条日志序列化并写盘。 + private static void WriteEntry(LogEntry entry) + { + if (Interlocked.Read(ref _storedCount) >= MaxStoredRequests) return; // 达到上限，跳过不再存储 + + string responseText = entry.ResponseBytes == null || entry.ResponseBytes.Length == 0 + ? "" + : Truncate(Desensitize(BytesToText(entry.ResponseBytes))); + + var json = JsonConvert.SerializeObject(new + { + id = entry.Record.id, + time = entry.Record.time.ToString("yyyy-MM-dd HH:mm:ss.fff"), + service = entry.Record.service, + action = entry.Record.action, + method = entry.Record.method, + url = entry.Record.url, + path = entry.Record.path, + query = entry.Record.query, + body = entry.Record.body, + response = responseText, + ip = entry.Record.ip, + ua = entry.Record.ua, + status = entry.StatusCode + }); + + lock (SyncRoot) + { + if (Interlocked.Read(ref _storedCount) >= MaxStoredRequests) return; // 双检，防止超限 + + string dir = GetRecordDirectory(entry.Record.path); + if (_createdDirs.Add(dir)) Directory.CreateDirectory(dir); + + string fileName = entry.Record.time.ToString("yyyyMMdd_HHmmss_fff") + "_" + entry.Record.id + ".json"; + File.WriteAllText(Path.Combine(dir, fileName), json, new UTF8Encoding(false)); // 不带 BOM 的 UTF-8 + Interlocked.Increment(ref _storedCount); + } + } + + /// 响应字节转文本；二进制内容只记录长度。 + private static string BytesToText(byte[] bytes) + { + if (bytes == null || bytes.Length == 0) return ""; + bool binary = false; + for (int i = 0; i < bytes.Length; i++) + { + byte b = bytes[i]; + if (b == 0 || b < 8) { binary = true; break; } + } + if (binary) return ""; + return Encoding.UTF8.GetString(bytes); + } + private static string ReadBody(HttpContext context) { try @@ -256,25 +392,17 @@ namespace KJD_Web.Utils _inner = inner ?? new MemoryStream(); } + public byte[] GetCapturedBytes() + { + return _buffer.ToArray(); + } + public string GetCapturedText() { if (_capturedText == null) _capturedText = BytesToText(_buffer.ToArray()); return _capturedText; } - private static string BytesToText(byte[] bytes) - { - if (bytes == null || bytes.Length == 0) return ""; - bool binary = false; - for (int i = 0; i < bytes.Length; i++) - { - byte b = bytes[i]; - if (b == 0 || b < 8) { binary = true; break; } - } - if (binary) return ""; - return Encoding.UTF8.GetString(bytes); - } - public override bool CanRead { get { return _inner.CanRead; } } public override bool CanSeek { get { return _inner.CanSeek; } } public override bool CanWrite { get { return _inner.CanWrite; } }