chfei/KJD_WebService
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

导入外部多多数据 增加新格式

Browse Source 
修改文件格式检测函数
This commit is contained in:
chfei committed 2026-08-11 16:32:42 +08:00
1 parent a568c685ee
commit 7d07c83579
6 files changed
+5090 -4535
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Work/Api/UploadPinduoduoMain.cs
+155 -104
View File
@@ -1174,120 +1174,171 @@ namespace Work.Api
public static System.Text.Encoding GetFileEncoding(string filename)
{
using (var fs = new System.IO.FileStream(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
using (var br = new System.IO.BinaryReader(fs))
{
var buffer = br.ReadBytes(2);
if (buffer[0] >= 0xEF)
{
if (buffer[0] == 0xEF && buffer[1] == 0xBB)
{
return System.Text.Encoding.UTF8;
}
byte[] bytes = File.ReadAllBytes(filename);
return DetectEncoding(bytes);
if (buffer[0] == 0xFE && buffer[1] == 0xFF)
{
return System.Text.Encoding.BigEndianUnicode;
}
}
/// <summary>
/// 检测字节数组的编码
/// </summary>
public static Encoding DetectEncoding(byte[] bytes)
{
if (bytes == null || bytes.Length == 0)
return Encoding.UTF8;
if (buffer[0] == 0xFF && buffer[1] == 0xFE)
{
return System.Text.Encoding.Unicode;
}
}
// ---------- 1. 优先判断 BOM ----------
Encoding bomEncoding = DetectByBom(bytes);
if (bomEncoding != null)
return bomEncoding;
return GetEncodingWithBomUtf8(fs, System.Text.Encoding.Default);
}
// ---------- 2. 判断是否为合法 UTF-8(无 BOM) ----------
if (IsValidUtf8(bytes))
return new UTF8Encoding(false); // 无 BOM 的 UTF-8
}
private static System.Text.Encoding GetEncodingWithBomUtf8(Stream fs, Encoding defaultEncoding)
{
byte[] unicode = new byte[] { 0xFF, 0xFE, 0x41 };
byte[] unicodeBig = new byte[] { 0xFE, 0xFF, 0x00 };
// ---------- 3. 判断是否符合 GBK/GB2312 双字节规则 ----------
if (IsLikelyGbk(bytes))
{
// GB18030 兼容 GBK/GB2312,是更安全的选择
try
{
return Encoding.GetEncoding("GB18030");
}
catch
{
// 某些精简运行时(如未注册 CodePages Provider)可能取不到,
// 需要在项目里加 <PackageReference Include="System.Text.Encoding.CodePages" />
// 并在启动时调用 Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
}
}
//带BOM
byte[] utf8 = new byte[] { 0xEF, 0xBB, 0xBF };
var reVal = defaultEncoding;
// ---------- 4. 兜底 ----------
return Encoding.Default; // 或者你想要的 fallback,比如 Encoding.UTF8
}
using (var r = new System.IO.BinaryReader(fs))
{
byte[] ss = r.ReadBytes(4);
if (ss[0] == 0xFE && ss[1] == 0xFF && ss[2] == 0x00)
{
reVal = Encoding.BigEndianUnicode;
}
else if (ss[0] == 0xFF && ss[1] == 0xFE && ss[2] == 0x41)
{
reVal = Encoding.Unicode;
}
else
{
if (ss[0] == 0xEF && ss[1] == 0xBB && ss[2] == 0xBF)
{
reVal = Encoding.UTF8;
}
else
{
int i;
int.TryParse(fs.Length.ToString(), out i);
ss = r.ReadBytes(i);
/// <summary>
/// 根据 BOM 判断编码,没有 BOM 返回 null
/// </summary>
private static Encoding DetectByBom(byte[] bytes)
{
if (bytes.Length >= 4 &&
bytes[0] == 0x00 && bytes[1] == 0x00 &&
bytes[2] == 0xFE && bytes[3] == 0xFF)
return new UTF32Encoding(true, true); // UTF-32 BE
if (IsUtf8Bytes(ss))
{
reVal = Encoding.UTF8;
}
}
}
if (bytes.Length >= 4 &&
bytes[0] == 0xFF && bytes[1] == 0xFE &&
bytes[2] == 0x00 && bytes[3] == 0x00)
return new UTF32Encoding(false, true); // UTF-32 LE
return reVal;
}
}
if (bytes.Length >= 3 &&
bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF)
return new UTF8Encoding(true); // UTF-8 with BOM
/// <summary>
/// 判断是否是不带 BOM 的 UTF8 格式
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
private static bool IsUtf8Bytes(byte[] data)
{
int charByteCounter = 1;　 //计算当前正分析的字符应还有的字节数
for (int i = 0; i < data.Length; i++)
{
var curByte = data[i]; //当前分析的字节.
if (charByteCounter == 1)
{
if (curByte >= 0x80)
{
//判断当前
while (((curByte <<= 1) & 0x80) != 0)
{
charByteCounter++;
}
//标记位首位若为非0 则至少以2个1开始 如:110XXXXX...........1111110X　
if (charByteCounter == 1 || charByteCounter > 6)
{
return false;
}
}
}
else
{
//若是UTF-8 此时第一位必须为1
if ((curByte & 0xC0) != 0x80)
{
return false;
}
charByteCounter--;
}
}
if (charByteCounter > 1)
{
throw new Exception("非预期的byte格式!");
}
return true;
}
if (bytes.Length >= 2 &&
bytes[0] == 0xFE && bytes[1] == 0xFF)
return new UnicodeEncoding(true, true); // UTF-16 BE
public void GetCat()
if (bytes.Length >= 2 &&
bytes[0] == 0xFF && bytes[1] == 0xFE)
return new UnicodeEncoding(false, true); // UTF-16 LE
return null;
}
/// <summary>
/// 严格校验字节序列是否为合法 UTF-8
/// </summary>
private static bool IsValidUtf8(byte[] bytes)
{
int i = 0;
int len = bytes.Length;
bool hasMultiByteSequence = false;
while (i < len)
{
byte b = bytes[i];
if (b <= 0x7F) // 单字节 ASCII
{
i++;
continue;
}
int extraBytes;
if ((b & 0xE0) == 0xC0) extraBytes = 1; // 110xxxxx
else if ((b & 0xF0) == 0xE0) extraBytes = 2; // 1110xxxx
else if ((b & 0xF8) == 0xF0) extraBytes = 3; // 11110xxx
else return false; // 不合法的起始字节
if (i + extraBytes >= len)
return false; // 长度不够
for (int j = 1; j <= extraBytes; j++)
{
if ((bytes[i + j] & 0xC0) != 0x80) // 后续字节必须是 10xxxxxx
return false;
}
hasMultiByteSequence = true;
i += extraBytes + 1;
}
// 纯 ASCII 内容严格来说 UTF-8/GBK/ASCII 都符合,
// 这里只有出现过合法多字节序列才更有把握判定为 UTF-8。
// 如果你希望纯 ASCII 也直接归为 UTF-8,可以把下面这行去掉判断直接 return true。
return hasMultiByteSequence || IsAllAscii(bytes);
}
private static bool IsAllAscii(byte[] bytes)
{
foreach (byte b in bytes)
{
if (b > 0x7F) return false;
}
return true;
}
/// <summary>
/// 启发式判断是否符合 GBK/GB2312 双字节编码规则
/// GBK: 首字节 0x81-0xFE,次字节 0x40-0xFE(不含 0x7F)
/// </summary>
private static bool IsLikelyGbk(byte[] bytes)
{
int i = 0;
int len = bytes.Length;
bool hasDoubleByte = false;
while (i < len)
{
byte b = bytes[i];
if (b <= 0x7F) // ASCII
{
i++;
continue;
}
if (b >= 0x81 && b <= 0xFE)
{
if (i + 1 >= len) return false;
byte b2 = bytes[i + 1];
if (b2 >= 0x40 && b2 <= 0xFE && b2 != 0x7F)
{
hasDoubleByte = true;
i += 2;
continue;
}
return false;
}
return false; // 不符合 GBK 规则的字节
}
return hasDoubleByte;
}
public void GetCat()
{
int CatThreadCount = 20;
if (FromTitleToCat.UserApi)