29Wx状态初始化
This commit is contained in:
1 parent
9fd5579fcd
commit
40c702883c
322 files changed
+155553
No files matched your search
@@ -0,0 +1,425 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace AllBase
|
||||
{
|
||||
public class Segment
|
||||
{
|
||||
private static Dictionary<int, List<string>> WordList = new Dictionary<int, List<string>>();
|
||||
|
||||
/// <summary>
|
||||
/// 关键词队列
|
||||
/// </summary>
|
||||
private string words = " ";
|
||||
|
||||
/// <summary>
|
||||
/// 获取绝对地址
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private static string GetPatch()
|
||||
{
|
||||
string patch = Process.GetCurrentProcess().MainModule.FileName;
|
||||
string[] l = patch.Split('\\');
|
||||
patch = "";
|
||||
for (int i = 0; i < l.Length - 1; i++)
|
||||
patch += l[i] + "\\";
|
||||
return patch;
|
||||
}
|
||||
/// <summary>
|
||||
/// 初始化
|
||||
/// </summary>
|
||||
public static void Int()
|
||||
{
|
||||
//MysqlHelper.ErrorString += "正在初始化分词库\r\n";
|
||||
string f = GetPatch() + "\\Segment.dll";
|
||||
if (!File.Exists(f))
|
||||
GetHtml.DownloadFile("http://dl.1n11.com/Segment.xml", f);
|
||||
if (!File.Exists(f))
|
||||
{
|
||||
BaseClass.ViewCode += "分词库加载失败,请前往官网下载完整版本";
|
||||
return;
|
||||
}
|
||||
if (WordList.Count > 0)
|
||||
return;
|
||||
for (int i = 2; i < 10; i++)
|
||||
{
|
||||
List<string> w = new List<string>();
|
||||
WordList.Add(i, w);
|
||||
}
|
||||
string[] l = File.ReadAllLines(f, Encoding.UTF8);
|
||||
for (int i = 0; i < l.Length; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(l[i]))
|
||||
continue;
|
||||
string key = l[i];
|
||||
if (key.Contains("\t") && key.IndexOf("\t")>0)
|
||||
key = key.Substring(0, key.IndexOf("\t"));
|
||||
if (key.Length < 2 || key.Length > 8)
|
||||
continue;
|
||||
WordList[key.Length].Add(key);
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// 二分法提取关键词
|
||||
/// </summary>
|
||||
/// <param name="len"></param>
|
||||
/// <param name="WordList"></param>
|
||||
/// <param name="html"></param>
|
||||
/// <param name="word"></param>
|
||||
private void GetKeywordList(int len, ref List<string> WordList, string html, ref List<string> word)
|
||||
{
|
||||
string temp = "";
|
||||
for (int i = 0; i <= html.Length - len; )
|
||||
{
|
||||
temp = html.Substring(i, len);
|
||||
if (!temp.Contains(" ") && Dichotomy(temp, ref WordList))
|
||||
{
|
||||
AddWord(temp, ref word);
|
||||
i += len;
|
||||
continue;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 分词
|
||||
/// </summary>
|
||||
/// <param name="html"></param>
|
||||
/// <returns></returns>
|
||||
public List<string> Get(string html)
|
||||
{
|
||||
if (string.IsNullOrEmpty(html))
|
||||
return null;
|
||||
words = " ";
|
||||
//WordList.Clear();
|
||||
html = html.ToLower();
|
||||
if (html.Length > 20000)
|
||||
html = html.Substring(0, 20000);
|
||||
BaseRegex.Relpace(ref html, @"(&[\w]+;)", " ");
|
||||
BaseRegex.Relpace(ref html, @"([ ]{2,2000})", " ");
|
||||
if (WordList.Count == 0)
|
||||
{
|
||||
BaseClass.ViewCode += "分词库为空,请检查系统是否正常";
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] b = Encoding.UTF8.GetBytes(html);
|
||||
//===============先处理特殊字符================//
|
||||
for (int i = 9; i > 1; i--)
|
||||
GetKeywords2(WordList[i], ref b, i * 3);
|
||||
|
||||
//GetKeywords2(ref WordList2, ref b, 6);
|
||||
string[] l = new[]
|
||||
{
|
||||
@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)",
|
||||
@"([-a-z0-9\.@]+)"
|
||||
|
||||
};
|
||||
for (int i = 0; i < l.Length; i++)
|
||||
{
|
||||
var m = BaseRegex.GetMatchRegex(ref html, l[i]);
|
||||
AddKeyword(ref m);
|
||||
}
|
||||
List<string> word = ReturnList();
|
||||
if (html.Length > 15000 || word.Count > 3000) //太长则不执行消耗资源的自动学习功能
|
||||
return word;
|
||||
|
||||
//==========自动学习关键词=========//
|
||||
for (int i = 0; i < word.Count; i++)
|
||||
html = html.Replace(word[i], " ");
|
||||
html = html.Replace("的", " ");
|
||||
BaseRegex.Relpace(ref html, @"([ ]+)", " ");
|
||||
char t;
|
||||
for (int i = 0; i < html.Length; i++)
|
||||
{
|
||||
t = char.Parse(html.Substring(i, 1));
|
||||
if (t != ' ' && !IsChineseWord(t))
|
||||
html = html.Replace(t.ToString(), " ");
|
||||
}
|
||||
BaseRegex.Relpace(ref html, @"([ ]+)", " ");
|
||||
if (string.IsNullOrEmpty(html))
|
||||
return word;
|
||||
string[] li = html.Split(' ');
|
||||
//========关键词长度倒排=========//
|
||||
for (int i = 0; i < li.Length; i++)
|
||||
{
|
||||
for (int j = i; j < li.Length; j++)
|
||||
{
|
||||
if (li[i].Length < li[j].Length)
|
||||
{
|
||||
string x = li[i];
|
||||
li[i] = li[j];
|
||||
li[j] = x;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < li.Length; i++)
|
||||
{
|
||||
string key = li[i];
|
||||
if (key.Length <= 1 || key.Length > 4)
|
||||
continue;
|
||||
if (!IsWordList(key))
|
||||
continue;
|
||||
AddWord(key, ref word);
|
||||
}
|
||||
return word;
|
||||
}
|
||||
/// <summary>
|
||||
/// 返回列表
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private List<string> ReturnList()
|
||||
{
|
||||
List<string> w = new List<string>();
|
||||
if (string.IsNullOrEmpty(words))
|
||||
return w;
|
||||
string[] l = words.Split(' ');
|
||||
foreach (string s in l)
|
||||
{
|
||||
if (string.IsNullOrEmpty(s))
|
||||
continue;
|
||||
w.Add(s);
|
||||
}
|
||||
return w;
|
||||
}
|
||||
/// <summary>
|
||||
/// 正则添加到队列
|
||||
/// </summary>
|
||||
/// <param name="m"></param>
|
||||
private void AddKeyword(ref MatchCollection m)
|
||||
{
|
||||
for (int i = 0; m != null && i < m.Count; i++)
|
||||
AddWordx(m[i].Groups[0].Value);
|
||||
}
|
||||
/// <summary>
|
||||
/// 字符串中是否包含中文
|
||||
/// </summary>
|
||||
/// <param name="words"></param>
|
||||
/// <returns></returns>
|
||||
public static bool HadChineseWord(ref string words)
|
||||
{
|
||||
if(string.IsNullOrEmpty(words))
|
||||
return false;
|
||||
var b = Encoding.UTF8.GetBytes(words);
|
||||
foreach (byte b1 in b)
|
||||
if(b1>227 && b1<235)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 软加速
|
||||
/// </summary>
|
||||
/// <param name="WordList">关键词队列</param>
|
||||
/// <param name="b">二进制文章</param>
|
||||
/// <param name="len">二进制长度</param>
|
||||
private void GetKeywords2(List<string> WordList, ref byte[] b, int len)
|
||||
{
|
||||
string t;
|
||||
for (int i = 0; i < b.Length - len; i++)
|
||||
{
|
||||
switch (len)
|
||||
{
|
||||
case 6:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 9:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 12:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
||||
&& b[i + 9] < 235 && b[i + 9] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 15:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
||||
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 18:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
||||
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 21:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
||||
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227
|
||||
&& b[i + 18] < 235 && b[i + 18] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
case 24:
|
||||
case 27:
|
||||
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
||||
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227
|
||||
&& b[i + 18] < 235 && b[i + 18] > 227 && b[i + 21] < 235 && b[i + 21] > 227)
|
||||
goto Add;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
Add:
|
||||
t = Encoding.UTF8.GetString(b, i, len);
|
||||
if (Dichotomy(t, ref WordList))
|
||||
{
|
||||
AddWordx(t);
|
||||
i += len - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// 添加关键词到列
|
||||
/// </summary>
|
||||
/// <param name="keyword"></param>
|
||||
private void AddWordx(string keyword)
|
||||
{
|
||||
if (words.Contains(" " + keyword + " "))
|
||||
return;
|
||||
words += keyword + " ";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 二分法关键词是否存在
|
||||
/// </summary>
|
||||
/// <param name="word">关键词</param>
|
||||
/// <returns></returns>
|
||||
public bool Dichotomy(string word, ref List<string> WordList)
|
||||
{
|
||||
if (WordList.Count < 1)
|
||||
return false;
|
||||
int start = 0, end = WordList.Count - 1, middle = 0, ok = 0;
|
||||
while (start != end)
|
||||
{
|
||||
middle = (start + end) / 2;
|
||||
ok = string.Compare(word, WordList[middle], true);
|
||||
if (ok == 0)
|
||||
return true;
|
||||
if (ok == 1)
|
||||
start = middle;
|
||||
else
|
||||
end = middle;
|
||||
if (start + 1 == end)
|
||||
{
|
||||
if (string.Compare(word, WordList[end], true) == 0)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 读取正则结果列表
|
||||
/// </summary>
|
||||
/// <param name="m"></param>
|
||||
/// <param name="word"></param>
|
||||
private void GetMatch(ref MatchCollection m, ref List<string> word, ref string html)
|
||||
{
|
||||
for (int i = 0; i < m.Count; i++)
|
||||
if (AddWord(m[i].Groups[0].Value, ref word))
|
||||
html = html.Replace(m[i].Groups[0].Value, " ");
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 添加关键词到列
|
||||
/// </summary>
|
||||
/// <param name="keyword"></param>
|
||||
/// <param name="word"></param>
|
||||
private bool AddWord(string keyword, ref List<string> word)
|
||||
{
|
||||
bool had = false;
|
||||
for (int i = 0; i < word.Count; i++)
|
||||
{
|
||||
if (word[i] == keyword)
|
||||
{
|
||||
had = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!had)
|
||||
{
|
||||
string b = keyword;
|
||||
word.Add(b);
|
||||
}
|
||||
return !had;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 是否是中文字符串列
|
||||
/// </summary>
|
||||
/// <param name="word"></param>
|
||||
/// <returns></returns>
|
||||
public static bool IsWordList(string word)
|
||||
{
|
||||
char[] xj = word.ToCharArray();
|
||||
for (int j = 0; j < xj.Length; j++)
|
||||
if (!IsWord(xj[j]))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
/// <summary>
|
||||
/// 是否是汉字串列
|
||||
/// </summary>
|
||||
/// <param name="word"></param>
|
||||
/// <returns></returns>
|
||||
public static bool IsChineseWord(string word)
|
||||
{
|
||||
char[] xj = word.ToCharArray();
|
||||
for (int j = 0; j < xj.Length; j++)
|
||||
if (!IsChineseWord(xj[j]))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 是否是字符
|
||||
/// </summary>
|
||||
/// <param name="ascii"></param>
|
||||
/// <returns></returns>
|
||||
private static bool IsWord(int ascii)
|
||||
{
|
||||
if (ascii == 32 || ascii == 35) //空格
|
||||
return true;
|
||||
if (ascii < 58 && ascii > 47) //数字
|
||||
return true;
|
||||
if (ascii < 91 && ascii > 64) //小写字母
|
||||
return true;
|
||||
if (ascii < 123 && ascii > 96) //大写字母
|
||||
return true;
|
||||
if (ascii < 40892 && ascii > 13311) //中文字符串
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
/// <summary>
|
||||
/// 是否是汉字
|
||||
/// </summary>
|
||||
/// <param name="ascii"></param>
|
||||
/// <returns></returns>
|
||||
private static bool IsChineseWord(int ascii)
|
||||
{
|
||||
if (ascii < 40892 && ascii > 13311) //中文字符串
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 清理多余空格
|
||||
/// </summary>
|
||||
/// <param name="html"></param>
|
||||
/// <returns></returns>
|
||||
private string ClearToMust(string html)
|
||||
{
|
||||
html = html.Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ");
|
||||
return html.Replace(" ", " ").Replace(" ", " ").Trim();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user