426 lines
15 KiB
C#
426 lines
15 KiB
C#
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
|
|
namespace AllBase
|
|
{
|
|
public class Segment
|
|
{
|
|
private static Dictionary<int, List<string>> WordList = new Dictionary<int, List<string>>();
|
|
|
|
/// <summary>
|
|
/// 关键词队列
|
|
/// </summary>
|
|
private string words = " ";
|
|
|
|
/// <summary>
|
|
/// 获取绝对地址
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
private static string GetPatch()
|
|
{
|
|
string patch = Process.GetCurrentProcess().MainModule.FileName;
|
|
string[] l = patch.Split('\\');
|
|
patch = "";
|
|
for (int i = 0; i < l.Length - 1; i++)
|
|
patch += l[i] + "\\";
|
|
return patch;
|
|
}
|
|
/// <summary>
|
|
/// 初始化
|
|
/// </summary>
|
|
public static void Int()
|
|
{
|
|
//MysqlHelper.ErrorString += "正在初始化分词库\r\n";
|
|
string f = GetPatch() + "\\Segment.dll";
|
|
if (!File.Exists(f))
|
|
GetHtml.DownloadFile("http://dl.1n11.com/Segment.xml", f);
|
|
if (!File.Exists(f))
|
|
{
|
|
BaseClass.ViewCode += "分词库加载失败，请前往官网下载完整版本";
|
|
return;
|
|
}
|
|
if (WordList.Count > 0)
|
|
return;
|
|
for (int i = 2; i < 10; i++)
|
|
{
|
|
List<string> w = new List<string>();
|
|
WordList.Add(i, w);
|
|
}
|
|
string[] l = File.ReadAllLines(f, Encoding.UTF8);
|
|
for (int i = 0; i < l.Length; i++)
|
|
{
|
|
if (string.IsNullOrEmpty(l[i]))
|
|
continue;
|
|
string key = l[i];
|
|
if (key.Contains("\t") && key.IndexOf("\t")>0)
|
|
key = key.Substring(0, key.IndexOf("\t"));
|
|
if (key.Length < 2 || key.Length > 8)
|
|
continue;
|
|
WordList[key.Length].Add(key);
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// 二分法提取关键词
|
|
/// </summary>
|
|
/// <param name="len"></param>
|
|
/// <param name="WordList"></param>
|
|
/// <param name="html"></param>
|
|
/// <param name="word"></param>
|
|
private void GetKeywordList(int len, ref List<string> WordList, string html, ref List<string> word)
|
|
{
|
|
string temp = "";
|
|
for (int i = 0; i <= html.Length - len; )
|
|
{
|
|
temp = html.Substring(i, len);
|
|
if (!temp.Contains(" ") && Dichotomy(temp, ref WordList))
|
|
{
|
|
AddWord(temp, ref word);
|
|
i += len;
|
|
continue;
|
|
}
|
|
i++;
|
|
}
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// 分词
|
|
/// </summary>
|
|
/// <param name="html"></param>
|
|
/// <returns></returns>
|
|
public List<string> Get(string html)
|
|
{
|
|
if (string.IsNullOrEmpty(html))
|
|
return null;
|
|
words = " ";
|
|
//WordList.Clear();
|
|
html = html.ToLower();
|
|
if (html.Length > 20000)
|
|
html = html.Substring(0, 20000);
|
|
BaseRegex.Relpace(ref html, @"(&[\w]+;)", " ");
|
|
BaseRegex.Relpace(ref html, @"([ ]{2,2000})", " ");
|
|
if (WordList.Count == 0)
|
|
{
|
|
BaseClass.ViewCode += "分词库为空，请检查系统是否正常";
|
|
return null;
|
|
}
|
|
|
|
byte[] b = Encoding.UTF8.GetBytes(html);
|
|
//===============先处理特殊字符================//
|
|
for (int i = 9; i > 1; i--)
|
|
GetKeywords2(WordList[i], ref b, i * 3);
|
|
|
|
//GetKeywords2(ref WordList2, ref b, 6);
|
|
string[] l = new[]
|
|
{
|
|
@"((http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)",
|
|
@"([-a-z0-9\.@]+)"
|
|
|
|
};
|
|
for (int i = 0; i < l.Length; i++)
|
|
{
|
|
var m = BaseRegex.GetMatchRegex(ref html, l[i]);
|
|
AddKeyword(ref m);
|
|
}
|
|
List<string> word = ReturnList();
|
|
if (html.Length > 15000 || word.Count > 3000) //太长则不执行消耗资源的自动学习功能
|
|
return word;
|
|
|
|
//==========自动学习关键词=========//
|
|
for (int i = 0; i < word.Count; i++)
|
|
html = html.Replace(word[i], " ");
|
|
html = html.Replace("的", " ");
|
|
BaseRegex.Relpace(ref html, @"([ ]+)", " ");
|
|
char t;
|
|
for (int i = 0; i < html.Length; i++)
|
|
{
|
|
t = char.Parse(html.Substring(i, 1));
|
|
if (t != ' ' && !IsChineseWord(t))
|
|
html = html.Replace(t.ToString(), " ");
|
|
}
|
|
BaseRegex.Relpace(ref html, @"([ ]+)", " ");
|
|
if (string.IsNullOrEmpty(html))
|
|
return word;
|
|
string[] li = html.Split(' ');
|
|
//========关键词长度倒排=========//
|
|
for (int i = 0; i < li.Length; i++)
|
|
{
|
|
for (int j = i; j < li.Length; j++)
|
|
{
|
|
if (li[i].Length < li[j].Length)
|
|
{
|
|
string x = li[i];
|
|
li[i] = li[j];
|
|
li[j] = x;
|
|
}
|
|
}
|
|
}
|
|
for (int i = 0; i < li.Length; i++)
|
|
{
|
|
string key = li[i];
|
|
if (key.Length <= 1 || key.Length > 4)
|
|
continue;
|
|
if (!IsWordList(key))
|
|
continue;
|
|
AddWord(key, ref word);
|
|
}
|
|
return word;
|
|
}
|
|
/// <summary>
|
|
/// 返回列表
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
private List<string> ReturnList()
|
|
{
|
|
List<string> w = new List<string>();
|
|
if (string.IsNullOrEmpty(words))
|
|
return w;
|
|
string[] l = words.Split(' ');
|
|
foreach (string s in l)
|
|
{
|
|
if (string.IsNullOrEmpty(s))
|
|
continue;
|
|
w.Add(s);
|
|
}
|
|
return w;
|
|
}
|
|
/// <summary>
|
|
/// 正则添加到队列
|
|
/// </summary>
|
|
/// <param name="m"></param>
|
|
private void AddKeyword(ref MatchCollection m)
|
|
{
|
|
for (int i = 0; m != null && i < m.Count; i++)
|
|
AddWordx(m[i].Groups[0].Value);
|
|
}
|
|
/// <summary>
|
|
/// 字符串中是否包含中文
|
|
/// </summary>
|
|
/// <param name="words"></param>
|
|
/// <returns></returns>
|
|
public static bool HadChineseWord(ref string words)
|
|
{
|
|
if(string.IsNullOrEmpty(words))
|
|
return false;
|
|
var b = Encoding.UTF8.GetBytes(words);
|
|
foreach (byte b1 in b)
|
|
if(b1>227 && b1<235)
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 软加速
|
|
/// </summary>
|
|
/// <param name="WordList">关键词队列</param>
|
|
/// <param name="b">二进制文章</param>
|
|
/// <param name="len">二进制长度</param>
|
|
private void GetKeywords2(List<string> WordList, ref byte[] b, int len)
|
|
{
|
|
string t;
|
|
for (int i = 0; i < b.Length - len; i++)
|
|
{
|
|
switch (len)
|
|
{
|
|
case 6:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227)
|
|
goto Add;
|
|
break;
|
|
case 9:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227)
|
|
goto Add;
|
|
break;
|
|
case 12:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
|
&& b[i + 9] < 235 && b[i + 9] > 227)
|
|
goto Add;
|
|
break;
|
|
case 15:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
|
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227)
|
|
goto Add;
|
|
break;
|
|
case 18:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
|
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227)
|
|
goto Add;
|
|
break;
|
|
case 21:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
|
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227
|
|
&& b[i + 18] < 235 && b[i + 18] > 227)
|
|
goto Add;
|
|
break;
|
|
case 24:
|
|
case 27:
|
|
if (b[i] < 235 && b[i] > 227 && b[i + 3] < 235 && b[i + 3] > 227 && b[i + 6] < 235 && b[i + 6] > 227
|
|
&& b[i + 9] < 235 && b[i + 9] > 227 && b[i + 12] < 235 && b[i + 12] > 227 && b[i + 15] < 235 && b[i + 15] > 227
|
|
&& b[i + 18] < 235 && b[i + 18] > 227 && b[i + 21] < 235 && b[i + 21] > 227)
|
|
goto Add;
|
|
break;
|
|
}
|
|
continue;
|
|
Add:
|
|
t = Encoding.UTF8.GetString(b, i, len);
|
|
if (Dichotomy(t, ref WordList))
|
|
{
|
|
AddWordx(t);
|
|
i += len - 1;
|
|
}
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// 添加关键词到列
|
|
/// </summary>
|
|
/// <param name="keyword"></param>
|
|
private void AddWordx(string keyword)
|
|
{
|
|
if (words.Contains(" " + keyword + " "))
|
|
return;
|
|
words += keyword + " ";
|
|
}
|
|
|
|
/// <summary>
|
|
/// 二分法关键词是否存在
|
|
/// </summary>
|
|
/// <param name="word">关键词</param>
|
|
/// <returns></returns>
|
|
public bool Dichotomy(string word, ref List<string> WordList)
|
|
{
|
|
if (WordList.Count < 1)
|
|
return false;
|
|
int start = 0, end = WordList.Count - 1, middle = 0, ok = 0;
|
|
while (start != end)
|
|
{
|
|
middle = (start + end) / 2;
|
|
ok = string.Compare(word, WordList[middle], true);
|
|
if (ok == 0)
|
|
return true;
|
|
if (ok == 1)
|
|
start = middle;
|
|
else
|
|
end = middle;
|
|
if (start + 1 == end)
|
|
{
|
|
if (string.Compare(word, WordList[end], true) == 0)
|
|
return true;
|
|
return false;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 读取正则结果列表
|
|
/// </summary>
|
|
/// <param name="m"></param>
|
|
/// <param name="word"></param>
|
|
private void GetMatch(ref MatchCollection m, ref List<string> word, ref string html)
|
|
{
|
|
for (int i = 0; i < m.Count; i++)
|
|
if (AddWord(m[i].Groups[0].Value, ref word))
|
|
html = html.Replace(m[i].Groups[0].Value, " ");
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// 添加关键词到列
|
|
/// </summary>
|
|
/// <param name="keyword"></param>
|
|
/// <param name="word"></param>
|
|
private bool AddWord(string keyword, ref List<string> word)
|
|
{
|
|
bool had = false;
|
|
for (int i = 0; i < word.Count; i++)
|
|
{
|
|
if (word[i] == keyword)
|
|
{
|
|
had = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!had)
|
|
{
|
|
string b = keyword;
|
|
word.Add(b);
|
|
}
|
|
return !had;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 是否是中文字符串列
|
|
/// </summary>
|
|
/// <param name="word"></param>
|
|
/// <returns></returns>
|
|
public static bool IsWordList(string word)
|
|
{
|
|
char[] xj = word.ToCharArray();
|
|
for (int j = 0; j < xj.Length; j++)
|
|
if (!IsWord(xj[j]))
|
|
return false;
|
|
return true;
|
|
}
|
|
/// <summary>
|
|
/// 是否是汉字串列
|
|
/// </summary>
|
|
/// <param name="word"></param>
|
|
/// <returns></returns>
|
|
public static bool IsChineseWord(string word)
|
|
{
|
|
char[] xj = word.ToCharArray();
|
|
for (int j = 0; j < xj.Length; j++)
|
|
if (!IsChineseWord(xj[j]))
|
|
return false;
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 是否是字符
|
|
/// </summary>
|
|
/// <param name="ascii"></param>
|
|
/// <returns></returns>
|
|
private static bool IsWord(int ascii)
|
|
{
|
|
if (ascii == 32 || ascii == 35) //空格
|
|
return true;
|
|
if (ascii < 58 && ascii > 47) //数字
|
|
return true;
|
|
if (ascii < 91 && ascii > 64) //小写字母
|
|
return true;
|
|
if (ascii < 123 && ascii > 96) //大写字母
|
|
return true;
|
|
if (ascii < 40892 && ascii > 13311) //中文字符串
|
|
return true;
|
|
return false;
|
|
}
|
|
/// <summary>
|
|
/// 是否是汉字
|
|
/// </summary>
|
|
/// <param name="ascii"></param>
|
|
/// <returns></returns>
|
|
private static bool IsChineseWord(int ascii)
|
|
{
|
|
if (ascii < 40892 && ascii > 13311) //中文字符串
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 清理多余空格
|
|
/// </summary>
|
|
/// <param name="html"></param>
|
|
/// <returns></returns>
|
|
private string ClearToMust(string html)
|
|
{
|
|
html = html.Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ");
|
|
return html.Replace(" ", " ").Replace(" ", " ").Trim();
|
|
}
|
|
|
|
|
|
|
|
}
|
|
}
|