Files
KJD/AllBase/MultiThreadClient.cs
2025-06-29 19:38:31 +08:00

694 lines
26 KiB
C#

/*
* 多线程模拟浏览器采集客户端
* 2013-7-24
* 肖俊
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
namespace AllBase
{
public class MultiThreadClient
{
/// <summary>
/// 附加Cookies
/// </summary>
public string AddCookies = "";
/// <summary>
/// 指定Cookies
/// </summary>
public string Cookies = "";
/// <summary>
/// 指定UserAgent
/// </summary>
public string UserAgent = "";
/// <summary>
/// 是否出错
/// </summary>
public bool IsError = false;
/// <summary>
/// 是否友情链接查询
/// </summary>
public bool FriendLink = false;
/// <summary>
/// 来路
/// </summary>
public string Referer = "";
/// <summary>
/// 是否手机头
/// </summary>
public bool IsMoblie = false;
/// <summary>
/// 是否手机头
/// </summary>
public bool IsWhoies = false;
/// <summary>
/// 强制定义为GBK编码
/// </summary>
public bool IsGbk = false;
/// <summary>
/// 强制定义为GBK编码
/// </summary>
public bool IsUtf8 = false;
/// <summary>
/// 强制不跳转
/// </summary>
public bool NoRedir = false;
/// <summary>
/// 是否代理模式
/// </summary>
public bool IsProxy = false;
/// <summary>
/// 百度搜索错误
/// </summary>
public static bool BaiduError = false;
/// <summary>
/// 百度错误时间
/// </summary>
public static DateTime BaiduErrorTime = DateTime.Now;
public string EndUrl = "";
/// <summary>
/// 超时
/// </summary>
public int TimeOut = 30;
public string GetHtml(string url)
{
return GetHtml(url, -1,null);
}
public string GetHtml(string url, byte[] post)
{
return GetHtml(url,-1, post);
}
/// <summary>
/// 采集器
/// </summary>
/// <param name="url">网址</param>
/// <returns></returns>
public string GetHtml(string url, int isUtf8, byte[] post)
{
IsError = false;
string cookies = "", html = "";
Random ra = new Random();
string domain = GetDomain(url);
if (domain == null)
return "";
if (domain.Length > 4 &&
(domain.Substring(0, 2) == "m." || domain.Substring(0, 2) == "wap." || domain.Substring(0, 2) == "3g." ||
domain.Substring(0, 2) == "4g."))
IsMoblie = true;
if (!string.IsNullOrEmpty(url))
url = url.Trim();
if (string.IsNullOrEmpty(url))
return "";
WebBrowserCoockies wc;
if (IsMoblie)
wc = WebCoockies.MobileCoockies.Count>0? WebCoockies.MobileCoockies[ra.Next(WebCoockies.MobileCoockies.Count)]:null;
else
wc = WebCoockies.Coockies.Count>0? WebCoockies.Coockies[ra.Next(WebCoockies.Coockies.Count)]:null;
if (wc == null)
wc = new WebBrowserCoockies();
for (int i = 0; wc.Coockies != null && i < wc.Coockies.Count; i++)
{
if (wc.Coockies[i] != null && wc.Coockies[i].Domain == domain)
{
cookies = wc.Coockies[i].Cookies;
break;
}
}
if (!string.IsNullOrEmpty(UserAgent))
wc.UserAgent = UserAgent;
if (!string.IsNullOrEmpty(Cookies))
cookies = Cookies;
WebClientx web = new WebClientx();
web.AllowAutoRedirect = false;
web.Timeout = TimeOut;
web.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8");
web.Headers.Add("Accept-Encoding", "gzip, deflate, sdch, br ");
web.Headers.Add("User-Agent", wc.UserAgent);
if (IsMoblie)
{
web.Headers.Add("X-Requested-With", "com.android.browser");
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
web.Headers.Add("x-wap-profile", "http://wap1.huawei.com/uaprof/HUAWEI_MT2_T00_UAProfile.xml");
}
else
{
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
web.Headers.Add("upgrade-insecure-requests", "1");
}
if (IsProxy)//是否是代理状态
web.Headers.Add("Add-UserAgent", wc.UserAgent);
if (!BaseClass.IsTest)
web.Proxy = null;
if (!string.IsNullOrEmpty(AddCookies) && string.IsNullOrEmpty(cookies))
cookies += AddCookies;
if (url.Contains("/url=") && IsMoblie)
web.Headers.Add("Add-Cookies", "Moblie");
if (url.Contains("/url=") && IsWhoies)
web.Headers.Add("Add-Cookies", "whois");
int retryTimes = 0;
GotoAgen:
EndUrl = url;
web.Headers.Remove("Cookie");
if (!string.IsNullOrEmpty(cookies))
web.Headers.Add("Cookie", cookies);
if (!string.IsNullOrEmpty(Referer))
web.Headers.Add("Referer", Referer);
try
{
byte[] b;
//if (post == null)
//{
// Stream receiveStream = web.OpenRead(url);
// MemoryStream ms = new MemoryStream();
// int num;
// byte[] buffer = new byte[10240];
// do
// {
// num = receiveStream.Read(buffer, 0, buffer.Length);
// if (num > 0)
// ms.Write(buffer, 0, num);
// } while (num > 0);
// b = ms.ToArray();
// ms.Close();
//}
//else
//{
// b = web.UploadData(url, post);
//}
b = post == null ? web.DownloadData(url) : web.UploadData(url, post);
string cookies1 = web.ResponseHeaders.Get("Set-Cookie"); //获取Cookies
string location = web.ResponseHeaders.Get("Location"); //跳转地址
if (!string.IsNullOrEmpty(cookies1))
{
cookies = UpCookies(cookies, cookies1);
if (!string.IsNullOrEmpty(Cookies))
Cookies = cookies;
bool ok = false;
foreach (var c in wc.Coockies)
{
if (c.Domain == domain)
{
c.Cookies = cookies;
ok = true;
}
}
if (!ok)
{
CoockiesBase c = new CoockiesBase();
c.Cookies = cookies;
c.Domain = domain;
wc.Coockies.Add(c);
}
}
if (string.IsNullOrEmpty(location))
{
string contentType = web.ResponseHeaders["Content-Type"];
if (b == null)
return "";
if (b.Length > 2 * 1024 * 1024)
return "文件太大";
b = ZipHelper.Decompress(b);
if (b.Length > 2 * 1024 * 1024)
return "文件太大";
if (!string.IsNullOrEmpty(contentType))
{
string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)");
if (!string.IsNullOrEmpty(c))
{
c = c.Trim().ToLower().Replace("utf8", "utf-8");
html = Encoding.GetEncoding(c).GetString(b);
}
}
if (string.IsNullOrEmpty(html) && isUtf8 == -1)
{
html = Encoding.GetEncoding("UTF-8").GetString(b);
string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
string c = GetRegexString(html, r).Trim();
if (c.IndexOf("'") > -1)
c = c.Substring(0, c.IndexOf("'"));
if (c.IndexOf("\"") > -1)
c = c.Substring(0, c.IndexOf("\""));
if (c.IndexOf(" ") > -1)
c = c.Substring(0, c.IndexOf(" "));
if (c.IndexOf(">") > -1)
c = c.Substring(0, c.IndexOf(">"));
if (c.ToLower() != "utf-8" && !string.IsNullOrEmpty(c))
html = Encoding.GetEncoding(c).GetString(b);
else if (string.IsNullOrEmpty(c))
html = Encoding.GetEncoding("UTF-8").GetString(b);
}
if (!string.IsNullOrEmpty(html) && isUtf8 == -1)
{
if (html.Substring(0, 1) == "锘")
html = Encoding.GetEncoding("UTF-8").GetString(b);
if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 20))
html = Encoding.GetEncoding("gbk").GetString(b);
}
if (isUtf8 == 0)
html = Encoding.GetEncoding("gbk").GetString(b);
if (isUtf8 == 1 && string.IsNullOrEmpty(html))
html = Encoding.UTF8.GetString(b);
if (IsGbk)
html = Encoding.GetEncoding("gbk").GetString(b);
if (IsUtf8)
html = Encoding.GetEncoding("UTF-8").GetString(b);
}
if (retryTimes < 3 && (!string.IsNullOrEmpty(location) || string.IsNullOrEmpty(html)))
{
if (NoRedir)
return location;
//下面是支持301和302跳转部分
if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
{
url = location;
}
else
{
if (location.Substring(0, 1) == "/")
{
string[] l = url.Split('/');
if (l.Length > 3)
{
url = "";
for (int i = 0; i < 3; i++)
url += l[i] + "/";
url += location.Substring(1, location.Length - 1);
}
}
else
{
if (url.Substring(url.Length - 1, 1) == "/")
url += location;
else
url += "/" + location;
}
}
retryTimes++;
goto GotoAgen;
}
}
catch (Exception e)
{
if (e.Message.Contains("(404)") || e.Message.Contains("(403)") || e.Message.Contains("(500)") || e.Message.Contains("指定的路径或文件名太长,"))
{
IsError = true;
return e.Message;
}
if (FriendLink)
{
IsError = true;
return e.Message;
}
if (retryTimes < 1)
{
Thread.Sleep(100);
retryTimes++;
goto GotoAgen;
}
IsError = true;
return e.Message;
}
if (html.Contains("您的访问出错了</title>"))
{
if (!BaiduError)
{
BaiduError = true;
BaiduErrorTime = DateTime.Now;
}
IsError = true;
}
if (BaiduError && BaiduErrorTime.AddMinutes(5) < DateTime.Now)
BaiduError = false;
return html;
}
/// <summary>
/// 返回Utf-8编码
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public string GetByte(string url, ref bool had, out bool isChinese)
{
string html = "";
had = false;
IsError = false;
isChinese = false;
string cookies = "";
Random ra = new Random();
string domain = AllBase.GetHtml.GetDomain(url);
if (string.IsNullOrEmpty(domain))
return "";
WebBrowserCoockies wc = WebCoockies.Coockies[ra.Next(WebCoockies.Coockies.Count)];
for (int i = 0; wc != null && wc.Coockies != null && i < wc.Coockies.Count; i++)
{
if (wc.Coockies[i] != null && wc.Coockies[i].Domain == domain)
{
cookies = wc.Coockies[i].Cookies;
break;
}
}
WebClientx web = new WebClientx();
web.Timeout = TimeOut;
web.AllowAutoRedirect = false;
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;");
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
web.Headers.Add("Accept-Encoding", "gzip,deflate");
// if (string.IsNullOrEmpty(UserAgent))
// web.Headers.Add("User-Agent", wc.UserAgent);
//else
// web.Headers.Add("User-Agent", UserAgent);
web.Headers.Add("User-Agent", "Mozilla/5.0+(compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)");
web.Proxy = null;
if (!string.IsNullOrEmpty(AddCookies) && string.IsNullOrEmpty(cookies))
cookies += AddCookies;
if (!string.IsNullOrEmpty(cookies))
web.Headers.Add("Cookie", cookies);
if (!string.IsNullOrEmpty(Referer))
web.Headers.Add("Referer", Referer);
if (url.Contains("/url=") && IsMoblie)
web.Headers.Add("Add-Cookies", "Moblie");
if (url.Contains("/url=") && IsWhoies)
web.Headers.Add("Add-Cookies", "whois");
int retryTimes = 0;
byte[] b = null;
GotoAgen:
try
{
int max = 3 * 1024, len = 0;//最大文件为3MB
//Stream receiveStream = web.OpenRead(url);
//MemoryStream ms = new MemoryStream();
//int num;
//byte[] buffer = new byte[1024];
//do
//{
// num = receiveStream.Read(buffer, 0, buffer.Length);
// if (num > 0)
// ms.Write(buffer, 0, num);
// len++;
//} while (num > 0 && len<max);
//if (len >= max)//太大的文件直接跳过
// return "";
//b = ms.ToArray();
//ms.Close();
b = web.DownloadData(url);
string cookies1 = web.ResponseHeaders.Get("Set-Cookie"); //获取Cookies
string location = web.ResponseHeaders.Get("Location"); //跳转地址
if (!string.IsNullOrEmpty(cookies1))
{
cookies = UpCookies(cookies, cookies1);
bool ok = false;
foreach (var c in wc.Coockies)
{
if (c.Domain == domain)
{
c.Cookies = cookies;
ok = true;
}
}
if (!ok)
{
CoockiesBase c = new CoockiesBase();
c.Cookies = cookies;
c.Domain = domain;
wc.Coockies.Add(c);
}
}
if (string.IsNullOrEmpty(location))
{
string contentType = web.ResponseHeaders["Content-Type"];
html = "";
b = ZipHelper.Decompress(b);
if (b.Length >= max*1024)//太大的文件直接跳过
return "文件太大";
//b = GzipDeCompress(b);
if (b == null || b.Length == 0 || b.Length > 2 * 1024 * 1024)
return "";
try
{
if (!string.IsNullOrEmpty(contentType))
{
string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)");
if (!string.IsNullOrEmpty(c))
{
c = c.Trim().ToLower().Replace("utf8", "utf-8").Replace(";", "");
html = Encoding.GetEncoding(c).GetString(b);
}
}
if (string.IsNullOrEmpty(html))
{
html = Encoding.GetEncoding("UTF-8").GetString(b);
string r = @"<meta [^>]*charset=[""']*([^>]+)[""']*[ />]";
string c = GetRegexString(html, r);
if (c.Contains("\""))
c = c.Substring(0, c.IndexOf("\""));
if (c.Contains("'"))
c = c.Substring(0, c.IndexOf("'"));
if (string.IsNullOrEmpty(c))
c = GetRegexString(html, @"<meta [^>]*charset=[""']([a-z0-9-]+)[""']>");
if (string.IsNullOrEmpty(c) && !string.IsNullOrEmpty(html) && html.Contains("�") &&html.Length- html.Replace("�","").Length>50)
{
html = Encoding.GetEncoding("gbk").GetString(b);
}
if (!string.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
{
html = Encoding.GetEncoding(c).GetString(b);
}
}
}
catch
{
}
if (IsUtf8|| !string.IsNullOrEmpty(html) && html.Substring(0, 1) == "锘")
{
html = Encoding.UTF8.GetString(b);
}
}
if (retryTimes < 3 && (!string.IsNullOrEmpty(location) || string.IsNullOrEmpty(html)))
{
//下面是支持301和302跳转部分
if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
{
url = location;
}
else
{
if (location.Substring(0, 1) == "/")
{
string[] l = url.Split('/');
if (l.Length > 3)
{
url = "";
for (int i = 0; i < 3; i++)
url += l[i] + "/";
url += location.Substring(1, location.Length - 1);
}
}
else
{
if (url.Substring(url.Length - 1, 1) == "/")
url += location;
else
url += "/" + location;
}
}
retryTimes++;
goto GotoAgen;
}
}
catch (Exception e)
{
if (e.Message.Contains("(404)") || e.Message.Contains("(403)") || e.Message.Contains("(500)"))
{
IsError = true;
html = e.Message;
}
if (!IsError && retryTimes < 1)
{
Thread.Sleep(2000);
retryTimes++;
goto GotoAgen;
}
IsError = true;
html = e.Message;
}
if (b == null || b.Length == 0)
return "";
foreach (byte b1 in b)
{
if (b1 > 227 && b1 < 235)
{
isChinese = true;
break;
}
}
if (html.Length > 2 * 1024 * 1024)
{
had = false;
return "";
}
had = true;
//if (b == null) //
return html;
}
/// <summary>
/// 获取域名
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
private string GetDomain(string url)
{
try
{
url = url.Substring(url.IndexOf("//") + 2, url.Length - url.IndexOf("//") - 2);
if (url.IndexOf("/") > -1)
url = url.Substring(0, url.IndexOf("/"));
return url;
}
catch
{
return null;
}
}
/// <summary>
/// 返回单个正则分析结果
/// </summary>
/// <param name="HtmlString"></param>
/// <param name="RegexString"></param>
/// <returns></returns>
private string GetRegexString(string HtmlString, string RegexString)
{
MatchCollection mc = Regex.Matches(HtmlString, RegexString, RegexOptions.IgnoreCase);
if (mc.Count > 0)
return mc[0].Groups[1].ToString();
return "";
}
/// <summary>
/// 更新cookies
/// </summary>
/// <param name="old"></param>
/// <param name="newc"></param>
/// <returns></returns>
public string UpCookies(string old, string newc)
{
if (string.IsNullOrEmpty(newc))
return old;
if (string.IsNullOrEmpty(old))
old = " ";
int io = 0;
old = old.Trim();
newc = newc.Trim();
string[] o = old.Split(';');
string[] n = newc.Split(';');
List<CoockiesValue> c = new List<CoockiesValue>();
for (int i = 0; i < o.Length; i++)
{
CoockiesValue cv = new CoockiesValue();
io = o[i].IndexOf("=");
if (io > -1)
{
cv.Title = o[i].Substring(0, io).Trim();
cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
}
else
{
if (!string.IsNullOrEmpty(o[i]))
cv.Title = o[i].Trim();
}
if (string.IsNullOrEmpty(cv.Title) || cv.Title == "domain" || cv.Title == "expires" || cv.Title == "path" || cv.Title.ToLower() == "httponly")
continue;
if (!string.IsNullOrEmpty(cv.Title))
c.Add(cv);
}
for (int i = 0; i < n.Length; i++)
{
CoockiesValue cv = new CoockiesValue();
io = n[i].IndexOf("=");
if (io > -1)
{
cv.Title = n[i].Substring(0, io).Trim();
cv.Value = n[i].Substring(io + 1, n[i].Length - io - 1).Trim();
}
else
{
if (!string.IsNullOrEmpty(n[i]))
cv.Title = n[i].Trim();
}
if (!string.IsNullOrEmpty(cv.Title))
{
bool isHad = false;
for (int j = 0; j < c.Count; j++)
{
if (cv.Title == c[j].Title)
{
c[j].Value = cv.Value;
isHad = true;
break;
}
}
if (string.IsNullOrEmpty(cv.Title) || string.IsNullOrEmpty(cv.Value) || cv.Title == "domain" || cv.Title == "expires"
|| cv.Title == "path"&& !cv.Value.Contains("/,") || cv.Title.ToLower() == "httponly")
continue;
if (!isHad)
{
if (cv.Title == "path" && cv.Value.Contains("/,")&&cv.Value.IndexOf("/,")==0 && cv.Value.Length>2)
{
string t = cv.Value.Substring(2);
if(!t.Contains("=")|| t.IndexOf("=")>t.Length-1)
continue;
cv.Title = t.Substring(0, t.IndexOf("="));
cv.Value = t.Substring(t.IndexOf("=") + 1);
}
c.Add(cv);
}
}
}
string temp = "";
for (int i = 0; i < c.Count; i++)
{
if (string.IsNullOrEmpty(c[i].Value))
temp += c[i].Title + "; ";
else
temp += c[i].Title + "=" + c[i].Value + "; ";
}
return temp;
}
}
}