1933 lines
74 KiB
C#
1933 lines
74 KiB
C#
/*
|
||
* 采集网页
|
||
* 肖俊
|
||
* 2013-11-12
|
||
*
|
||
*/
|
||
|
||
using System;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.Drawing;
|
||
using System.Globalization;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Net;
|
||
using System.Runtime.InteropServices;
|
||
using System.Text;
|
||
using System.Text.RegularExpressions;
|
||
using System.Threading;
|
||
using System.Web;
|
||
|
||
namespace AllBase
|
||
{
|
||
public class GetHtml
|
||
{
|
||
/// <summary>
|
||
/// 加载远程图片
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static Image LoadImage(string url)
|
||
{
|
||
try
|
||
{
|
||
WebRequest webreq = WebRequest.Create(url);
|
||
WebResponse webres = webreq.GetResponse();
|
||
Stream stream = webres.GetResponseStream();
|
||
return Image.FromStream(stream);
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
public static Icon LoadIcon(string url)
|
||
{
|
||
try
|
||
{
|
||
Bitmap bmp = new Bitmap(LoadImage(url));
|
||
IntPtr h = bmp.GetHicon();
|
||
System.Drawing.Icon icon = System.Drawing.Icon.FromHandle(h);
|
||
DeleteObject(h);
|
||
return icon;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
[DllImport("gdi32.dll")]
|
||
public static extern bool DeleteObject(IntPtr hObject);
|
||
|
||
/// <summary>
|
||
/// 代理抓取
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByProxy(string url)
|
||
{
|
||
string html = "", newurl = GetProxyUrlx(url);
|
||
Random ro = new Random();
|
||
int i = 0;
|
||
MultiThreadClient mc = new MultiThreadClient();
|
||
ReTry:
|
||
html = mc.GetHtml(newurl);
|
||
if (IsError(ref html))
|
||
{
|
||
Thread.Sleep(ro.Next(2000, 6000));
|
||
if (i < 4)
|
||
{
|
||
i++;
|
||
goto ReTry;
|
||
}
|
||
newurl = GetProxyUrlx(url);
|
||
html = mc.GetHtml(newurl);
|
||
if (IsError(ref html))
|
||
return mc.GetHtml(url);
|
||
return html;
|
||
}
|
||
Thread.Sleep(100 + BaseClass.WaitTime / 2 + ro.Next(BaseClass.WaitTime));
|
||
Clear(ref html);
|
||
return html;
|
||
}
|
||
/// <summary>
|
||
/// 清理、处理代码
|
||
/// </summary>
|
||
/// <param name="html"></param>
|
||
private static void Clear(ref string html)
|
||
{
|
||
html = UrlDecode(html);
|
||
html = html.Replace("\\", "");
|
||
}
|
||
/// <summary>
|
||
/// 是否错误
|
||
/// </summary>
|
||
/// <param name="html"></param>
|
||
/// <returns></returns>
|
||
public static bool IsError(ref string html)
|
||
{
|
||
return String.IsNullOrEmpty(html) || html.Length < 100 || html.Contains("您的访问出错了</title>") ||
|
||
html.Contains("访问异常出错</title>");
|
||
}
|
||
/// <summary>
|
||
/// 获取代理地址
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetProxyUrlx(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return null;
|
||
if (BaseClass.ProxyOut.Count == 0)
|
||
{
|
||
BaseClass.ViewCode += "代理异常";
|
||
return url;
|
||
}
|
||
Random ra = new Random();
|
||
string p = BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
|
||
return "http://" + p + "/url=" + url.Replace(".google.", ".goo00gle.")
|
||
.Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
|
||
}
|
||
/// <summary>
|
||
/// ascii码转译为汉字
|
||
/// </summary>
|
||
/// <param name="ascii">带ASCII码的字符串</param>
|
||
/// <returns>还原字符串</returns>
|
||
public static void AsciiToString(ref string temp)
|
||
{
|
||
if (String.IsNullOrEmpty(temp))
|
||
return;
|
||
//return;
|
||
try
|
||
{
|
||
if (temp.Contains("&#"))
|
||
{
|
||
Regex reUnicode = new Regex(@"&#[x]([0-9A-Fa-f]{4,5});", RegexOptions.Compiled);
|
||
temp = reUnicode.Replace(temp, m =>
|
||
{
|
||
short c;
|
||
if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
|
||
return "" + (char)c;
|
||
return m.Value;
|
||
});
|
||
}
|
||
if (temp.Contains("\\u"))
|
||
{
|
||
Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4,5})", RegexOptions.Compiled);
|
||
temp = reUnicode.Replace(temp, m =>
|
||
{
|
||
short c;
|
||
if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber,
|
||
CultureInfo.InvariantCulture, out c))
|
||
return "" + (char)c;
|
||
return m.Value;
|
||
});
|
||
}
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
//string html = "´,´©,©>,>µ,µ®,®°,°¡,¡, »,»" +
|
||
// "¦,¦÷,÷¿,¿¬,¬§,§•,•½,½«,«¶,¶¨,¨" +
|
||
// "¸,¸¼,¼<,<±,±×,×¢,¢¾,¾¯,¯“,"" +
|
||
// "™,™€,€£,£¥,¥„,„…,…·,·›,›ª,ª" +
|
||
// "ˆ,ˆ“,“—,—’,’º,º†,†‹,‹–,–" +
|
||
// "‚,‚”,”‡,‡‘,‘‰,‰,­˜,˜≈,≈" +
|
||
// "⁄,⁄←,←∂,∂♠,♠∩,∩≥,≥≤,≤″,″∑,∑♣,♣" +
|
||
// "↔,↔◊,◊′,′↑,↑↓,↓♥,♥−,−∏,∏,‍♦,♦" +
|
||
// "∞,∞≠,≠√,√,‌≡,≡∫,∫‾,‾→,→α,αη,η" +
|
||
// "μ,μπ,πθ,θβ,βγ,γν,νψ,ψυ,υχ,χι,ι" +
|
||
// "ω,ωρ,ρξ,ξδ,δκ,κο,οσ,σζ,ζε,ε" +
|
||
// "λ,λφ,φτ,τΑ,ΑΗ,ΗΜ,ΜΠ,ΠΘ,ΘΒ,ΒΓ,Γ" +
|
||
// "Ν,ΝΨ,ΨΥ,ΥΧ,ΧΙ,ΙΩ,ΩΡ,ΡΞ,ΞΔ,ΔΚ,Κ" +
|
||
// "Ο,ΟΣ,ΣΖ,ΖΕ,ΕΛ,ΛΦ,ΦΤ,Τς,ς&,&";
|
||
//string[] h = html.Split(';');
|
||
//string s1;
|
||
//string s2;
|
||
//foreach (string s in h)
|
||
//{
|
||
// s2 = s.Substring(2, s.Length - 2) + ";";
|
||
// if (!temp.Contains(s2))
|
||
// continue;
|
||
// s1 = s.Substring(0, 1);
|
||
// temp = temp.Replace(s2, s1);
|
||
//}
|
||
//return temp;
|
||
}
|
||
/// <summary>
|
||
/// html解码，\u解码
|
||
/// </summary>
|
||
/// <param name="html"></param>
|
||
/// <returns></returns>
|
||
public static string UrlDecode(string html)
|
||
{
|
||
string temp = HttpUtility.UrlDecode(html);
|
||
Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4})", RegexOptions.Compiled);
|
||
return reUnicode.Replace(temp, m =>
|
||
{
|
||
short c;
|
||
if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
|
||
return "" + (char)c;
|
||
return m.Value;
|
||
});
|
||
}
|
||
|
||
/// <summary>
|
||
/// 本地+国内+美国
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string GetByMyself(string url, out bool isError, bool isServer)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = isServer ? 40 : 20;
|
||
string Html = mu.GetHtml(url);
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(ProxyIpInUrl(url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(ProxyIpOutUrl(url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
/// <summary>
|
||
/// 国内+本地+美国
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string GetByIn(string url, out bool isError, bool isServer)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = isServer ? 40 : 20; ;
|
||
string Html = mu.GetHtml(ProxyIpInUrl(url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(url);
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
mu.TimeOut = 30;
|
||
Html = mu.GetHtml(ProxyIpOutUrl(url));
|
||
}
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
/// <summary>
|
||
/// 国内+本地+美国
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string PostByIn(string url, out bool isError, bool isServer, string data)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = isServer ? 40 : 20; ;
|
||
string Html = mu.GetHtml(ProxyIpInUrl(url), Encoding.UTF8.GetBytes(data));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(url, Encoding.UTF8.GetBytes(data));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(ProxyIpOutUrl(url), Encoding.UTF8.GetBytes(data));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
/// <summary>
|
||
/// 美国+国内+本地
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string GetByOut(string url, out bool isError, bool isServer)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = isServer ? 60 : 30; ;
|
||
string Html = mu.GetHtml(ProxyIpOutUrl(url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(ProxyIpInUrl(url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(url);
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsa(string url)
|
||
{
|
||
bool isError;
|
||
return ByUsa(url, out isError, false, false, false, false);
|
||
}
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsa(string url, bool mobile)
|
||
{
|
||
bool isError;
|
||
return ByUsa(url, out isError, mobile, false, false, false);
|
||
}
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsa(bool isUtf8, string url)
|
||
{
|
||
bool isError;
|
||
return ByUsa(url, out isError, false, false, isUtf8, false);
|
||
}
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsa(string url, bool mobile, bool isGbk)
|
||
{
|
||
bool isError;
|
||
return ByUsa(url, out isError, mobile, isGbk, false, false);
|
||
}
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="domain"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsaWhoies(string domain)
|
||
{
|
||
bool isError;
|
||
return ByUsa(domain, out isError, false, false, false, true);
|
||
}
|
||
|
||
/// <summary>
|
||
/// 美国服务器出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string ByUsa(string url, out bool isError, bool mobile, bool isGbk, bool isUtf8, bool isWhois)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = 50;
|
||
mu.IsGbk = isGbk;
|
||
mu.IsUtf8 = isUtf8;
|
||
if (mobile)
|
||
mu.AddCookies = "Moblie";
|
||
mu.IsWhoies = isWhois;
|
||
string Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyOut, url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyOut, url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 局域网出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string ByLocation(string url)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = 30;
|
||
string Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
return null;
|
||
}
|
||
return Html;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 远程下载文件
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="path"></param>
|
||
/// <returns></returns>
|
||
public static bool ByUsaDownload(string url, string path)
|
||
{
|
||
string newUrl = GetProxyUrl(BaseClass.ProxyOut, url);
|
||
bool ok = DownloadFile(newUrl, path);
|
||
if (!ok)
|
||
DownloadFile(GetProxyUrl(BaseClass.ProxyOut, url), path);
|
||
if (!ok)
|
||
DownloadFile(GetProxyUrl(BaseClass.ProxyOut, url), path);
|
||
return ok;
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// HTTPS专用出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByHttps(string url)
|
||
{
|
||
bool isError;
|
||
return ByHttps(url, out isError, false);
|
||
}
|
||
/// <summary>
|
||
/// HTTPS专用出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByHttps(string url, bool mobile)
|
||
{
|
||
bool isError;
|
||
return ByHttps(url, out isError, mobile);
|
||
}
|
||
/// <summary>
|
||
/// HTTPS专用出口
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="isError"></param>
|
||
/// <returns></returns>
|
||
public static string ByHttps(string url, out bool isError, bool mobile)
|
||
{
|
||
MultiThreadClient mu = new MultiThreadClient();
|
||
mu.TimeOut = 60;
|
||
if (mobile)
|
||
mu.IsMoblie = true;
|
||
string urlx = GetProxyUrl(BaseClass.ProxyHttps, url);
|
||
string Html = mu.GetHtml(urlx);
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
urlx = GetProxyUrl(BaseClass.ProxyHttps, url);
|
||
Html = mu.GetHtml(urlx);
|
||
}
|
||
if (String.IsNullOrEmpty(Html) || mu.IsError)
|
||
{
|
||
isError = true;
|
||
return null;
|
||
}
|
||
isError = false;
|
||
return Html;
|
||
}
|
||
/// <summary>
|
||
/// 转换URL
|
||
/// </summary>
|
||
/// <param name="ips"></param>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetProxyUrl(List<string> ips, string url)
|
||
{
|
||
if (ips.Count == 0)
|
||
return url;
|
||
Random ra = new Random(Guid.NewGuid().GetHashCode());
|
||
string ip = ips[ra.Next(ips.Count)];
|
||
|
||
return "http://" + (ip.Contains(":") ? ip : ip + ":6000") + "/url=" + ReplaceUrl(url);
|
||
}
|
||
|
||
/// <summary>
|
||
/// utf8 下载页面
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string Utf8(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
try
|
||
{
|
||
WebClientx web = new WebClientx();
|
||
web.Timeout = 60;
|
||
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
|
||
web.Proxy = null;
|
||
byte[] b = web.DownloadData(url);
|
||
return Encoding.UTF8.GetString(b);
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
} /// <summary>
|
||
/// utf8 下载页面
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string Utf8(string url, int timeOut = 6)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
try
|
||
{
|
||
WebClientx web = new WebClientx();
|
||
web.Timeout = timeOut;
|
||
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
|
||
web.Proxy = null;
|
||
byte[] b = web.DownloadData(url);
|
||
return Encoding.UTF8.GetString(b);
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// Whois 查询
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string Whois(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
try
|
||
{
|
||
WebClientx web = new WebClientx();
|
||
web.Timeout = 30;
|
||
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
|
||
web.Headers.Add("Add-Cookies", "whois");
|
||
web.Proxy = null;
|
||
byte[] b = web.DownloadData(url);
|
||
return Encoding.UTF8.GetString(b);
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 模拟百度抓取
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByBaidu(string url)
|
||
{
|
||
string userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
|
||
return ByBaidu(url, userAgent);
|
||
}
|
||
/// <summary>
|
||
/// 模拟百度蜘蛛下载页面
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByBaidu(string url, bool redirect)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
||
request.Method = "GET";
|
||
request.Accept = "*/*";
|
||
request.Headers.Add("Accept-Language", "zh-CN");
|
||
request.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
request.UserAgent =
|
||
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1";
|
||
request.AllowAutoRedirect = redirect;
|
||
request.Timeout = 20000;
|
||
request.ReadWriteTimeout = 20000;
|
||
if (!BaseClass.IsTest)
|
||
request.Proxy = null;
|
||
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
||
Stream receiveStream = response.GetResponseStream();
|
||
MemoryStream ms = new MemoryStream();
|
||
int num = 0;
|
||
byte[] buffer = new byte[10240];
|
||
do
|
||
{
|
||
num = receiveStream.Read(buffer, 0, buffer.Length);
|
||
if (num > 0)
|
||
ms.Write(buffer, 0, num);
|
||
} while (num > 0);
|
||
byte[] b = ms.ToArray();
|
||
ms.Close();
|
||
b = ZipHelper.Decompress(b);
|
||
string contentType = response.Headers["Content-Type"];
|
||
string newurl = response.Headers["Location"];
|
||
if (!String.IsNullOrEmpty(newurl))
|
||
return "重定向：" + newurl;
|
||
string html = "";
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
|
||
if (!String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
|
||
string c = GetRegexString(html.ToLower(), r);
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (c.ToLower() != "utf-8" && !String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
else if (String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding("gbk").GetString(b);
|
||
}
|
||
if (!String.IsNullOrEmpty(html))
|
||
{
|
||
if (html.Substring(0, 1) == "锘")
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
//if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 3))
|
||
// html = Encoding.GetEncoding("gbk").GetString(b);
|
||
}
|
||
if (String.IsNullOrEmpty(html) && !String.IsNullOrEmpty(newurl))
|
||
html = "重定向：" + newurl;
|
||
return html;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
return e.Message;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 自动识别代码模拟浏览器
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetUrl(string getUrl, ref string cookies, ref string codeType)
|
||
{
|
||
if (String.IsNullOrEmpty(getUrl))
|
||
return "";
|
||
try
|
||
{
|
||
int retryTimes = 0;
|
||
GotoAgen:
|
||
if (String.IsNullOrEmpty(cookies))
|
||
cookies = null;
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(getUrl);
|
||
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0";
|
||
request.Method = "GET";
|
||
request.Accept = "*/*";
|
||
request.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
request.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
request.Headers.Add("Cookie", cookies);
|
||
if (!BaseClass.IsTest)
|
||
request.Proxy = null;
|
||
request.AllowAutoRedirect = false;
|
||
request.Timeout = 10000;
|
||
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
||
Stream receiveStream = response.GetResponseStream();
|
||
MemoryStream ms = new MemoryStream();
|
||
int num = 0;
|
||
byte[] buffer = new byte[20480];
|
||
do
|
||
{
|
||
try
|
||
{
|
||
num = receiveStream.Read(buffer, 0, buffer.Length);
|
||
if (num > 0)
|
||
ms.Write(buffer, 0, num);
|
||
}
|
||
catch
|
||
{
|
||
num = 0;
|
||
}
|
||
}
|
||
while (num > 0);
|
||
byte[] b = ms.ToArray();
|
||
ms.Close();
|
||
|
||
b = ZipHelper.Decompress(b);
|
||
string cookies1 = response.Headers.Get("Set-Cookie");
|
||
string location = response.Headers.Get("Location"); //跳转地址
|
||
if (!String.IsNullOrEmpty(cookies1))
|
||
cookies = UpCookies(cookies, cookies1);
|
||
string contentType = response.Headers["Content-Type"];
|
||
string html = "";
|
||
if (String.IsNullOrEmpty(location)) //不是跳转
|
||
{
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
codeType = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
|
||
if (!String.IsNullOrEmpty(codeType))
|
||
html = Encoding.GetEncoding(codeType).GetString(b);
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
codeType = "UTF-8";
|
||
html = Encoding.GetEncoding(codeType).GetString(b);
|
||
string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
|
||
string c = GetRegexString(html, r).Trim();
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
|
||
{
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
codeType = c;
|
||
}
|
||
}
|
||
}
|
||
if (retryTimes < 4 && (!String.IsNullOrEmpty(location) || String.IsNullOrEmpty(html)))
|
||
{
|
||
//下面是支持301和302跳转部分
|
||
if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
|
||
{
|
||
getUrl = location;
|
||
}
|
||
else
|
||
{
|
||
if (location.Substring(0, 1) == "/")
|
||
{
|
||
string[] l = getUrl.Split('/');
|
||
if (l.Length > 3)
|
||
{
|
||
getUrl = "";
|
||
for (int i = 0; i < 3; i++)
|
||
getUrl += l[i] + "/";
|
||
getUrl += location.Substring(1, location.Length - 1);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (getUrl.Substring(getUrl.Length - 1, 1) == "/")
|
||
getUrl += location;
|
||
else
|
||
getUrl += "/" + location;
|
||
}
|
||
}
|
||
retryTimes++;
|
||
goto GotoAgen;
|
||
}
|
||
return html;
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 更新cookies
|
||
/// </summary>
|
||
/// <param name="old"></param>
|
||
/// <param name="newc"></param>
|
||
/// <returns></returns>
|
||
private static string UpCookies(string old, string newc)
|
||
{
|
||
if (String.IsNullOrEmpty(old))
|
||
{
|
||
return newc;
|
||
}
|
||
if (String.IsNullOrEmpty(newc))
|
||
{
|
||
return old;
|
||
}
|
||
int io = 0;
|
||
old = old.Trim();
|
||
newc = newc.Trim();
|
||
string[] o = old.Split(';');
|
||
string[] n = newc.Split(';');
|
||
List<CoockiesValue> c = new List<CoockiesValue>();
|
||
for (int i = 0; i < o.Length; i++)
|
||
{
|
||
CoockiesValue cv = new CoockiesValue();
|
||
io = o[i].IndexOf("=");
|
||
if (io > -1)
|
||
{
|
||
cv.Title = o[i].Substring(0, io).Trim();
|
||
cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
|
||
}
|
||
else
|
||
{
|
||
if (!String.IsNullOrEmpty(o[i]))
|
||
cv.Title = o[i].Trim();
|
||
}
|
||
if (!String.IsNullOrEmpty(cv.Title))
|
||
c.Add(cv);
|
||
}
|
||
for (int i = 0; i < n.Length; i++)
|
||
{
|
||
CoockiesValue cv = new CoockiesValue();
|
||
io = o[i].IndexOf("=");
|
||
if (io > -1)
|
||
{
|
||
cv.Title = o[i].Substring(0, io).Trim();
|
||
cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
|
||
}
|
||
else
|
||
{
|
||
if (!String.IsNullOrEmpty(o[i]))
|
||
cv.Title = o[i].Trim();
|
||
}
|
||
if (!String.IsNullOrEmpty(cv.Title))
|
||
{
|
||
bool isHad = false;
|
||
for (int j = 0; j < c.Count; j++)
|
||
{
|
||
if (cv.Title == c[j].Title)
|
||
{
|
||
c[j].Value = cv.Value;
|
||
isHad = true;
|
||
break;
|
||
}
|
||
}
|
||
if (!isHad)
|
||
c.Add(cv);
|
||
}
|
||
}
|
||
string temp = "";
|
||
for (int i = 0; i < c.Count; i++)
|
||
{
|
||
if (String.IsNullOrEmpty(c[i].Value))
|
||
temp += c[i].Title + "; ";
|
||
else
|
||
temp += c[i].Title + "=" + c[i].Value + "; ";
|
||
}
|
||
return temp;
|
||
}
|
||
/// <summary>
|
||
/// cookies值
|
||
/// </summary>
|
||
public class CoockiesValue
|
||
{
|
||
/// <summary>
|
||
/// 名称
|
||
/// </summary>
|
||
public string Title { get; set; }
|
||
/// <summary>
|
||
/// 值
|
||
/// </summary>
|
||
public string Value { get; set; }
|
||
}
|
||
/// <summary>
|
||
/// 模拟百度蜘蛛下载页面
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ByBaidu(string url, string userAgent)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
{
|
||
return "";
|
||
}
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
||
request.Method = "GET";
|
||
request.Accept = "*/*";
|
||
request.Headers.Add("Accept-Language", "zh-CN");
|
||
request.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
request.UserAgent = userAgent;
|
||
request.AllowAutoRedirect = false;
|
||
request.Timeout = 10000;
|
||
request.ReadWriteTimeout = 10000;
|
||
if (!BaseClass.IsTest)
|
||
request.Proxy = null;
|
||
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
||
Stream receiveStream = response.GetResponseStream();
|
||
MemoryStream ms = new MemoryStream();
|
||
int num = 0;
|
||
byte[] buffer = new byte[0x400];
|
||
do
|
||
{
|
||
num = receiveStream.Read(buffer, 0, buffer.Length);
|
||
if (num > 0)
|
||
ms.Write(buffer, 0, num);
|
||
}
|
||
while (num > 0);
|
||
byte[] b = ms.ToArray();
|
||
ms.Close();
|
||
|
||
b = ZipHelper.Decompress(b);
|
||
string contentType = response.Headers["Content-Type"];
|
||
string html = "";
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
|
||
if (!String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
|
||
string c = GetRegexString(html.ToLower(), r);
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (c.ToLower() != "utf-8" && !String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
else if (String.IsNullOrEmpty(c))
|
||
html = Encoding.GetEncoding("gbk").GetString(b);
|
||
}
|
||
if (!String.IsNullOrEmpty(html))
|
||
{
|
||
if (html.Substring(0, 1) == "锘")
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
//if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 3))
|
||
// html = Encoding.GetEncoding("gbk").GetString(b);
|
||
}
|
||
return html;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
return e.Message;
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 返回单个正则分析结果
|
||
/// </summary>
|
||
/// <param name="HtmlString"></param>
|
||
/// <param name="RegexString"></param>
|
||
/// <returns></returns>
|
||
public static string GetRegexString(string HtmlString, string RegexString)
|
||
{
|
||
MatchCollection mc = Regex.Matches(HtmlString, RegexString, RegexOptions.IgnoreCase);
|
||
if (mc.Count > 0)
|
||
return mc[0].Groups[1].ToString();
|
||
return "";
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gbk 下载页面
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string Gbk(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
try
|
||
{
|
||
WebClientx web = new WebClientx();
|
||
web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
|
||
web.Proxy = null;
|
||
byte[] b = web.DownloadData(url);
|
||
return Encoding.GetEncoding("gbk").GetString(b);
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// post 数据
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="postData"></param>
|
||
/// <returns></returns>
|
||
public static string PostUtf8(string url, StringBuilder postData)
|
||
{
|
||
try
|
||
{
|
||
WebClientx wc = new WebClientx();
|
||
byte[] sendData = Encoding.GetEncoding("utf-8").GetBytes(postData.ToString());
|
||
wc.Headers.Add("ContentLength", sendData.Length.ToString());
|
||
wc.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
|
||
wc.Proxy = null;
|
||
return Encoding.UTF8.GetString(wc.UploadData(url, "POST", sendData));
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// post 数据
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="byteData"></param>
|
||
/// <returns></returns>
|
||
public static string PostUtf8(string url, byte[] byteData, int time)
|
||
{
|
||
WebClientx webClient = new WebClientx();
|
||
try
|
||
{
|
||
webClient.Timeout = time;
|
||
webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
|
||
webClient.Proxy = null;
|
||
byte[] b = webClient.UploadData(url, byteData);
|
||
string contentType = webClient.ResponseHeaders["Content-Type"];
|
||
webClient.Dispose();
|
||
webClient = null;
|
||
string html = "";
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
|
||
if (!String.IsNullOrEmpty(c))
|
||
{
|
||
c = c.Trim();
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
|
||
string temp = BaseRegex.GetMatchRegexString(html.ToLower(), r);
|
||
if (String.IsNullOrEmpty(temp))
|
||
return html;
|
||
string c = temp.Trim();
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
return html;
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 浏览器标识
|
||
/// </summary>
|
||
public static string UserAgent = @"Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+6.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/54.0.2840.99+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/31.0.1650.63+Safari/537.36
|
||
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+Win64;+x64;+Trident/5.0)
|
||
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+6.3;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko
|
||
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+WOW64;+Trident/5.0)
|
||
Mozilla/5.0+(Windows+NT+6.3;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1
|
||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||
Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2372.400 QQBrowser/9.5.10548.400";
|
||
|
||
/// <summary>
|
||
/// post 数据
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="postData"></param>
|
||
/// <returns></returns>
|
||
public static string PostUtf8(string url, string postData, string contentType = "")
|
||
{
|
||
//return PostUtf8(url, Encoding.UTF8.GetBytes(postData));
|
||
|
||
bool isError = false;
|
||
string[] l = UserAgent.Split('\n');
|
||
reTry:
|
||
try
|
||
{
|
||
string codeType = "utf-8";
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
||
request.UserAgent = l[new Random(Guid.NewGuid().GetHashCode()).Next(l.Length)].Trim();
|
||
byte[] byteData = Encoding.GetEncoding(codeType).GetBytes(postData);
|
||
request.Method = "POST";
|
||
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
||
request.Timeout = BaseClass.TimeOut * 1000;
|
||
request.ReadWriteTimeout = BaseClass.TimeOut * 1000;
|
||
request.Proxy = null;
|
||
request.ContentType = "application/x-www-form-urlencoded; charset=" + codeType;
|
||
if (url.StartsWith("https://openapi.jinritemai.com/"))
|
||
{
|
||
request.Headers.Add("X-USE-BOE", "1");
|
||
request.Headers.Add("Origin-From", "djt_prod");
|
||
request.Headers.Add("djt_prod", "boe_time_sku");
|
||
}
|
||
if (!string.IsNullOrEmpty(contentType))
|
||
{
|
||
request.ContentType = contentType;
|
||
}
|
||
request.ContentLength = byteData.Length;
|
||
request.Referer = url;
|
||
request.CookieContainer = new CookieContainer();
|
||
request.AllowAutoRedirect = false;
|
||
using (Stream requestStream = request.GetRequestStream())
|
||
{
|
||
requestStream.Write(byteData, 0, byteData.Length);
|
||
}
|
||
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
||
|
||
Stream receiveStream = response.GetResponseStream();
|
||
StreamReader readStream = new StreamReader(receiveStream, Encoding.GetEncoding(codeType));
|
||
return readStream.ReadToEnd().Trim();
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
LogSave.Add(e.ToString());
|
||
//if (!isError)
|
||
//{
|
||
// isError = true;
|
||
// goto reTry;
|
||
//}
|
||
return "";
|
||
}
|
||
|
||
}
|
||
/// <summary>
|
||
/// post 数据
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <param name="byteData"></param>
|
||
/// <returns></returns>
|
||
public static string PostUtf8(string url, byte[] byteData)
|
||
{
|
||
WebClientx webClient = new WebClientx();
|
||
try
|
||
{
|
||
webClient.Timeout = 70;
|
||
webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
|
||
webClient.Proxy = null;
|
||
byte[] b = webClient.UploadData(url, byteData);
|
||
string contentType = webClient.ResponseHeaders["Content-Type"];
|
||
webClient.Dispose();
|
||
webClient = null;
|
||
string html = "";
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
|
||
if (!String.IsNullOrEmpty(c))
|
||
{
|
||
c = c.Trim();
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
|
||
string temp = BaseRegex.GetMatchRegexString(html.ToLower(), r);
|
||
if (String.IsNullOrEmpty(temp))
|
||
return html;
|
||
string c = temp.Trim();
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
return html;
|
||
}
|
||
catch
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// POST读取详细
|
||
/// </summary>
|
||
/// <param name="strUrl">提交地址</param>
|
||
/// <param name="data">提交数据</param>
|
||
/// <param name="userAgen">浏览器头</param>
|
||
/// <returns></returns>
|
||
public static string GetUtf8(string strUrl, string userAgen, string cookie = "", string accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||
{
|
||
try
|
||
{
|
||
WebClientx web = new WebClientx();
|
||
web.Headers.Add("Accept", accept);
|
||
web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
web.Headers.Add("User-Agent", userAgen);
|
||
if (!string.IsNullOrEmpty(cookie))
|
||
web.Headers.Add("Cookie", cookie);
|
||
web.Proxy = null;
|
||
byte[] b = web.DownloadData(strUrl);
|
||
return Encoding.UTF8.GetString(b);
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
return "";
|
||
}
|
||
}
|
||
|
||
public static bool DownloadFileNew(string Urlstring, string FilePath, string cookies)
|
||
{
|
||
try
|
||
{
|
||
WebClientx wb = new WebClientx();
|
||
if (File.Exists(FilePath))
|
||
File.Delete(FilePath);
|
||
string userAgent = BaseClass.ChromeUserAgent;
|
||
wb.Timeout = 30;
|
||
wb.AllowAutoRedirect = true;
|
||
wb.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
|
||
wb.Headers.Add("X-Requested-With", "XMLHttpRequest");
|
||
wb.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
//if (!string.IsNullOrEmpty(cookies) && cookies.Contains("PDDAccessToken") && BaseClass.CookiesAll.ContainsKey("yangkeduo.com") && BaseClass.CookiesAll["yangkeduo.com"].ContainsKey("PDDAccessToken"))
|
||
// wb.Headers.Add("accesstoken", BaseClass.CookiesAll["yangkeduo.com"]["PDDAccessToken"]);
|
||
wb.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
wb.Headers.Add("User-Agent", userAgent);
|
||
wb.Proxy = null;
|
||
//if (!string.IsNullOrEmpty(refere))
|
||
// wb.Headers.Add("Referer", refere);
|
||
cookies = GetCookies(Urlstring, cookies);
|
||
if (!String.IsNullOrEmpty(cookies))
|
||
wb.Headers.Add("Cookie", cookies);
|
||
wb.DownloadFile(Urlstring, FilePath);
|
||
return true;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
public static string Resove(string html, string prefix, string subfix)
|
||
{
|
||
int inl = html.IndexOf(prefix);
|
||
if (inl == -1)
|
||
{
|
||
prefix = prefix.Replace("\"", "'");
|
||
subfix = subfix.Replace("\"", "'");
|
||
inl = html.IndexOf(prefix);
|
||
}
|
||
if (inl == -1)
|
||
{
|
||
return null;
|
||
}
|
||
inl += prefix.Length;
|
||
if (!string.IsNullOrEmpty(subfix))
|
||
{
|
||
int inl2 = html.IndexOf(subfix, inl);
|
||
if (inl2 == -1)
|
||
{
|
||
return null;
|
||
}
|
||
string result = html.Substring(inl, inl2 - inl);
|
||
return result;
|
||
}
|
||
else
|
||
{
|
||
string result = html.Substring(inl);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
public static string GetCookies(string url, string cookies)
|
||
{
|
||
if (!String.IsNullOrEmpty(cookies))
|
||
return cookies;
|
||
string mainDomain = GetHtml.GetMainDomain(GetHtml.GetDomain(url));
|
||
if (BaseClass.CookiesAllString.ContainsKey(mainDomain))
|
||
cookies = BaseClass.CookiesAllString[mainDomain];
|
||
return cookies;
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// URL GB2312转码
|
||
/// </summary>
|
||
/// <param name="IndexKey"></param>
|
||
/// <returns></returns>
|
||
public static string UrlEncodeGb2312(string IndexKey)
|
||
{
|
||
if (String.IsNullOrEmpty(IndexKey))
|
||
return "";
|
||
return HttpUtility.UrlEncode(IndexKey, Encoding.GetEncoding("GB2312"));
|
||
|
||
byte[] sarr = Encoding.GetEncoding("gb2312").GetBytes(IndexKey);
|
||
IndexKey = "";
|
||
for (int i = 0; i < sarr.Length; i++)
|
||
{
|
||
IndexKey += "%";
|
||
IndexKey += sarr[i].ToString("X2");
|
||
}
|
||
return IndexKey;
|
||
}
|
||
/// <summary>
|
||
/// URL UTF-8转码
|
||
/// </summary>
|
||
/// <param name="IndexKey"></param>
|
||
/// <returns></returns>
|
||
public static string UrlEncodeUtf8(string IndexKey)
|
||
{
|
||
if (String.IsNullOrEmpty(IndexKey))
|
||
return "";
|
||
return HttpUtility.UrlEncode(IndexKey, Encoding.UTF8);
|
||
byte[] sarr = Encoding.UTF8.GetBytes(IndexKey);
|
||
IndexKey = "";
|
||
for (int i = 0; i < sarr.Length; i++)
|
||
{
|
||
IndexKey += "%";
|
||
IndexKey += sarr[i].ToString("X2");
|
||
}
|
||
return IndexKey;
|
||
}
|
||
/// <summary>
|
||
/// 用IE打开网址
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
public static void OpenUrl(string url)
|
||
{
|
||
try
|
||
{
|
||
//if (url.Contains("http"))
|
||
// Process.Start("iexplore.exe", url);
|
||
//else
|
||
Process.Start(url);
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 下载文件
|
||
/// </summary>
|
||
/// <param name="Urlstring">下载地址</param>
|
||
/// <param name="FilePath">保存目录</param>
|
||
/// <returns></returns>
|
||
public static bool DownloadFile(string Urlstring, string FilePath)
|
||
{
|
||
try
|
||
{
|
||
WebClientx mywebclient = new WebClientx();
|
||
mywebclient.Timeout = 300;
|
||
if (File.Exists(FilePath))
|
||
File.Delete(FilePath);
|
||
mywebclient.DownloadFile(Urlstring, FilePath);
|
||
return true;
|
||
}
|
||
catch
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 下载文件
|
||
/// </summary>
|
||
/// <param name="Urlstring">下载地址</param>
|
||
/// <param name="b">保存目录</param>
|
||
/// <returns></returns>
|
||
public static bool DownloadFile(string Urlstring, out byte[] b)
|
||
{
|
||
b = new byte[0];
|
||
int times = 0;
|
||
Next:
|
||
try
|
||
{
|
||
WebClientx mywebclient = new WebClientx();
|
||
b = mywebclient.DownloadData(Urlstring);
|
||
return true;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
if (e.Message.Contains("404"))
|
||
return false;
|
||
if (times < 4)
|
||
{
|
||
times++;
|
||
Thread.Sleep(3000);
|
||
goto Next;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// 读取网址，兼容https，兼容任意编码
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetUrl(string url)
|
||
{
|
||
WebClientx webClient = new WebClientx();
|
||
try
|
||
{
|
||
webClient.Timeout = 30;
|
||
webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
|
||
webClient.Proxy = null;
|
||
byte[] b = webClient.DownloadData(url);
|
||
b = ZipHelper.Decompress(b);
|
||
string contentType = webClient.ResponseHeaders["Content-Type"];
|
||
webClient.Dispose();
|
||
webClient = null;
|
||
string html = "";
|
||
if (!String.IsNullOrEmpty(contentType))
|
||
{
|
||
string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
|
||
if (!String.IsNullOrEmpty(c))
|
||
{
|
||
c = c.Trim();
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
}
|
||
if (String.IsNullOrEmpty(html))
|
||
{
|
||
html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
|
||
string c = BaseRegex.GetMatchRegexString(html.ToLower(), r).Trim();
|
||
if (c.IndexOf("'") > -1)
|
||
c = c.Substring(0, c.IndexOf("'"));
|
||
if (c.IndexOf("\"") > -1)
|
||
c = c.Substring(0, c.IndexOf("\""));
|
||
if (c.IndexOf(" ") > -1)
|
||
c = c.Substring(0, c.IndexOf(" "));
|
||
if (c.IndexOf(">") > -1)
|
||
c = c.Substring(0, c.IndexOf(">"));
|
||
if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
|
||
html = Encoding.GetEncoding(c).GetString(b);
|
||
}
|
||
return html;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
LogSave.AddView(url + " 错误：" + e.Message.Replace("\r\n", " "));
|
||
return null;
|
||
}
|
||
finally
|
||
{
|
||
try
|
||
{
|
||
if (webClient != null)
|
||
{
|
||
webClient.Dispose();
|
||
webClient = null;
|
||
}
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// 获取域名
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetDomain(string url)
|
||
{
|
||
try
|
||
{
|
||
url = url.ToLower();
|
||
string temp = BaseRegex.GetMatchRegexString(url, @"([a-z0-9-\.]+\.[a-z]{2,8})");
|
||
if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
|
||
return null;
|
||
if (!String.IsNullOrEmpty(temp))
|
||
return temp;
|
||
if (url.Contains("//") && url.IndexOf("//") + 2 < url.Length)
|
||
url = url.Substring(url.IndexOf("//") + 2, url.Length - url.IndexOf("//") - 2);
|
||
if (url.Contains("/") && url.IndexOf("/") > 0)
|
||
url = url.Substring(0, url.IndexOf("/"));
|
||
return url;
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// 获取域名&ip
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetDomainIp(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return null;
|
||
url = url.ToLower();
|
||
string temp = BaseRegex.GetMatchRegexString(url, @"http[s]{0,1}://([a-z0-9-\.]+\.[a-z]{2,8}|[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})");
|
||
if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
|
||
return null;
|
||
if (!String.IsNullOrEmpty(temp))
|
||
return temp;
|
||
return null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 解析域名ip
|
||
/// </summary>
|
||
/// <param name="domain"></param>
|
||
/// <returns></returns>
|
||
public static string GetIp(string domain)
|
||
{
|
||
try
|
||
{
|
||
for (int i = 0; i < 2; i++)
|
||
{
|
||
IPAddress[] aryIP = Dns.GetHostEntry(domain).AddressList;
|
||
if (aryIP.Length > 0)
|
||
return aryIP[0].ToString();
|
||
Thread.Sleep(1000); //重试，防止DNS缓存
|
||
}
|
||
return null;
|
||
}
|
||
catch
|
||
{
|
||
//SocketServer.ViewCode += "域名：" + domain + "无法解析\r\n";
|
||
return null;
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 是否是域名
|
||
/// </summary>
|
||
/// <param name="domain"></param>
|
||
/// <returns></returns>
|
||
public static bool IsDomain(string domain)
|
||
{
|
||
string temp = BaseRegex.GetMatchRegexString(domain, @"([a-z0-9-\.]+\.[a-z]{2,8})");
|
||
if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
|
||
return false;
|
||
if (temp == domain)
|
||
return true;
|
||
return false;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 获取顶级域名
|
||
/// </summary>
|
||
/// <param name="domain"></param>
|
||
/// <returns></returns>
|
||
public static string GetMainDomain(string domain)
|
||
{
|
||
if (String.IsNullOrEmpty(domain))
|
||
return "";
|
||
domain = domain.ToLower();
|
||
if (!domain.Contains(".") || domain.Length < 4)
|
||
return "";
|
||
string[] d = domain.Split('.');
|
||
string main = d[d.Length - 2] + "." + d[d.Length - 1]; //原始域名
|
||
if (d.Length > 2 &&
|
||
(d[d.Length - 2] == "com" || d[d.Length - 2] == "net" || d[d.Length - 2] == "org" ||
|
||
d[d.Length - 2] == "gov" || d[d.Length - 2] == "edu" || d[d.Length - 2] == "co"))
|
||
main = d[d.Length - 3] + "." + main;
|
||
return main;
|
||
}
|
||
/// <summary>
|
||
/// 解密Jsion
|
||
/// </summary>
|
||
/// <param name="s"></param>
|
||
/// <returns></returns>
|
||
public static string Decode(string s)
|
||
{
|
||
s = WebUtility.HtmlDecode(s);
|
||
Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4})", RegexOptions.Compiled);
|
||
return reUnicode.Replace(s, m =>
|
||
{
|
||
short c;
|
||
if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
|
||
{
|
||
return "" + (char)c;
|
||
}
|
||
return m.Value;
|
||
});
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// 获取国内IP
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public static string ProxyIpIn()
|
||
{
|
||
if (BaseClass.ProxyIn.Count == 0)
|
||
return "";
|
||
Random ra = new Random(Guid.NewGuid().GetHashCode());
|
||
return BaseClass.ProxyIn[ra.Next(BaseClass.ProxyIn.Count)];
|
||
}
|
||
/// <summary>
|
||
/// 获取美国IP
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public static string ProxyIpOut()
|
||
{
|
||
if (BaseClass.ProxyOut.Count == 0)
|
||
return "";
|
||
Random ra = new Random(Guid.NewGuid().GetHashCode());
|
||
return BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
|
||
}
|
||
/// <summary>
|
||
/// 获取国内IP代理URL地址
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public static string ProxyIpInUrl(string url)
|
||
{
|
||
if (BaseClass.ProxyIn.Count == 0)
|
||
return url;
|
||
Random ra = new Random(Guid.NewGuid().GetHashCode());
|
||
string ip = BaseClass.ProxyIn[ra.Next(BaseClass.ProxyIn.Count)];
|
||
return "http://" + ip + ":6000/url=" + ReplaceUrl(url);
|
||
}
|
||
/// <summary>
|
||
/// 获取美国IP代理URL地址
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
public static string ProxyIpOutUrl(string url)
|
||
{
|
||
if (BaseClass.ProxyOut.Count == 0)
|
||
return url;
|
||
Random ra = new Random(Guid.NewGuid().GetHashCode());
|
||
string ip = BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
|
||
return "http://" + ip + ":6000/url=" + ReplaceUrl(url);
|
||
}
|
||
/// <summary>
|
||
/// 替换链接，防止防火墙
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ReplaceUrl(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
string newurl = "";
|
||
string[] l = url.Split('.');
|
||
for (int i = 0; i < l.Length; i++)
|
||
{
|
||
if (l[i].Length > 2)
|
||
newurl += l[i].Substring(0, l[i].Length - 2) + "***ooo***" + l[i].Substring(l[i].Length - 2, 2);
|
||
else
|
||
newurl += l[i];
|
||
if (i < l.Length - 1)
|
||
newurl += ".";
|
||
}
|
||
return newurl.Replace(".google.", ".goo00gle.").Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
|
||
}
|
||
/// <summary>
|
||
/// 还原链接
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string ReplaceUrlBak(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return "";
|
||
string newurl = url.Replace("***ooo***", "");
|
||
if (String.IsNullOrEmpty(newurl))
|
||
return "";
|
||
return newurl.Replace(".goo00gle.", ".google.").Replace("%3f", "?").Replace("%3d", "=").Replace("%26", "&");
|
||
}
|
||
/// <summary>
|
||
/// 生成URL绝对地址
|
||
/// </summary>
|
||
/// <param name="url">新URL</param>
|
||
/// <param name="oldurl">源地址</param>
|
||
/// <returns></returns>
|
||
public static string BuildUrl(string url, string oldurl)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return oldurl;
|
||
if (url.Contains("tencent:/"))
|
||
return url;
|
||
if (url.Length > 8 && url.Substring(0, 8).Contains("://"))
|
||
{
|
||
if (url.Length - url.Replace("/", "").Length == 2)
|
||
url += "/";
|
||
return url;
|
||
}
|
||
|
||
string homeurl = BaseRegex.GetMatchRegexString(oldurl.ToLower(), "(http://[^/]*?)/");
|
||
if (String.IsNullOrEmpty(homeurl))
|
||
{
|
||
homeurl = BaseRegex.GetMatchRegexString(oldurl.ToLower(), "(http://[^/]+)") + "/";
|
||
if (String.IsNullOrEmpty(homeurl))
|
||
{
|
||
//BaseClass.ViewCode += "网址：" + oldurl + "识别失败\r\n";
|
||
return url;
|
||
}
|
||
}
|
||
if (url.Substring(0, 1) == "/")
|
||
{
|
||
string newurlx = homeurl + url;
|
||
newurlx = newurlx.Replace("//", "/").Replace("http:/", "http://");
|
||
return newurlx;
|
||
}
|
||
string[] l = oldurl.Split('/');
|
||
int cute = l.Length;
|
||
if (l.Length > 3)
|
||
cute--;
|
||
while (url.Contains("../"))
|
||
{
|
||
int s = url.IndexOf("../");
|
||
if (s == 0)
|
||
url = url.Substring(3);
|
||
else
|
||
url = url.Substring(0, s) + url.Substring(s + 3);
|
||
cute--;
|
||
if (cute < 3)
|
||
cute = 3;
|
||
}
|
||
string newurl = "";
|
||
for (int i = 0; i < cute; i++)
|
||
newurl += l[i] + "/";
|
||
newurl = newurl + url;
|
||
return newurl.Replace("//", "/").Replace("http:/", "http://").Replace("/./", "/");
|
||
}
|
||
/// <summary>
|
||
/// 获取代理地址
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetProxyUrl(string url)
|
||
{
|
||
if (String.IsNullOrEmpty(url))
|
||
return null;
|
||
if (BaseClass.Proxy.Count == 0)
|
||
{
|
||
BaseClass.ViewCode += "代理异常";
|
||
return url;
|
||
}
|
||
Random ra = new Random();
|
||
string p = BaseClass.Proxy[ra.Next(BaseClass.Proxy.Count)];
|
||
return "http://" + p + "/url=" + url.Replace(".google.", ".goo00gle.")
|
||
.Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
|
||
}
|
||
|
||
|
||
public static List<string> ProxyIpList = new List<string>();
|
||
|
||
/// <summary>
|
||
/// 服务器代理采集
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static string GetProxyServer(string url, string needString)
|
||
{
|
||
string html = GetProxyServerx(url);
|
||
if (string.IsNullOrEmpty(html) || !html.Contains(needString))
|
||
html = GetProxyServerx(url);
|
||
if (string.IsNullOrEmpty(html) || !html.Contains(needString))
|
||
html = GetProxyServerx(url);
|
||
if (string.IsNullOrEmpty(html) || !html.Contains(needString))
|
||
html = GetProxyServerx(url);
|
||
return html;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 服务器代理采集
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
private static string GetProxyServerx(string url)
|
||
{
|
||
if (ProxyIpList.Count < 10)
|
||
return "";
|
||
string ip = ProxyIpList[Randomx.GetId(ProxyIpList.Count)];
|
||
WebClientx webClient = new WebClientx();
|
||
try
|
||
{
|
||
webClient.Timeout = 10;
|
||
webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
|
||
webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
|
||
webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
|
||
webClient.Proxy = new WebProxy(ip);
|
||
byte[] b = webClient.DownloadData(url);
|
||
b = ZipHelper.Decompress(b);
|
||
string html = Encoding.GetEncoding("UTF-8").GetString(b);
|
||
return html;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
LogSave.AddView(url + " 错误：" + e.Message.Replace("\r\n", " "));
|
||
return null;
|
||
}
|
||
finally
|
||
{
|
||
try
|
||
{
|
||
webClient.Dispose();
|
||
webClient = null;
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
}
|
||
}
|
||
public static string RequestImg(string url, byte[] imgData)
|
||
{
|
||
HttpWebResponse response = null;
|
||
var responseText = "";
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
||
request.KeepAlive = true;
|
||
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
||
request.Headers.Add("Origin", url);
|
||
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
||
request.ContentType = "multipart/form-data; boundary=----WebKitFormBoundaryg0ATB1h6Pa505OBi";
|
||
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36";
|
||
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
|
||
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
|
||
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
||
request.Method = "POST";
|
||
request.ServicePoint.Expect100Continue = false;
|
||
string body = "------WebKitFormBoundaryg0ATB1h6Pa505OBi\r\n" +
|
||
"Content-Disposition: form-data; name=\"file\"; filename=\"search.jpg\"\r\n" +
|
||
"Content-Type: image/jpeg\r\n\r\n" +
|
||
"<!>filedata<!>\r\n" +
|
||
"------WebKitFormBoundaryg0ATB1h6Pa505OBi--\r\n";
|
||
WriteMultipartBodyToRequest(request, body, imgData);
|
||
|
||
response = (HttpWebResponse)request.GetResponse();
|
||
}
|
||
catch (WebException e)
|
||
{
|
||
if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
||
else return "error: " + e.Message;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
if (response != null) response.Close();
|
||
return "error: " + ex.Message;
|
||
}
|
||
if (response != null)
|
||
{
|
||
responseText = ReadResponse(response);
|
||
response.Close();
|
||
}
|
||
return responseText;
|
||
}
|
||
/// <summary>
|
||
/// 读取返回值
|
||
/// </summary>
|
||
/// <param name="response"></param>
|
||
/// <returns></returns>
|
||
private static string ReadResponse(HttpWebResponse response)
|
||
{
|
||
using (Stream responseStream = response.GetResponseStream())
|
||
{
|
||
Stream streamToRead = responseStream;
|
||
if (response.ContentEncoding.ToLower().Contains("gzip"))
|
||
{
|
||
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
|
||
}
|
||
else if (response.ContentEncoding.ToLower().Contains("deflate"))
|
||
{
|
||
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
|
||
}
|
||
|
||
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
|
||
{
|
||
return streamReader.ReadToEnd();
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void WriteMultipartBodyToRequest(HttpWebRequest request, string body, byte[] data)
|
||
{
|
||
string[] multiparts = Regex.Split(body, @"<!>");
|
||
byte[] bytes;
|
||
using (MemoryStream ms = new MemoryStream())
|
||
{
|
||
foreach (string part in multiparts)
|
||
{
|
||
if (part == "filedata") // 文件信息
|
||
{
|
||
bytes = data;
|
||
}
|
||
else
|
||
{
|
||
bytes = System.Text.Encoding.UTF8.GetBytes(part.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n"));
|
||
}
|
||
ms.Write(bytes, 0, bytes.Length);
|
||
}
|
||
|
||
request.ContentLength = ms.Length;
|
||
using (Stream stream = request.GetRequestStream())
|
||
{
|
||
ms.WriteTo(stream);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|