/*
 * 采集网页
 * 肖俊
 * 2013-11-12
 * 
 */

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;

namespace AllBase
{
    public class GetHtml
    {
        /// <summary>
        /// 加载远程图片
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static Image LoadImage(string url)
        {
            try
            {
                WebRequest webreq = WebRequest.Create(url);
                WebResponse webres = webreq.GetResponse();
                Stream stream = webres.GetResponseStream();
                return Image.FromStream(stream);
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// 读取网络图片转为Img以在DataGridView中显示，用于HttpWebRequest直接读取可能403的情况
        /// </summary>
        /// <param name="imgUrl"></param>
        /// <param name="rst"></param>
        /// <returns></returns>
        public static Image GetUrlImg(string imgUrl, out string rst)
        {
            rst = "";
            //return LoadImage(imgUrl);
            Image image = null;
            HttpWebResponse response = null;
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(imgUrl);

                request.KeepAlive = true;
                request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
                request.Headers.Add("Upgrade-Insecure-Requests", @"1");
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
                request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
                request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
                //request.IfModifiedSince = DateTime.Parse("Fri, 18 Jul 2025 06:00:49 GMT"); // 没改动不更新，CS程序无缓存不使用

                response = (HttpWebResponse)request.GetResponse();

                using (Stream stream = response.GetResponseStream())
                {
                    image = Image.FromStream(stream);
                }
            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.ProtocolError)
                {
                    response = (HttpWebResponse)ex.Response;
                    rst = GetProxyHtml.ReadResponse(response);
                }
                if (string.IsNullOrEmpty(rst))
                    rst = ex.Message;
            }
            catch (Exception e)
            {
                rst = e.Message;
            }
            if (response != null)
            {
                response.Close();
                response.Dispose();
            }
            return image;

            //try
            //{
            //    if (!string.IsNullOrEmpty(prodSelect?.main_img))
            //    {
            //        using (Stream stream = WebRequest.Create(prodSelect.main_img.Replace("https", "http")).GetResponse().GetResponseStream())
            //        {
            //            image = Image.FromStream(stream);
            //        }
            //    }
            //}
            //catch (Exception ex)
            //{
            //}
        }

        public static Icon LoadIcon(string url)
        {
            try
            {
                Bitmap bmp = new Bitmap(LoadImage(url));
                IntPtr h = bmp.GetHicon();
                System.Drawing.Icon icon = System.Drawing.Icon.FromHandle(h);
                DeleteObject(h);
                return icon;
            }
            catch (Exception ex)
            {
                return null;
            }
        }
        [DllImport("gdi32.dll")]
        public static extern bool DeleteObject(IntPtr hObject);

        /// <summary>
        /// 代理抓取
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByProxy(string url)
        {
            string html = "", newurl = GetProxyUrlx(url);
            Random ro = new Random();
            int i = 0;
            MultiThreadClient mc = new MultiThreadClient();
        ReTry:
            html = mc.GetHtml(newurl);
            if (IsError(ref html))
            {
                Thread.Sleep(ro.Next(2000, 6000));
                if (i < 4)
                {
                    i++;
                    goto ReTry;
                }
                newurl = GetProxyUrlx(url);
                html = mc.GetHtml(newurl);
                if (IsError(ref html))
                    return mc.GetHtml(url);
                return html;
            }
            Thread.Sleep(100 + BaseClass.WaitTime / 2 + ro.Next(BaseClass.WaitTime));
            Clear(ref html);
            return html;
        }
        /// <summary>
        /// 清理、处理代码
        /// </summary>
        /// <param name="html"></param>
        private static void Clear(ref string html)
        {
            html = UrlDecode(html);
            html = html.Replace("\\", "");
        }
        /// <summary>
        /// 是否错误
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public static bool IsError(ref string html)
        {
            return String.IsNullOrEmpty(html) || html.Length < 100 || html.Contains("您的访问出错了</title>") ||
                   html.Contains("访问异常出错</title>");
        }
        /// <summary>
        /// 获取代理地址
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetProxyUrlx(string url)
        {
            if (String.IsNullOrEmpty(url))
                return null;
            if (BaseClass.ProxyOut.Count == 0)
            {
                BaseClass.ViewCode += "代理异常";
                return url;
            }
            Random ra = new Random();
            string p = BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
            return "http://" + p + "/url=" + url.Replace(".google.", ".goo00gle.")
                .Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
        }
        /// <summary>
        /// ascii码转译为汉字
        /// </summary>
        /// <param name="ascii">带ASCII码的字符串</param>
        /// <returns>还原字符串</returns>
        public static void AsciiToString(ref string temp)
        {
            if (String.IsNullOrEmpty(temp))
                return;
            //return;
            try
            {
                if (temp.Contains("&#"))
                {
                    Regex reUnicode = new Regex(@"&#[x]([0-9A-Fa-f]{4,5});", RegexOptions.Compiled);
                    temp = reUnicode.Replace(temp, m =>
                    {
                        short c;
                        if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
                            return "" + (char)c;
                        return m.Value;
                    });
                }
                if (temp.Contains("\\u"))
                {
                    Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4,5})", RegexOptions.Compiled);
                    temp = reUnicode.Replace(temp, m =>
                    {
                        short c;
                        if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber,
                            CultureInfo.InvariantCulture, out c))
                            return "" + (char)c;
                        return m.Value;
                    });
                }
            }
            catch
            {
            }
            //string html = "´,&acute;©,&copy;&gt;,&gt;µ,&micro;®,&reg;°,&deg;¡,&iexcl;,&nbsp;»,&raquo;" +
            //    "¦,&brvbar;÷,&divide;¿,&iquest;¬,&not;§,&sect;•,&bull;½,&frac12;«,&laquo;¶,&para;¨,&uml;" +
            //    "¸,&cedil;¼,&frac14;&lt;,&lt;±,&plusmn;×,&times;¢,&cent;¾,&frac34;¯,&macr;&ldquo;,&quot;" +
            //    "™,&trade;€,&euro;£,&pound;¥,&yen;&bdquo;,&bdquo;…,&hellip;·,&middot;›,&rsaquo;ª,&ordf;" +
            //    "ˆ,&circ;&ldquo;,&ldquo;—,&mdash;&rsquo;,&rsquo;º,&ordm;†,&dagger;‹,&lsaquo;–,&ndash;" +
            //    "‚,&sbquo;&rdquo;,&rdquo;‡,&Dagger;&lsquo;,&lsquo;‰,&permil;­,&shy;˜,&tilde;≈,&asymp;" +
            //    "⁄,&frasl;←,&larr;∂,&part;♠,&spades;∩,&cap;≥,&ge;≤,&le;″,&Prime;∑,&sum;♣,&clubs;" +
            //    "↔,&harr;◊,&loz;′,&prime;↑,&uarr;↓,&darr;♥,&hearts;−,&minus;∏,&prod;‍,&zwj;♦,&diams;" +
            //    "∞,&infin;≠,&ne;√,&radic;‌,&zwnj;≡,&equiv;∫,&int;‾,&oline;→,&rarr;α,&alpha;η,&eta;" +
            //    "μ,&mu;π,&pi;θ,&theta;β,&beta;γ,&gamma;ν,&nu;ψ,&psi;υ,&upsilon;χ,&chi;ι,&iota;" +
            //    "ω,&omega;ρ,&rho;ξ,&xi;δ,&delta;κ,&kappa;ο,&omicron;σ,&sigma;ζ,&zeta;ε,&epsilon;" +
            //    "λ,&lambda;φ,&phi;τ,&tau;Α,&Alpha;Η,&Eta;Μ,&Mu;Π,&Pi;Θ,&Theta;Β,&Beta;Γ,&Gamma;" +
            //    "Ν,&Nu;Ψ,&Psi;Υ,&Upsilon;Χ,&Chi;Ι,&Iota;Ω,&Omega;Ρ,&Rho;Ξ,&Xi;Δ,&Delta;Κ,&Kappa;" +
            //    "Ο,&Omicron;Σ,&Sigma;Ζ,&Zeta;Ε,&Epsilon;Λ,&Lambda;Φ,&Phi;Τ,&Tau;ς,&sigmaf;&,&amp";
            //string[] h = html.Split(';');
            //string s1;
            //string s2;
            //foreach (string s in h)
            //{
            //    s2 = s.Substring(2, s.Length - 2) + ";";
            //    if (!temp.Contains(s2))
            //        continue;
            //    s1 = s.Substring(0, 1);
            //    temp = temp.Replace(s2, s1);
            //}
            //return temp;
        }
        /// <summary>
        /// html解码，\u解码
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public static string UrlDecode(string html)
        {
            if (html == null)
                return html;

            string temp = HttpUtility.UrlDecode(html);
            Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4})", RegexOptions.Compiled);
            return reUnicode.Replace(temp, m =>
            {
                short c;
                if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
                    return "" + (char)c;
                return m.Value;
            });
        }

        /// <summary>
        /// 本地+国内+美国
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string GetByMyself(string url, out bool isError, bool isServer)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = isServer ? 40 : 20;
            string Html = mu.GetHtml(url);
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(ProxyIpInUrl(url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(ProxyIpOutUrl(url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }
        /// <summary>
        /// 国内+本地+美国
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string GetByIn(string url, out bool isError, bool isServer)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = isServer ? 40 : 20; ;
            string Html = mu.GetHtml(ProxyIpInUrl(url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(url);
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                mu.TimeOut = 30;
                Html = mu.GetHtml(ProxyIpOutUrl(url));
            }
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }
        /// <summary>
        /// 国内+本地+美国
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string PostByIn(string url, out bool isError, bool isServer, string data)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = isServer ? 40 : 20; ;
            string Html = mu.GetHtml(ProxyIpInUrl(url), Encoding.UTF8.GetBytes(data));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(url, Encoding.UTF8.GetBytes(data));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(ProxyIpOutUrl(url), Encoding.UTF8.GetBytes(data));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }
        /// <summary>
        /// 美国+国内+本地
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string GetByOut(string url, out bool isError, bool isServer)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = isServer ? 60 : 30; ;
            string Html = mu.GetHtml(ProxyIpOutUrl(url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(ProxyIpInUrl(url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(url);
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }
        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByUsa(string url)
        {
            bool isError;
            return ByUsa(url, out isError, false, false, false, false);
        }
        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByUsa(string url, bool mobile)
        {
            bool isError;
            return ByUsa(url, out isError, mobile, false, false, false);
        }
        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByUsa(bool isUtf8, string url)
        {
            bool isError;
            return ByUsa(url, out isError, false, false, isUtf8, false);
        }
        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByUsa(string url, bool mobile, bool isGbk)
        {
            bool isError;
            return ByUsa(url, out isError, mobile, isGbk, false, false);
        }
        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="domain"></param>
        /// <returns></returns>
        public static string ByUsaWhoies(string domain)
        {
            bool isError;
            return ByUsa(domain, out isError, false, false, false, true);
        }

        /// <summary>
        /// 美国服务器出口
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string ByUsa(string url, out bool isError, bool mobile, bool isGbk, bool isUtf8, bool isWhois)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = 50;
            mu.IsGbk = isGbk;
            mu.IsUtf8 = isUtf8;
            if (mobile)
                mu.AddCookies = "Moblie";
            mu.IsWhoies = isWhois;
            string Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyOut, url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyOut, url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }

        /// <summary>
        /// 局域网出口
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string ByLocation(string url)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = 30;
            string Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
                Html = mu.GetHtml(GetProxyUrl(BaseClass.ProxyIn, url));
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                return null;
            }
            return Html;
        }

        /// <summary>
        /// 远程下载文件
        /// </summary>
        /// <param name="url"></param>
        /// <param name="path"></param>
        /// <returns></returns>
        public static bool ByUsaDownload(string url, string path)
        {
            string newUrl = GetProxyUrl(BaseClass.ProxyOut, url);
            bool ok = DownloadFile(newUrl, path);
            if (!ok)
                DownloadFile(GetProxyUrl(BaseClass.ProxyOut, url), path);
            if (!ok)
                DownloadFile(GetProxyUrl(BaseClass.ProxyOut, url), path);
            return ok;
        }


        /// <summary>
        /// HTTPS专用出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByHttps(string url)
        {
            bool isError;
            return ByHttps(url, out isError, false);
        }
        /// <summary>
        /// HTTPS专用出口
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByHttps(string url, bool mobile)
        {
            bool isError;
            return ByHttps(url, out isError, mobile);
        }
        /// <summary>
        /// HTTPS专用出口
        /// </summary>
        /// <param name="url"></param>
        /// <param name="isError"></param>
        /// <returns></returns>
        public static string ByHttps(string url, out bool isError, bool mobile)
        {
            MultiThreadClient mu = new MultiThreadClient();
            mu.TimeOut = 60;
            if (mobile)
                mu.IsMoblie = true;
            string urlx = GetProxyUrl(BaseClass.ProxyHttps, url);
            string Html = mu.GetHtml(urlx);
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                urlx = GetProxyUrl(BaseClass.ProxyHttps, url);
                Html = mu.GetHtml(urlx);
            }
            if (String.IsNullOrEmpty(Html) || mu.IsError)
            {
                isError = true;
                return null;
            }
            isError = false;
            return Html;
        }
        /// <summary>
        /// 转换URL
        /// </summary>
        /// <param name="ips"></param>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetProxyUrl(List<string> ips, string url)
        {
            if (ips.Count == 0)
                return url;
            Random ra = new Random(Guid.NewGuid().GetHashCode());
            string ip = ips[ra.Next(ips.Count)];

            return "http://" + (ip.Contains(":") ? ip : ip + ":6000") + "/url=" + ReplaceUrl(url);
        }

        /// <summary>
        /// utf8 下载页面
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string Utf8(string url)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            try
            {
                WebClientx web = new WebClientx();
                web.Timeout = 60;
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
                web.Proxy = null;
                byte[] b = web.DownloadData(url);
                return Encoding.UTF8.GetString(b);
            }
            catch
            {
                return "";
            }
        }        /// <summary>
                 /// utf8 下载页面
                 /// </summary>
                 /// <param name="url"></param>
                 /// <returns></returns>
        public static string Utf8(string url, int timeOut = 6)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            try
            {
                WebClientx web = new WebClientx();
                web.Timeout = timeOut;
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
                web.Proxy = null;
                byte[] b = web.DownloadData(url);
                return Encoding.UTF8.GetString(b);
            }
            catch
            {
                return "";
            }
        }
        /// <summary>
        /// Whois 查询
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string Whois(string url)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            try
            {
                WebClientx web = new WebClientx();
                web.Timeout = 30;
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
                web.Headers.Add("Add-Cookies", "whois");
                web.Proxy = null;
                byte[] b = web.DownloadData(url);
                return Encoding.UTF8.GetString(b);
            }
            catch
            {
                return "";
            }
        }

        /// <summary>
        /// 模拟百度抓取
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByBaidu(string url)
        {
            string userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
            return ByBaidu(url, userAgent);
        }
        /// <summary>
        /// 模拟百度蜘蛛下载页面
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByBaidu(string url, bool redirect)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.Method = "GET";
                request.Accept = "*/*";
                request.Headers.Add("Accept-Language", "zh-CN");
                request.Headers.Add("Accept-Encoding", "gzip,deflate");
                request.UserAgent =
                    "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1";
                request.AllowAutoRedirect = redirect;
                request.Timeout = 20000;
                request.ReadWriteTimeout = 20000;
                if (!BaseClass.IsTest)
                    request.Proxy = null;
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream receiveStream = response.GetResponseStream();
                MemoryStream ms = new MemoryStream();
                int num = 0;
                byte[] buffer = new byte[10240];
                do
                {
                    num = receiveStream.Read(buffer, 0, buffer.Length);
                    if (num > 0)
                        ms.Write(buffer, 0, num);
                } while (num > 0);
                byte[] b = ms.ToArray();
                ms.Close();
                b = ZipHelper.Decompress(b);
                string contentType = response.Headers["Content-Type"];
                string newurl = response.Headers["Location"];
                if (!String.IsNullOrEmpty(newurl))
                    return "重定向：" + newurl;
                string html = "";
                if (!String.IsNullOrEmpty(contentType))
                {
                    string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
                    if (!String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding(c).GetString(b);
                }
                if (String.IsNullOrEmpty(html))
                {
                    html = Encoding.GetEncoding("UTF-8").GetString(b);
                    string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
                    string c = GetRegexString(html.ToLower(), r);
                    if (c.IndexOf("'") > -1)
                        c = c.Substring(0, c.IndexOf("'"));
                    if (c.IndexOf("\"") > -1)
                        c = c.Substring(0, c.IndexOf("\""));
                    if (c.IndexOf(" ") > -1)
                        c = c.Substring(0, c.IndexOf(" "));
                    if (c.IndexOf(">") > -1)
                        c = c.Substring(0, c.IndexOf(">"));
                    if (c.ToLower() != "utf-8" && !String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding(c).GetString(b);
                    else if (String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding("gbk").GetString(b);
                }
                if (!String.IsNullOrEmpty(html))
                {
                    if (html.Substring(0, 1) == "锘")
                        html = Encoding.GetEncoding("UTF-8").GetString(b);
                    //if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 3))
                    //   html = Encoding.GetEncoding("gbk").GetString(b);
                }
                if (String.IsNullOrEmpty(html) && !String.IsNullOrEmpty(newurl))
                    html = "重定向：" + newurl;
                return html;
            }
            catch (Exception e)
            {
                return e.Message;
            }
        }

        /// <summary>
        /// 自动识别代码模拟浏览器
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetUrl(string getUrl, ref string cookies, ref string codeType)
        {
            if (String.IsNullOrEmpty(getUrl))
                return "";
            try
            {
                int retryTimes = 0;
            GotoAgen:
                if (String.IsNullOrEmpty(cookies))
                    cookies = null;
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(getUrl);
                request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0";
                request.Method = "GET";
                request.Accept = "*/*";
                request.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                request.Headers.Add("Accept-Encoding", "gzip,deflate");
                request.Headers.Add("Cookie", cookies);
                if (!BaseClass.IsTest)
                    request.Proxy = null;
                request.AllowAutoRedirect = false;
                request.Timeout = 10000;
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream receiveStream = response.GetResponseStream();
                MemoryStream ms = new MemoryStream();
                int num = 0;
                byte[] buffer = new byte[20480];
                do
                {
                    try
                    {
                        num = receiveStream.Read(buffer, 0, buffer.Length);
                        if (num > 0)
                            ms.Write(buffer, 0, num);
                    }
                    catch
                    {
                        num = 0;
                    }
                }
                while (num > 0);
                byte[] b = ms.ToArray();
                ms.Close();

                b = ZipHelper.Decompress(b);
                string cookies1 = response.Headers.Get("Set-Cookie");
                string location = response.Headers.Get("Location"); //跳转地址
                if (!String.IsNullOrEmpty(cookies1))
                    cookies = UpCookies(cookies, cookies1);
                string contentType = response.Headers["Content-Type"];
                string html = "";
                if (String.IsNullOrEmpty(location))     //不是跳转
                {
                    if (!String.IsNullOrEmpty(contentType))
                    {
                        codeType = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
                        if (!String.IsNullOrEmpty(codeType))
                            html = Encoding.GetEncoding(codeType).GetString(b);
                    }
                    if (String.IsNullOrEmpty(html))
                    {
                        codeType = "UTF-8";
                        html = Encoding.GetEncoding(codeType).GetString(b);
                        string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
                        string c = GetRegexString(html, r).Trim();
                        if (c.IndexOf("'") > -1)
                            c = c.Substring(0, c.IndexOf("'"));
                        if (c.IndexOf("\"") > -1)
                            c = c.Substring(0, c.IndexOf("\""));
                        if (c.IndexOf(" ") > -1)
                            c = c.Substring(0, c.IndexOf(" "));
                        if (c.IndexOf(">") > -1)
                            c = c.Substring(0, c.IndexOf(">"));
                        if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
                        {
                            html = Encoding.GetEncoding(c).GetString(b);
                            codeType = c;
                        }
                    }
                }
                if (retryTimes < 4 && (!String.IsNullOrEmpty(location) || String.IsNullOrEmpty(html)))
                {
                    //下面是支持301和302跳转部分
                    if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
                    {
                        getUrl = location;
                    }
                    else
                    {
                        if (location.Substring(0, 1) == "/")
                        {
                            string[] l = getUrl.Split('/');
                            if (l.Length > 3)
                            {
                                getUrl = "";
                                for (int i = 0; i < 3; i++)
                                    getUrl += l[i] + "/";
                                getUrl += location.Substring(1, location.Length - 1);
                            }
                        }
                        else
                        {
                            if (getUrl.Substring(getUrl.Length - 1, 1) == "/")
                                getUrl += location;
                            else
                                getUrl += "/" + location;
                        }
                    }
                    retryTimes++;
                    goto GotoAgen;
                }
                return html;
            }
            catch
            {
                return "";
            }
        }
        /// <summary>
        /// 更新cookies
        /// </summary>
        /// <param name="old"></param>
        /// <param name="newc"></param>
        /// <returns></returns>
        private static string UpCookies(string old, string newc)
        {
            if (String.IsNullOrEmpty(old))
            {
                return newc;
            }
            if (String.IsNullOrEmpty(newc))
            {
                return old;
            }
            int io = 0;
            old = old.Trim();
            newc = newc.Trim();
            string[] o = old.Split(';');
            string[] n = newc.Split(';');
            List<CoockiesValue> c = new List<CoockiesValue>();
            for (int i = 0; i < o.Length; i++)
            {
                CoockiesValue cv = new CoockiesValue();
                io = o[i].IndexOf("=");
                if (io > -1)
                {
                    cv.Title = o[i].Substring(0, io).Trim();
                    cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
                }
                else
                {
                    if (!String.IsNullOrEmpty(o[i]))
                        cv.Title = o[i].Trim();
                }
                if (!String.IsNullOrEmpty(cv.Title))
                    c.Add(cv);
            }
            for (int i = 0; i < n.Length; i++)
            {
                CoockiesValue cv = new CoockiesValue();
                io = o[i].IndexOf("=");
                if (io > -1)
                {
                    cv.Title = o[i].Substring(0, io).Trim();
                    cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
                }
                else
                {
                    if (!String.IsNullOrEmpty(o[i]))
                        cv.Title = o[i].Trim();
                }
                if (!String.IsNullOrEmpty(cv.Title))
                {
                    bool isHad = false;
                    for (int j = 0; j < c.Count; j++)
                    {
                        if (cv.Title == c[j].Title)
                        {
                            c[j].Value = cv.Value;
                            isHad = true;
                            break;
                        }
                    }
                    if (!isHad)
                        c.Add(cv);
                }
            }
            string temp = "";
            for (int i = 0; i < c.Count; i++)
            {
                if (String.IsNullOrEmpty(c[i].Value))
                    temp += c[i].Title + "; ";
                else
                    temp += c[i].Title + "=" + c[i].Value + "; ";
            }
            return temp;
        }
        /// <summary>
        /// cookies值
        /// </summary>
        public class CoockiesValue
        {
            /// <summary>
            /// 名称
            /// </summary>
            public string Title { get; set; }
            /// <summary>
            /// 值
            /// </summary>
            public string Value { get; set; }
        }
        /// <summary>
        /// 模拟百度蜘蛛下载页面
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ByBaidu(string url, string userAgent)
        {
            if (String.IsNullOrEmpty(url))
            {
                return "";
            }
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.Method = "GET";
                request.Accept = "*/*";
                request.Headers.Add("Accept-Language", "zh-CN");
                request.Headers.Add("Accept-Encoding", "gzip,deflate");
                request.UserAgent = userAgent;
                request.AllowAutoRedirect = false;
                request.Timeout = 10000;
                request.ReadWriteTimeout = 10000;
                if (!BaseClass.IsTest)
                    request.Proxy = null;
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream receiveStream = response.GetResponseStream();
                MemoryStream ms = new MemoryStream();
                int num = 0;
                byte[] buffer = new byte[0x400];
                do
                {
                    num = receiveStream.Read(buffer, 0, buffer.Length);
                    if (num > 0)
                        ms.Write(buffer, 0, num);
                }
                while (num > 0);
                byte[] b = ms.ToArray();
                ms.Close();

                b = ZipHelper.Decompress(b);
                string contentType = response.Headers["Content-Type"];
                string html = "";
                if (!String.IsNullOrEmpty(contentType))
                {
                    string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)").Trim();
                    if (!String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding(c).GetString(b);
                }
                if (String.IsNullOrEmpty(html))
                {
                    html = Encoding.GetEncoding("UTF-8").GetString(b);
                    string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
                    string c = GetRegexString(html.ToLower(), r);
                    if (c.IndexOf("'") > -1)
                        c = c.Substring(0, c.IndexOf("'"));
                    if (c.IndexOf("\"") > -1)
                        c = c.Substring(0, c.IndexOf("\""));
                    if (c.IndexOf(" ") > -1)
                        c = c.Substring(0, c.IndexOf(" "));
                    if (c.IndexOf(">") > -1)
                        c = c.Substring(0, c.IndexOf(">"));
                    if (c.ToLower() != "utf-8" && !String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding(c).GetString(b);
                    else if (String.IsNullOrEmpty(c))
                        html = Encoding.GetEncoding("gbk").GetString(b);
                }
                if (!String.IsNullOrEmpty(html))
                {
                    if (html.Substring(0, 1) == "锘")
                        html = Encoding.GetEncoding("UTF-8").GetString(b);
                    //if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 3))
                    //   html = Encoding.GetEncoding("gbk").GetString(b);
                }
                return html;
            }
            catch (Exception e)
            {
                return e.Message;
            }
        }
        /// <summary>
        /// 返回单个正则分析结果
        /// </summary>
        /// <param name="HtmlString"></param>
        /// <param name="RegexString"></param>
        /// <returns></returns>
        public static string GetRegexString(string HtmlString, string RegexString)
        {
            MatchCollection mc = Regex.Matches(HtmlString, RegexString, RegexOptions.IgnoreCase);
            if (mc.Count > 0)
                return mc[0].Groups[1].ToString();
            return "";
        }

        /// <summary>
        /// Gbk 下载页面
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string Gbk(string url)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            try
            {
                WebClientx web = new WebClientx();
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                web.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)");
                web.Proxy = null;
                byte[] b = web.DownloadData(url);
                return Encoding.GetEncoding("gbk").GetString(b);
            }
            catch
            {
                return "";
            }
        }
        /// <summary>
        /// post 数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="postData"></param>
        /// <returns></returns>
        public static string PostUtf8(string url, StringBuilder postData)
        {
            try
            {
                WebClientx wc = new WebClientx();
                byte[] sendData = Encoding.GetEncoding("utf-8").GetBytes(postData.ToString());
                wc.Headers.Add("ContentLength", sendData.Length.ToString());
                wc.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
                wc.Proxy = null;
                return Encoding.UTF8.GetString(wc.UploadData(url, "POST", sendData));
            }
            catch
            {
                return "";
            }
        }
        /// <summary>
        /// post 数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="byteData"></param>
        /// <returns></returns>
        public static string PostUtf8(string url, byte[] byteData, int time)
        {
            WebClientx webClient = new WebClientx();
            try
            {
                webClient.Timeout = time;
                webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
                webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
                webClient.Proxy = null;
                byte[] b = webClient.UploadData(url, byteData);
                string contentType = webClient.ResponseHeaders["Content-Type"];
                webClient.Dispose();
                webClient = null;
                string html = "";
                if (!String.IsNullOrEmpty(contentType))
                {
                    string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
                    if (!String.IsNullOrEmpty(c))
                    {
                        c = c.Trim();
                        html = Encoding.GetEncoding(c).GetString(b);
                    }
                }
                if (String.IsNullOrEmpty(html))
                {
                    html = Encoding.GetEncoding("UTF-8").GetString(b);
                    string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
                    string temp = BaseRegex.GetMatchRegexString(html.ToLower(), r);
                    if (String.IsNullOrEmpty(temp))
                        return html;
                    string c = temp.Trim();
                    if (c.IndexOf("'") > -1)
                        c = c.Substring(0, c.IndexOf("'"));
                    if (c.IndexOf("\"") > -1)
                        c = c.Substring(0, c.IndexOf("\""));
                    if (c.IndexOf(" ") > -1)
                        c = c.Substring(0, c.IndexOf(" "));
                    if (c.IndexOf(">") > -1)
                        c = c.Substring(0, c.IndexOf(">"));
                    if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
                        html = Encoding.GetEncoding(c).GetString(b);
                }
                return html;
            }
            catch
            {
                return "";
            }
        }
        /// <summary>
        /// 浏览器标识
        /// </summary>
        public static string UserAgent = @"Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
Mozilla/5.0+(Windows+NT+6.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/54.0.2840.99+Safari/537.36
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/31.0.1650.63+Safari/537.36
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+Win64;+x64;+Trident/5.0)
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
Mozilla/5.0+(Windows+NT+6.3;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+WOW64;+Trident/5.0)
Mozilla/5.0+(Windows+NT+6.3;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2372.400 QQBrowser/9.5.10548.400";

        /// <summary>
        /// post 数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="postData"></param>
        /// <returns></returns>
        public static string PostUtf8(string url, string postData, string contentType = "")
        {
            //return PostUtf8(url, Encoding.UTF8.GetBytes(postData));

            bool isError = false;
            string[] l = UserAgent.Split('\n');
        reTry:
            try
            {
                string codeType = "utf-8";
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.UserAgent = l[new Random(Guid.NewGuid().GetHashCode()).Next(l.Length)].Trim();
                byte[] byteData = Encoding.GetEncoding(codeType).GetBytes(postData);
                request.Method = "POST";
                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                request.Timeout = BaseClass.TimeOut * 1000;
                request.ReadWriteTimeout = BaseClass.TimeOut * 1000;
                request.Proxy = null;
                request.ContentType = "application/x-www-form-urlencoded; charset=" + codeType;
                //if(BaseClass.SaLevel > 0)
                {
                    //260306 采用GZip/deflate压缩
                    //request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br, zstd");
                    request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
                }
                if (url.StartsWith("https://openapi.jinritemai.com/"))
                {
                    request.Headers.Add("X-USE-BOE", "1");
                    request.Headers.Add("Origin-From", "djt_prod");
                    request.Headers.Add("djt_prod", "boe_time_sku");
                }
                if (!string.IsNullOrEmpty(contentType))
                {
                    request.ContentType = contentType;
                }
                request.ContentLength = byteData.Length;
                request.Referer = url;
                request.CookieContainer = new CookieContainer();
                request.AllowAutoRedirect = false;
                using (Stream requestStream = request.GetRequestStream())
                {
                    requestStream.Write(byteData, 0, byteData.Length);
                }
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                Stream receiveStream = response.GetResponseStream();
                StreamReader readStream = new StreamReader(receiveStream, Encoding.GetEncoding(codeType));

                //if (BaseClass.SaLevel == 0)
                //{
                //    return readStream.ReadToEnd().Trim();
                //}
                //else
                {
                    //260306 超管采用GZip/deflate压缩
                    //Thread.Sleep(50);
                    return GetProxyHtml.ReadResponse(response);
                }
            }
            catch (Exception e)
            {
                LogSave.Add(e.ToString());
                //if (!isError)
                //{
                //    isError = true;
                //    goto reTry;
                //}
                Thread.Sleep(50);
                return "";
            }

        }
        /// <summary>
        /// post 数据
        /// </summary>
        /// <param name="url"></param>
        /// <param name="byteData"></param>
        /// <returns></returns>
        public static string PostUtf8(string url, byte[] byteData)
        {
            WebClientx webClient = new WebClientx();
            try
            {
                webClient.Timeout = 70;
                webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
                webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
                webClient.Proxy = null;
                byte[] b = webClient.UploadData(url, byteData);
                string contentType = webClient.ResponseHeaders["Content-Type"];
                webClient.Dispose();
                webClient = null;
                string html = "";
                if (!String.IsNullOrEmpty(contentType))
                {
                    string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
                    if (!String.IsNullOrEmpty(c))
                    {
                        c = c.Trim();
                        html = Encoding.GetEncoding(c).GetString(b);
                    }
                }
                if (String.IsNullOrEmpty(html))
                {
                    html = Encoding.GetEncoding("UTF-8").GetString(b);
                    string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
                    string temp = BaseRegex.GetMatchRegexString(html.ToLower(), r);
                    if (String.IsNullOrEmpty(temp))
                        return html;
                    string c = temp.Trim();
                    if (c.IndexOf("'") > -1)
                        c = c.Substring(0, c.IndexOf("'"));
                    if (c.IndexOf("\"") > -1)
                        c = c.Substring(0, c.IndexOf("\""));
                    if (c.IndexOf(" ") > -1)
                        c = c.Substring(0, c.IndexOf(" "));
                    if (c.IndexOf(">") > -1)
                        c = c.Substring(0, c.IndexOf(">"));
                    if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
                        html = Encoding.GetEncoding(c).GetString(b);
                }
                return html;
            }
            catch
            {
                return "";
            }
        }

        /// <summary>
        /// POST读取详细
        /// </summary>
        /// <param name="strUrl">提交地址</param>
        /// <param name="data">提交数据</param>
        /// <param name="userAgen">浏览器头</param>
        /// <returns></returns>
        public static string GetUtf8(string strUrl, string userAgen, string cookie = "", string accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
        {
            try
            {
                WebClientx web = new WebClientx();
                web.Headers.Add("Accept", accept);
                web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                web.Headers.Add("User-Agent", userAgen);
                if (!string.IsNullOrEmpty(cookie))
                    web.Headers.Add("Cookie", cookie);
                web.Proxy = null;
                byte[] b = web.DownloadData(strUrl);
                return Encoding.UTF8.GetString(b);
            }
            catch (Exception e)
            {
                return "";
            }
        }

        public static bool DownloadFileNew(string Urlstring, string FilePath, string cookies)
        {
            try
            {
                WebClientx wb = new WebClientx();
                if (File.Exists(FilePath))
                    File.Delete(FilePath);
                string userAgent = BaseClass.ChromeUserAgent;
                wb.Timeout = 30;
                wb.AllowAutoRedirect = true;
                wb.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
                wb.Headers.Add("X-Requested-With", "XMLHttpRequest");
                wb.Headers.Add("Accept-Encoding", "gzip,deflate");
                //if (!string.IsNullOrEmpty(cookies) && cookies.Contains("PDDAccessToken") && BaseClass.CookiesAll.ContainsKey("yangkeduo.com") && BaseClass.CookiesAll["yangkeduo.com"].ContainsKey("PDDAccessToken"))
                //    wb.Headers.Add("accesstoken", BaseClass.CookiesAll["yangkeduo.com"]["PDDAccessToken"]);
                wb.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                wb.Headers.Add("User-Agent", userAgent);
                wb.Proxy = null;
                //if (!string.IsNullOrEmpty(refere))
                //    wb.Headers.Add("Referer", refere);
                cookies = GetCookies(Urlstring, cookies);
                if (!String.IsNullOrEmpty(cookies))
                    wb.Headers.Add("Cookie", cookies);
                wb.DownloadFile(Urlstring, FilePath);
                return true;
            }
            catch (Exception e)
            {
                return false;
            }
        }

        public static string Resove(string html, string prefix, string subfix)
        {
            int inl = html.IndexOf(prefix);
            if (inl == -1)
            {
                prefix = prefix.Replace("\"", "'");
                subfix = subfix.Replace("\"", "'");
                inl = html.IndexOf(prefix);
            }
            if (inl == -1)
            {
                return null;
            }
            inl += prefix.Length;
            if (!string.IsNullOrEmpty(subfix))
            {
                int inl2 = html.IndexOf(subfix, inl);
                if (inl2 == -1)
                {
                    return null;
                }
                string result = html.Substring(inl, inl2 - inl);
                return result;
            }
            else
            {
                string result = html.Substring(inl);
                return result;
            }
        }

        public static string GetCookies(string url, string cookies)
        {
            if (!String.IsNullOrEmpty(cookies))
                return cookies;
            string mainDomain = GetHtml.GetMainDomain(GetHtml.GetDomain(url));
            if (BaseClass.CookiesAllString.ContainsKey(mainDomain))
                cookies = BaseClass.CookiesAllString[mainDomain];
            return cookies;
        }


        /// <summary>
        /// URL GB2312转码
        /// </summary>
        /// <param name="IndexKey"></param>
        /// <returns></returns>
        public static string UrlEncodeGb2312(string IndexKey)
        {
            if (String.IsNullOrEmpty(IndexKey))
                return "";
            return HttpUtility.UrlEncode(IndexKey, Encoding.GetEncoding("GB2312"));

            byte[] sarr = Encoding.GetEncoding("gb2312").GetBytes(IndexKey);
            IndexKey = "";
            for (int i = 0; i < sarr.Length; i++)
            {
                IndexKey += "%";
                IndexKey += sarr[i].ToString("X2");
            }
            return IndexKey;
        }
        /// <summary>
        /// URL UTF-8转码
        /// </summary>
        /// <param name="IndexKey"></param>
        /// <returns></returns>
        public static string UrlEncodeUtf8(string IndexKey)
        {
            if (String.IsNullOrEmpty(IndexKey))
                return "";
            return HttpUtility.UrlEncode(IndexKey, Encoding.UTF8);
            byte[] sarr = Encoding.UTF8.GetBytes(IndexKey);
            IndexKey = "";
            for (int i = 0; i < sarr.Length; i++)
            {
                IndexKey += "%";
                IndexKey += sarr[i].ToString("X2");
            }
            return IndexKey;
        }
        /// <summary>
        /// 用IE打开网址
        /// </summary>
        /// <param name="url"></param>
        public static void OpenUrl(string url)
        {
            try
            {
                //if (url.Contains("http"))
                //    Process.Start("iexplore.exe", url);
                //else
                Process.Start(url);
            }
            catch
            {
            }
        }

        /// <summary>
        /// 下载文件
        /// </summary>
        /// <param name="Urlstring">下载地址</param>
        /// <param name="FilePath">保存目录</param>
        /// <returns></returns>
        public static bool DownloadFile(string Urlstring, string FilePath)
        {
            try
            {
                WebClientx mywebclient = new WebClientx();
                mywebclient.Timeout = 300;
                if (File.Exists(FilePath))
                    File.Delete(FilePath);
                mywebclient.DownloadFile(Urlstring, FilePath);
                return true;
            }
            catch(Exception ex)
            {
                return false;
            }
        }
        /// <summary>
        /// 下载文件
        /// </summary>
        /// <param name="Urlstring">下载地址</param>
        /// <param name="b">保存目录</param>
        /// <returns></returns>
        public static bool DownloadFile(string Urlstring, out byte[] b)
        {
            b = new byte[0];
            int times = 0;
        Next:
            try
            {
                WebClientx mywebclient = new WebClientx();
                b = mywebclient.DownloadData(Urlstring);
                return true;
            }
            catch (Exception e)
            {
                if (e.Message.Contains("404"))
                    return false;
                if (times < 4)
                {
                    times++;
                    Thread.Sleep(3000);
                    goto Next;
                }
                return false;
            }

        }


        /// <summary>
        /// 读取网址，兼容https，兼容任意编码
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetUrl(string url)
        {
            WebClientx webClient = new WebClientx();
            try
            {
                webClient.Timeout = 30;
                webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
                webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
                webClient.Proxy = null;
                byte[] b = webClient.DownloadData(url);
                b = ZipHelper.Decompress(b);
                string contentType = webClient.ResponseHeaders["Content-Type"];
                webClient.Dispose();
                webClient = null;
                string html = "";
                if (!String.IsNullOrEmpty(contentType))
                {
                    string c = BaseRegex.GetMatchRegexString(contentType, "charset\\s*=\\s*(\\S+)");
                    if (!String.IsNullOrEmpty(c))
                    {
                        c = c.Trim();
                        html = Encoding.GetEncoding(c).GetString(b);
                    }
                }
                if (String.IsNullOrEmpty(html))
                {
                    html = Encoding.GetEncoding("UTF-8").GetString(b);
                    string r = @"<\s*meta.+charset\s*=\s*[""']*(\S+)[""'\s>]*";
                    string c = BaseRegex.GetMatchRegexString(html.ToLower(), r).Trim();
                    if (c.IndexOf("'") > -1)
                        c = c.Substring(0, c.IndexOf("'"));
                    if (c.IndexOf("\"") > -1)
                        c = c.Substring(0, c.IndexOf("\""));
                    if (c.IndexOf(" ") > -1)
                        c = c.Substring(0, c.IndexOf(" "));
                    if (c.IndexOf(">") > -1)
                        c = c.Substring(0, c.IndexOf(">"));
                    if (!String.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
                        html = Encoding.GetEncoding(c).GetString(b);
                }
                return html;
            }
            catch (Exception e)
            {
                LogSave.AddView(url + " 错误：" + e.Message.Replace("\r\n", " "));
                return null;
            }
            finally
            {
                try
                {
                    if (webClient != null)
                    {
                        webClient.Dispose();
                        webClient = null;
                    }
                }
                catch
                {
                }
            }
        }


        /// <summary>
        /// 获取域名
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetDomain(string url)
        {
            try
            {
                url = url.ToLower();
                string temp = BaseRegex.GetMatchRegexString(url, @"([a-z0-9-\.]+\.[a-z]{2,8})");
                if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
                    return null;
                if (!String.IsNullOrEmpty(temp))
                    return temp;
                if (url.Contains("//") && url.IndexOf("//") + 2 < url.Length)
                    url = url.Substring(url.IndexOf("//") + 2, url.Length - url.IndexOf("//") - 2);
                if (url.Contains("/") && url.IndexOf("/") > 0)
                    url = url.Substring(0, url.IndexOf("/"));
                return url;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// 获取域名&ip
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetDomainIp(string url)
        {
            if (String.IsNullOrEmpty(url))
                return null;
            url = url.ToLower();
            string temp = BaseRegex.GetMatchRegexString(url, @"http[s]{0,1}://([a-z0-9-\.]+\.[a-z]{2,8}|[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3})");
            if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
                return null;
            if (!String.IsNullOrEmpty(temp))
                return temp;
            return null;
        }

        /// <summary>
        /// 解析域名ip
        /// </summary>
        /// <param name="domain"></param>
        /// <returns></returns>
        public static string GetIp(string domain)
        {
            try
            {
                for (int i = 0; i < 2; i++)
                {
                    IPAddress[] aryIP = Dns.GetHostEntry(domain).AddressList;
                    if (aryIP.Length > 0)
                        return aryIP[0].ToString();
                    Thread.Sleep(1000); //重试，防止DNS缓存
                }
                return null;
            }
            catch
            {
                //SocketServer.ViewCode += "域名：" + domain + "无法解析\r\n";
                return null;
            }
        }
        /// <summary>
        /// 是否是域名
        /// </summary>
        /// <param name="domain"></param>
        /// <returns></returns>
        public static bool IsDomain(string domain)
        {
            string temp = BaseRegex.GetMatchRegexString(domain, @"([a-z0-9-\.]+\.[a-z]{2,8})");
            if (!String.IsNullOrEmpty(temp) && temp.Length > 1 && (temp.Substring(0, 1) == "." || temp.Substring(0, 1) == "-"))
                return false;
            if (temp == domain)
                return true;
            return false;
        }

        /// <summary>
        /// 获取顶级域名
        /// </summary>
        /// <param name="domain"></param>
        /// <returns></returns>
        public static string GetMainDomain(string domain)
        {
            if (String.IsNullOrEmpty(domain))
                return "";
            domain = domain.ToLower();
            if (!domain.Contains(".") || domain.Length < 4)
                return "";
            string[] d = domain.Split('.');
            string main = d[d.Length - 2] + "." + d[d.Length - 1]; //原始域名
            if (d.Length > 2 &&
                (d[d.Length - 2] == "com" || d[d.Length - 2] == "net" || d[d.Length - 2] == "org" ||
                 d[d.Length - 2] == "gov" || d[d.Length - 2] == "edu" || d[d.Length - 2] == "co"))
                main = d[d.Length - 3] + "." + main;
            return main;
        }
        /// <summary>
        /// 解密Jsion
        /// </summary>
        /// <param name="s"></param>
        /// <returns></returns>
        public static string Decode(string s)
        {
            s = WebUtility.HtmlDecode(s);
            Regex reUnicode = new Regex(@"\\u([0-9a-fA-F]{4})", RegexOptions.Compiled);
            return reUnicode.Replace(s, m =>
            {
                short c;
                if (Int16.TryParse(m.Groups[1].Value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out c))
                {
                    return "" + (char)c;
                }
                return m.Value;
            });
        }


        /// <summary>
        /// 获取国内IP
        /// </summary>
        /// <returns></returns>
        public static string ProxyIpIn()
        {
            if (BaseClass.ProxyIn.Count == 0)
                return "";
            Random ra = new Random(Guid.NewGuid().GetHashCode());
            return BaseClass.ProxyIn[ra.Next(BaseClass.ProxyIn.Count)];
        }
        /// <summary>
        /// 获取美国IP
        /// </summary>
        /// <returns></returns>
        public static string ProxyIpOut()
        {
            if (BaseClass.ProxyOut.Count == 0)
                return "";
            Random ra = new Random(Guid.NewGuid().GetHashCode());
            return BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
        }
        /// <summary>
        /// 获取国内IP代理URL地址
        /// </summary>
        /// <returns></returns>
        public static string ProxyIpInUrl(string url)
        {
            if (BaseClass.ProxyIn.Count == 0)
                return url;
            Random ra = new Random(Guid.NewGuid().GetHashCode());
            string ip = BaseClass.ProxyIn[ra.Next(BaseClass.ProxyIn.Count)];
            return "http://" + ip + ":6000/url=" + ReplaceUrl(url);
        }
        /// <summary>
        /// 获取美国IP代理URL地址
        /// </summary>
        /// <returns></returns>
        public static string ProxyIpOutUrl(string url)
        {
            if (BaseClass.ProxyOut.Count == 0)
                return url;
            Random ra = new Random(Guid.NewGuid().GetHashCode());
            string ip = BaseClass.ProxyOut[ra.Next(BaseClass.ProxyOut.Count)];
            return "http://" + ip + ":6000/url=" + ReplaceUrl(url);
        }
        /// <summary>
        /// 替换链接，防止防火墙
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ReplaceUrl(string url)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            string newurl = "";
            string[] l = url.Split('.');
            for (int i = 0; i < l.Length; i++)
            {
                if (l[i].Length > 2)
                    newurl += l[i].Substring(0, l[i].Length - 2) + "***ooo***" + l[i].Substring(l[i].Length - 2, 2);
                else
                    newurl += l[i];
                if (i < l.Length - 1)
                    newurl += ".";
            }
            return newurl.Replace(".google.", ".goo00gle.").Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
        }
        /// <summary>
        /// 还原链接
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string ReplaceUrlBak(string url)
        {
            if (String.IsNullOrEmpty(url))
                return "";
            string newurl = url.Replace("***ooo***", "");
            if (String.IsNullOrEmpty(newurl))
                return "";
            return newurl.Replace(".goo00gle.", ".google.").Replace("%3f", "?").Replace("%3d", "=").Replace("%26", "&");
        }
        /// <summary>
        /// 生成URL绝对地址
        /// </summary>
        /// <param name="url">新URL</param>
        /// <param name="oldurl">源地址</param>
        /// <returns></returns>
        public static string BuildUrl(string url, string oldurl)
        {
            if (String.IsNullOrEmpty(url))
                return oldurl;
            if (url.Contains("tencent:/"))
                return url;
            if (url.Length > 8 && url.Substring(0, 8).Contains("://"))
            {
                if (url.Length - url.Replace("/", "").Length == 2)
                    url += "/";
                return url;
            }

            string homeurl = BaseRegex.GetMatchRegexString(oldurl.ToLower(), "(http://[^/]*?)/");
            if (String.IsNullOrEmpty(homeurl))
            {
                homeurl = BaseRegex.GetMatchRegexString(oldurl.ToLower(), "(http://[^/]+)") + "/";
                if (String.IsNullOrEmpty(homeurl))
                {
                    //BaseClass.ViewCode += "网址：" + oldurl + "识别失败\r\n";
                    return url;
                }
            }
            if (url.Substring(0, 1) == "/")
            {
                string newurlx = homeurl + url;
                newurlx = newurlx.Replace("//", "/").Replace("http:/", "http://");
                return newurlx;
            }
            string[] l = oldurl.Split('/');
            int cute = l.Length;
            if (l.Length > 3)
                cute--;
            while (url.Contains("../"))
            {
                int s = url.IndexOf("../");
                if (s == 0)
                    url = url.Substring(3);
                else
                    url = url.Substring(0, s) + url.Substring(s + 3);
                cute--;
                if (cute < 3)
                    cute = 3;
            }
            string newurl = "";
            for (int i = 0; i < cute; i++)
                newurl += l[i] + "/";
            newurl = newurl + url;
            return newurl.Replace("//", "/").Replace("http:/", "http://").Replace("/./", "/");
        }
        /// <summary>
        /// 获取代理地址
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetProxyUrl(string url)
        {
            if (String.IsNullOrEmpty(url))
                return null;
            if (BaseClass.Proxy.Count == 0)
            {
                BaseClass.ViewCode += "代理异常";
                return url;
            }
            Random ra = new Random();
            string p = BaseClass.Proxy[ra.Next(BaseClass.Proxy.Count)];
            return "http://" + p + "/url=" + url.Replace(".google.", ".goo00gle.")
                .Replace("?", "%3f").Replace("=", "%3d").Replace("&", "%26");
        }


        public static List<string> ProxyIpList = new List<string>();

        /// <summary>
        /// 服务器代理采集
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetProxyServer(string url, string needString)
        {
            string html = GetProxyServerx(url);
            if (string.IsNullOrEmpty(html) || !html.Contains(needString))
                html = GetProxyServerx(url);
            if (string.IsNullOrEmpty(html) || !html.Contains(needString))
                html = GetProxyServerx(url);
            if (string.IsNullOrEmpty(html) || !html.Contains(needString))
                html = GetProxyServerx(url);
            return html;
        }

        /// <summary>
        /// 服务器代理采集
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private static string GetProxyServerx(string url)
        {
            if (ProxyIpList.Count < 10)
                return "";
            string ip = ProxyIpList[Randomx.GetId(ProxyIpList.Count)];
            WebClientx webClient = new WebClientx();
            try
            {
                webClient.Timeout = 10;
                webClient.Headers.Add("Accept-Encoding", "gzip,deflate");
                webClient.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0)");
                webClient.Proxy = new WebProxy(ip);
                byte[] b = webClient.DownloadData(url);
                b = ZipHelper.Decompress(b);
                string html = Encoding.GetEncoding("UTF-8").GetString(b);
                return html;
            }
            catch (Exception e)
            {
                LogSave.AddView(url + " 错误：" + e.Message.Replace("\r\n", " "));
                return null;
            }
            finally
            {
                try
                {
                    webClient.Dispose();
                    webClient = null;
                }
                catch
                {
                }
            }
        }
        public static string RequestImg(string url, byte[] imgData)
        {
            HttpWebResponse response = null;
            var responseText = "";
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.KeepAlive = true;
                request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
                request.Headers.Add("Origin", url);
                request.Headers.Add("Upgrade-Insecure-Requests", @"1");
                request.ContentType = "multipart/form-data; boundary=----WebKitFormBoundaryg0ATB1h6Pa505OBi";
                request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36";
                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
                request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
                request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
                request.Method = "POST";
                request.ServicePoint.Expect100Continue = false;
                string body = "------WebKitFormBoundaryg0ATB1h6Pa505OBi\r\n" +
"Content-Disposition: form-data; name=\"file\"; filename=\"search.jpg\"\r\n" +
"Content-Type: image/jpeg\r\n\r\n" +
"<!>filedata<!>\r\n" +
"------WebKitFormBoundaryg0ATB1h6Pa505OBi--\r\n";
                WriteMultipartBodyToRequest(request, body, imgData);

                response = (HttpWebResponse)request.GetResponse();
            }
            catch (WebException e)
            {
                if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
                else return "error: " + e.Message;
            }
            catch (Exception ex)
            {
                if (response != null) response.Close();
                return "error: " + ex.Message;
            }
            if (response != null)
            {
                responseText = ReadResponse(response);
                response.Close();
            }
            return responseText;
        }
        /// <summary>
        /// 读取返回值
        /// </summary>
        /// <param name="response"></param>
        /// <returns></returns>
        private static string ReadResponse(HttpWebResponse response)
        {
            using (Stream responseStream = response.GetResponseStream())
            {
                Stream streamToRead = responseStream;
                if (response.ContentEncoding.ToLower().Contains("gzip"))
                {
                    streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                {
                    streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
                }

                using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
                {
                    return streamReader.ReadToEnd();
                }
            }
        }

        private static void WriteMultipartBodyToRequest(HttpWebRequest request, string body, byte[] data)
        {
            string[] multiparts = Regex.Split(body, @"<!>");
            byte[] bytes;
            using (MemoryStream ms = new MemoryStream())
            {
                foreach (string part in multiparts)
                {
                    if (part == "filedata") // 文件信息
                    {
                        bytes = data;
                    }
                    else
                    {
                        bytes = System.Text.Encoding.UTF8.GetBytes(part.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n"));
                    }
                    ms.Write(bytes, 0, bytes.Length);
                }

                request.ContentLength = ms.Length;
                using (Stream stream = request.GetRequestStream())
                {
                    ms.WriteTo(stream);
                }
            }
        }

        /// <summary>
        /// 字符串转Unicode
        /// </summary>
        /// <param name="source">源字符串</param>
        /// <returns>Unicode编码后的字符串</returns>
        public static string String2Unicode(string source)
        {
            var bytes = Encoding.Unicode.GetBytes(source);
            var stringBuilder = new StringBuilder();
            for (var i = 0; i < bytes.Length; i += 2)
            {
                stringBuilder.AppendFormat("\\u{0}{1}", bytes[i + 1].ToString("x").PadLeft(2, '0'), bytes[i].ToString("x").PadLeft(2, '0'));
            }
            return stringBuilder.ToString();
        }

        /// <summary>
        /// Unicode转字符串
        /// </summary>
        /// <param name="source">经过Unicode编码的字符串</param>
        /// <returns>正常字符串</returns>
        public static string Unicode2String(string source)
        {
            return new Regex(@"\\u([0-9A-F]{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled).Replace(source, x => Convert.ToChar(Convert.ToUInt16(x.Result("$1"), 16)).ToString());
        }

        public static string ReadResponse(HttpWebResponse response, Encoding encoding = null)
        {
            return GetProxyHtml.ReadResponse(response, encoding);
        }
    }
}