/*
 * 多线程模拟浏览器采集客户端
 * 2013-7-24
 * 肖俊
 */
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

namespace AllBase
{
    public class MultiThreadClient
    {
        /// <summary>
        /// 附加Cookies
        /// </summary>
        public string AddCookies = "";
        /// <summary>
        /// 指定Cookies
        /// </summary>
        public string Cookies = "";
        /// <summary>
        /// 指定UserAgent
        /// </summary>
        public string UserAgent = "";
        /// <summary>
        /// 是否出错
        /// </summary>
        public bool IsError = false;
        /// <summary>
        /// 是否友情链接查询
        /// </summary>
        public bool FriendLink = false;
        /// <summary>
        /// 来路
        /// </summary>
        public string Referer = "";
        /// <summary>
        /// 是否手机头
        /// </summary>
        public bool IsMoblie = false;
        /// <summary>
        /// 是否手机头
        /// </summary>
        public bool IsWhoies = false;
        /// <summary>
        /// 强制定义为GBK编码
        /// </summary>
        public bool IsGbk = false;
        /// <summary>
        /// 强制定义为GBK编码
        /// </summary>
        public bool IsUtf8 = false;
        /// <summary>
        /// 强制不跳转
        /// </summary>
        public bool NoRedir = false;
        /// <summary>
        /// 是否代理模式
        /// </summary>
        public bool IsProxy = false;
        /// <summary>
        /// 百度搜索错误
        /// </summary>
        public static bool BaiduError = false;
        /// <summary>
        /// 百度错误时间
        /// </summary>
        public static DateTime BaiduErrorTime = DateTime.Now;

        public string EndUrl = "";
        /// <summary>
        /// 超时
        /// </summary>
        public int TimeOut = 30;
        public string GetHtml(string url)
        {
            return GetHtml(url, -1,null);
        }

        public string GetHtml(string url, byte[] post)
        {
            return GetHtml(url,-1, post);
        }


        /// <summary>
        /// 采集器
        /// </summary>
        /// <param name="url">网址</param>
        /// <returns></returns>
        public string GetHtml(string url, int isUtf8, byte[] post)
        {
            IsError = false;
            string cookies = "", html = "";
            Random ra = new Random();
            string domain = GetDomain(url);
            if (domain == null)
                return "";
            if (domain.Length > 4 &&
                (domain.Substring(0, 2) == "m." || domain.Substring(0, 2) == "wap." || domain.Substring(0, 2) == "3g." ||
                 domain.Substring(0, 2) == "4g."))
                IsMoblie = true;
            if (!string.IsNullOrEmpty(url))
                url = url.Trim();
            if (string.IsNullOrEmpty(url))
                return "";
            WebBrowserCoockies wc;
            if (IsMoblie)
                wc = WebCoockies.MobileCoockies[ra.Next(WebCoockies.MobileCoockies.Count)];
            else
                wc = WebCoockies.Coockies[ra.Next(WebCoockies.Coockies.Count)];
            if (wc == null)
                wc = new WebBrowserCoockies();
            for (int i = 0; wc.Coockies != null && i < wc.Coockies.Count; i++)
            {
                if (wc.Coockies[i] != null && wc.Coockies[i].Domain == domain)
                {
                    cookies = wc.Coockies[i].Cookies;
                    break;
                }
            }
            if (!string.IsNullOrEmpty(UserAgent))
                wc.UserAgent = UserAgent;
            if (!string.IsNullOrEmpty(Cookies))
                cookies = Cookies;
            WebClientx web = new WebClientx();
            web.AllowAutoRedirect = false;
            web.Timeout = TimeOut;
            web.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8");
            web.Headers.Add("Accept-Encoding", "gzip, deflate, sdch, br ");
            web.Headers.Add("User-Agent", wc.UserAgent);

            if (IsMoblie)
            {
                web.Headers.Add("X-Requested-With", "com.android.browser");
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                web.Headers.Add("x-wap-profile", "http://wap1.huawei.com/uaprof/HUAWEI_MT2_T00_UAProfile.xml");
            }
            else
            {
                web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                web.Headers.Add("upgrade-insecure-requests", "1");
            }

            if (IsProxy)//是否是代理状态
                web.Headers.Add("Add-UserAgent", wc.UserAgent);
 
                web.Proxy = null;
            if (!string.IsNullOrEmpty(AddCookies) && string.IsNullOrEmpty(cookies))
                cookies += AddCookies;
            if (url.Contains("/url=") && IsMoblie)
                web.Headers.Add("Add-Cookies", "Moblie");
            if (url.Contains("/url=") && IsWhoies)
                web.Headers.Add("Add-Cookies", "whois");
            int retryTimes = 0;
        GotoAgen:
            EndUrl = url;
            web.Headers.Remove("Cookie");
            if (!string.IsNullOrEmpty(cookies))
                web.Headers.Add("Cookie", cookies);
            if (!string.IsNullOrEmpty(Referer))
                web.Headers.Add("Referer", Referer);
            try
            {
                byte[] b;
                //if (post == null)
                //{
                //    Stream receiveStream = web.OpenRead(url);
                //    MemoryStream ms = new MemoryStream();
                //    int num;
                //    byte[] buffer = new byte[10240];
                //    do
                //    {
                //        num = receiveStream.Read(buffer, 0, buffer.Length);
                //        if (num > 0)
                //            ms.Write(buffer, 0, num);
                //    } while (num > 0);
                //    b = ms.ToArray();
                //    ms.Close();
                //}
                //else
                //{
                //    b = web.UploadData(url, post);
                //}
                b = post == null ? web.DownloadData(url) : web.UploadData(url, post);
                string cookies1 = web.ResponseHeaders.Get("Set-Cookie"); //获取Cookies
                string location = web.ResponseHeaders.Get("Location"); //跳转地址
                if (!string.IsNullOrEmpty(cookies1))
                {
                    cookies = UpCookies(cookies, cookies1);
                    if (!string.IsNullOrEmpty(Cookies))
                        Cookies = cookies;
                    bool ok = false;
                    foreach (var c in wc.Coockies)
                    {
                        if (c.Domain == domain)
                        {
                            c.Cookies = cookies;
                            ok = true;
                        }
                    }
                    if (!ok)
                    {
                        CoockiesBase c = new CoockiesBase();
                        c.Cookies = cookies;
                        c.Domain = domain;
                        wc.Coockies.Add(c);
                    }
                }
                if (string.IsNullOrEmpty(location))
                {
                    string contentType = web.ResponseHeaders["Content-Type"];
                    if (b == null)
                        return "";
                    if (b.Length > 2 * 1024 * 1024)
                        return "文件太大";
                    b = ZipHelper.Decompress(b);
                    if (b.Length > 2 * 1024 * 1024)
                        return "文件太大";
                    if (!string.IsNullOrEmpty(contentType))
                    {
                        string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)");
                        if (!string.IsNullOrEmpty(c))
                        {
                            c = c.Trim().ToLower().Replace("utf8", "utf-8");
                            html = Encoding.GetEncoding(c).GetString(b);
                        }
                    }
                    if (string.IsNullOrEmpty(html) && isUtf8 == -1)
                    {
                        html = Encoding.GetEncoding("UTF-8").GetString(b);
                        string r = @"<meta [^>]*?charset=[""']*([\w-]+)[""']*";
                        string c = GetRegexString(html, r).Trim();
                        if (c.IndexOf("'") > -1)
                            c = c.Substring(0, c.IndexOf("'"));
                        if (c.IndexOf("\"") > -1)
                            c = c.Substring(0, c.IndexOf("\""));
                        if (c.IndexOf(" ") > -1)
                            c = c.Substring(0, c.IndexOf(" "));
                        if (c.IndexOf(">") > -1)
                            c = c.Substring(0, c.IndexOf(">"));
                        if (c.ToLower() != "utf-8" && !string.IsNullOrEmpty(c))
                            html = Encoding.GetEncoding(c).GetString(b);
                        else if (string.IsNullOrEmpty(c))
                            html = Encoding.GetEncoding("UTF-8").GetString(b);
                    }
                    if (!string.IsNullOrEmpty(html) && isUtf8 == -1)
                    {
                        if (html.Substring(0, 1) == "锘")
                            html = Encoding.GetEncoding("UTF-8").GetString(b);
                        if (html.Contains("�") && (html.Length - html.Replace("�", "").Length > 20))
                            html = Encoding.GetEncoding("gbk").GetString(b);
                    }
                    if (isUtf8 == 0)
                        html = Encoding.GetEncoding("gbk").GetString(b);
                    if (isUtf8 == 1 && string.IsNullOrEmpty(html))
                        html = Encoding.UTF8.GetString(b);
                    if (IsGbk)
                        html = Encoding.GetEncoding("gbk").GetString(b);
                    if (IsUtf8)
                        html = Encoding.GetEncoding("UTF-8").GetString(b);
                }
                if (retryTimes < 3 && (!string.IsNullOrEmpty(location) || string.IsNullOrEmpty(html)))
                {
                    if (NoRedir)
                        return location;

                    //下面是支持301和302跳转部分
                    if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
                    {
                        url = location;

                    }
                    else
                    {
                        if (location.Substring(0, 1) == "/")
                        {
                            string[] l = url.Split('/');
                            if (l.Length > 3)
                            {
                                url = "";
                                for (int i = 0; i < 3; i++)
                                    url += l[i] + "/";
                                url += location.Substring(1, location.Length - 1);
                            }
                        }
                        else
                        {
                            if (url.Substring(url.Length - 1, 1) == "/")
                                url += location;
                            else
                                url += "/" + location;
                        }
                    }
                    retryTimes++;
                    goto GotoAgen;
                }
            }
            catch (Exception e)
            {
                if (e.Message.Contains("(404)") || e.Message.Contains("(403)") || e.Message.Contains("(500)") || e.Message.Contains("指定的路径或文件名太长，"))
                {
                    IsError = true;
                    return e.Message;
                }
                if (FriendLink)
                {
                    IsError = true;
                    return e.Message;
                }
                if (retryTimes < 1)
                {
                    Thread.Sleep(100);
                    retryTimes++;
                    goto GotoAgen;
                }
                IsError = true;
                return e.Message;
            }
            if (html.Contains("您的访问出错了</title>"))
            {
                if (!BaiduError)
                {
                    BaiduError = true;
                    BaiduErrorTime = DateTime.Now;
                }
                IsError = true;
            }
            if (BaiduError && BaiduErrorTime.AddMinutes(5) < DateTime.Now)
                BaiduError = false;
            return html;
        }
















        /// <summary>
        /// 返回Utf-8编码
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string GetByte(string url, ref bool had, out bool isChinese)
        {
            string html = "";
            had = false;
            IsError = false;
            isChinese = false;
            string cookies = "";
            Random ra = new Random();
            string domain = AllBase.GetHtml.GetDomain(url);
            if (string.IsNullOrEmpty(domain))
                return "";
            WebBrowserCoockies wc = WebCoockies.Coockies[ra.Next(WebCoockies.Coockies.Count)];

            for (int i = 0; wc != null && wc.Coockies != null && i < wc.Coockies.Count; i++)
            {
                if (wc.Coockies[i] != null && wc.Coockies[i].Domain == domain)
                {
                    cookies = wc.Coockies[i].Cookies;
                    break;
                }
            }
            WebClientx web = new WebClientx();
            web.Timeout = TimeOut;
            web.AllowAutoRedirect = false;
            web.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;");
            web.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
            web.Headers.Add("Accept-Encoding", "gzip,deflate");
            // if (string.IsNullOrEmpty(UserAgent))
            //     web.Headers.Add("User-Agent", wc.UserAgent);
            //else
            //    web.Headers.Add("User-Agent", UserAgent);
            web.Headers.Add("User-Agent", "Mozilla/5.0+(compatible;+Baiduspider/2.0;++http://www.baidu.com/search/spider.html)");
            web.Proxy = null;
            if (!string.IsNullOrEmpty(AddCookies) && string.IsNullOrEmpty(cookies))
                cookies += AddCookies;
            if (!string.IsNullOrEmpty(cookies))
                web.Headers.Add("Cookie", cookies);
            if (!string.IsNullOrEmpty(Referer))
                web.Headers.Add("Referer", Referer);
            if (url.Contains("/url=") && IsMoblie)
                web.Headers.Add("Add-Cookies", "Moblie");
            if (url.Contains("/url=") && IsWhoies)
                web.Headers.Add("Add-Cookies", "whois");
            int retryTimes = 0;
            byte[] b = null;
        GotoAgen:
            try
            {
                int max = 3 * 1024, len = 0;//最大文件为3MB
                //Stream receiveStream = web.OpenRead(url);
                //MemoryStream ms = new MemoryStream();
                //int num;
                //byte[] buffer = new byte[1024];
                //do
                //{
                //    num = receiveStream.Read(buffer, 0, buffer.Length);
                //    if (num > 0)
                //        ms.Write(buffer, 0, num);
                //    len++;
                //} while (num > 0 && len<max);
                //if (len >= max)//太大的文件直接跳过
                //    return "";
                //b = ms.ToArray();
                //ms.Close();
                b = web.DownloadData(url);
                string cookies1 = web.ResponseHeaders.Get("Set-Cookie"); //获取Cookies
                string location = web.ResponseHeaders.Get("Location"); //跳转地址
                if (!string.IsNullOrEmpty(cookies1))
                {
                    cookies = UpCookies(cookies, cookies1);
                    bool ok = false;
                    foreach (var c in wc.Coockies)
                    {
                        if (c.Domain == domain)
                        {
                            c.Cookies = cookies;
                            ok = true;
                        }
                    }
                    if (!ok)
                    {
                        CoockiesBase c = new CoockiesBase();
                        c.Cookies = cookies;
                        c.Domain = domain;
                        wc.Coockies.Add(c);
                    }
                }
                if (string.IsNullOrEmpty(location))
                {
                    string contentType = web.ResponseHeaders["Content-Type"];
                    html = "";
                    b = ZipHelper.Decompress(b);
                    if (b.Length >= max*1024)//太大的文件直接跳过
                        return "文件太大";
                    //b = GzipDeCompress(b);
                    if (b == null || b.Length == 0 || b.Length > 2 * 1024 * 1024)
                        return "";
                    try
                    {
                        if (!string.IsNullOrEmpty(contentType))
                        {
                            string c = GetRegexString(contentType, "charset\\s*=\\s*(\\S+)");
                            if (!string.IsNullOrEmpty(c))
                            {
                                c = c.Trim().ToLower().Replace("utf8", "utf-8").Replace(";", "");
                                html = Encoding.GetEncoding(c).GetString(b);
                            }
                        }
                        if (string.IsNullOrEmpty(html))
                        {
                            html = Encoding.GetEncoding("UTF-8").GetString(b);
                            string r = @"<meta [^>]*charset=[""']*([^>]+)[""']*[ />]";
                            string c = GetRegexString(html, r);
                            if (c.Contains("\""))
                                c = c.Substring(0, c.IndexOf("\""));
                            if (c.Contains("'"))
                                c = c.Substring(0, c.IndexOf("'"));
                            if (string.IsNullOrEmpty(c))
                                c = GetRegexString(html, @"<meta [^>]*charset=[""']([a-z0-9-]+)[""']>");
                            if (string.IsNullOrEmpty(c) && !string.IsNullOrEmpty(html) && html.Contains("�") &&html.Length- html.Replace("�","").Length>50)
                            {
                                html = Encoding.GetEncoding("gbk").GetString(b);
                            }
                            if (!string.IsNullOrEmpty(c) && c.ToLower() != "utf-8")
                            {
                                html = Encoding.GetEncoding(c).GetString(b);
                            }
                        }
                    }
                    catch
                    {
                    }
                    if (IsUtf8|| !string.IsNullOrEmpty(html) && html.Substring(0, 1) == "锘")
                    {
                        html = Encoding.UTF8.GetString(b);
                    }
                    
                }
                if (retryTimes < 3 && (!string.IsNullOrEmpty(location) || string.IsNullOrEmpty(html)))
                {
                    //下面是支持301和302跳转部分
                    if (location.Length > 8 && (location.Substring(0, 7) == "http://" || location.Substring(0, 8) == "https://" || location.Substring(0, 6) == "ftp://"))
                    {
                        url = location;
                    }
                    else
                    {
                        if (location.Substring(0, 1) == "/")
                        {
                            string[] l = url.Split('/');
                            if (l.Length > 3)
                            {
                                url = "";
                                for (int i = 0; i < 3; i++)
                                    url += l[i] + "/";
                                url += location.Substring(1, location.Length - 1);
                            }
                        }
                        else
                        {
                            if (url.Substring(url.Length - 1, 1) == "/")
                                url += location;
                            else
                                url += "/" + location;
                        }
                    }
                    retryTimes++;
                    goto GotoAgen;
                }
            }
            catch (Exception e)
            {
                if (e.Message.Contains("(404)") || e.Message.Contains("(403)") || e.Message.Contains("(500)"))
                {
                    IsError = true;
                    html = e.Message;
                }
                if (!IsError && retryTimes < 1)
                {
                    Thread.Sleep(2000);
                    retryTimes++;
                    goto GotoAgen;
                }
                IsError = true;
                html = e.Message;
            }
            if (b == null || b.Length == 0)
                return "";
            foreach (byte b1 in b)
            {
                if (b1 > 227 && b1 < 235)
                {
                    isChinese = true;
                    break;
                }
            }
            if (html.Length > 2 * 1024 * 1024)
            {
                had = false;
                return "";
            }
            had = true;
            //if (b == null) //
            return html;
            
        }


        /// <summary>
        /// 获取域名
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private string GetDomain(string url)
        {
            try
            {
                url = url.Substring(url.IndexOf("//") + 2, url.Length - url.IndexOf("//") - 2);
                if (url.IndexOf("/") > -1)
                    url = url.Substring(0, url.IndexOf("/"));
                return url;
            }
            catch
            {
                return null;
            }
        }


        /// <summary>
        /// 返回单个正则分析结果
        /// </summary>
        /// <param name="HtmlString"></param>
        /// <param name="RegexString"></param>
        /// <returns></returns>
        private string GetRegexString(string HtmlString, string RegexString)
        {
            MatchCollection mc = Regex.Matches(HtmlString, RegexString, RegexOptions.IgnoreCase);
            if (mc.Count > 0)
                return mc[0].Groups[1].ToString();
            return "";
        }

        /// <summary>
        /// 更新cookies
        /// </summary>
        /// <param name="old"></param>
        /// <param name="newc"></param>
        /// <returns></returns>
        public string UpCookies(string old, string newc)
        {
            if (string.IsNullOrEmpty(newc))
                return old;
            if (string.IsNullOrEmpty(old))
                old = " ";
            int io = 0;
            old = old.Trim();
            newc = newc.Trim();
            string[] o = old.Split(';');
            string[] n = newc.Split(';');
            List<CoockiesValue> c = new List<CoockiesValue>();
            for (int i = 0; i < o.Length; i++)
            {
                CoockiesValue cv = new CoockiesValue();
                io = o[i].IndexOf("=");
                if (io > -1)
                {
                    cv.Title = o[i].Substring(0, io).Trim();
                    cv.Value = o[i].Substring(io + 1, o[i].Length - io - 1).Trim();
                }
                else
                {
                    if (!string.IsNullOrEmpty(o[i]))
                        cv.Title = o[i].Trim();
                }
                if (string.IsNullOrEmpty(cv.Title) || cv.Title == "domain" || cv.Title == "expires" || cv.Title == "path" || cv.Title.ToLower() == "httponly")
                    continue;
                if (!string.IsNullOrEmpty(cv.Title))
                    c.Add(cv);
            }
            for (int i = 0; i < n.Length; i++)
            {
                CoockiesValue cv = new CoockiesValue();
                io = n[i].IndexOf("=");
                if (io > -1)
                {
                    cv.Title = n[i].Substring(0, io).Trim();
                    cv.Value = n[i].Substring(io + 1, n[i].Length - io - 1).Trim();
                }
                else
                {
                    if (!string.IsNullOrEmpty(n[i]))
                        cv.Title = n[i].Trim();
                }
                if (!string.IsNullOrEmpty(cv.Title))
                {
                    bool isHad = false;
                    for (int j = 0; j < c.Count; j++)
                    {
                        if (cv.Title == c[j].Title)
                        {
                            c[j].Value = cv.Value;
                            isHad = true;
                            break;
                        }
                    }
                    if (string.IsNullOrEmpty(cv.Title) || string.IsNullOrEmpty(cv.Value) || cv.Title == "domain" || cv.Title == "expires"
                        || cv.Title == "path"&& !cv.Value.Contains("/,") || cv.Title.ToLower() == "httponly")
                        continue;
                    if (!isHad)
                    {
                        if (cv.Title == "path" && cv.Value.Contains("/,")&&cv.Value.IndexOf("/,")==0 && cv.Value.Length>2)
                        {
                            string t = cv.Value.Substring(2);
                            if(!t.Contains("=")|| t.IndexOf("=")>t.Length-1)
                                continue;
                            cv.Title = t.Substring(0, t.IndexOf("="));
                            cv.Value = t.Substring(t.IndexOf("=") + 1);
                        }
                        c.Add(cv);
                    }
                }
            }
            string temp = "";
            for (int i = 0; i < c.Count; i++)
            {
                if (string.IsNullOrEmpty(c[i].Value))
                    temp += c[i].Title + "; ";
                else
                    temp += c[i].Title + "=" + c[i].Value + "; ";
            }
            return temp;
        }
    }
}