789 lines
35 KiB
C#
789 lines
35 KiB
C#
using NaiBa.Util;
|
|
using NaiBa.Util.Extension;
|
|
using NaiBa.Util.Helper;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace Work.BLL.Collect
|
|
{
|
|
public class PddSameScreeningBLL
|
|
{
|
|
#region http请求
|
|
|
|
//Returns the text contained in the response. For example, the page HTML. Only handles the most common HTTP encodings.
|
|
public static string ReadResponse(HttpWebResponse response)
|
|
{
|
|
using (Stream responseStream = response.GetResponseStream())
|
|
{
|
|
Stream streamToRead = responseStream;
|
|
if (response.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
|
|
}
|
|
else if (response.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
|
|
}
|
|
|
|
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
|
|
{
|
|
return streamReader.ReadToEnd();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// 请求当前店铺销量
|
|
/// </summary>
|
|
/// <param name="response">After the function has finished, will possibly contain the response to the request.</param>
|
|
/// <returns>True if the request was successful; false otherwise.</returns>
|
|
public static string RequestShopSold(string url, string Token)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
//Create request to URL.
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
|
|
|
//Set request headers.
|
|
request.KeepAlive = true;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
|
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36";
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
request.Headers.Set(HttpRequestHeader.Cookie, Token);
|
|
|
|
//Get response to request.
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
//ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
|
|
if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
|
else return "";
|
|
}
|
|
catch (Exception)
|
|
{
|
|
if (response != null) response.Close();
|
|
return "";
|
|
}
|
|
|
|
return ReadResponse(response);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
/// 获取宝贝数量
|
|
/// </summary>
|
|
/// <param name="response">After the function has finished, will possibly contain the response to the request.</param>
|
|
/// <returns>True if the request was successful; false otherwise.</returns>
|
|
public static string RequestShopCount(string url, string Token)
|
|
{
|
|
HttpWebResponse response = null;
|
|
|
|
try
|
|
{
|
|
//Create request to URL.
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://yangkeduo.com/proxy/api/api/turing/mall/query_mall_category_list?pdduid=2423695749878&mall_id=" + url);
|
|
|
|
//Set request headers.
|
|
request.KeepAlive = true;
|
|
request.Accept = "application/json, text/plain, */*";
|
|
request.Headers.Add("AccessToken", Token);
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36";
|
|
request.Headers.Add("VerifyAuthToken", @"kUo_K6VOliEhlNZNorECKQ");
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
//Get response to request.
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
//ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
|
|
if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
|
else return "";
|
|
}
|
|
catch (Exception)
|
|
{
|
|
if (response != null) response.Close();
|
|
return "";
|
|
}
|
|
|
|
return ReadResponse(response);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tries to request the URL: http://api.yangkeduo.com/mall/412236912/info
|
|
/// </summary>
|
|
/// <param name="response">After the function has finished, will possibly contain the response to the request.</param>
|
|
/// <returns>True if the request was successful; false otherwise.</returns>
|
|
public static string Request_api_yangkeduo_com(string url, string Token)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
//Create request to URL.
|
|
string requestUrl = "http://api.yangkeduo.com/mall/" + url + "/info";
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(requestUrl);
|
|
//Set request headers.
|
|
request.Referer = "Android";
|
|
request.ContentType = "application/json;charset=UTF-8";
|
|
request.Headers.Add("AccessToken", Token);
|
|
request.UserAgent = "android Mozilla/5.0 (Linux; Android 5.1.1; m1 metal Build/LMY49I; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.100 Safari/537.36 phh_android_version/4.17.1 phh_android_build/1a7ec8b149 phh_android_channel/gw";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip");
|
|
|
|
//Get response to request.
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
//ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
|
|
if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
|
else return "";
|
|
}
|
|
catch (Exception)
|
|
{
|
|
if (response != null) response.Close();
|
|
return "";
|
|
}
|
|
|
|
return ReadResponse(response);
|
|
}
|
|
|
|
public static string Request_api_yangkeduo_Ip(string url, string[] UserAgentList)
|
|
{
|
|
string result = "";
|
|
try
|
|
{
|
|
string requestUrl = url;
|
|
HttpParam httpParam = new HttpParam();
|
|
httpParam.Url = requestUrl;
|
|
httpParam.Referer = url;
|
|
string UserAgentTemp = UserAgentList[new Random().Next(UserAgentList.Count())];
|
|
if (!string.IsNullOrEmpty(UserAgentTemp) && !UserAgentTemp.Contains("MicroMessenger"))
|
|
{
|
|
UserAgentTemp = UserAgentTemp.Insert(new Random().Next(UserAgentTemp.Length), "MicroMessenger" + Guid.NewGuid().ToString().Replace("-", "").Substring(0, new Random().Next(5)));
|
|
}
|
|
httpParam.UserAgent = UserAgentTemp;
|
|
httpParam.TryTimes = 60;
|
|
httpParam.ProxySleep = 500;
|
|
if (url.Contains("mall_page.html"))
|
|
{
|
|
result = GlobalProxyHelper.Request(httpParam, (response) => !string.IsNullOrEmpty(response) && !response.Contains("<title>店铺主页</title>"));
|
|
if (result.Contains(("<title>店铺主页</title>")))
|
|
{
|
|
Thread.Sleep(10000);
|
|
httpParam.TryTimes = 60;
|
|
httpParam.Timeout = 10000;
|
|
result = GlobalProxyHelper.Request(httpParam, (response) => !string.IsNullOrEmpty(response) && !response.Contains("<title>店铺主页</title>"));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
result = GlobalProxyHelper.Request(httpParam, (response) => !string.IsNullOrEmpty(response) && !response.Contains("error_code"));
|
|
if (result.Contains(("error_code")))
|
|
{
|
|
Thread.Sleep(10000);
|
|
httpParam.TryTimes = 60;
|
|
result = GlobalProxyHelper.Request(httpParam, (response) => !string.IsNullOrEmpty(response) && !response.Contains("error_code"));
|
|
|
|
}
|
|
}
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return "";
|
|
}
|
|
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
public static List<string> listIp = new List<string>();
|
|
|
|
private static readonly object obj = new object();
|
|
private static int UseCount = 0;
|
|
public static string Request_yangkeduo_com(string sIpAndPort, long lProdId, string[] UserAgentList, out bool IsOverdue, bool IsBreak = false)
|
|
{
|
|
IsOverdue = false;
|
|
try
|
|
{
|
|
if (IsBreak)
|
|
{
|
|
return "";
|
|
}
|
|
if (UseCount >= 150 && listIp.Count > 0)//代理ip 使用超过150次 才去请求一次ip
|
|
{
|
|
listIp.Clear();
|
|
}
|
|
lock (obj)
|
|
{
|
|
//Thread.Sleep(3000);
|
|
if (listIp.Count() <= 0)
|
|
{
|
|
//Thread.Sleep(1000);
|
|
listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
UseCount = 0;
|
|
}
|
|
}
|
|
if (IsOverdue)
|
|
{
|
|
return "";
|
|
}
|
|
string ProxyIP = listIp[new Random().Next(0, listIp.Count)];
|
|
UseCount++;
|
|
//listIp.Remove(listIp[0]);
|
|
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
string UserAgentTemp = UserAgentList[new Random().Next(UserAgentList.Count())];
|
|
if (!string.IsNullOrEmpty(UserAgentTemp) && !UserAgentTemp.Contains("MicroMessenger"))
|
|
{
|
|
UserAgentTemp = UserAgentTemp.Insert(new Random().Next(UserAgentTemp.Length), "MicroMessenger" + Guid.NewGuid().ToString().Replace("-", "").Substring(0, new Random().Next(5)));
|
|
}
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://mobile.yangkeduo.com/goods.html?goods_id=" + lProdId);
|
|
request.Timeout = 5000;
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9,en;q=0.8");
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
|
|
request.UserAgent = UserAgentTemp;
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "deflate");
|
|
System.Net.WebProxy proxy = GetWebProxy(ProxyIP);
|
|
request.Proxy = proxy;
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
Thread.Sleep(3000);
|
|
LogHelper.Write(e.Message);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
|
|
}
|
|
|
|
if (response != null)
|
|
{
|
|
|
|
string result = ReadResponse(response);
|
|
if (result.Contains("mallID"))
|
|
{
|
|
//GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return HtmlHelper.UnicodeDecode(result);
|
|
}
|
|
|
|
}
|
|
return "";
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return "";
|
|
}
|
|
}
|
|
|
|
public static string RequestUrl_yangkeduo_com(long lProdId, string[] UserAgentList, string orderId, out bool IsOverdue)
|
|
{
|
|
IsOverdue = false;
|
|
int count = 0;
|
|
try
|
|
{
|
|
Again:
|
|
count++;
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
string UserAgentTemp = UserAgentList[new Random().Next(UserAgentList.Count())];
|
|
if (!string.IsNullOrEmpty(UserAgentTemp) && !UserAgentTemp.Contains("MicroMessenger"))
|
|
{
|
|
UserAgentTemp = UserAgentTemp.Insert(new Random().Next(UserAgentTemp.Length), "MicroMessenger" + Guid.NewGuid().ToString().Replace("-", "").Substring(0, new Random().Next(5)));
|
|
}
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://goods.66daili.cn/api/goods/get/?orderid=" + orderId + "&token=5AF433ED506FA475&goodID=" + lProdId);
|
|
request.Timeout = 5000;
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9,en;q=0.8");
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
|
|
request.UserAgent = UserAgentTemp;
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "deflate");
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Thread.Sleep(1000);
|
|
LogHelper.Write(ex.Message);
|
|
}
|
|
if (response != null)
|
|
{
|
|
string result = ReadResponse(response);
|
|
if (result.Contains("order expire or not exist"))
|
|
{
|
|
IsOverdue = true;
|
|
return "";
|
|
}
|
|
if (result.Contains("mall_id") || result.Contains("mallId") || result.Contains("mallID"))
|
|
{
|
|
return HtmlHelper.UnicodeDecode(result);
|
|
}
|
|
else
|
|
{
|
|
if (count < 10)
|
|
{
|
|
Thread.Sleep(new Random().Next(50, 200));
|
|
goto Again;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (count < 10)
|
|
{
|
|
Thread.Sleep(new Random().Next(50, 200));
|
|
goto Again;
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
return "";
|
|
}
|
|
}
|
|
|
|
public static string MakeRequestForGetPddProdDetailByWeixin(string sIpAndPort, long lProdId)
|
|
{
|
|
while (true)
|
|
{
|
|
bool IsOverdue = false;
|
|
List<string> listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
//if(listIp!=null&&listIp.Count>0)
|
|
for (int i = 0; i < listIp.Count; i++)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://wxapp.yangkeduo.com/compose/vtx/goods/detail/first/v3?pdduid=4014246301&xcx=20161201&xcx_version=v5.7.6.1&xcx_hash=" + GetTimeStampMilliSec() + GetRandomString(16));
|
|
request.KeepAlive = true;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "no-cache");
|
|
request.Headers.Add("GZToken", Guid.NewGuid().ToString().Replace("-", "").Substring(0, 32));
|
|
//request.Headers.Add("GZToken", @"8dc2715af33d80577efe1cccd945fd38");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat";
|
|
request.ContentType = "application/json;charset=UTF-8";
|
|
request.Referer = "https://servicewechat.com/wx32540bd863b27570/1116/page-frame.html";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
|
|
System.Net.WebProxy proxy = GetWebProxy(listIp[i]);
|
|
request.Proxy = proxy;
|
|
|
|
request.Method = "POST";
|
|
request.ServicePoint.Expect100Continue = false;
|
|
|
|
string body = @"{""goods_id"":""" + lProdId + @""",""from"":""24"",""has_rec"":true,""address_list"":[{""address_id"":""19159230174"",""province_id"":""3"",""city_id"":""3401"",""district_id"":""4124"",""is_default"":""1"",""is_top"":0}],""page_version"":7,""show_type"":1,""xcx_trace_id"":""" + GetRandrom(16) + @""",""client_time"":" + GetTimeStampMilliSec() + @",""anti_content"":""3aje" + GetRandomString(122) + @""",""xcx_version"":""v5.7.6.1""}";
|
|
//string body = @"{""goods_id"":""" + lProdId + @""",""from"":""24"",""has_rec"":true,""address_list"":[{""address_id"":""19159230174"",""province_id"":""3"",""city_id"":""3401"",""district_id"":""4124"",""is_default"":""1"",""is_top"":0}],""page_version"":7,""show_type"":1,""xcx_trace_id"":""4277241276553842"",""client_time"":" + GetTimeStampMilliSec() + @",""anti_content"":""3ajeJxjZGRgYAhjYGJMBOIkIE4G4hQgTmVQZEyTMzEwMDAzsDCxMLYwNDY2MdU1NLU0MDW1NDUyNjM2ZeBiTGdnYCwK2WOnxMDKmMEENIyZMZMByM5iYmAEADpcDLo"",""xcx_version"":""v5.7.6.1""}";
|
|
byte[] postBytes = System.Text.Encoding.UTF8.GetBytes(body);
|
|
request.ContentLength = postBytes.Length;
|
|
Stream stream = request.GetRequestStream();
|
|
stream.Write(postBytes, 0, postBytes.Length);
|
|
stream.Close();
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
//return "";
|
|
}
|
|
catch (Exception)
|
|
{
|
|
//return "";
|
|
}
|
|
if (response == null) continue;
|
|
string result = ReadResponse(response);
|
|
if (result.Contains("\"success\":true"))
|
|
{
|
|
//GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return result;
|
|
}
|
|
else
|
|
{
|
|
|
|
}
|
|
Thread.Sleep(500);
|
|
}
|
|
GlobalProxyHelper.CloseIp(listIp);//没找到也关闭
|
|
Thread.Sleep(3000);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// 整店采集
|
|
/// </summary>
|
|
/// <param name="sIpAndPort"></param>
|
|
/// <param name="lProdId"></param>
|
|
/// <param name="UserAgent"></param>
|
|
/// <returns></returns>
|
|
public static string RequestAllShopList(string sIpAndPort, string url, Func<string, string> antiContentFunc, string UserAgent)
|
|
{
|
|
while (true)
|
|
{
|
|
bool IsOverdue = false;
|
|
List<string> listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
string NewUrl = "";
|
|
for (int i = 0; i < listIp.Count; i++)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
NewUrl = url + "&anticontent=" + antiContentFunc(UserAgent);
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(NewUrl);
|
|
request.KeepAlive = true;
|
|
request.Accept = "application/json, text/plain, */*";
|
|
request.UserAgent = UserAgent;
|
|
//request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 QBCore/4.0.1301.400 QQBrowser/9.0.2524.400 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2875.116 Safari/537.36 NetType/WIFI MicroMessenger/7.0.5 WindowsWechat";
|
|
request.Referer = "https://mobile.yangkeduo.com/";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.5;q=0.4");
|
|
System.Net.WebProxy proxy = GetWebProxy(listIp[i]);
|
|
request.Proxy = proxy;
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
|
|
}
|
|
if (response == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string result = ReadResponse(response);
|
|
if (result.Contains("goods_list"))
|
|
{
|
|
//GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return HtmlHelper.UnicodeDecode(result);
|
|
}
|
|
Thread.Sleep(new Random().Next(500, 1500));
|
|
}
|
|
GlobalProxyHelper.CloseIp(listIp);//没找到也关闭
|
|
Thread.Sleep(3000);
|
|
}
|
|
}
|
|
|
|
|
|
private static long GetRandrom(int len)
|
|
{
|
|
Random rand = new Random();
|
|
string strNum = "";
|
|
int i = 0;
|
|
while (i < len)
|
|
{
|
|
i++;
|
|
int randomNum = rand.Next(0, 10);
|
|
strNum += randomNum.ToString();
|
|
}
|
|
|
|
return strNum.ParseToLong();
|
|
}
|
|
|
|
|
|
public static string Request_weixinKweWords(string sIpAndPort, string keyword, int page)
|
|
{
|
|
while (true)
|
|
{
|
|
bool IsOverdue = false;
|
|
List<string> listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
//if(listIp!=null&&listIp.Count>0)
|
|
for (int i = 0; i < listIp.Count; i++)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://wxapp.yangkeduo.com/compose/vtx/search?pdduid=4014246301&xcx=20161201&xcx_version=v5.7.9&xcx_hash=1590732694274NkDSTBGS61cRPBiD");
|
|
|
|
request.KeepAlive = true;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "no-cache");
|
|
request.Headers.Add("GZToken", @"7933a2eca48857c7640ce14ff17d4bbb");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat";
|
|
request.ContentType = "application/json;charset=UTF-8";
|
|
request.Referer = "https://servicewechat.com/wx32540bd863b27570/1125/page-frame.html";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
|
|
System.Net.WebProxy proxy = GetWebProxy(listIp[i]);
|
|
request.Proxy = proxy;
|
|
|
|
request.Method = "POST";
|
|
request.ServicePoint.Expect100Continue = false;
|
|
|
|
//string body = "{\"\"is_form_wallet\"\":false,\"\"query\"\":\"\"" + keyword + "\"\",\"\"new_search\"\":true,\"\"search\"\":{\"\"page\"\":" + page + ",\"\"size\"\":20,\"\"requery\"\":0,\"\"list_id\"\":\"\"xcx_search_ZN6Xie\"\",\"\"source\"\":\"\"index\"\",\"\"sort\"\":\"\"default\"\"},\"\"anti_content\"\":\"\"3ajeJxjZGRgYAhjYGJMBOIkIE4G4hQgTmVQZEyTMzYzMzU0trAwNzU1NbI0NdU1NLU0MDc2MrM0NTA0Z-BiTGdnYCyK51vMwMDKmMEENIyZMRPEzgKyATcqDHE\"\",\"\"xcx_version\"\":\"\"v5.7.9\"\"}";
|
|
string body = @"{""is_form_wallet"":false,""query"":""连衣裙"",""new_search"":true,""search"":{""page"":1,""size"":20,""requery"":0,""list_id"":""xcx_search_ZN6Xie"",""source"":""index"",""sort"":""default""},""anti_content"":""3ajeJxjZGRgYAhjYGJMBOIkIE4G4hQgTmVQZEyTMzYzMzU0trAwNzU1NbI0NdU1NLU0MDc2MrM0NTA0Z-BiTGdnYCyK51vMwMDKmMEENIyZMRPEzgKyATcqDHE"",""xcx_version"":""v5.7.9""}";
|
|
byte[] postBytes = System.Text.Encoding.UTF8.GetBytes(body);
|
|
request.ContentLength = postBytes.Length;
|
|
Stream stream = request.GetRequestStream();
|
|
stream.Write(postBytes, 0, postBytes.Length);
|
|
stream.Close();
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
|
|
}
|
|
catch (Exception)
|
|
{
|
|
|
|
}
|
|
|
|
string result = ReadResponse(response);
|
|
//if (result.Contains("mallID"))
|
|
{
|
|
GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return result;
|
|
}
|
|
Thread.Sleep(500);
|
|
}
|
|
GlobalProxyHelper.CloseIp(listIp);//没找到也关闭
|
|
Thread.Sleep(3000);
|
|
}
|
|
}
|
|
|
|
private static char[] constant =
|
|
{
|
|
'0','1','2','3','4','5','6','7','8','9',
|
|
'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',
|
|
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'
|
|
};
|
|
private static string GetRandomString(int iLength)
|
|
{
|
|
System.Text.StringBuilder newRandom = new System.Text.StringBuilder(62);
|
|
Random rd = new Random();
|
|
for (int i = 0; i < iLength; i++)
|
|
{
|
|
newRandom.Append(constant[rd.Next(62)]);
|
|
}
|
|
return newRandom.ToString();
|
|
|
|
}
|
|
public static string GetTimeStampMilliSec()
|
|
{
|
|
TimeSpan ts = DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0);
|
|
return Convert.ToInt64(ts.TotalMilliseconds).ToString();
|
|
}
|
|
|
|
public static List<string> InitIp(string ProxyUrl, out bool IsOverdue)
|
|
{
|
|
IsOverdue = false;
|
|
if (listIp.Count > 0)
|
|
{
|
|
return listIp;
|
|
}
|
|
HttpParam httpParm = new HttpParam { Url = ProxyUrl };
|
|
GlobalProxyHelper.SetAuthorization(httpParm);
|
|
string proxyIpHtml = string.Empty;
|
|
List<string> proxyIpList = new List<string>();
|
|
int tryTimes = 0;
|
|
int maxTryTimes = 10;
|
|
while (true && tryTimes <= maxTryTimes)
|
|
{
|
|
proxyIpHtml = HttpHelper.Get(httpParm);
|
|
if (proxyIpHtml.Contains("过期") || proxyIpHtml.Contains("套餐不存在"))
|
|
{
|
|
IsOverdue = true;
|
|
break;
|
|
}
|
|
if (!string.IsNullOrEmpty(proxyIpHtml) && !proxyIpHtml.Contains("后获取") && !proxyIpHtml.Contains("过期") && !proxyIpHtml.Contains("套餐不存在"))
|
|
{
|
|
proxyIpList = proxyIpHtml.Split(new string[] { "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToList();
|
|
break;
|
|
}
|
|
tryTimes++;
|
|
Thread.Sleep(3000);
|
|
}
|
|
return proxyIpList;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public static string RequestTest(string sIpAndPort, string url)
|
|
{
|
|
|
|
while (true)
|
|
{
|
|
bool IsOverdue = false;
|
|
List<string> listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
//if(listIp!=null&&listIp.Count>0)
|
|
for (int i = 0; i < listIp.Count; i++)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
|
request.KeepAlive = true;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
|
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.2444.133 Safari/537.36";
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
|
|
request.Referer = url;
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
System.Net.WebProxy proxy = GetWebProxy(listIp[i]);
|
|
request.Proxy = proxy;
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
|
|
}
|
|
catch (Exception)
|
|
{
|
|
|
|
}
|
|
|
|
string result = ReadResponse(response);
|
|
if (!result.Contains("<title>店铺主页</title>"))
|
|
{
|
|
GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return result;
|
|
}
|
|
Thread.Sleep(500);
|
|
}
|
|
GlobalProxyHelper.CloseIp(listIp);//没找到也关闭
|
|
Thread.Sleep(3000);
|
|
}
|
|
|
|
//HttpWebResponse response = null;
|
|
|
|
//try
|
|
//{
|
|
// HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
|
|
|
// request.KeepAlive = true;
|
|
// request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
|
// request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
|
// request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36";
|
|
// request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
|
|
// request.Referer = url;
|
|
// request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
// request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
// System.Net.WebProxy proxy = GetWebProxy("120.79.214.87 11318 test200420-1 gu05VREpn349KSiT");
|
|
// request.Proxy = proxy;
|
|
|
|
// response = (HttpWebResponse)request.GetResponse();
|
|
//}
|
|
//catch (WebException e)
|
|
//{
|
|
// //ProtocolError indicates a valid HTTP response, but with a non-200 status code (e.g. 304 Not Modified, 404 Not Found)
|
|
// if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
|
// else return "";
|
|
//}
|
|
//catch (Exception)
|
|
//{
|
|
// if (response != null) response.Close();
|
|
// return "";
|
|
//}
|
|
|
|
//return ReadResponse(response);
|
|
}
|
|
public static string RequestTestA(string sIpAndPort, string url)
|
|
{
|
|
|
|
while (true)
|
|
{
|
|
bool IsOverdue = false;
|
|
List<string> listIp = InitIp(sIpAndPort, out IsOverdue);
|
|
//if(listIp!=null&&listIp.Count>0)
|
|
for (int i = 0; i < listIp.Count; i++)
|
|
{
|
|
HttpWebResponse response = null;
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
|
request.KeepAlive = true;
|
|
request.Accept = "application/json, text/plain, */*";
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36";
|
|
//request.Headers.Add("VerifyAuthToken", @"ymXDWelTcWGRcf9dQBsVgw");
|
|
request.Referer = url;
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
System.Net.WebProxy proxy = GetWebProxy(listIp[i]);
|
|
request.Proxy = proxy;
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
|
|
}
|
|
catch (Exception)
|
|
{
|
|
|
|
}
|
|
|
|
string result = ReadResponse(response);
|
|
if (!result.Contains("error_code"))
|
|
{
|
|
GlobalProxyHelper.CloseIp(listIp);//找到了 把当前获取到的ip关闭
|
|
return result;
|
|
}
|
|
Thread.Sleep(500);
|
|
}
|
|
GlobalProxyHelper.CloseIp(listIp);//没找到也关闭
|
|
Thread.Sleep(3000);
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
public static WebProxy GetWebProxy(string ip = "")
|
|
{
|
|
WebProxy webProxy = null;
|
|
try
|
|
{
|
|
if (!string.IsNullOrEmpty(ip))
|
|
{
|
|
string[] ipArr = ip.Split(new string[] { " ", ":" }, StringSplitOptions.RemoveEmptyEntries);
|
|
switch (ipArr.Length)
|
|
{
|
|
case 2:
|
|
webProxy = new WebProxy(ipArr[0], ipArr[1].ParseToInt());
|
|
break;
|
|
|
|
case 4:
|
|
webProxy = new WebProxy(ipArr[0], ipArr[1].ParseToInt());
|
|
webProxy.Credentials = new NetworkCredential(ipArr[2], ipArr[3]);
|
|
break;
|
|
}
|
|
return webProxy;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogHelper.WriteWithTime(ex);
|
|
}
|
|
return webProxy;
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
}
|