783 lines
38 KiB
C#
783 lines
38 KiB
C#
using AllBase;
|
|
using AllBase.Helper;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Work.Api;
|
|
|
|
namespace Work.Business.ProductMappingBLL
|
|
{
|
|
|
|
public class WeiDianCollectBLL
|
|
{
|
|
/// <summary>
|
|
/// 微店采集是否进行
|
|
/// </summary>
|
|
public static bool WdCollecting { get; set; } = false;
|
|
|
|
//伪装成百度蜘蛛访问
|
|
private static string sUserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36";
|
|
|
|
/// <summary>
|
|
/// 微店采集进行中
|
|
/// </summary>
|
|
internal static bool BWDCollecting;
|
|
|
|
/// <summary>
|
|
/// 微店店铺ID
|
|
/// </summary>
|
|
private static long lWdShopId;
|
|
|
|
/// <summary>
|
|
/// 同行店铺Ids
|
|
/// </summary>
|
|
private static List<string> lstColleagueShopId = new List<string>();
|
|
|
|
/// <summary>
|
|
/// 采集区号
|
|
/// </summary>
|
|
private static int SectionNo;
|
|
|
|
/// <summary>
|
|
/// 采集客户端的公网IP
|
|
/// </summary>
|
|
private static string PubIp;
|
|
|
|
/// <summary>
|
|
/// 服务调用时间间隔基准(实际时间以此为基本随机上下浮动)
|
|
/// </summary>
|
|
private static int iTimeSpan;
|
|
|
|
private static readonly object _locker = new object();
|
|
|
|
public static void StartWeiDianCollect()
|
|
{
|
|
//读取服务端的采集设置 1-功能开关； 2-请求间隔(s)
|
|
string sServerCfg = ArchiveWdCollectionConfigFromServer();
|
|
if (string.IsNullOrEmpty(sServerCfg)) return;
|
|
string[] sConfig = sServerCfg.Split(',');
|
|
if (sConfig.Length != 2) return;
|
|
if (sConfig[0].Equals("0")) return;
|
|
iTimeSpan = int.Parse(sConfig[1]);
|
|
|
|
//获取公网IP
|
|
PubIp = GetPublicIp();
|
|
if (string.IsNullOrEmpty(PubIp)) return;
|
|
|
|
//根据公网IP去服务端取采集区号(1-20亿，分2000段，每段100万笔采集量)
|
|
string sSecNoAndCollectCount = GetCollectSectionNoFromServer();
|
|
if (string.IsNullOrEmpty(sSecNoAndCollectCount)) return;
|
|
string[] sRes = sSecNoAndCollectCount.Split(',');
|
|
SectionNo = int.Parse(sRes[0]);
|
|
//如果此公网IP采集状态为1-采集中 2-限制或已经采集完毕，则返回
|
|
if (SectionNo < 0)
|
|
{
|
|
//MessageBox.Show(PubIp + " 不可进行采集！");
|
|
return;
|
|
}
|
|
if (sRes.Length != 2) return;
|
|
WdCollecting = true;
|
|
int iCollectedCount = int.Parse(sRes[1]);
|
|
if (iCollectedCount >= 1000000)
|
|
{
|
|
return;
|
|
}
|
|
iTimeSpanMin = iTimeSpan;
|
|
iTimeSpanMax = iTimeSpan + 3;
|
|
Thread thread = new Thread(AutoUpdateCollectInfo)
|
|
{
|
|
IsBackground = true
|
|
};
|
|
thread.Start();
|
|
|
|
Task.Factory.StartNew(() =>
|
|
{
|
|
Random random = new Random();
|
|
string sWdRes = string.Empty;
|
|
|
|
//MessageBox.Show("已开启微店采集，将从" + (SectionNo * 1000000 + iCollectedCount + 1) + "开始");
|
|
for (lWdShopId = SectionNo * 1000000 + iCollectedCount + 1; lWdShopId <= (SectionNo + 1) * 1000000 && WdCollecting; lWdShopId++)
|
|
{
|
|
int iRetryCnt = 0;
|
|
MakeReq:
|
|
Thread.Sleep(Randomx.GetId(iTimeSpanMin, iTimeSpanMax) * 100);
|
|
sWdRes = MakeRequestWd4GetColleagueShops(lWdShopId);
|
|
if (sWdRes.Length == 0)
|
|
{
|
|
//此IP被限制，更新服务端采集状态为2
|
|
UpdateCollectStatusFromServer(2);
|
|
WdCollecting = false;
|
|
return;
|
|
}
|
|
if (!sWdRes.Contains("shopItems\""))
|
|
{
|
|
if (sWdRes.Contains("请稍后重试"))
|
|
{
|
|
sUserAgent = GetHtml.UserAgent.Split('\n')[new Random(Guid.NewGuid().GetHashCode()).Next(GetHtml.UserAgent.Split('\n').Length)].Trim();
|
|
}
|
|
iRetryCnt++;
|
|
if (iRetryCnt > 10)
|
|
{
|
|
//此IP被限制，更新服务端采集状态为2
|
|
UpdateCollectStatusFromServer(2);
|
|
WdCollecting = false;
|
|
return;
|
|
}
|
|
Thread.Sleep(Randomx.GetId(iTimeSpanMin, iTimeSpanMax) * 100);
|
|
goto MakeReq;
|
|
}
|
|
if (!sWdRes.Contains("shopItems\":{\"items\":[{\""))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
lock (_locker)
|
|
{
|
|
lstColleagueShopId.Add(lWdShopId.ToString());
|
|
}
|
|
|
|
}
|
|
WdCollecting = false;
|
|
UpdateCollectStatusFromServer(0);
|
|
});
|
|
}
|
|
|
|
|
|
private static int iTimeSpanMin;
|
|
private static int iTimeSpanMax;
|
|
private static void AutoUpdateCollectInfo()
|
|
{
|
|
|
|
while (true)
|
|
{
|
|
if (lstColleagueShopId.Count == 0 && !WdCollecting)
|
|
{
|
|
return;
|
|
}
|
|
if (lWdShopId > 0)
|
|
{
|
|
try
|
|
{
|
|
UpdateColleagueShopIdsAndCollectCountFromServer();
|
|
lock (_locker)
|
|
{
|
|
lstColleagueShopId.Clear();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
|
|
}
|
|
}
|
|
Thread.Sleep(Randomx.GetId(5, 10) * 1000);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// 更新服务端的同行店铺Ids以及已采集数量
|
|
/// </summary>
|
|
private static void UpdateColleagueShopIdsAndCollectCountFromServer()
|
|
{
|
|
string requestData = "shopIds=" + GetHtml.UrlEncodeUtf8(string.Join(",", lstColleagueShopId)) + "&collect_ip=" + PubIp + "&collect_number=" + (lWdShopId - SectionNo * 1000000);
|
|
string url = BaseClass.LoginApiUrl + "?obj=product/collect&act=add_collect_data";
|
|
GoAheadSupportRetry(url, requestData);
|
|
}
|
|
|
|
/// <summary>
|
|
/// 更新服务端的同行店铺Ids以及已采集数量
|
|
/// </summary>
|
|
private static string ArchiveWdCollectionConfigFromServer()
|
|
{
|
|
string url = BaseClass.LoginApiUrl + "?obj=product/collect&act=get_collect_config";
|
|
return GoAheadSupportRetry(url, "");
|
|
}
|
|
|
|
/// <summary>
|
|
/// 更新指定IP的采集状态
|
|
/// </summary>
|
|
/// <param name="sPubIp">公网IP</param>
|
|
/// <param name="iStatus">采集状态</param>
|
|
public static void UpdateCollectStatusFromServer(int iStatus)
|
|
{
|
|
//调用php后台更新指定IP的采集状态
|
|
string requestData = "collect_ip=" + PubIp + "&collect_status=" + iStatus;
|
|
string url = BaseClass.LoginApiUrl + "?obj=product/collect&act=update_collect";
|
|
GoAheadSupportRetry(url, requestData);
|
|
}
|
|
|
|
private static string GetCollectSectionNoFromServer()
|
|
{
|
|
//调用php后台获取指定IP的采集区号
|
|
string requestData = "collect_ip=" + PubIp;
|
|
string url = BaseClass.LoginApiUrl + "?obj=product/collect&act=get_collect_code";
|
|
return GoAheadSupportRetry(url, requestData);
|
|
}
|
|
|
|
private static string GoAheadSupportRetry(string sUrl, string sReqData, int iRetryMax = 3)
|
|
{
|
|
int iRetryCnt = 0;
|
|
string result;
|
|
GoAhead:
|
|
result = GetHtml.PostUtf8(sUrl, sReqData);
|
|
if (string.IsNullOrEmpty(result) && iRetryCnt < iRetryMax)
|
|
{
|
|
iRetryCnt++;
|
|
Thread.Sleep(Randomx.GetId(50, 500));
|
|
goto GoAhead;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 获取公网IP
|
|
/// </summary>
|
|
/// <returns>公网IP地址</returns>
|
|
private static string GetPublicIp()
|
|
{
|
|
HttpWebResponse response;
|
|
string sRes = string.Empty;
|
|
if (Request_2000019_ip138_com(out response))
|
|
{
|
|
sRes = ReadResponse(response);
|
|
if (sRes.Length > 0)
|
|
{
|
|
Match mh = Regex.Match(sRes, "您的IP地址是：([\\w\\W]+?)</title>");
|
|
if (mh.Length > 0)
|
|
{
|
|
sRes = mh.Groups[1].Value;
|
|
}
|
|
else
|
|
{
|
|
sRes = string.Empty;
|
|
}
|
|
}
|
|
response.Close();
|
|
}
|
|
return sRes;
|
|
}
|
|
|
|
private static string ReadResponse(HttpWebResponse response)
|
|
{
|
|
using (Stream responseStream = response.GetResponseStream())
|
|
{
|
|
Stream streamToRead = responseStream;
|
|
if (response.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
|
|
}
|
|
else if (response.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
|
|
}
|
|
|
|
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
|
|
{
|
|
return streamReader.ReadToEnd();
|
|
}
|
|
}
|
|
}
|
|
|
|
private static bool Request_2000019_ip138_com(out HttpWebResponse response)
|
|
{
|
|
response = null;
|
|
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://2000019.ip138.com/");
|
|
|
|
request.KeepAlive = true;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
|
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
|
request.UserAgent = sUserAgent;
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9";
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9,en;q=0.8");
|
|
request.Headers.Set(HttpRequestHeader.Cookie, @"PHPSESSID=gouadr6cgch4h4st8pdi1gavp3; ASPSESSIONIDASBCSTBS=JOPDALEBHONGIBJHJPPJFPDJ");
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response;
|
|
else return false;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
if (response != null) response.Close();
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private static string MakeRequestWd4GetColleagueShops(long lShopId)
|
|
{
|
|
HttpWebResponse response;
|
|
var v = "";
|
|
if (Request_weidian_com(lShopId, out response))
|
|
{
|
|
try
|
|
{
|
|
using (StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8))
|
|
{
|
|
v = sr.ReadToEnd();
|
|
}
|
|
response.Close();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
|
|
}
|
|
}
|
|
else
|
|
{
|
|
return "";
|
|
}
|
|
return v;
|
|
}
|
|
|
|
private static bool Request_weidian_com(long lShopId, out HttpWebResponse response)
|
|
{
|
|
response = null;
|
|
MakeRequestWd:
|
|
try
|
|
{
|
|
//获取店铺第1000+商品信息
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://thor.weidian.com/ares/shop.getItemsInShop/1.0?param={\"shopId\":\"" + lShopId + "\",\"page\":11,\"limit\":100}");
|
|
request.KeepAlive = true;
|
|
request.Timeout = 15000;
|
|
request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0");
|
|
request.Headers.Add("Upgrade-Insecure-Requests", @"1");
|
|
request.UserAgent = sUserAgent;
|
|
request.Headers.Add("Sec-Fetch-User", @"?1");
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9";
|
|
request.Headers.Add("Sec-Fetch-Site", @"none");
|
|
request.Headers.Add("Sec-Fetch-Mode", @"navigate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "deflate, br");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9,en;q=0.8");
|
|
//request.Headers.Set(HttpRequestHeader.Cookie, @"__spider__visitorid=99847ca38c8d6d84; vc_fpcookie=d09a621b-a4c6-d905-0177-9da2ce436be1; vc_device_id=d496a1adb4f8b88d65cb30de7264031a; login_token=_EwWqqVIQEOZk3K4Xlw_Jg9LlPshZBhdbAvfITT9_RjRD4NcDyFJyjJbYfnhizpflOAvHfG7PSMovzOj4ejBjwzfzrKMKMUcsDhgsUlZEVouSxvo1LW0VS9eKDZ828OLKQsajaJG78ddJmfGxxrUPHJEU84juJHaEPl_TuzFNJSLgN9_LqOI; is_login=true; login_type=LOGIN_USER_TYPE_MASTER; login_source=LOGIN_USER_SOURCE_MASTER; uid=316846450; sid=316846450; token=; isLogin=; loginUserType=; loginUserSource=; WD_b_wduss=; WD_s_wduss=; WD_b_id=; WD_s_id=; WD_b_country=; WD_b_tele=; WD_s_tele=; WD_seller=; vc_token_id=8903a2e4-00d2-220c-8967-a4acf4fbcce1-16f6563b2b8-3a46; WD_wfr=shop; unreadMsg=0; WD_guid=1577962261222_1977684674589264; v-components/cpn-coupon-dialog@iwdDefault=1; CartreferrerURl=shopid_212239496; __spider__sessionid=455f3f04f7aaf1bb; wdtoken=c511ac4f; is_follow_mp=0; cart_inner_item_count=0");
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException e)
|
|
{
|
|
if (e.Message.Contains("超时"))
|
|
{
|
|
Thread.Sleep(Randomx.GetId(1, 5) * 1000);
|
|
goto MakeRequestWd;
|
|
}
|
|
return false;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message.Contains("超时"))
|
|
{
|
|
Thread.Sleep(Randomx.GetId(1, 5) * 1000);
|
|
goto MakeRequestWd;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private static Random random = new Random();
|
|
public static string[] arrSignSvrWeidian;
|
|
//private static string[] arrSignSvrWeidian = "26t606j345.zicp.vip:48932;2664638a6k.qicp.vip;2664638a6k.qicp.vip:12393".Split(';');
|
|
|
|
/// <summary>
|
|
/// WeiDian Context集合
|
|
/// </summary>
|
|
private static string[] arrWeiDianContext = null;
|
|
/// <summary>
|
|
/// 微店app接口api版本
|
|
/// </summary>
|
|
private static List<string> thorVers = "2.4,2.6".Split(',').ToList();
|
|
|
|
/// <summary>
|
|
/// 通过关键词(或标题)搜索微店商品
|
|
/// </summary>
|
|
/// <param name="sSearchKeyword">搜索关键词或标题</param>
|
|
/// <param name="dPriceMin">价格最低价,不限制传0</param>
|
|
/// <param name="dPriceMax">价格最高价,不限制传0</param>
|
|
/// <param name="iPage">搜索页数,从1开始</param>
|
|
/// <param name="bEncryptResponse">是否调用加密接口</param>
|
|
/// <param name="bHookMode">是否通过hook方式</param>
|
|
/// <returns></returns>
|
|
public static string SearchWdProductsByKeywords(string sSearchKeyword, double dPriceMin = 0, double dPriceMax = 0, int iPage = 1, bool bEncryptResponse = false, bool bHookMode = false)
|
|
{
|
|
#region Random params Prevent being restricted
|
|
long lTimeStampMilli = BaseClass.ConvertDateTimeLong();
|
|
string sSign = Md5Base.GetMd5(lTimeStampMilli / 1000 + "nbytht");
|
|
int index = random.Next(0, arrSignSvrWeidian.Length);
|
|
string sSignSvrWeidian = arrSignSvrWeidian[index];
|
|
//随机参数全部移至hook App端完成以节省传输带宽
|
|
//string sAndroidId = BaseClass.GetRandomAlphanumeric(16).ToLower();
|
|
//string sBrand = "Huawei,HONOR,Xiaomi,Meizu,samsung,Lenovo,zte,htc,nubia".Split(',')[random.Next(0, 9)];
|
|
string sCuid = Guid.NewGuid().ToString().Replace("-", "");
|
|
//string sImei = BaseClass.GetRandromNumeral(15);
|
|
//经纬度随机
|
|
//string sLat = (random.NextDouble() * (53.550000 - 3.860000) + 3.860000).ToString("f6");
|
|
//string sLon = (random.NextDouble() * (135.050000 - 73.660000) + 73.660000).ToString("f6");
|
|
//string sMac = BaseClass.RandomMacAddress().ToUpper();
|
|
//string sWmac = BaseClass.RandomMacAddress().ToLower();
|
|
string sFootprintId = sCuid + "_" + lTimeStampMilli;
|
|
string sChoicePrices = string.Empty;
|
|
if (dPriceMax > 0 || dPriceMin > 0)
|
|
{
|
|
sChoicePrices = (dPriceMin > 0 ? dPriceMin.ToString() : " ") + "-" + (dPriceMax > 0 ? dPriceMax.ToString() : " ");
|
|
}
|
|
#endregion
|
|
|
|
#region 构造PostData原文并通过hook得到PostData密文
|
|
//先从后台获取WdContext集合
|
|
if (arrWeiDianContext == null)
|
|
{
|
|
arrWeiDianContext = GetWdContextService();
|
|
}
|
|
int iTryTimes = 0;
|
|
LbHookPostData:
|
|
index++;
|
|
if (index > 2)
|
|
index = 0;
|
|
sSignSvrWeidian = arrSignSvrWeidian[index];
|
|
string sContext = arrWeiDianContext[Randomx.GetId(arrWeiDianContext.Length)];
|
|
string sParam = @"{""area"":"""",""brands"":[],""chosePoints"":"""",""chosePrices"":""" + sChoicePrices + @""",""coreWord"":"""",""footprintId"":""" + sFootprintId + @""",""id"":""" + sSearchKeyword + @""",""increServices"":[],""keyword"":""" + sSearchKeyword + @""",""limit"":16,""order"":""desc"",""page"":" + iPage + @",""remind"":1,""search_wfr"":""search_input"",""sortKey"":""price"",""source"":""1"",""tab"":""all"",""tags"":[],""wdServices"":[]}";
|
|
string sPostData = @"v=2.0×tamp=" + lTimeStampMilli + @"&appkey=06475281&context=" + HttpHelper.UrlEncode(sContext) + "¶m=" + HttpHelper.UrlEncode(sParam);
|
|
|
|
//不走hook方式，走直调接口方式
|
|
if (!bHookMode)
|
|
{
|
|
string platForm = new string[] { "ios", "android" }.ToList()[random.Next(2)];
|
|
string thorVer = thorVers[random.Next(thorVers.Count)];
|
|
string userAgent = $"{platForm.ToUpper()}/{random.Next(5, 8)}.{random.Next(0, 3)}.{random.Next(5)} WDAPP(WDBuyer/{random.Next(5, 8)}.{random.Next(0, 3)}.{random.Next(5)}) Thor/{thorVer}.6.6";
|
|
return HttpHelper.Post(new AllBase.Helper.HttpParam() { Url = $"http://thor.weidian.com/faith/search.item/{thorVer}", Method = "POST", PostData = sPostData, Referer = $"https://{platForm}.weidian.com", UserAgent = userAgent });
|
|
}
|
|
|
|
//走hook方式
|
|
AllBase.Helper.HttpParam httpParamRequest = new AllBase.Helper.HttpParam
|
|
{
|
|
Url = $"http://{sSignSvrWeidian}/sign/do.json?action=request&nb_timestamp={lTimeStampMilli / 1000}&nb_sign={sSign}&mode=0",
|
|
PostData = sPostData,
|
|
Timeout = 10,
|
|
ContentType = "application/x-www-form-urlencoded; charset=utf-8"
|
|
};
|
|
string requestResult = string.Empty;
|
|
try
|
|
{
|
|
Thread.Sleep(random.Next(1, 3) * 1000);
|
|
requestResult = HttpHelper.Post(httpParamRequest);
|
|
if (string.IsNullOrEmpty(requestResult) )
|
|
{
|
|
if (ProductMapping.ischeck)
|
|
UploadPinduoduoMain.CheckSearch = false;
|
|
LogSave.Add("标题搜索sign为空"+ sSignSvrWeidian);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogSave.Add("微店标题搜索接口请求异常:" + ex.Message);
|
|
}
|
|
|
|
if (!requestResult.Contains("data") && iTryTimes < 1)
|
|
{
|
|
Thread.Sleep(random.Next(2, 3) * 1000);
|
|
iTryTimes++;
|
|
goto LbHookPostData;
|
|
}
|
|
if (!requestResult.Contains("data")) return "";
|
|
#endregion
|
|
|
|
#region 发送PostData密文去请求微店接口 可得到ResponseData明文，也可得到ResponseData密文
|
|
AllBase.Helper.HttpParam httpParamWeiDian = new AllBase.Helper.HttpParam
|
|
{
|
|
Url = "http://thor.weidian.com/faith/search.item/2.4",
|
|
UserAgent = "Android/6.0.1 WDAPP(WDBuyer/6.0.3) Thor/2.3.6.3",
|
|
Timeout = 6,
|
|
Referer = "https://android.weidian.com",
|
|
ContentType = "application/x-www-form-urlencoded; charset=utf-8",
|
|
Header = new Dictionary<string, string>()
|
|
};
|
|
httpParamWeiDian.Header.Add("accept-encoding", bEncryptResponse ? "GLZip" : "Deflate");
|
|
httpParamWeiDian.Header.Add("x-origin", "thor");
|
|
httpParamWeiDian.Header.Add("x-schema", "https");
|
|
httpParamWeiDian.Header.Add("x-encrypt", bEncryptResponse ? "1" : "0");
|
|
httpParamWeiDian.Header.Add("origin", "android.weidian.com");
|
|
httpParamWeiDian.PostData = HtmlHelper.Resove(requestResult, "\"data\":\"", "\"");
|
|
if (!bEncryptResponse)
|
|
{
|
|
try
|
|
{
|
|
Thread.Sleep(random.Next(1, 6) * 1000);
|
|
requestResult = HttpHelper.Post(httpParamWeiDian);
|
|
if (string.IsNullOrEmpty(requestResult))
|
|
{
|
|
if (ProductMapping.ischeck)
|
|
UploadPinduoduoMain.CheckSearch = false;
|
|
LogSave.Add("标题搜索返回数据为空");
|
|
}
|
|
else
|
|
{
|
|
if (ProductMapping.ischeck)
|
|
UploadPinduoduoMain.CheckSearch = true;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogSave.Add("微店标题搜索接口请求异常:" + ex.Message);
|
|
return "";
|
|
}
|
|
return requestResult;
|
|
}
|
|
|
|
var result = PostByte(httpParamWeiDian);
|
|
#endregion
|
|
|
|
#region 将ResponseData密文通过hook得到ResponseData原文
|
|
string base64 = Convert.ToBase64String(result);
|
|
AllBase.Helper.HttpParam httpParamResponse = new AllBase.Helper.HttpParam
|
|
{
|
|
Url = $"http://{sSignSvrWeidian}/sign/do.json?action=response&nb_timestamp={lTimeStampMilli / 1000}&nb_sign={sSign}",
|
|
PostData = "data=" + HttpHelper.UrlEncode(base64),
|
|
ContentType = "application/x-www-form-urlencoded; charset=utf-8"
|
|
};
|
|
try
|
|
{
|
|
Thread.Sleep(random.Next(1, 3) * 1000);
|
|
requestResult = HttpHelper.Post(httpParamResponse);
|
|
if (string.IsNullOrEmpty(requestResult))
|
|
LogSave.Add("标题搜索返回数据为空");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogSave.Add("微店标题搜索接口请求异常:" + ex.Message);
|
|
return "";
|
|
}
|
|
return requestResult;
|
|
#endregion
|
|
|
|
//微店不加密接口已失效
|
|
//return HttpHelper.PostUtf8(new HttpParam() { Url = "http://thor.weidian.com/faith/search.item/2.6", Method = "POST", PostData = sPostData, Referer = "https://android.weidian.com", UserAgent = "Android/6.0.1 WDAPP(WDBuyer/6.0.6) Thor/2.3.6.6"});
|
|
}
|
|
|
|
/// <summary>
|
|
/// 获取WeiDian Context集合
|
|
/// </summary>
|
|
private static string[] GetWdContextService()
|
|
{
|
|
string url = BaseClass.LoginApiUrl + "/?obj=api/config&act=wd_context_v2";
|
|
string res = GetHtml.PostUtf8(url, UserLogin.GetSignaturex());
|
|
if (!string.IsNullOrEmpty(res))
|
|
{
|
|
return res.Split(';');
|
|
}
|
|
else //默认值
|
|
{
|
|
return "{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQEyZKqM2PZUQcHsBRwE4X41-sxHF1SqJBMKW2-Ep2_ZxzVdoS3mvrvYTCcedcJUxCdAZgIkypkz3F811inqWCXWSaHf0yZytHxQQPF0aHUeF769XwaWw9YTrae-VxRE3RW69O6yqFWrj5qamlgfD3SpACdiOJvjxUfdXWKNPP75I\",\"duid\":\"1315506113\",\"uss\":\"_EwWqqVIQEyZKqM2PZUQcHsBRwE4X41-sxHF1SqJBMKW2-Ep2_ZxzVdoS3mvrvYTCcedcJUxCdAZgIkypkz3F811inqWCXWSaHf0yZytHxQQPF0aHUeF769XwaWw9YTrae-VxRE3RW69O6yqFWrj5qamlgfD3SpACdiOJvjxUfdXWKNPP75I\",\"anonymousId\":\"0b40d738ec8016f5ef0e833bf66b8aa87099dfb6\",\"userID\":\"1315506113\",\"wduserID\":\"1315506113\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQImnSvEqukYADvVMSBDbMsIttEmXS8YwnHdmlBEG8kLxGxWInk0EXrYqljlRQ5lp2uJUhT0NPcSWsHqVVElpJmXUqiopvGs2HmBWjtr678V0yprAkS9OP3pVsdutyQgzbmYIUhl6PFs7VUEeFM0pNqHBYf_G7TYDljpjFRRG_Cbc\",\"duid\":\"1613253916\",\"uss\":\"_EwWqqVIQImnSvEqukYADvVMSBDbMsIttEmXS8YwnHdmlBEG8kLxGxWInk0EXrYqljlRQ5lp2uJUhT0NPcSWsHqVVElpJmXUqiopvGs2HmBWjtr678V0yprAkS9OP3pVsdutyQgzbmYIUhl6PFs7VUEeFM0pNqHBYf_G7TYDljpjFRRG_Cbc\",\"anonymousId\":\"1b28adc9e6113131422488e3a97f7d0f6df2c833\",\"userID\":\"1613253916\",\"wduserID\":\"1823050762\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQa4fD09LpoVv8vECPm18ebpjpxNzDXTBQV2dzCBTLwDU6B8KhNZFIsNJg2Mi5AZxMWSnQkfQvLaPodjEYfXVnXjwBIjKff_iD4TiuEQF6gObB1Dx12jkBLQ2Ztfuz_Wys_n9m0_gDsBfb_QLJp8f9jd8pbx_-6oxm4kMQkq9vaxs\",\"duid\":\"1590515991\",\"uss\":\"_EwWqqVIQa4fD09LpoVv8vECPm18ebpjpxNzDXTBQV2dzCBTLwDU6B8KhNZFIsNJg2Mi5AZxMWSnQkfQvLaPodjEYfXVnXjwBIjKff_iD4TiuEQF6gObB1Dx12jkBLQ2Ztfuz_Wys_n9m0_gDsBfb_QLJp8f9jd8pbx_-6oxm4kMQkq9vaxs\",\"anonymousId\":\"fe0168956c78bc160aa51cadeab4fbf0e841b88a\",\"userID\":\"1590515991\",\"wduserID\":\"1720909970\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQWKgZwEXoPyaPyTxVrRnpBvbA4ub2vlaYOIf3sQutH5kaxt5C-eV-srUxI-hDSV5WjqoZc_hAIy7v6KAWaCr1r0SGB91ysDkKLNdqNao-aP9j0P4IZ4m7kBrc4vr0TCOVUi8L_JpEZ32v-HH6ONT6oHd--aAfOqF6JmJAR3J-OZI\",\"duid\":\"1458041550\",\"uss\":\"_EwWqqVIQWKgZwEXoPyaPyTxVrRnpBvbA4ub2vlaYOIf3sQutH5kaxt5C-eV-srUxI-hDSV5WjqoZc_hAIy7v6KAWaCr1r0SGB91ysDkKLNdqNao-aP9j0P4IZ4m7kBrc4vr0TCOVUi8L_JpEZ32v-HH6ONT6oHd--aAfOqF6JmJAR3J-OZI\",\"anonymousId\":\"9740e2814bdda65f15fe00b6baaa20516a6b9ebb\",\"userID\":\"1458041550\",\"wduserID\":\"1775289160\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQAbBHv33Hb8h4BTk6D3xDuZ2xb3M_3Qx9BwNiUgBZD5YU_sN-Z5WxWqifRhJpnIBo02A4a0Gd9BhKGlQXVHrjhlqIzWU3t7gOvfAtTkF6zY7lV_hULG3jZ-XxkgmV8m9jKYQMdGw8jw_XqRvUuHZeTW3o1ZifXyPwEp0w1DIYBjo\",\"duid\":\"1457886926\",\"uss\":\"_EwWqqVIQAbBHv33Hb8h4BTk6D3xDuZ2xb3M_3Qx9BwNiUgBZD5YU_sN-Z5WxWqifRhJpnIBo02A4a0Gd9BhKGlQXVHrjhlqIzWU3t7gOvfAtTkF6zY7lV_hULG3jZ-XxkgmV8m9jKYQMdGw8jw_XqRvUuHZeTW3o1ZifXyPwEp0w1DIYBjo\",\"anonymousId\":\"c946abdbbf6d2b176c5dafc115ccfa4407cc9b96\",\"userID\":\"1457886926\",\"wduserID\":\"1728949064\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQW0SLhMjMF--dXY8z4sLNA3i5yMCTOiAPdb8FGVrzqA_AxJ5aE-DtSbMhMEDgz2Pp59KBt26jK9WkaJPGfGtc8kRu8s40RdEhBk2i4rIKIPBAXSl6-jS-4NH0PZe00q3ppBYuvjfFfoeZeBfCdiA4K4G5G4NvExeEm1RXBLeGz8c\",\"duid\":\"1457941037\",\"uss\":\"_EwWqqVIQW0SLhMjMF--dXY8z4sLNA3i5yMCTOiAPdb8FGVrzqA_AxJ5aE-DtSbMhMEDgz2Pp59KBt26jK9WkaJPGfGtc8kRu8s40RdEhBk2i4rIKIPBAXSl6-jS-4NH0PZe00q3ppBYuvjfFfoeZeBfCdiA4K4G5G4NvExeEm1RXBLeGz8c\",\"anonymousId\":\"ef346e978e0e576758509467fadbc4d7a8c3c7d0\",\"userID\":\"1457941037\",\"wduserID\":\"1746010952\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQfrm41rMjm-1cmkdyFH2WD0Rd8omOQkZMWyMXqTTIzXPGF_kh1kdAGULBIQn6IM_82tKHzxCpzQvpTpVpNPBY5xBrrc04ZmKq4lgi3f-hs16Nlay9KHB5x4AmHES3zNHL2NrHODzriqgnQrMN-tp1m0k-aU4SzIjjrIrWhvvKx48\",\"duid\":\"1457948637\",\"uss\":\"_EwWqqVIQfrm41rMjm-1cmkdyFH2WD0Rd8omOQkZMWyMXqTTIzXPGF_kh1kdAGULBIQn6IM_82tKHzxCpzQvpTpVpNPBY5xBrrc04ZmKq4lgi3f-hs16Nlay9KHB5x4AmHES3zNHL2NrHODzriqgnQrMN-tp1m0k-aU4SzIjjrIrWhvvKx48\",\"anonymousId\":\"d5e95317c5e65351a64b4cffd9af0d51b60f7102\",\"userID\":\"1457948637\",\"wduserID\":\"1716544328\",\"wxappid\":\"wx4b74228baa15489a\"};{\"appid\":\"wxbuyer\",\"platform\":\"windows\",\"token\":\"_EwWqqVIQIx4ntFzDm50GX5WS1aWQuasH1x-Epv5HAlG60iSG6OdlnxQ_FspOnIq4FFQXgCtMoOCDzuTPFLZ3l0oLQ2ewticmOLLd90KgCkt0cUNELEen2nrrT06kmvij2yjFAWGiHqZRp2aGXAVm_OTW40ktzSFEShdjZIvgM02mS00ZDlI\",\"duid\":\"1613513032\",\"uss\":\"_EwWqqVIQIx4ntFzDm50GX5WS1aWQuasH1x-Epv5HAlG60iSG6OdlnxQ_FspOnIq4FFQXgCtMoOCDzuTPFLZ3l0oLQ2ewticmOLLd90KgCkt0cUNELEen2nrrT06kmvij2yjFAWGiHqZRp2aGXAVm_OTW40ktzSFEShdjZIvgM02mS00ZDlI\",\"anonymousId\":\"f0cd1119c8cf0cecce0fa6af90531a7ffc258e6f\",\"userID\":\"1613513032\",\"wduserID\":\"1794619402\",\"wxappid\":\"wx4b74228baa15489a\"}".Split(';'); ;
|
|
}
|
|
}
|
|
|
|
private static string userAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36";
|
|
|
|
/// <summary>
|
|
/// 获取指定微店商品标题_商品网页获取模式(目前暂未遇到采集限制，故无法确切做限制过滤)
|
|
/// </summary>
|
|
/// <param name="itemId">微店商品id</param>
|
|
/// <param name="iMaxTryTimes">最大重试次数</param>
|
|
/// <returns>商品标题(为空则表示商品不存在)</returns>
|
|
public static string GetProductTitleByHtml(string itemId, int iMaxTryTimes = 1)
|
|
{
|
|
int iTryTimes = 0;
|
|
while(iTryTimes++ < iMaxTryTimes)
|
|
{
|
|
string wdToken = BaseClass.GetRandomAlphanumeric(8).ToLower(); //随机参数 wdtoken=f1565ee9
|
|
AllBase.Helper.HttpParam httpParam = new AllBase.Helper.HttpParam
|
|
{
|
|
Url = $"https://weidian.com/item.html?itemID={itemId}",
|
|
Referer = $"https://weidian.com/item.html?itemID={itemId}&spider_token={BaseClass.GetRandomAlphanumeric(4).ToLower()}",
|
|
UserAgent = GetProxyHtml.UserAgentList?.Count > 0
|
|
? GetProxyHtml.UserAgentList[Randomx.GetId(GetProxyHtml.UserAgentList.Count)]
|
|
: userAgent,
|
|
Timeout = 10,
|
|
Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
|
Cookie = $"wdtoken={wdToken}; __spider__visitorid={BaseClass.GetRandomAlphanumeric(16).ToLower()}; visitor_id={BaseClass.GetRandomAlphanumeric(8).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(12).ToLower()}"
|
|
};
|
|
string res = HttpHelper.Get(httpParam);
|
|
string itemTitle = BaseRegex.GetMatchMaxRegexString(ref res, "<span[\\s]*class=\"item-name\">([\\w\\W]+?)</span>");
|
|
if(!string.IsNullOrEmpty(itemTitle))
|
|
{
|
|
return itemTitle;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// 获取指定微店商品标题_商品Api获取模式(目前暂未遇到采集限制，故无法确切做限制过滤)
|
|
/// </summary>
|
|
/// <param name="itemId">微店商品id</param>
|
|
/// <returns>商品详情数据</returns>
|
|
public static string GetProductDetailByApi(string itemId)
|
|
{
|
|
string wdToken = BaseClass.GetRandomAlphanumeric(8).ToLower(); //随机参数 wdtoken=f1565ee9
|
|
AllBase.Helper.HttpParam httpParam = new AllBase.Helper.HttpParam
|
|
{
|
|
Url = $"https://thor.weidian.com/detail/getItemSkuInfo/1.0?param=%7B%22itemId%22%3A%22{itemId}%22%7D&wdtoken={wdToken}&_={BaseClass.ConvertDateTimeLong()}",
|
|
Referer = $"https://weidian.com/item.html?itemID={itemId}&spider_token=9d2e",
|
|
UserAgent = GetProxyHtml.UserAgentList?.Count > 0
|
|
? GetProxyHtml.UserAgentList[Randomx.GetId(GetProxyHtml.UserAgentList.Count)]
|
|
: userAgent,
|
|
Timeout = 10,
|
|
Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
|
Cookie = $"wdtoken={wdToken}; __spider__visitorid={BaseClass.GetRandomAlphanumeric(16).ToLower()}; visitor_id={BaseClass.GetRandomAlphanumeric(8).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(4).ToLower()}-{BaseClass.GetRandomAlphanumeric(12).ToLower()}"
|
|
};
|
|
return HttpHelper.Get(httpParam);
|
|
}
|
|
|
|
public static byte[] PostByte(AllBase.Helper.HttpParam httpParam, bool throwException = false)
|
|
{
|
|
if (string.IsNullOrEmpty(httpParam.Url))
|
|
{
|
|
return default(byte[]);
|
|
}
|
|
try
|
|
{
|
|
HttpWebRequest request = GetHttpWebRequest(httpParam);
|
|
request.Method = "Post";
|
|
SetPostData(request, httpParam.PostData);
|
|
return ReadResponseByte(request);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
LogHelper.WriteWithTime(ex);
|
|
if (!ex.Message.Contains("404"))
|
|
{
|
|
LogHelper.WriteWithTime(ex);
|
|
if (httpParam.TryTimes > 0 && httpParam.ActualTryTimes < httpParam.TryTimes)
|
|
{
|
|
httpParam.ActualTryTimes++;
|
|
PostByte(httpParam, throwException);
|
|
}
|
|
if (throwException)
|
|
{
|
|
throw ex;
|
|
}
|
|
}
|
|
}
|
|
return default(byte[]);
|
|
}
|
|
|
|
public static void SetPostData(HttpWebRequest request, string postData)
|
|
{
|
|
request.ServicePoint.Expect100Continue = false;
|
|
byte[] postDataByte = Encoding.UTF8.GetBytes(postData);
|
|
request.ContentLength = postDataByte.Length;
|
|
request.AllowAutoRedirect = false;
|
|
using (Stream requestStream = request.GetRequestStream())
|
|
{
|
|
requestStream.Write(postDataByte, 0, postDataByte.Length);
|
|
}
|
|
}
|
|
|
|
public static byte[] ReadResponseByte(HttpWebRequest request)
|
|
{
|
|
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
|
{
|
|
using (BinaryReader binaryReader = new BinaryReader(response.GetResponseStream()))
|
|
{
|
|
using (MemoryStream ms = new MemoryStream())
|
|
{
|
|
byte[] buffer = binaryReader.ReadBytes(1024);
|
|
while (buffer.Length > 0)
|
|
{
|
|
ms.Write(buffer, 0, buffer.Length);
|
|
buffer = binaryReader.ReadBytes(1024);
|
|
}
|
|
byte[] fileByte = new byte[ms.Length];
|
|
ms.Position = 0;
|
|
ms.Read(fileByte, 0, fileByte.Length);
|
|
return fileByte;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public static HttpWebRequest GetHttpWebRequest(AllBase.Helper.HttpParam param)
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(param.Url);
|
|
if (param.Proxy != null)
|
|
{
|
|
request.Proxy = param.Proxy;
|
|
}
|
|
else
|
|
{
|
|
request.Proxy = NaiBaGlobalConfig.GetDevelopProxy();
|
|
}
|
|
if (!string.IsNullOrEmpty(param.UserAgent))
|
|
{
|
|
request.UserAgent = param.UserAgent;
|
|
}
|
|
else
|
|
{
|
|
request.UserAgent = "";
|
|
}
|
|
if (!string.IsNullOrEmpty(param.Accept))
|
|
{
|
|
request.Accept = param.Accept;
|
|
}
|
|
else
|
|
{
|
|
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
}
|
|
if (string.IsNullOrEmpty(param.AcceptLanguage))
|
|
{
|
|
request.Headers.Add("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
|
|
}
|
|
else
|
|
{
|
|
request.Headers.Add("Accept-Language", param.AcceptLanguage);
|
|
}
|
|
if (!string.IsNullOrEmpty(param.ContentType))
|
|
{
|
|
request.ContentType = param.ContentType;
|
|
}
|
|
else
|
|
{
|
|
request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
|
|
}
|
|
if (!string.IsNullOrEmpty(param.Referer))
|
|
{
|
|
request.Referer = param.Referer;
|
|
}
|
|
request.Headers.Remove("Cookie");
|
|
if (!string.IsNullOrEmpty(param.Cookie))
|
|
{
|
|
request.Headers.Add("Cookie", param.Cookie);
|
|
}
|
|
if (param.Header != null)
|
|
{
|
|
foreach (KeyValuePair<string, string> header in param.Header)
|
|
{
|
|
request.Headers.Add(header.Key, header.Value);
|
|
}
|
|
}
|
|
if (param.Timeout > 0)
|
|
{
|
|
request.Timeout = param.Timeout * 1000;
|
|
}
|
|
else
|
|
{
|
|
request.Timeout = 60 * 1000;
|
|
}
|
|
return request;
|
|
}
|
|
}
|
|
}
|