401 lines
18 KiB
C#
401 lines
18 KiB
C#
using AllBase;
|
|
using NaiBa.Util.Extension;
|
|
using Newtonsoft.Json.Linq;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Work.Api;
|
|
|
|
namespace Work.CollectClass
|
|
{
|
|
public class CollectDoyin
|
|
{
|
|
public static bool isGeting = false;
|
|
|
|
public static Dictionary<long, ProductCollectBase> ProductTemp = new Dictionary<long, ProductCollectBase>();
|
|
public static DateTime lastGetTime = DateTime.Now;
|
|
|
|
public bool GetSuccess = false;
|
|
public ProductCollectBase Product;
|
|
|
|
public string Message = "";
|
|
|
|
public CollectDoyin(ProductCollectBase product)
|
|
{
|
|
Product = product;
|
|
}
|
|
|
|
|
|
public void GetProduct()
|
|
{
|
|
while (isGeting)
|
|
{
|
|
Thread.Sleep(50);
|
|
}
|
|
isGeting = true;
|
|
Thread.Sleep(Randomx.GetId(3500, 4500));
|
|
try
|
|
{
|
|
this.GetProd();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Message = ex.Message;
|
|
}
|
|
isGeting = false;
|
|
}
|
|
|
|
public void GetProd()
|
|
{
|
|
if (ProductTemp.ContainsKey(Product.from_id))
|
|
{
|
|
GetSuccess = true;
|
|
Product = ProductTemp[Product.from_id].Clone();
|
|
return;
|
|
}
|
|
|
|
GetSuccess = false;
|
|
string html = "";
|
|
string sku = "";
|
|
bool canUse = false;
|
|
if (UploadPinduoduoMain.isUseDouYinCache)
|
|
html = ServerTemp.Get(6, Product.from_id.ToString(), out canUse);
|
|
|
|
// 使用缓存数据
|
|
if (!string.IsNullOrEmpty(html) && canUse)
|
|
{
|
|
var dtString = BaseRegex.GetMatchRegexString(ref html, @"(\d{4}-\d{2}-\d{2})");
|
|
while (!string.IsNullOrEmpty(dtString) && html.StartsWith(dtString))
|
|
{
|
|
html = html.Substring(dtString.Length).TrimStart(' ', ',');
|
|
dtString = BaseRegex.GetMatchRegexString(ref html, @"(\d{4}-\d{2}-\d{2})");
|
|
}
|
|
try
|
|
{
|
|
//Product = JsonHelper.DeserializeJsonToObject<ProductCollectBase>(html);
|
|
Product = Newtonsoft.Json.JsonConvert.DeserializeObject<ProductCollectBase>(html);
|
|
Product.cat_id = Product.from_cat_id = Product.cat4 == 0 ? Product.cat3 : Product.cat4;
|
|
List<KeyValueBase> kvList = new List<KeyValueBase>();
|
|
foreach (var kv in Product.Prop)
|
|
{
|
|
if (!kvList.Any(k => k.Key == kv.Key))
|
|
kvList.Add(kv);
|
|
}
|
|
Product.Prop.Clear();
|
|
Product.Prop.AddRange(kvList);
|
|
|
|
if (Product.cat_id == 0 && Product.cat2 != 0)
|
|
Product.cat_id = Product.cat2;
|
|
ProductTemp.Add(Product.from_id, Product);
|
|
GetSuccess = true;
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Message = ex.Message;
|
|
}
|
|
}
|
|
|
|
// 缓存不可用，采集最新数据
|
|
while (DateTime.Now < lastGetTime.AddSeconds(5))
|
|
{
|
|
Thread.Sleep(50);
|
|
}
|
|
html = "";
|
|
for (int r = 0; !html.Contains("\"st\":0") && r < 5; r++)
|
|
{
|
|
if (r > 0)
|
|
Thread.Sleep(Randomx.GetId(3500, 4500));
|
|
|
|
// 采集商品详情
|
|
html = RequestProduct(Product.from_id.ToString());
|
|
if (html.Contains("商品已失效"))
|
|
{
|
|
GetSuccess = true;
|
|
Product.IsOnSale = false;
|
|
return;
|
|
}
|
|
}
|
|
if (!html.Contains("\"st\":0"))
|
|
return;
|
|
|
|
// 采集SKU
|
|
sku = RequestSku(Product.from_id.ToString());
|
|
for (int r = 0; !sku.Contains("\"st\":0") && r < 5; r++)
|
|
{
|
|
Thread.Sleep(Randomx.GetId(3500, 4500));
|
|
sku = RequestSku(Product.from_id.ToString());
|
|
}
|
|
if (!sku.Contains("\"st\":0"))
|
|
return;
|
|
|
|
GetSuccess = true;
|
|
try
|
|
{
|
|
JObject jProd = JObject.Parse(html);
|
|
JObject jSku = JObject.Parse(sku);
|
|
Product.from_site_id = 7;
|
|
Product.from = "抖音";
|
|
Product.name = jProd["data"]["name"].ToString();
|
|
string sellNumStr = jProd["data"]["sell_num"]?.ToString().Replace("+", "").Replace("万", "0000").Replace("千", "000");
|
|
if (!string.IsNullOrEmpty(sellNumStr))
|
|
Product.saleCount = sellNumStr.ParseToInt();
|
|
else
|
|
Product.saleCount = 0;
|
|
Product.price = jProd["data"]["discount_price"].ParseToDecimal() / 100;
|
|
Product.express_price = 0;
|
|
Product.num = jSku["data"]["total_stock_num"].ParseToInt();
|
|
Product.from_shop_name = jProd["data"]["shop_name"].ToString();
|
|
Product.main_cid = jProd["data"]["first_cid"].ParseToInt();
|
|
Product.IsOnSale = jProd["data"]["status"] != null && jProd["data"]["status"].ParseToInt() == 0;
|
|
var salesStr = jProd["data"]["sell_num"].ToString();
|
|
var salesInt = salesStr.Replace("+", "").Replace("万", "").Replace("千", "").ParseToDecimal();
|
|
if (salesStr.Contains("千"))
|
|
salesInt = salesInt * 1000;
|
|
if (salesStr.Contains("万"))
|
|
salesInt = salesInt * 10000;
|
|
Product.saleCount = (int)salesInt;
|
|
if (!Product.IsOnSale)
|
|
Message = "商品已下架";
|
|
try
|
|
{
|
|
Product.main_cat_name = jProd["data"]?["extra"]?["category_detail"]?["first_cname"]?.ToString();
|
|
Product.cat1 = (jProd["data"]?["extra"]?["category_detail"]?["first_cid"]).ParseToInt();
|
|
Product.cat2 = (jProd["data"]?["extra"]?["category_detail"]?["second_cid"]).ParseToInt();
|
|
Product.cat3 = (jProd["data"]?["extra"]?["category_detail"]?["third_cid"]).ParseToInt();
|
|
Product.cat4 = (jProd["data"]?["extra"]?["category_detail"]?["fourth_cid"]).ParseToInt();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Message = ex.Message;
|
|
}
|
|
if (string.IsNullOrEmpty(Product.main_cat_name))
|
|
Product.main_cat_name = BaseRegex.GetStringJson(ref html, "first_cname");
|
|
Product.cat_id = Product.from_cat_id = Product.cat4 == 0 ? Product.cat3 : Product.cat4;
|
|
if (Product.cat_id == 0 && Product.cat2 != 0)
|
|
Product.cat_id = Product.cat2;
|
|
if (!string.IsNullOrEmpty(jProd["data"]["freight1"]?.ToString()) && jProd["data"]["freight"]["product_province_name"] != null)
|
|
Product.state = jProd["data"]["freight"]["product_province_name"].ToString();
|
|
Product.item_size = "";
|
|
Product.item_weight = (int)jProd["data"]["weight"].ParseToDecimal();
|
|
if (Product.item_weight == 0)
|
|
Product.item_weight = 1;
|
|
|
|
JArray jImgArr = jProd["data"]["img_list"] as JArray;
|
|
if (jImgArr != null)
|
|
{
|
|
foreach (var jImg in jImgArr)
|
|
Product.Images.Add(jImg.ToString());
|
|
}
|
|
Product.image = jProd["data"]["img"].ToString();
|
|
if (string.IsNullOrEmpty(Product.image) && Product.Images.Count > 0)
|
|
Product.image = Product.Images[0];
|
|
var description = jProd["data"]["description"].ToString();
|
|
var contentMatch = BaseRegex.GetMatchRegex(ref description, "<img[^>]*?>([^>]+?)</img>");
|
|
if (contentMatch.Count == 0)
|
|
contentMatch = BaseRegex.GetMatchRegex(ref description, "<img[^>]*?src=\"([^>]+?)\"([^>]*?)>");
|
|
var contentStr = "";
|
|
for (int i = 0; contentMatch != null && i < contentMatch.Count; i++)
|
|
{
|
|
contentStr += "<img src=\"" + contentMatch[i].Groups[1].Value + "\" />";
|
|
}
|
|
Product.content = contentStr;
|
|
|
|
JObject jPropArr = jProd["data"]["product_format"] as JObject;
|
|
if (jPropArr != null)
|
|
{
|
|
foreach (var jProp in jPropArr)
|
|
{
|
|
Product.Prop.Add(new KeyValueBase(jProp.Key, jProp.Value.ToString()));
|
|
}
|
|
}
|
|
|
|
// 读取商品的sku属性和sku
|
|
// 读取sku属性
|
|
var propValueBaseList = new List<PropValueBase>();
|
|
|
|
var propsObjs = jProd["data"]["spec_info"];
|
|
foreach (var propsObj in propsObjs)
|
|
{
|
|
PropBase pb = new PropBase();
|
|
pb.name = propsObj["name"].ToString();
|
|
foreach (var prObj in propsObj["list"])
|
|
{
|
|
PropValueBase pr = new PropValueBase();
|
|
pr.name = prObj["name"].ToString();
|
|
pr.id = prObj["id"].ToString();
|
|
pb.Values.Add(pr);
|
|
propValueBaseList.Add(pr);
|
|
}
|
|
Product.Props.Add(pb);
|
|
}
|
|
|
|
// 读取属性图片
|
|
var skuImages = jSku["data"]["pic_map"] as JObject;
|
|
foreach (var imageObj in skuImages)
|
|
{
|
|
var imagePropValueId = imageObj.Key.ToString();
|
|
var propValue = propValueBaseList.FirstOrDefault(pr => pr.id == imagePropValueId);
|
|
if (propValue != null)
|
|
propValue.image = "https://sf1-ttcdn-tos.pstatp.com/obj/" + imageObj.Value.ToString();
|
|
}
|
|
|
|
// 读取sku
|
|
var skuObjs = jSku["data"]["stock"] as JObject;
|
|
foreach (var skuObj in skuObjs)
|
|
{
|
|
var skuPropIdsStr = skuObj.Key.ToString();
|
|
var skuPropIdsArr = skuPropIdsStr.Split(';');
|
|
var skuValueObj = skuObj.Value;
|
|
|
|
var skuId = skuValueObj["id"].ToString();
|
|
var num = skuValueObj["stock_num"].ParseToInt();
|
|
if (num > 90)
|
|
num = 99;
|
|
var price = skuValueObj["discount_price"].ParseToDecimal() / 100;
|
|
var skuName = "";
|
|
foreach (var propId in skuPropIdsArr)
|
|
{
|
|
var propValue = propValueBaseList.FirstOrDefault(pr => pr.id == propId);
|
|
if (propValue != null)
|
|
{
|
|
var propGroup = Product.Props.FirstOrDefault(pg => pg.Values.Contains(propValue));
|
|
if (propGroup != null)
|
|
{
|
|
skuName += propGroup.name + ":" + propValue.name + ";";
|
|
};
|
|
}
|
|
}
|
|
skuName = skuName.TrimEnd(';');
|
|
if (string.IsNullOrEmpty(skuName))
|
|
continue;
|
|
|
|
SkuMapBase sm = new SkuMapBase();
|
|
sm.id = long.Parse(skuId);
|
|
sm.name = skuName;
|
|
sm.price = price;
|
|
sm.number = num;
|
|
Product.SkuMaps.Add(sm);
|
|
}
|
|
|
|
Product.from_title = Product.name;
|
|
ProductTemp.Add(Product.from_id, Product);
|
|
|
|
string saveHtml = JsonHelper.SerializeObject(Product);
|
|
IdFessionTaobao.ConcurrentQueueHtmlSaveServer.Enqueue(new SaveServerParm() { SaveType = 6, ProuctId = Product.from_id.ToString(), SaveHtml = saveHtml });
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Message = ex.Message;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// 商品主图、属性、详情采集
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
private string RequestProduct(string productId = "3496306606084372244")
|
|
{
|
|
HttpWebResponse response = null;
|
|
string result;
|
|
|
|
try
|
|
{
|
|
//https://ec.snssdk.com/product/fxgajaxstaticitem?&b_type_new=0&device_id=0&is_outside=1&id=3508075071942062941&preview=0&is_native_h5=1&is_in_app=0
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://ec.snssdk.com/product/fxgajaxstaticitem?&b_type_new=0&device_id=0&is_outside=1&id=" + productId + "&preview=0&is_native_h5=1&is_in_app=0");
|
|
|
|
request.KeepAlive = true;
|
|
request.Headers.Add("sec-ch-ua", @"""Google Chrome"";v=""105"", ""Not)A;Brand"";v=""8"", ""Chromium"";v=""105""");
|
|
request.Accept = "application/json, text/plain, */*";
|
|
request.Headers.Add("sec-ch-ua-mobile", @"?0");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36";
|
|
request.Headers.Add("sec-ch-ua-platform", @"""Windows""");
|
|
request.Headers.Add("Origin", @"https://haohuo.jinritemai.com");
|
|
request.Headers.Add("Sec-Fetch-Site", @"cross-site");
|
|
request.Headers.Add("Sec-Fetch-Mode", @"cors");
|
|
request.Headers.Add("Sec-Fetch-Dest", @"empty");
|
|
request.Referer = "https://haohuo.jinritemai.com/";
|
|
//request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
//request.Headers.Set(HttpRequestHeader.Cookie, AllBase.Helper.GlobalConfig.DouShopLoginList.FirstOrDefault(s => s.LoginStatus == "登录成功")?.Cookies);
|
|
//request.Headers.Set(HttpRequestHeader.Cookie, "msToken=SyvmaX114hG0QqSd-cZ3ZaS_-fgnxjUfBRR3HR8LN4YNU3L91jWVqXH6oWylsN6u7DPpNkIaUdiPSiOXjSfBgxdkuk-GYlahGDbT9We4WPry");
|
|
//request.Headers.Set(HttpRequestHeader.Cookie, @"passport_csrf_token=89e087e1541315f257f2439fdc939606");
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException ex)
|
|
{
|
|
if (ex.Status == WebExceptionStatus.ProtocolError)
|
|
{
|
|
response = (HttpWebResponse)ex.Response;
|
|
result = GetProxyHtml.ReadResponse(response);
|
|
if (response != null) response.Close();
|
|
return result;
|
|
}
|
|
else return ex.Message;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
if (response != null) response.Close();
|
|
return e.Message;
|
|
}
|
|
result = GetProxyHtml.ReadResponse(response);
|
|
if (response != null) response.Close();
|
|
return result;
|
|
}
|
|
|
|
// Sku采集
|
|
private string RequestSku(string productId)
|
|
{
|
|
HttpWebResponse response = null;
|
|
string result = "";
|
|
|
|
try
|
|
{
|
|
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://ec.snssdk.com/product/getstock?id=" + productId + "&add_cart=0&b_type_new=0&device_id=0&is_outside=1");
|
|
|
|
request.KeepAlive = true;
|
|
request.Headers.Add("sec-ch-ua", @"""Chromium"";v=""92"", "" Not A;Brand"";v=""99"", ""Google Chrome"";v=""92""");
|
|
request.Accept = "application/json, text/plain, */*";
|
|
request.Headers.Add("sec-ch-ua-mobile", @"?0");
|
|
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36";
|
|
request.Headers.Add("Origin", @"https://haohuo.jinritemai.com");
|
|
request.Headers.Add("Sec-Fetch-Site", @"cross-site");
|
|
request.Headers.Add("Sec-Fetch-Mode", @"cors");
|
|
request.Headers.Add("Sec-Fetch-Dest", @"empty");
|
|
request.Referer = "https://haohuo.jinritemai.com/";
|
|
//request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
|
|
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
|
request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9");
|
|
//request.Headers.Set(HttpRequestHeader.Cookie, @"passport_csrf_token=89e087e1541315f257f2439fdc939606");
|
|
|
|
response = (HttpWebResponse)request.GetResponse();
|
|
}
|
|
catch (WebException ex)
|
|
{
|
|
if (ex.Status == WebExceptionStatus.ProtocolError)
|
|
{
|
|
response = (HttpWebResponse)ex.Response;
|
|
result = GetProxyHtml.ReadResponse(response);
|
|
if (response != null) response.Close();
|
|
return result;
|
|
}
|
|
else return ex.Message;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
if (response != null) response.Close();
|
|
return e.Message;
|
|
}
|
|
result = GetProxyHtml.ReadResponse(response);
|
|
if (response != null) response.Close();
|
|
return result;
|
|
}
|
|
}
|
|
}
|