using AllBase; using NaiBa.Util.Extension; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using Work.Api; namespace Work.CollectClass { public class CollectDoyin { public static bool isGeting = false; public static Dictionary ProductTemp = new Dictionary(); public static DateTime lastGetTime = DateTime.Now; public bool GetSuccess = false; public ProductCollectBase Product; public string Message = ""; public CollectDoyin(ProductCollectBase product) { Product = product; } public void GetProduct() { while (isGeting) { Thread.Sleep(50); } isGeting = true; Thread.Sleep(Randomx.GetId(3500, 4500)); try { this.GetProd(); } catch (Exception ex) { Message = ex.Message; } isGeting = false; } public void GetProd() { if (ProductTemp.ContainsKey(Product.from_id)) { GetSuccess = true; Product = ProductTemp[Product.from_id].Clone(); return; } GetSuccess = false; string html = ""; string sku = ""; bool canUse = false; if (UploadPinduoduoMain.isUseDouYinCache) html = ServerTemp.Get(6, Product.from_id.ToString(), out canUse); // 使用缓存数据 if (!string.IsNullOrEmpty(html) && canUse) { var dtString = BaseRegex.GetMatchRegexString(ref html, @"(\d{4}-\d{2}-\d{2})"); while (!string.IsNullOrEmpty(dtString) && html.StartsWith(dtString)) { html = html.Substring(dtString.Length).TrimStart(' ', ','); dtString = BaseRegex.GetMatchRegexString(ref html, @"(\d{4}-\d{2}-\d{2})"); } try { //Product = JsonHelper.DeserializeJsonToObject(html); Product = Newtonsoft.Json.JsonConvert.DeserializeObject(html); Product.cat_id = Product.from_cat_id = Product.cat4 == 0 ? Product.cat3 : Product.cat4; List kvList = new List(); foreach (var kv in Product.Prop) { if (!kvList.Any(k => k.Key == kv.Key)) kvList.Add(kv); } Product.Prop.Clear(); Product.Prop.AddRange(kvList); if (Product.cat_id == 0 && Product.cat2 != 0) Product.cat_id = Product.cat2; ProductTemp.Add(Product.from_id, Product); GetSuccess = true; return; } catch (Exception ex) { Message = ex.Message; } } // 缓存不可用,采集最新数据 while (DateTime.Now < lastGetTime.AddSeconds(5)) { Thread.Sleep(50); } html = ""; for (int r = 0; !html.Contains("\"st\":0") && r < 5; r++) { if (r > 0) Thread.Sleep(Randomx.GetId(3500, 4500)); // 采集商品详情 html = RequestProduct(Product.from_id.ToString()); if (html.Contains("商品已失效")) { GetSuccess = true; Product.IsOnSale = false; return; } } if (!html.Contains("\"st\":0")) return; // 采集SKU sku = RequestSku(Product.from_id.ToString()); for (int r = 0; !sku.Contains("\"st\":0") && r < 5; r++) { Thread.Sleep(Randomx.GetId(3500, 4500)); sku = RequestSku(Product.from_id.ToString()); } if (!sku.Contains("\"st\":0")) return; GetSuccess = true; try { JObject jProd = JObject.Parse(html); JObject jSku = JObject.Parse(sku); Product.from_site_id = 7; Product.from = "抖音"; Product.name = jProd["data"]["name"].ToString(); string sellNumStr = jProd["data"]["sell_num"]?.ToString().Replace("+", "").Replace("万", "0000").Replace("千", "000"); if (!string.IsNullOrEmpty(sellNumStr)) Product.saleCount = sellNumStr.ParseToInt(); else Product.saleCount = 0; Product.price = jProd["data"]["discount_price"].ParseToDecimal() / 100; Product.express_price = 0; Product.num = jSku["data"]["total_stock_num"].ParseToInt(); Product.from_shop_name = jProd["data"]["shop_name"].ToString(); Product.main_cid = jProd["data"]["first_cid"].ParseToInt(); Product.IsOnSale = jProd["data"]["status"] != null && jProd["data"]["status"].ParseToInt() == 0; var salesStr = jProd["data"]["sell_num"].ToString(); var salesInt = salesStr.Replace("+", "").Replace("万", "").Replace("千", "").ParseToDecimal(); if (salesStr.Contains("千")) salesInt = salesInt * 1000; if (salesStr.Contains("万")) salesInt = salesInt * 10000; Product.saleCount = (int)salesInt; if (!Product.IsOnSale) Message = "商品已下架"; try { Product.main_cat_name = jProd["data"]?["extra"]?["category_detail"]?["first_cname"]?.ToString(); Product.cat1 = (jProd["data"]?["extra"]?["category_detail"]?["first_cid"]).ParseToInt(); Product.cat2 = (jProd["data"]?["extra"]?["category_detail"]?["second_cid"]).ParseToInt(); Product.cat3 = (jProd["data"]?["extra"]?["category_detail"]?["third_cid"]).ParseToInt(); Product.cat4 = (jProd["data"]?["extra"]?["category_detail"]?["fourth_cid"]).ParseToInt(); } catch (Exception ex) { Message = ex.Message; } if (string.IsNullOrEmpty(Product.main_cat_name)) Product.main_cat_name = BaseRegex.GetStringJson(ref html, "first_cname"); Product.cat_id = Product.from_cat_id = Product.cat4 == 0 ? Product.cat3 : Product.cat4; if (Product.cat_id == 0 && Product.cat2 != 0) Product.cat_id = Product.cat2; if (!string.IsNullOrEmpty(jProd["data"]["freight1"]?.ToString()) && jProd["data"]["freight"]["product_province_name"] != null) Product.state = jProd["data"]["freight"]["product_province_name"].ToString(); Product.item_size = ""; Product.item_weight = (int)jProd["data"]["weight"].ParseToDecimal(); if (Product.item_weight == 0) Product.item_weight = 1; JArray jImgArr = jProd["data"]["img_list"] as JArray; if (jImgArr != null) { foreach (var jImg in jImgArr) Product.Images.Add(jImg.ToString()); } Product.image = jProd["data"]["img"].ToString(); if (string.IsNullOrEmpty(Product.image) && Product.Images.Count > 0) Product.image = Product.Images[0]; var description = jProd["data"]["description"].ToString(); var contentMatch = BaseRegex.GetMatchRegex(ref description, "]*?>([^>]+?)"); if (contentMatch.Count == 0) contentMatch = BaseRegex.GetMatchRegex(ref description, "]*?src=\"([^>]+?)\"([^>]*?)>"); var contentStr = ""; for (int i = 0; contentMatch != null && i < contentMatch.Count; i++) { contentStr += ""; } Product.content = contentStr; JObject jPropArr = jProd["data"]["product_format"] as JObject; if (jPropArr != null) { foreach (var jProp in jPropArr) { Product.Prop.Add(new KeyValueBase(jProp.Key, jProp.Value.ToString())); } } // 读取商品的sku属性和sku // 读取sku属性 var propValueBaseList = new List(); var propsObjs = jProd["data"]["spec_info"]; foreach (var propsObj in propsObjs) { PropBase pb = new PropBase(); pb.name = propsObj["name"].ToString(); foreach (var prObj in propsObj["list"]) { PropValueBase pr = new PropValueBase(); pr.name = prObj["name"].ToString(); pr.id = prObj["id"].ToString(); pb.Values.Add(pr); propValueBaseList.Add(pr); } Product.Props.Add(pb); } // 读取属性图片 var skuImages = jSku["data"]["pic_map"] as JObject; foreach (var imageObj in skuImages) { var imagePropValueId = imageObj.Key.ToString(); var propValue = propValueBaseList.FirstOrDefault(pr => pr.id == imagePropValueId); if (propValue != null) propValue.image = "https://sf1-ttcdn-tos.pstatp.com/obj/" + imageObj.Value.ToString(); } // 读取sku var skuObjs = jSku["data"]["stock"] as JObject; foreach (var skuObj in skuObjs) { var skuPropIdsStr = skuObj.Key.ToString(); var skuPropIdsArr = skuPropIdsStr.Split(';'); var skuValueObj = skuObj.Value; var skuId = skuValueObj["id"].ToString(); var num = skuValueObj["stock_num"].ParseToInt(); if (num > 90) num = 99; var price = skuValueObj["discount_price"].ParseToDecimal() / 100; var skuName = ""; foreach (var propId in skuPropIdsArr) { var propValue = propValueBaseList.FirstOrDefault(pr => pr.id == propId); if (propValue != null) { var propGroup = Product.Props.FirstOrDefault(pg => pg.Values.Contains(propValue)); if (propGroup != null) { skuName += propGroup.name + ":" + propValue.name + ";"; }; } } skuName = skuName.TrimEnd(';'); if (string.IsNullOrEmpty(skuName)) continue; SkuMapBase sm = new SkuMapBase(); sm.id = long.Parse(skuId); sm.name = skuName; sm.price = price; sm.number = num; Product.SkuMaps.Add(sm); } Product.from_title = Product.name; ProductTemp.Add(Product.from_id, Product); string saveHtml = JsonHelper.SerializeObject(Product); IdFessionTaobao.ConcurrentQueueHtmlSaveServer.Enqueue(new SaveServerParm() { SaveType = 6, ProuctId = Product.from_id.ToString(), SaveHtml = saveHtml }); } catch (Exception ex) { Message = ex.Message; } } /// /// 商品主图、属性、详情采集 /// /// private string RequestProduct(string productId = "3496306606084372244") { HttpWebResponse response = null; string result; try { //https://ec.snssdk.com/product/fxgajaxstaticitem?&b_type_new=0&device_id=0&is_outside=1&id=3508075071942062941&preview=0&is_native_h5=1&is_in_app=0 HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://ec.snssdk.com/product/fxgajaxstaticitem?&b_type_new=0&device_id=0&is_outside=1&id=" + productId + "&preview=0&is_native_h5=1&is_in_app=0"); request.KeepAlive = true; request.Headers.Add("sec-ch-ua", @"""Google Chrome"";v=""105"", ""Not)A;Brand"";v=""8"", ""Chromium"";v=""105"""); request.Accept = "application/json, text/plain, */*"; request.Headers.Add("sec-ch-ua-mobile", @"?0"); request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"; request.Headers.Add("sec-ch-ua-platform", @"""Windows"""); request.Headers.Add("Origin", @"https://haohuo.jinritemai.com"); request.Headers.Add("Sec-Fetch-Site", @"cross-site"); request.Headers.Add("Sec-Fetch-Mode", @"cors"); request.Headers.Add("Sec-Fetch-Dest", @"empty"); request.Referer = "https://haohuo.jinritemai.com/"; //request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br"); request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate"); request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9"); //request.Headers.Set(HttpRequestHeader.Cookie, AllBase.Helper.GlobalConfig.DouShopLoginList.FirstOrDefault(s => s.LoginStatus == "登录成功")?.Cookies); //request.Headers.Set(HttpRequestHeader.Cookie, "msToken=SyvmaX114hG0QqSd-cZ3ZaS_-fgnxjUfBRR3HR8LN4YNU3L91jWVqXH6oWylsN6u7DPpNkIaUdiPSiOXjSfBgxdkuk-GYlahGDbT9We4WPry"); //request.Headers.Set(HttpRequestHeader.Cookie, @"passport_csrf_token=89e087e1541315f257f2439fdc939606"); response = (HttpWebResponse)request.GetResponse(); } catch (WebException ex) { if (ex.Status == WebExceptionStatus.ProtocolError) { response = (HttpWebResponse)ex.Response; result = GetProxyHtml.ReadResponse(response); if (response != null) response.Close(); return result; } else return ex.Message; } catch (Exception e) { if (response != null) response.Close(); return e.Message; } result = GetProxyHtml.ReadResponse(response); if (response != null) response.Close(); return result; } // Sku采集 private string RequestSku(string productId) { HttpWebResponse response = null; string result = ""; try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://ec.snssdk.com/product/getstock?id=" + productId + "&add_cart=0&b_type_new=0&device_id=0&is_outside=1"); request.KeepAlive = true; request.Headers.Add("sec-ch-ua", @"""Chromium"";v=""92"", "" Not A;Brand"";v=""99"", ""Google Chrome"";v=""92"""); request.Accept = "application/json, text/plain, */*"; request.Headers.Add("sec-ch-ua-mobile", @"?0"); request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"; request.Headers.Add("Origin", @"https://haohuo.jinritemai.com"); request.Headers.Add("Sec-Fetch-Site", @"cross-site"); request.Headers.Add("Sec-Fetch-Mode", @"cors"); request.Headers.Add("Sec-Fetch-Dest", @"empty"); request.Referer = "https://haohuo.jinritemai.com/"; //request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br"); request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate"); request.Headers.Set(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9"); //request.Headers.Set(HttpRequestHeader.Cookie, @"passport_csrf_token=89e087e1541315f257f2439fdc939606"); response = (HttpWebResponse)request.GetResponse(); } catch (WebException ex) { if (ex.Status == WebExceptionStatus.ProtocolError) { response = (HttpWebResponse)ex.Response; result = GetProxyHtml.ReadResponse(response); if (response != null) response.Close(); return result; } else return ex.Message; } catch (Exception e) { if (response != null) response.Close(); return e.Message; } result = GetProxyHtml.ReadResponse(response); if (response != null) response.Close(); return result; } } }