using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using AllBase; using Work.Api; using Work.CollectClass; namespace Work.CollectClass { public class DetailMobileCollect { public string Name, Content; public bool IsUrlCollect = false, Pinduoduo = false; public bool IsUpdateSku = false, ThisShopId = false; public bool GetSuccess = false, NoTemp = false, Delete = false, NoContent = false, IsDown = false, IsExist = true, NoUpToServer = false; public static bool LastError = false; public decimal BeforePrice = 0, MinPrice = 0, MaxPrice = 0; //public static Dictionary ProductTemp = new Dictionary(); public ProductCollectBase Product; public DetailMobileCollect(ProductCollectBase product) { Product = product; } public void Get() { //if (TaobaoMobileCollect.ProductTemp.ContainsKey(Product.from_id)) //{ // GetSuccess = true; // Product = TaobaoMobileCollect.ProductTemp[Product.from_id].Clone(); // return; //} GetSuccess = false; Delete = false; bool useTemp = false; string urlProduct = "https://detail.m.tmall.com/item.htm?id=" + (ThisShopId ? Product.id : Product.from_id); bool nokey = false, hadFile = false; string content = "", newContent = ""; string refere = ""; //string html =NoTemp?"": TempFileReadWrite.Read(Product.id, "content", out hadFile); //if(string.IsNullOrEmpty(html)) // html = GetPostTaobaoHtml.UploadTaobao(urlProduct, refere, true); string html = ""; bool useUsa = false; //if (LastError) //{ // LastError = false; // html = GetHtml.ByUsa(urlProduct, true, false); // useUsa = true; //} //else //{ //} bool canUse; string serverTempHtml = ServerTemp.Get(2, "main-tm-" + Product.from_id.ToString(), out canUse); if (!canUse) { if (GetProxyHtml.ips != null && GetProxyHtml.ips.Count > 0) { html = GetProxyHtml.UploadProxyTaobao(urlProduct, refere, true, ""); } if (string.IsNullOrEmpty(html) || !html.Contains("天猫商品详情")) { html = GetPostTaobaoHtml.UploadTaobao(urlProduct, refere, true); if (html.Contains("宝贝不存在")) { IsExist = false; LogSave.Add(Name, "宝贝不存在:" + urlProduct); BigDataBuild.Delete(Product.from_id, 1); return; } } if (string.IsNullOrEmpty(html) || !html.Contains("天猫商品详情")) { html = GetHtml.ByUsa(urlProduct, true, false); useUsa = true; } if (string.IsNullOrEmpty(html) || !html.Contains("天猫商品详情")) { Thread.Sleep(10000); html = GetPostTaobaoHtml.UploadTaobao(urlProduct, refere, true); useUsa = false; } if (string.IsNullOrEmpty(html) || !html.Contains("天猫商品详情")) { LastError = true; LogSave.Add(Name, "读取页面失败:" + urlProduct); return; } if (!string.IsNullOrEmpty(html) && (html.Contains("天猫商品详情") || html.Contains("expired.html?isInvalid=true") || html.Contains("\\\"已下架\\\""))) ServerTemp.Set(2, "main-tm-" + Product.from_id.ToString(), html); } else { html = serverTempHtml; } if (!string.IsNullOrEmpty(html) && (html.Contains("expired.html?isInvalid=true") || html.Contains("\\\"已下架\\\""))) IsDown = true; if (string.IsNullOrEmpty(html) || html.Contains("redirectUrl") && !html.Contains("快递")) { LogSave.Add(Name, "宝贝已经下架:" + urlProduct); BigDataBuild.Delete(Product.from_id, 1); return; } GetSuccess = true; html = GetHtml.Decode(html).Replace("\\\"", "\"").Replace("\\\\", "\\"); Product.name = BaseRegex.GetMatchRegexString(ref html, @"""item"":\{[^\}]*?""title"":""([^""]+)"""); if (string.IsNullOrEmpty(Product.name)) { //BigDataBuild.Delete(Product.from_id, 1); if (IsUpdateSku) { Delete = true; LogSave.Add(Name, "上家已经删除:" + (ThisShopId ? Product.id : Product.from_id)); } else { LogSave.Add(Name, "标题获取错误:" + Product.from_url); } return; } string pricex = BaseRegex.GetMatchRegexString(ref html, @"""transmitPrice"":\{""priceText"":""([\d\.]+)"); if (string.IsNullOrEmpty(pricex)) { pricex = BaseRegex.GetMatchRegexString(ref html, @"""price"":\{""price"":\{""priceMoney"":""[\d]+"",""priceText"":""([\d\.]+)"""); } if (string.IsNullOrEmpty(pricex)) pricex = BaseRegex.GetMatchRegexString(ref html, @"""price"":\{""price"":\{""priceText"":""([\d\.]+)"); string price_beforex = BaseRegex.GetMatchRegexString(ref html, @"""extraPrices"":\[\{""priceMoney"":""[\d]+"",""priceText"":""([\d\.]+)"""); if (string.IsNullOrEmpty(price_beforex)) { price_beforex = pricex; } if (price_beforex == "0" || string.IsNullOrEmpty(price_beforex)) { LogSave.Add("价格为空,退出:" + Product.from_url); return; } Product.saleCount = BaseRegex.GetIntJson(ref html, "sellCount"); string eprice = BaseRegex.GetMatchRegexString(ref html, @"""postage"":""快递[::][\s]*([\d\.]+)"); Product.express_price = string.IsNullOrEmpty(eprice) ? 0 : decimal.Parse(eprice); Product.price = decimal.Parse(pricex); MinPrice = Product.price; MaxPrice = Product.price; Product.num = BaseRegex.GetIntJson(ref html, "num"); Product.from_shop_name = BaseRegex.GetStringJson(ref html, "sellerNick"); if (Product.num == 0) { string num = BaseRegex.GetMatchRegexString(ref html, @"""skuCore"":[\w\W]+?""quantity"":""([\d]+)"""); if (string.IsNullOrEmpty(num)) num = BaseRegex.GetMatchRegexString(ref html, @"]},""skuCore"":[\w\W]+?""quantity"":([\d]+)}"); if (!string.IsNullOrEmpty(num)) Product.num = int.Parse(num); } Product.discountPrice = decimal.Parse(pricex); Product.before_price = decimal.Parse(price_beforex); Product.discount = Product.discountPrice / Product.before_price;//折扣比例 Product.onlinePrice = Product.discountPrice * Product.shopAddPercent / 100 + new Random(Guid.NewGuid().GetHashCode()).Next((int)(Product.shopAddPriceMin * 100), (int)(Product.shopAddPriceMax * 100)) / 100; if (IsUrlCollect) { Product.onlinePrice = GetOnlinePrice(Product.price); } if (IsUpdateSku)//如果是更新SKU则直接跳转到SKU部分 { goto GetSku; } string catId = BaseRegex.GetStringJson(ref html, @"categoryId"); catId = string.IsNullOrEmpty(catId) ? BaseRegex.GetMatchRegexString(ref html, @"categoryId:([\d]+)") : catId; Product.cat_id = int.Parse(catId); Product.main_cid = BaseRegex.GetIntJson(ref html, @"rootCategoryId"); Product.from_cat_id = Product.cat_id; Product.from_site_id = 1; if (Product.main_cid == 0) { LogSave.Add(Name, "没有找到主分类:" + Product.from_url); return; } //===================标题过滤====================== string title = Product.name.ToLower(); long md5 = BaseClass.GetLong(Product.name); foreach (string s in BaseClass.CantTitleKeywords) { if (title.Contains(s)) { nokey = true; LogSave.Add(Name, s + " : " + title); break; } } foreach (string s in BaseClass.CantTitle) { if (title.Contains(s)) { nokey = true; LogSave.Add(Name, s + " : " + title); break; } } foreach (string s in BaseClass.CantCatId) { if (Product.cat_id.ToString() == s || Product.main_cid.ToString() == s) { nokey = true; LogSave.Add(Name, s + " : " + title); break; } } string tempx = ""; var tempmc = BaseRegex.GetMatchRegex(ref html, @"{""component"":""mui/wpimagetext/index""[\w\W]+?""image_text""}"); for (int i = 0; i < tempmc.Count; i++) { string item= tempmc[i].Groups[0].Value; var mm = BaseRegex.GetMatchRegex(ref item, @"""img[^>]*?"":""([^>]+?)"""); for (int j = 0; j < mm.Count; j++) { tempx += ""; } } content = tempx; if (string.IsNullOrEmpty(tempx)) { LogSave.Add(Name, "描述正文正则失败,跳过 " ); return; } newContent = ClearContent(content); if (string.IsNullOrEmpty(content) || string.IsNullOrEmpty(newContent)) { LogSave.Add(Name, "描述正文为空,跳过 " ); return; } Content = newContent; Product.content = Content; Product.item_size = ""; Product.item_weight = 1; Product.state = "浙江"; Product.city = "杭州"; string address = BaseRegex.GetMatchRegexString(ref html, @"""delivery"":\{""from"":""([^""]+?)"""); if (string.IsNullOrEmpty(address)) address = BaseRegex.GetStringJson(ref html, @"location"); string[] provice = "上海,北京,天津,重庆,黑龙江,辽宁,内蒙古,吉林,河北,河南,山西,山东,陕西,青海,宁夏,甘肃,新疆,西藏,四川,云南,贵州,广西,湖南,湖北,安徽,江苏,浙江,广东,福建,海南,台湾,香港,澳门".Split(','); string[] city = "上海,北京,天津,重庆,哈尔滨,沈阳,呼和浩特,长春,石家庄,郑州,太远,济南,西安,西宁,银川,兰州,乌鲁木齐,拉萨,程度,昆明,贵阳,南宁,长沙,武汉,合肥,南京,杭州,广州,福州,海口,台北,香港,澳门".Split(','); if (!string.IsNullOrEmpty(address)) { for (int i = 0; i < provice.Length; i++) { if (address.Contains(provice[i])) { Product.state = provice[i]; Product.city = city[i]; break; } } } string temp = BaseRegex.GetMatchRegexString(ref html, @",""images"":\[([^\]]+?)\]"); if (string.IsNullOrEmpty(temp)) { LogSave.Add(Name, "图片错误:" + Product.from_url); return; } var m = BaseRegex.GetMatchRegex(ref temp, @"""([^""]+)"""); if (m.Count == 0) { LogSave.Add(Name, "图片错误:" + Product.from_url); return; } for (int i = 0; i < m.Count; i++) { Product.Images.Add(m[i].Groups[1].Value); } Product.image = Product.Images[0]; Product.image_from_url = Product.Images[0]; temp = BaseRegex.GetMatchRegexString(ref html, @"""props"":\{[\w\W]+?""groupProps"":([\w\W]+?)\]\}]"); if (!string.IsNullOrEmpty(temp)) { m = BaseRegex.GetMatchRegex(ref temp, @"\{""([^""]+)"":""([^""]+)""\}"); if (m.Count == 0) { LogSave.Add("参数错误:" + Product.from_url); return; } for (int i = 0; i < m.Count; i++) { string name = m[i].Groups[1].Value; string value = m[i].Groups[2].Value; if (name.Length > 50 || value.Length > 50) continue; Product.Prop.Add(new KeyValueBase(name, value)); } } GetSku: bool hadPrice = false; m =BaseRegex.GetMatchRegex(ref html, @"\{""name"":""([^""]+)"",""pid"":([\d]+),""values"":\[([\w\W]+?)\]\}"); for (int i = 0; i < m.Count; i++) { PropBase p = new PropBase(); p.id = long.Parse(m[i].Groups[2].Value); p.name = m[i].Groups[1].Value; p.name = BaseRegex.Relpace(p.name, "(&[a-zA-z0-9]+?;)", ""); bool had = false; for (int j = 0; j < Product.Props.Count; j++) { if (Product.Props[j].name == p.name) { p = Product.Props[j]; had = true; break; } } temp = m[i].Groups[3].Value; var mm = BaseRegex.GetMatchRegex(ref temp, @"""image"":""([^""]+?)"",""name"":""([^""]+?)"",""vid"":([\d]+?)\}"); for (int j = 0; j < mm.Count; j++) { string v = mm[j].Groups[3].Value; string vi = mm[j].Groups[0].Value; string image = BaseRegex.GetStringJson(ref vi, "image"); PropValueBase pr = new PropValueBase(); pr.value = v; pr.name = mm[j].Groups[2].Value; pr.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000); pr.image = image; pr.image_from_url = image; bool hadx = false; for (int k = 0; k < p.Values.Count; k++) { if (p.Values[k].name == pr.name) { hadx = true; break; } } if (!hadx) p.Values.Add(pr); } if (!had) { p.sort = i + 1; Product.Props.Add(p); } } if (Product.Props.Count > 2) { Product.Props = Product.Props.OrderBy(u => u.Values.Count).ToList(); int count = Product.Props.Count - 2; Product.Props.RemoveRange(0, count); Product.Props = Product.Props.OrderByDescending(u => u.sort).ToList(); } m = BaseRegex.GetMatchRegex(ref html, @"""propPath"":""([-\d:;]+)"",""skuId"":([\d]+)"); for (int i = 0; i < m.Count; i++) { string skuid = m[i].Groups[2].Value; string skuValues = m[i].Groups[1].Value; var mm = BaseRegex.GetMatchRegex(ref html, @"""" + skuid + @""":[^\}]+?""priceText"":""([\d.]+)""[^\}]*?},""quantity"":([\d]+)\}"); if (mm.Count == 0) { LogSave.Add("SKU识别错误:" + Product.from_url); continue; } string priceText = mm[0].Groups[1].Value; decimal before_price = decimal.Parse(priceText); decimal price = before_price; string numberString = BaseRegex.GetMatchRegexString(ref html, @"]},""skuCore""[\w\W]+?""" + skuid + @""":[\w\W]+?""quantity"":([\d]+)"); int number = 0; if (!string.IsNullOrEmpty(numberString)) number = int.Parse(numberString); if (number > 100) number = 100; string name = BuildName(skuValues); if (string.IsNullOrEmpty(name)) continue; bool had = false; for (int j = 0; j < Product.SkuMaps.Count; j++) { if (Product.SkuMaps[j].name == name) { had = true; break; } } if (had) { continue; } mm = BaseRegex.GetMatchRegex(ref html, @"""" + skuid + @""":{""price"":{""priceMoney"":([\d.]+?),""priceText"":""([\d.]+)""}"); if (mm.Count != 0) { price = decimal.Parse(mm[0].Groups[2].Value);//这是折扣后的价格 decimal dic = before_price > 0 ? price / before_price : 1; if (dic < Product.discount) Product.discount = dic; } SkuMapBase sm = new SkuMapBase(); sm.id = long.Parse(skuid); sm.name = name; sm.price = price; if (MaxPrice < price) MaxPrice = price; if (MinPrice > price) MinPrice = price; sm.online_price = (int)(sm.price * Product.shopAddPercent / 100 + new Random(Guid.NewGuid().GetHashCode()).Next((int)(Product.shopAddPriceMin * 100), (int)(Product.shopAddPriceMax * 100)) / 100); if (!Pinduoduo && (price < BaseClass.MinPrice || price > BaseClass.MaxPrice)) number = 0; else hadPrice = true; if (IsUrlCollect) { sm.online_price = (int)GetOnlinePrice(sm.price); } sm.number = number; sm.before_price = before_price; sm.sale_count = 0; Product.SkuMaps.Add(sm); } for (int i = 0; Product.discount < 1 && i < Product.SkuMaps.Count; i++) { if (Product.SkuMaps[i].number == 0) Product.SkuMaps[i].price = Decimal.Round(Product.discount * Product.SkuMaps[i].before_price, 2); } try { if (TaobaoMobileCollect.ProductTemp.Count > 1000) { TaobaoMobileCollect.ProductTemp.Clear(); } if (!TaobaoMobileCollect.ProductTemp.ContainsKey(Product.from_id)) TaobaoMobileCollect.ProductTemp.Add(Product.from_id, Product); } catch { } if (!hadPrice && (Product.price < (IsUpdateSku ? BaseClass.MinPrice * 0.8m : BaseClass.MinPrice) || Product.price > BaseClass.MaxPrice)) { //LogSave.Add(Name, "价格太低:" + Product.from_url); //if (IsUpdateSku) //{ // SetSku( 0, 0, 5); //} //else //{ // return; //} } if (NoUpToServer) return; if (IsUpdateSku || Pinduoduo) //如果是更新上家价格,则执行此代码 { if (Product.cat1 == 0) { FromTitleToCat.UserApi = true; FromTitleToCat ft = new FromTitleToCat(); ft.TaobaoProduct = new TaobaoProductBase(); ft.Title = Product.name; ft.TaobaoProduct.Titlte = Product.name; ft.TaobaoProduct.FromId = Product.from_id; ft.TaobaoCatId = Product.from_cat_id; ft.JustGetCat = true; ft.Get(); if (!string.IsNullOrEmpty(ft.Cat1) && ft.Cat1 != "0") { Product.cat1 = int.Parse(ft.Cat1); Product.cat2 = int.Parse(ft.Cat2); Product.cat3 = int.Parse(ft.Cat3); Product.cat4 = int.Parse(ft.Cat4); } } var ftc = new FromTitleToCat(); Product.main_cid = ftc.GetMainCatId(Product.cat1); Product.main_cat_name = ftc.MainCatName; return; } //淘宝大数据收集 //string json = JsonHelper.SerializeObject(Product); //string data = "id=" + Product.id + "&site_id=" + Product.from_site_id + "&" + UserLogin.GetSignaturex() + // "&json=" + GetHtml.UrlEncodeUtf8(json); //string upUrl = BaseClass.BuildObjString("upload_data"); //html = GetHtml.PostUtf8(upUrl, data); if (nokey)//过滤数据 return; if (html == "Ok" || html == "Had") { LogSave.Add(Name, "采集成功:" + Product.id); } Product.from_title = Product.name; } /// /// 生成名称 /// /// /// private string BuildName(string name) { string temp = ""; for (int i = 0; i < Product.Props.Count; i++) { for (int j = 0; j < Product.Props[i].Values.Count; j++) { if (name.Contains(Product.Props[i].Values[j].value)) { temp += Product.Props[i].name + ":" + Product.Props[i].Values[j].name + ";"; break; } } } if (string.IsNullOrEmpty(temp)) return ""; return temp.Substring(0, temp.Length - 1); } /// /// 网址采集计算价格 /// /// /// public decimal GetOnlinePrice(decimal price) { if (price < 40) { price = price * 100 + new Random(Guid.NewGuid().GetHashCode()).Next(1500, 2000); } else if (price < 100) { price = price * 100 + new Random(Guid.NewGuid().GetHashCode()).Next(2500, 3000); } else if (price < 200) { price = price * 100 + new Random(Guid.NewGuid().GetHashCode()).Next(3000, 5000); } else { price = price * 100 + new Random(Guid.NewGuid().GetHashCode()).Next(4000, 6000); } return price / 100; } /// /// 获取上家ID /// /// /// public static long GetUpId(ref string content) { string upShopId = BaseRegex.GetMatchRegexString(ref content, @"//item\.taobao\.com/item\.htm\?id=([345][\d]{11})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"<[^>]*#fff[^>]*>[^<]*([345][\d]{11})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"<[^>]*white[^>]*>[^<]*([345][\d]{11})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"<[^>]*#ffffff[^>]*>[^<]*([345][\d]{11})[\D]"); if (string.IsNullOrEmpty(upShopId) && content.Length > 100) upShopId = BaseRegex.GetMatchRegexString(content.Substring(0, 100), @"[\D]([345][\d]{11})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"//item\.taobao\.com/item\.htm\?id=[\D]([\d]{11,12})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"[\D]([\d]{11,12})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = BaseRegex.GetMatchRegexString(ref content, @"[\D]([\d]{6,12})[\D]"); if (string.IsNullOrEmpty(upShopId)) upShopId = "0"; long up; long.TryParse(upShopId, out up); return up; } /// /// 清理正文 /// /// /// public static string ClearContent(string content) { string newContent = BaseRegex.Relpace(content, @"(]*assets\.alicdn\.com[^>]*>)", ""); ClearTable(ref newContent); BaseRegex.Relpace(ref newContent, @"(<[aA] [\w\W]+?)", ""); BaseRegex.Relpace(ref newContent, @"(]*>[^<]*[\d]{10,15}[^<]*

)", ""); BaseRegex.Relpace(ref newContent, @"(<[^>]*#ffffff[^>]*>[^<]*[\d]{10,15}[^<]*<[^>]*>)", ""); BaseRegex.Relpace(ref newContent, @"((<[pdiv]+[^>]*width:[\s]*0;[\s]*height:[\s]*0;[^>]*>[\w\W]*?))", ""); BaseRegex.Relpace(ref newContent, @"((<[pdiv]+[^>]*width:[\s]*0px;[\s]*height:[\s]*0px;[^>]*>[\w\W]*?))", ""); BaseRegex.Relpace(ref newContent, @"(http[\s]{0,1}://item\.taobao\.com/item\.htm\?id=[\d]+)", ""); newContent = newContent.Replace("

 

", ""); if (!newContent.Contains(" /// 清除表格广告代码 /// /// /// public static void ClearTable(ref string html) { if (string.IsNullOrEmpty(html)) return; int i = html.IndexOf(" 0 && i >= 0) { end = html.IndexOf("= 0 && (t1 < t2 || t2 < 0)) i = t1; if (t2 >= 0 && (t2 < t1 || t1 < 0)) i = t2; } string re = html.Substring(first, end - first + 8); if (re.Contains("