950 lines
43 KiB
C#
950 lines
43 KiB
C#
using System;
|
||
using System.Threading;
|
||
using AllBase;
|
||
using Work.ManageClass;
|
||
using Work.CollectClass;
|
||
using System.Net;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Text;
|
||
using Newtonsoft.Json.Linq;
|
||
using System.Linq;
|
||
using NaiBa.Util.Extension;
|
||
|
||
namespace Work.Api
|
||
{
|
||
public class AlibabaProductGet
|
||
{
|
||
public ProductCollectBase Product;
|
||
public bool IsPinduoduo = false;
|
||
public bool OnSale = true;
|
||
public AlibabaProductGet(ProductCollectBase p)
|
||
{
|
||
Product = p;
|
||
}
|
||
|
||
public void Get()
|
||
{
|
||
if (TaobaoMobileCollect.ProductTemp.ContainsKey(Product.from_id))
|
||
{
|
||
Product = TaobaoMobileCollect.ProductTemp[Product.from_id].Clone();
|
||
return;
|
||
}
|
||
string url = Product.from_url;
|
||
string refere = "http://page.1688.com/";
|
||
|
||
bool canUse;
|
||
string serverTempHtml = ServerTemp.Get(3, "main-" + Product.from_id.ToString(), out canUse);
|
||
string html = serverTempHtml;
|
||
|
||
if (!string.IsNullOrEmpty(html) && !html.Contains("调用成功"))
|
||
{
|
||
if (string.IsNullOrEmpty(html))
|
||
{
|
||
html = GetPostTaobaoHtml.UploadTaobao(url, refere, false);
|
||
}
|
||
if (string.IsNullOrEmpty(html) || !html.Contains("currentTime"))
|
||
{
|
||
Thread.Sleep(60 * 1000);
|
||
html = GetPostTaobaoHtml.UploadTaobao(url, refere, false);
|
||
}
|
||
if (string.IsNullOrEmpty(html) || !html.Contains("currentTime"))
|
||
{
|
||
LogSave.Add("获取产品详细页面失败，请检查程序 " + url);
|
||
OnSale = false;
|
||
return;
|
||
}
|
||
if (html.Contains("no-price-txt"))
|
||
{
|
||
LogSave.Add("获取产品详细页面发现没有价格，跳过 " + url);
|
||
OnSale = false;
|
||
return;
|
||
}
|
||
if (!canUse)
|
||
ServerTemp.Set(3, "main-" + Product.from_id.ToString(), html);
|
||
Product.from_cat_id = BaseRegex.GetIntJson(ref html, @"catid");
|
||
if (Product.from_cat_id == 0)
|
||
{
|
||
int.TryParse(BaseRegex.GetMatchRegexString(ref html, "catid:'([^}]+?)'"), out Product.from_cat_id);
|
||
}
|
||
Product.item_size = "";
|
||
Product.item_weight = 1;
|
||
Product.price = BaseRegex.GetDecimalJson(ref html, @"refPrice");
|
||
if (Product.price == 0)
|
||
{
|
||
decimal.TryParse(BaseRegex.GetMatchRegexString(ref html, "refPrice:'([^}]+?)'"), out Product.price);
|
||
}
|
||
Product.discount = BaseRegex.GetDecimalJson(ref html, @"discount");
|
||
Product.discountPrice = BaseRegex.GetDecimalJson(ref html, @"discountPrice");
|
||
if (Product.discount == 0)
|
||
{
|
||
Product.discount = 1;
|
||
Product.discountPrice = Product.price;
|
||
}
|
||
|
||
Product.name = BaseRegex.GetMatchRegexString(html, "<h1[^>]*?>([^<]+)</h1>");
|
||
foreach (string s in AlibabaShop.NoTitleWords)
|
||
{
|
||
Product.name = Product.name.Replace(s, "");
|
||
}
|
||
|
||
Product.name = BaseRegex.Relpace(Product.name.Replace("2016", "2017").Replace("2015", "2017"), @"([a-zA-Z]{1,2}[\d]{1,5})", "").Trim();
|
||
bool had2017 = Product.name.Contains("2017");
|
||
Product.name = "'" + Product.name + "\"";
|
||
Product.name = BaseRegex.Relpace(Product.name, @"([a-zA-Z0-9# -]{2,20}?)""", "").Replace("\"", "");//去掉结尾的数字和字母下划线
|
||
Product.name = BaseRegex.Relpace(Product.name, @"'([a-zA-Z0-9# -]{2,20})", "").Replace("'", "");//去掉开始的数字和字母下划线
|
||
if (had2017 && !Product.name.Contains("2017"))
|
||
Product.name = "2017" + Product.name;
|
||
|
||
bool titleHad = false;
|
||
//foreach (string s in AlibabaShop.TitleCantHad)
|
||
//{
|
||
// if (Product.name.Contains(s))
|
||
// titleHad = true;
|
||
//}
|
||
if (Product.name.Length < 12 || titleHad)
|
||
{
|
||
LogSave.Add("获取产品详细页面发现标题 " + Product.name + " 异常，跳过 " + url);
|
||
return;
|
||
}
|
||
Product.sell_pt = "";
|
||
var m = BaseRegex.GetMatchRegex(ref html, @"data-imgs='{""preview"":""([^']+?)"",""original"":""([^']+?)""");
|
||
if (m.Count == 0)
|
||
{
|
||
LogSave.Add("获取图片为空，请检查程序 " + url);
|
||
return;
|
||
}
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
Product.Images.Add(m[i].Groups[2].Value);
|
||
}
|
||
Product.image = Product.Images[0];
|
||
Product.image_from_url = Product.image;
|
||
Product.image_path = "";
|
||
SetSkuMap(ref html, url);
|
||
string area = BaseRegex.GetMatchRegexString(ref html, @"<span class=""disc"">([^<]+)</span>");
|
||
if (!string.IsNullOrEmpty(area) && area.Contains(" "))
|
||
{
|
||
Product.state = area.Substring(0, area.IndexOf(" "));
|
||
Product.city = area.Substring(area.IndexOf(" ") + 1);
|
||
}
|
||
else
|
||
{
|
||
Product.state = BaseRegex.GetMatchRegexString(ref html, "province=([^;]+?);");
|
||
Product.city = BaseRegex.GetMatchRegexString(ref html, "city=([^;]+?)\"");
|
||
}
|
||
m = BaseRegex.GetMatchRegex(ref html,
|
||
@"<td class=""de-feature"">([^<]+?)</td>[\s]+<td class=""de-value"">([^<]+?)</td>");
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
string name = m[i].Groups[1].Value;
|
||
string value = m[i].Groups[2].Value;
|
||
bool had = false;
|
||
foreach (string s in AlibabaShop.NoPropWords)
|
||
{
|
||
if (name.Contains(s))
|
||
{
|
||
had = true;
|
||
break;
|
||
}
|
||
}
|
||
if (had)
|
||
continue;
|
||
if (name == "品牌")
|
||
value = "十四楼";
|
||
Product.Prop.Add(new KeyValueBase(name, value));
|
||
}
|
||
|
||
string memberId = BaseRegex.GetMatchRegexString(ref html, @" data-tfs-url=""([^""]+)""");
|
||
string desc =
|
||
GetPostTaobaoHtml.UploadTaobao(memberId, Product.from_url, false);
|
||
string content = BaseRegex.GetMatchRegexString(ref desc, @"""content"":""([\w\W]+?)""};");
|
||
if (string.IsNullOrEmpty(content))
|
||
{
|
||
LogSave.Add("获取描述异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
content = content.Replace("\\\"", "\"").Replace("\\\\", "\\");
|
||
BaseRegex.Relpace(ref content, @"<table background=([\w\W]+?)>超级快车", "");
|
||
string newContentx = BaseRegex.Relpace(content, @"<table ([\w\W]+)</table>", "");
|
||
BaseRegex.Relpace(ref newContentx, @"(<[aA] [^>]+?>([\w\W]*?)</[aA]>)", "");
|
||
if (string.IsNullOrEmpty(newContentx) || !newContentx.ToLower().Contains("<img "))
|
||
newContentx = content;
|
||
string newContent = "";
|
||
m = BaseRegex.GetMatchRegex(ref newContentx, @"(<img [^>]+?>)");
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
if (!m[i].Groups[1].Value.Contains(".alicdn.com") || m[i].Groups[0].Value.Contains("display: none") || m[i].Groups[0].Value.Contains("display:none"))
|
||
continue;
|
||
newContent += m[i].Groups[1].Value + "<br /><br />";
|
||
}
|
||
if (string.IsNullOrEmpty(newContent))
|
||
{
|
||
LogSave.Add("描述正文为空，跳过 " + url);
|
||
return;
|
||
}
|
||
html = GetPostTaobaoHtml.UploadTaobao(
|
||
"https://laputa.1688.com/offer/ajax/widgetList.do?callback=jQuery172025158196459054927_1533388414900&blocks=&data=offerdetail_ditto" +
|
||
"_title%2Cofferdetail_common_report%2Cofferdetail_ditto_serviceDesc%2Cofferdetail_ditto_preferential%2Cofferdetail_ditto_postage%2Coff" +
|
||
"erdetail_ditto_offerSatisfaction%2Cofferdetail_w1190_guarantee%2Cofferdetail_w1190_tradeWay%2Cofferdetail_w1190_samplePromotion%2Coffer" +
|
||
"detail_ditto_whosaleself&offerId=" + Product.from_id + "&pageId=laputa20140721212446", Product.from_url, false);
|
||
Product.express_price = BaseRegex.GetDecimalJson(ref html, "totalCost");
|
||
Product.content = newContent;
|
||
Product.content_alibaba = newContent;
|
||
Product.main_cid = 0;
|
||
Product.from_title = Product.name;
|
||
Product.from_site_id = 3;
|
||
}
|
||
else
|
||
{
|
||
if (string.IsNullOrEmpty(html))
|
||
{
|
||
string data = "{\"responseClass\":\"\\\"com.alibaba.wireless.detail.netdata.offerdatanet.OfferDetailDataResponse\\\"\",\"RESPONSE_DATA_KEY\":\"\",\"MOCK_DATA\":\"false\",\"offerId\":\"" + Product.from_id + "\"}";
|
||
//先去取数据
|
||
html = MakeRequestThroughTradeAscM(data);
|
||
if (!html.Contains("调用成功"))
|
||
{
|
||
return;
|
||
}
|
||
}
|
||
bool bo = AliBaBaModle(html);
|
||
if (!bo)
|
||
{
|
||
return;
|
||
}
|
||
}
|
||
if (Product.cat3 == 3109)
|
||
{
|
||
var pro = Product.Props.Where(s => s.name.Contains("容量")).FirstOrDefault();
|
||
if (pro == null)
|
||
{
|
||
string mah = BaseRegex.GetMatchRegexString(ref Product.name, @"([\d]+)毫安");
|
||
PropBase p = new PropBase();
|
||
p.pinduoduo_id = "140871455";
|
||
p.id = 140871455;
|
||
p.name = "电池容量";
|
||
PropValueBase pr = new PropValueBase();
|
||
pr.id = PinduoduoCatList.GetDianYuanId(int.Parse(mah)).ToString();
|
||
pr.value = pr.id;
|
||
pr.name = mah + "毫安";
|
||
pr.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000);
|
||
pr.image = "";
|
||
pr.image_from_url = "";
|
||
p.Values.Add(pr);
|
||
Product.Props.Add(p);
|
||
for (int i = 0; i < Product.SkuMaps.Count; i++)
|
||
{
|
||
var sku = Product.SkuMaps[i];
|
||
sku.name = sku.name + ";" + mah + "毫安";
|
||
sku.nameLite = sku.nameLite + mah + "毫安";
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (string.IsNullOrEmpty(pro.pinduoduo_id))
|
||
{
|
||
pro.pinduoduo_id = "140871455";
|
||
pro.id = 140871455;
|
||
}
|
||
for (int i = 0; i < pro.Values.Count; i++)
|
||
{
|
||
var pr = pro.Values[i];
|
||
string mah= BaseRegex.GetMatchRegexString(ref pr.name, @"([\d]+)毫安");
|
||
pr.id= PinduoduoCatList.GetDianYuanId(int.Parse(mah)).ToString();
|
||
pr.value = pr.id;
|
||
}
|
||
}
|
||
}
|
||
if (IsPinduoduo)//如果是更新上家价格，则执行此代码
|
||
{
|
||
Product.main_cid = new FromTitleToCat().GetMainCatId(Product.cat1);
|
||
if (Product.main_cid == 0)
|
||
return;
|
||
//url = BaseClass.BigDataApiUrl + "?obj=product/product";
|
||
//string datax = "&act=update&site_id=3&shop_name=" + GetHtml.UrlEncodeUtf8(Product.from_shop_name) + "&json=" + GetHtml.UrlEncodeUtf8(JsonHelper.SerializeObject(Product));
|
||
//html = GetHtml.PostUtf8(url, UserLogin.GetSignaturex() + datax);
|
||
try
|
||
{
|
||
if (TaobaoMobileCollect.ProductTemp.Count > 1000)
|
||
{
|
||
TaobaoMobileCollect.ProductTemp.Clear();
|
||
}
|
||
|
||
if (!TaobaoMobileCollect.ProductTemp.ContainsKey(Product.from_id))
|
||
TaobaoMobileCollect.ProductTemp.Add(Product.from_id, Product);
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
//后期加代码
|
||
return;
|
||
}
|
||
if (Product.from_cat_id == 0)
|
||
{
|
||
Product.message = "分类识别错误：" + Product.name;
|
||
LogSave.Add(Product.message);
|
||
return;
|
||
}
|
||
LogSave.Add("获取成功：" + Product.name);
|
||
}
|
||
public void AslasyHtml(TaobaoProductBase t, string html)
|
||
{
|
||
if (string.IsNullOrEmpty(html) || html == "null")
|
||
return;
|
||
try
|
||
{
|
||
//string title = BaseRegex.GetStringJson(ref html, "fession_title");
|
||
//string[] goodTitle = title.Split(';');
|
||
//主图
|
||
string mainPic = BaseRegex.GetStringJson(ref html, "main_pic");
|
||
string[] images = mainPic.Split(';');
|
||
Product.Images.Clear();
|
||
for (int i = 0; i < images.Length; i++)
|
||
{
|
||
string img = images[i];
|
||
if (img.Contains("?"))
|
||
img = img.Substring(0, img.IndexOf("?"));
|
||
Product.Images.Add(img);
|
||
}
|
||
//详情图
|
||
string container_info = BaseRegex.GetMatchRegexString(ref html, @"""container_info"":""(.*?)""");
|
||
string[] contentimage = container_info.Split(';');
|
||
var m = BaseRegex.GetMatchRegex(ref container_info, @"<img([^""]+?)>");
|
||
string content = "";
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
string img = m[i].Groups[0].Value;
|
||
if (img.Contains("?"))
|
||
img = img.Substring(0, img.IndexOf("?"));
|
||
content += img;
|
||
}
|
||
Product.content = content;
|
||
Product.Props.Clear();
|
||
string temp = "";
|
||
m = BaseRegex.GetMatchRegex(ref html,
|
||
@"\{""id"":([-\d]+),""name"":""([^}]+)"",""pinduoduo_name"":[^}]+?,""Values"":\[([\w\W]+?)\]\}");
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
PropBase p = new PropBase();
|
||
p.id = long.Parse(m[i].Groups[1].Value);
|
||
p.name = m[i].Groups[2].Value;
|
||
p.name = BaseRegex.Relpace(p.name, "(&[a-zA-z0-9]+?;)", "");
|
||
bool had = false;
|
||
for (int j = 0; j < Product.Props.Count; j++)
|
||
{
|
||
if (Product.Props[j].name == p.name)
|
||
{
|
||
p = Product.Props[j];
|
||
had = true;
|
||
break;
|
||
}
|
||
}
|
||
temp = m[i].Groups[3].Value;
|
||
var mm = BaseRegex.GetMatchRegex(ref temp, @"""value"":""([-\d]*?)"",""name"":""([^""]+?)""[^\}]*\}");
|
||
for (int j = 0; j < mm.Count; j++)
|
||
{
|
||
string v = mm[j].Groups[1].Value;
|
||
string vi = mm[j].Groups[0].Value;
|
||
string image = BaseRegex.GetStringJson(ref vi, "image");
|
||
if (image == "null")
|
||
image = "";
|
||
PropValueBase pr = new PropValueBase();
|
||
pr.value = v;
|
||
pr.name = mm[j].Groups[2].Value;
|
||
pr.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000);
|
||
pr.image = image;
|
||
pr.image_from_url = image;
|
||
bool hadx = false;
|
||
for (int k = 0; k < p.Values.Count; k++)
|
||
{
|
||
if (p.Values[k].name == pr.name)
|
||
{
|
||
hadx = true;
|
||
break;
|
||
}
|
||
}
|
||
if (!hadx)
|
||
p.Values.Add(pr);
|
||
}
|
||
if (!had)
|
||
{
|
||
p.sort = i + 1;
|
||
Product.Props.Add(p);
|
||
}
|
||
}
|
||
//Product.name = goodTitle[0];
|
||
|
||
|
||
//for (int i = 1; i < goodTitle.Length; i++)
|
||
//{
|
||
// var p = t.Clone();
|
||
// t.Titlte = goodTitle[i];
|
||
// t.FromType = "阿里巴巴";
|
||
// t.Uped = false;
|
||
// UploadPinduoduoMain.ProductStock.Insert(0, p);
|
||
//}
|
||
}
|
||
catch
|
||
{
|
||
}
|
||
}
|
||
|
||
private string MakeRequestThroughTradeAscM(string data)
|
||
{
|
||
HttpWebResponse response = null;
|
||
string url = "http://acs.m.taobao.com/gw/mtop.1688.wosc.cbu.offerdetailservice.queryofferdetail/1.0/?data=";
|
||
try
|
||
{
|
||
string api = "mtop.1688.wosc.cbu.offerdetailservice.queryofferdetail";
|
||
string v = "1.0";
|
||
|
||
var t = BaseClass.GetTime(BaseClass.GetServerTimeString()).ToString();
|
||
string utdid = "XUEI6Cqy4UgDAHCsEH0z5one";
|
||
string uid = "";
|
||
string appKey = "25443018";
|
||
string lat = "";
|
||
string lng = "";
|
||
string ttid = "219201@taobao_android_7.6.0";
|
||
string sid = "";
|
||
string deviceId = "AqjuNSqg5WtUMlGAgV2ZsudCBHdVINyeYYmyjrge4w2P";
|
||
string features = "27";
|
||
string x_pv = "5.1";
|
||
string url_getSign = BaseClass.BuildObjString("get_taobao_sign&act=get_tb_sign");// "http://new.qixiaoke.com/?obj=littlecow/user&act=get_tb_sign";
|
||
string url_data = UserLogin.GetSignaturex("", "");
|
||
string sign = GetHtml.PostUtf8(url_getSign, url_data + "&t=" + t + "&api=" + api + "&v=" + v + "&data=" + data);
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url + GetHtml.UrlEncodeUtf8(data));
|
||
request.Headers.Add("x-features", features);
|
||
request.UserAgent = "MTOPSDK%2F3.1.1.7+%28Android%3B5.1%3BOPPO%3BOPPO+R9m%29";
|
||
request.Headers.Add("x-ttid", "219201@taobao_android_7.6.0");
|
||
request.Headers.Set(HttpRequestHeader.CacheControl, "no-cache");
|
||
request.Headers.Add("x-appkey", appKey);
|
||
request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
|
||
request.Headers.Add("x-pv", x_pv);
|
||
request.Headers.Add("x-t", t.ToString());
|
||
request.Headers.Add("x-utdid", utdid);
|
||
request.Headers.Add("x-devid", deviceId);
|
||
request.Headers.Add("x-sign", sign);
|
||
request.Headers.Add("x-uid", uid);
|
||
request.Headers.Add("x-sid", sid);
|
||
request.Headers.Add("x-page-url", "http%3A%2F%2Fitem.taobao.com%2Fitem.htm");
|
||
request.Headers.Add("x-nettype", "WIFI");
|
||
request.Headers.Set(HttpRequestHeader.AcceptEncoding, "deflate");
|
||
response = (HttpWebResponse)request.GetResponse();
|
||
}
|
||
catch (WebException e)
|
||
{
|
||
bool flag = e.Status == WebExceptionStatus.ProtocolError;
|
||
if (!flag)
|
||
{
|
||
return "";
|
||
}
|
||
response = (HttpWebResponse)e.Response;
|
||
}
|
||
catch (Exception)
|
||
{
|
||
bool flag2 = response != null;
|
||
if (flag2)
|
||
{
|
||
response.Close();
|
||
}
|
||
return "";
|
||
}
|
||
return ReadResponse(response);
|
||
}
|
||
private static string ReadResponse(HttpWebResponse response)
|
||
{
|
||
using (Stream responseStream = response.GetResponseStream())
|
||
{
|
||
Stream streamToRead = responseStream;
|
||
if (response.ContentEncoding.ToLower().Contains("gzip"))
|
||
{
|
||
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
|
||
}
|
||
else if (response.ContentEncoding.ToLower().Contains("deflate"))
|
||
{
|
||
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
|
||
}
|
||
|
||
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
|
||
{
|
||
return streamReader.ReadToEnd();
|
||
}
|
||
}
|
||
}
|
||
private bool AliBaBaModle(string html)
|
||
{
|
||
if (html.Contains("商品不存在或不可展示"))
|
||
{
|
||
//IsOnSale = false;
|
||
return false;
|
||
}
|
||
Product.item_weight = 1;
|
||
var priceInfo = BaseRegex.GetMatchRegexString(ref html, "\"currentPrices\":([\\w\\W]+?)],");
|
||
if (priceInfo.Contains("price"))
|
||
{
|
||
var sPriceTemp = BaseRegex.GetMatchRegex(ref priceInfo, "\"price\":\"([\\w\\W]+?)\"");
|
||
if (sPriceTemp.Count > 0)
|
||
{
|
||
string sPrice = "";
|
||
foreach (var item in sPriceTemp)
|
||
{
|
||
var v = item.ToString();
|
||
sPrice = BaseRegex.GetMatchRegexString(ref v, "\"price\":\"([\\w\\W]+?)\"");
|
||
if (!string.IsNullOrEmpty(Product.price.ToString()) && sPrice.ParseToDecimal() > Product.price)
|
||
{
|
||
if (sPrice.IndexOf("-") > 0)
|
||
{
|
||
sPrice = sPrice.Substring(0, sPrice.IndexOf('-'));
|
||
}
|
||
Product.price = Convert.ToDecimal(sPrice);
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
}
|
||
var saleCount = BaseRegex.GetMatchRegexString(ref html, "\"saledCount\":\"([\\w\\W]+?)\"");
|
||
int iSaleCount = 0;
|
||
int.TryParse(saleCount, out iSaleCount);
|
||
Product.saleCount = iSaleCount;
|
||
Product.discountPrice = Product.price;
|
||
|
||
|
||
Product.name = BaseRegex.GetMatchRegexString(html, "\"subject\":\"([\\w\\W]+?)\"");
|
||
foreach (string s in AlibabaShop.NoTitleWords)
|
||
{
|
||
Product.name = Product.name.Replace(s, "");
|
||
}
|
||
|
||
Product.name = BaseRegex.Relpace(Product.name.Replace("2018", "2020").Replace("2019", "2020"), @"([a-zA-Z]{1,2}[\d]{1,5})", "").Trim();
|
||
bool had2017 = Product.name.Contains("2019");
|
||
//Product.name = "'" + Product.name + "\"";
|
||
//Product.name = BaseRegex.Relpace(Product.name, @"([a-zA-Z0-9# -]{2,20}?)""", "").Replace("\"", "");//去掉结尾的数字和字母下划线
|
||
//Product.name = BaseRegex.Relpace(Product.name, @"'([a-zA-Z0-9# -]{2,20})", "").Replace("'", "");//去掉开始的数字和字母下划线
|
||
if (had2017 && !Product.name.Contains("2019"))
|
||
Product.name = "2019" + Product.name;
|
||
|
||
if (Product.name.Length < 12)
|
||
{
|
||
LogSave.Add("获取产品详细页面发现标题 " + Product.name + " 异常，跳过 " + Product.from_url);
|
||
return false;
|
||
}
|
||
Product.sell_pt = "";
|
||
var m = BaseRegex.GetMatchRegex(ref html, "\"originalImageURI\":\"([\\w\\W]+?)\"");
|
||
if (m.Count == 0)
|
||
{
|
||
LogSave.Add("获取图片为空，请检查程序 " + Product.from_url);
|
||
return false;
|
||
}
|
||
if (Product.Images.Count <= 0)//如果主图有了 则不需要从页面拿了（文本备份里面有了） updata by tl 20190716
|
||
{
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
Product.Images.Add(m[i].Groups[1].Value);
|
||
}
|
||
}
|
||
Product.image = Product.Images[0];
|
||
Product.image_from_url = Product.image;
|
||
Product.image_path = "";
|
||
SetSkuMapMoble(ref html);
|
||
|
||
string area = BaseRegex.GetMatchRegexString(ref html, "\"location\":\"([\\w\\W]+?)\"");
|
||
if (!string.IsNullOrEmpty(area) && area.Contains(" "))
|
||
{
|
||
Product.state = area.Substring(0, area.IndexOf(" "));
|
||
Product.city = area.Substring(area.IndexOf(" ") + 1);
|
||
}
|
||
var prop = BaseRegex.GetMatchRegexString(ref html, "\"productFeatureList\":\\[([\\w\\W]+?)\\],");
|
||
m = BaseRegex.GetMatchRegex(ref prop, "\"name\":\"([\\w\\W]+?)}");
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
string name = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, "\"name\":\"([\\w\\W]+?)\"");
|
||
string value = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, "\"value\":\"([\\w\\W]+?)\"");
|
||
bool had = false;
|
||
foreach (string s in AlibabaShop.NoPropWords)
|
||
{
|
||
if (name.Contains(s))
|
||
{
|
||
had = true;
|
||
break;
|
||
}
|
||
}
|
||
if (had)
|
||
continue;
|
||
if (name == "品牌")
|
||
value = "other/其他";
|
||
Product.Prop.Add(new KeyValueBase(name, value));
|
||
}
|
||
|
||
string contentUrl = BaseRegex.GetMatchRegexString(ref html, "\"detailUrl\":\"([\\w\\W]+?)\"");
|
||
string desc =
|
||
GetPostTaobaoHtml.UploadTaobao(contentUrl, Product.from_url, false);
|
||
string content = BaseRegex.GetMatchRegexString(ref desc, "\"content\":\"([\\w\\W]+?)\"};");
|
||
if (string.IsNullOrEmpty(content))
|
||
{
|
||
LogSave.Add("获取描述异常，请检查程序 " + Product.from_url);
|
||
return false;
|
||
}
|
||
content = content.Replace("\\\"", "\"").Replace("\\\\", "\\");
|
||
BaseRegex.Relpace(ref content, @"<table background=([\w\W]+?)>超级快车", "");
|
||
string newContentx = content;
|
||
BaseRegex.Relpace(ref newContentx, @"(<[aA] [^>]+?>([\w\W]*?)</[aA]>)", "");
|
||
if (string.IsNullOrEmpty(newContentx) || !newContentx.ToLower().Contains("<img "))
|
||
newContentx = content;
|
||
string newContent = "";
|
||
m = BaseRegex.GetMatchRegex(ref newContentx, @"(<img [^>]+?>)");
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
if (!m[i].Groups[1].Value.Contains(".alicdn.com") || m[i].Groups[0].Value.Contains("display: none") || m[i].Groups[0].Value.Contains("display:none"))
|
||
continue;
|
||
newContent += m[i].Groups[1].Value + "<br /><br />";
|
||
}
|
||
if (string.IsNullOrEmpty(newContent))
|
||
{
|
||
LogSave.Add("描述正文为空，跳过 " + Product.from_url);
|
||
return false;
|
||
}
|
||
|
||
string offerFare = BaseRegex.GetMatchRegexString(ref html, "\"offerFareModel\":{([\\w\\W]+?)}");
|
||
string IsHasFare = BaseRegex.GetStringJson(ref html, "hasFare");
|
||
if (IsHasFare == "true")
|
||
{
|
||
Product.express_price = BaseRegex.GetDecimalJson(ref html, "cost");
|
||
}
|
||
|
||
Product.from_cat_id = BaseRegex.GetIntJson(ref html, @"topCategoryId");
|
||
|
||
|
||
//Product.content = newContent;
|
||
if (string.IsNullOrEmpty(Product.content))//加这个判断是为了有可能文件里面有了详情图 updata by tl 20190716
|
||
{
|
||
Product.content = newContent;
|
||
}
|
||
|
||
Product.content_alibaba = newContent;
|
||
Product.main_cid = 0;
|
||
Product.from_title = Product.name;
|
||
Product.from_site_id = 3;
|
||
return true;
|
||
}
|
||
private string UnTransferred(string Meaning)
|
||
{
|
||
//转义字符变换成普通字符
|
||
Meaning = Meaning.Replace("<", "<");
|
||
Meaning = Meaning.Replace(">", ">");
|
||
Meaning = Meaning.Replace("'", "'");
|
||
Meaning = Meaning.Replace(""", "\"");
|
||
Meaning = Meaning.Replace("&", "&");
|
||
|
||
return Meaning;
|
||
}
|
||
/// <summary>
|
||
/// 设置skumap
|
||
/// </summary>
|
||
/// <param name="product"></param>
|
||
/// <param name="html"></param>
|
||
/// <param name="url"></param>
|
||
public void SetSkuMapMoble(ref string html)
|
||
{
|
||
string prop = BaseRegex.GetMatchRegexString(ref html, "\"skuProps\":([\\w\\W]+?)],");
|
||
string skuMap = string.Empty;
|
||
System.Text.RegularExpressions.MatchCollection mc = System.Text.RegularExpressions.Regex.Matches(html, "\"skuInfoMap\":([\\w\\W]+?)}},", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||
if (mc.Count > 0)
|
||
{
|
||
skuMap = mc[0].Groups[0].Value.ToString();
|
||
}
|
||
skuMap = skuMap.Replace("\"skuInfoMap\":", "").Trim(',');
|
||
if (string.IsNullOrEmpty(skuMap) || string.IsNullOrEmpty(prop))
|
||
{
|
||
return;
|
||
}
|
||
var m = BaseRegex.GetMatchRegex(ref prop, @"""prop"":""([^""]+?)"",""value"":\[([\w\W]+?\})\]\}");
|
||
if (m == null || m.Count == 0)
|
||
{
|
||
LogSave.Add("获取Prop异常，请检查程序 " + Product.from_url);
|
||
return;
|
||
}
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
PropBase p = new PropBase();
|
||
string values = m[i].Groups[2].Value;
|
||
p.name = m[i].Groups[1].Value;
|
||
p.name = BaseRegex.Relpace(p.name, "(&[a-zA-z0-9]+?;)", "");
|
||
bool had = false;
|
||
for (int j = 0; j < Product.Props.Count; j++)
|
||
{
|
||
if (Product.Props[j].name == p.name)
|
||
{
|
||
p = Product.Props[j];
|
||
had = true;
|
||
break;
|
||
}
|
||
}
|
||
var mm = BaseRegex.GetMatchRegex(ref values, @"\{([^\}]+?)\}");
|
||
if (mm.Count == 0)
|
||
{
|
||
LogSave.Add("获取Prop数值异常，请检查程序 " + Product.from_url);
|
||
return;
|
||
}
|
||
for (int j = 0; j < mm.Count; j++)
|
||
{
|
||
PropValueBase pr = new PropValueBase();
|
||
string h = mm[j].Groups[1].Value;
|
||
h = UnTransferred(h).Replace(";", "").Replace(",", "");
|
||
//h = BaseRegex.Relpace(h, "(&[a-zA-z0-9]+?;)", "").Replace(";", "").Replace(",", "");
|
||
pr.name = BaseRegex.GetStringJson(ref h, "name");
|
||
pr.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000);
|
||
pr.image = BaseRegex.GetStringJson(ref h, "imageUrl").Replace("null",null);
|
||
pr.image_from_url = BaseRegex.GetStringJson(ref h, "imageUrl").Replace("null", null);
|
||
p.Values.Add(pr);
|
||
}
|
||
if (!had)
|
||
{
|
||
p.sort = i + 1;
|
||
Product.Props.Add(p);
|
||
}
|
||
}
|
||
try
|
||
{
|
||
JObject ja = JObject.Parse(skuMap);
|
||
string temp = ja.ToString();
|
||
string price = Product.price.ToString();
|
||
foreach (var item in ja)
|
||
{
|
||
string result = item.ToString();
|
||
string pri = BaseRegex.GetStringJson(ref result, "discountPrice");
|
||
if (string.IsNullOrEmpty(pri))
|
||
pri = BaseRegex.GetStringJson(ref result, "price");
|
||
if (!string.IsNullOrEmpty(pri))
|
||
price = pri;
|
||
string name = item.Key;
|
||
|
||
string sale_count = BaseRegex.GetStringJson(ref result, "saleCount");
|
||
string number = BaseRegex.GetStringJson(ref result, "canBookCount");
|
||
name = UnTransferred(name);
|
||
name = BuildName(name);
|
||
if (string.IsNullOrEmpty(name))
|
||
{
|
||
LogSave.Add("获取SKUmap名称异常，请检查程序 " + Product.from_url);
|
||
return;
|
||
}
|
||
SkuMapBase sm = new SkuMapBase();
|
||
sm.name = name;
|
||
sm.price = decimal.Parse(price);
|
||
sm.number = int.Parse(number);
|
||
sm.sale_count = int.Parse(sale_count);
|
||
Product.SkuMaps.Add(sm);
|
||
}
|
||
}
|
||
catch
|
||
{
|
||
|
||
}
|
||
//m = BaseRegex.GetMatchRegex(ref skuMap, "\"([\\s\\S]+?\":\\{\"specId\":[\\s\\S]+?skuId\":\"[\\d]+?\")");
|
||
//if (m.Count == 0)
|
||
//{
|
||
// LogSave.Add("获取skuMap异常，请检查程序 " + Product.from_url);
|
||
// return;
|
||
//}
|
||
//string price = BaseRegex.GetMatchRegexString(ref html, @"""priceRangeOriginal"":\[\[[\d]+, ([\d\.]+)?\]");
|
||
//if (string.IsNullOrEmpty(price))
|
||
//{
|
||
// price = Product.price.ToString();
|
||
//}
|
||
//for (int i = 0; i < m.Count; i++)
|
||
//{
|
||
// string pri = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, @"""discountPrice"":""([\d\.]+)""");
|
||
// if (string.IsNullOrEmpty(pri))
|
||
// pri = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, @"""price"":""([\d\.]+)""");
|
||
// if (!string.IsNullOrEmpty(pri))
|
||
// price = pri;
|
||
// string name = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, "\"([\\w\\W]+?)\":{");
|
||
|
||
// string sale_count = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, "\"saleCount\":\"([\\w\\W]+?)\"");
|
||
// string number = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, "\"canBookCount\":\"([\\w\\W]+?)\"");
|
||
// name = UnTransferred(name);
|
||
// name = BuildName(name);
|
||
|
||
// //name = BaseRegex.Relpace(name, "(&[a-zA-z0-9]+?;)", "").Replace(",", "");
|
||
// if (string.IsNullOrEmpty(name))
|
||
// {
|
||
// LogSave.Add("获取SKUmap名称异常，请检查程序 " + Product.from_url);
|
||
// return;
|
||
// }
|
||
// SkuMapBase sm = new SkuMapBase();
|
||
// sm.name = name;
|
||
// sm.price = decimal.Parse(price);
|
||
// sm.number = int.Parse(number);
|
||
// sm.sale_count = int.Parse(sale_count);
|
||
// Product.SkuMaps.Add(sm);
|
||
//}
|
||
|
||
|
||
}
|
||
|
||
/// <summary>
|
||
/// 设置skumap
|
||
/// </summary>
|
||
/// <param name="product"></param>
|
||
/// <param name="html"></param>
|
||
/// <param name="url"></param>
|
||
public void SetSkuMap(ref string html, string url)
|
||
{
|
||
string prop = BaseRegex.GetMatchRegexString(ref html, "\"skuProps\":([^\n]+?)\n");
|
||
if (string.IsNullOrEmpty(prop))
|
||
{
|
||
prop = BaseRegex.GetMatchRegexString(ref html, "skuProps:([^\n]+?)\n");
|
||
}
|
||
string skuMap = BaseRegex.GetMatchRegexString(ref html, "\"skuMap\":([^\n]+?)\n");
|
||
if (string.IsNullOrEmpty(skuMap))
|
||
{
|
||
skuMap = BaseRegex.GetMatchRegexString(ref html, "skuMap:([^\n]+?)\n");
|
||
}
|
||
if (string.IsNullOrEmpty(skuMap) || string.IsNullOrEmpty(prop))
|
||
{
|
||
//LogSave.Add("获取Prop或者skuMap异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
var m = BaseRegex.GetMatchRegex(ref prop, @"""prop"":""([^""]+?)"",""value"":\[([\w\W]+?\})\]\}");
|
||
if (m == null || m.Count == 0)
|
||
{
|
||
LogSave.Add("获取Prop异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
PropBase p = new PropBase();
|
||
string values = m[i].Groups[2].Value;
|
||
p.name = m[i].Groups[1].Value;
|
||
p.name = BaseRegex.Relpace(p.name, "(&[a-zA-z0-9]+?;)", "");
|
||
bool had = false;
|
||
for (int j = 0; j < Product.Props.Count; j++)
|
||
{
|
||
if (Product.Props[j].name == p.name)
|
||
{
|
||
p = Product.Props[j];
|
||
had = true;
|
||
break;
|
||
}
|
||
}
|
||
var mm = BaseRegex.GetMatchRegex(ref values, @"\{([^\}]+?)\}");
|
||
//var mmm = BaseRegex.GetMatchRegex(ref values, @"""imageUrl"":""([^""]+?)""");
|
||
if (mm.Count == 0)
|
||
{
|
||
LogSave.Add("获取Prop数值异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
for (int j = 0; j < mm.Count; j++)
|
||
{
|
||
PropValueBase pr = new PropValueBase();
|
||
string h = mm[j].Groups[1].Value;
|
||
h = BaseRegex.Relpace(h, "(&[a-zA-z0-9]+?;)", "").Replace(";", "").Replace(",", "");
|
||
pr.name = BaseRegex.GetStringJson(ref h, "name");
|
||
pr.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000);
|
||
pr.image = BaseRegex.GetStringJson(ref h, "imageUrl");
|
||
pr.image_from_url = BaseRegex.GetStringJson(ref h, "imageUrl");
|
||
p.Values.Add(pr);
|
||
}
|
||
if (!had)
|
||
{
|
||
p.sort = i + 1;
|
||
Product.Props.Add(p);
|
||
}
|
||
}
|
||
m = BaseRegex.GetMatchRegex(ref skuMap, @"""([^""]+?)"":{""price"":""([^""]+?)""");
|
||
if (m.Count == 0)
|
||
{
|
||
m = BaseRegex.GetMatchRegex(ref skuMap, @"""([^""]+?)"":\{[^\}]*""canBookCount"":([\d]+),[^\}]+?""saleCount"":([\d]+)");
|
||
if (m.Count == 0)
|
||
{
|
||
m = BaseRegex.GetMatchRegex(ref skuMap, @"""([^""]+?)"":[\w\W]+?,""saleCount"":([\d]+)[^\}]+?,""canBookCount"":([\d]+),[^\}]+?\}");
|
||
}
|
||
if (m.Count == 0)
|
||
{
|
||
m = BaseRegex.GetMatchRegex(ref skuMap, @"""([^""]+?)"":[\w\W]+?,""saleCount"":([\d]+),""canBookCount"":([\d]+),[^\}]+?\}");
|
||
}
|
||
if (m.Count == 0)
|
||
{
|
||
LogSave.Add("获取skuMap异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
string price = BaseRegex.GetMatchRegexString(ref html, @"""priceRangeOriginal"":\[\[[\d]+, ([\d\.]+)?\]");
|
||
if (string.IsNullOrEmpty(price))
|
||
{
|
||
price = Product.price.ToString();
|
||
//LogSave.Add("获取统一批发价异常，请检查程序 " + url);
|
||
//return;
|
||
}
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
string pri = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, @"""discountPrice"":""([\d\.]+)""");
|
||
if (string.IsNullOrEmpty(pri))
|
||
pri = BaseRegex.GetMatchRegexString(m[i].Groups[0].Value, @"""price"":""([\d\.]+)""");
|
||
if (!string.IsNullOrEmpty(pri))
|
||
price = pri;
|
||
string name = m[i].Groups[1].Value;
|
||
string number = m[i].Groups[2].Value=="0"?"1": m[i].Groups[2].Value;
|
||
string sale_count = m[i].Groups[3].Value;
|
||
name = BuildName(name);
|
||
name = BaseRegex.Relpace(name, "(&[a-zA-z0-9]+?;)", "").Replace(",", "");
|
||
if (string.IsNullOrEmpty(name))
|
||
{
|
||
LogSave.Add("获取SKUmap名称异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
SkuMapBase sm = new SkuMapBase();
|
||
sm.name = name;
|
||
sm.price = decimal.Parse(price);
|
||
sm.number = int.Parse(number);
|
||
sm.sale_count = int.Parse(sale_count);
|
||
Product.SkuMaps.Add(sm);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (int i = 0; i < m.Count; i++)
|
||
{
|
||
string price = m[i].Groups[2].Value;
|
||
string name = m[i].Groups[1].Value;
|
||
name = BuildName(name);
|
||
name = BaseRegex.Relpace(name, "(&[a-zA-z0-9]+?;)", "").Replace(",", "");
|
||
if (string.IsNullOrEmpty(name))
|
||
{
|
||
LogSave.Add("获取SKUmap名称异常，请检查程序 " + url);
|
||
return;
|
||
}
|
||
SkuMapBase sm = new SkuMapBase();
|
||
sm.name = name;
|
||
sm.price = decimal.Parse(price);
|
||
sm.number = new Random(Guid.NewGuid().GetHashCode()).Next(100, 1000);
|
||
Product.SkuMaps.Add(sm);
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// 生成名称
|
||
/// </summary>
|
||
/// <param name="name"></param>
|
||
/// <returns></returns>
|
||
private string BuildName(string name)
|
||
{
|
||
if (!name.Contains(">") && !name.Contains(">"))
|
||
return Product.Props[0].name + ":" + name;
|
||
name = name.Replace(">", ">");
|
||
string[] l = name.Split('>');
|
||
if (l.Length != Product.Props.Count)
|
||
{
|
||
return ""; //数量不匹配可能有错误
|
||
}
|
||
string temp = "";
|
||
for (int i = 0; i < l.Length; i++)
|
||
{
|
||
temp += ";" + Product.Props[i].name + ":" + l[i];
|
||
}
|
||
return temp.Substring(1);
|
||
}
|
||
}
|
||
}
|