using AllBase; using AllBase.Helper; using AllBase.Model.Result; using TemuKJD.KJDForms.ToolForms; using NaiBa.Util.Extension; using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Drawing; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; using Work.BLL.Collect; namespace TemuKJD.KJDForms.CollectForms { public partial class HaoWuCollectForm : Form { CancellationTokenSource cancellationTokenSource = null; /// /// 商品标题不包含的关键词列表 /// private List lstBanKeyText = new List(); List lstProduct = new List(); /// /// 商品过滤的最低销量 /// private string minSaleCount = "0"; private string maxSaleCount = "99999999"; private string minPrice = "0"; private string maxPrice = "99999999"; /// /// 好评率 /// private double hplMin = -1; private double hplMax = 100; /// /// 采集详情推荐中的商品 /// private bool isXqtj = false; /// /// 是否显示图片 /// private bool isShowImg = false; private string exportFileName = "douyinhaowu_collect.csv"; private string[] cates = new string[] { "精选", "食品饮料", "生鲜", "女装", "鞋靴", "彩妆", "护肤", "珠宝文玩", "家电", "数码办公", "男装", "家纺", "家装", "厨具", "图书", "配饰", "母婴", "运动", "百货", "洗护", "票务卡券", "车品", "绿植农资" }; public HaoWuCollectForm() { InitializeComponent(); } private void HaoWuCollectForm_Load(object sender, EventArgs e) { foreach (var item in cates) { this.cmbCategory.Items.Add(item); } this.cmbCategory.SelectedIndex = 0; this.splitContainer2.SplitterWidth = 1; } public void Collect(string dyCookies, int totalPage, string cateName, string cateIndex, ref List lstGoods, ref int currentPage) { int total = totalPage; string str = ""; try { if (!Directory.Exists("采集导出")) Directory.CreateDirectory("采集导出"); exportFileName = "采集导出\\抖音好物采集" + DateTime.Now.ToString("yyyyMMddHHmmss") + ".csv"; StreamWriter writer = new StreamWriter(this.exportFileName, true, Encoding.GetEncoding("gb2312")); object[] objArray1 = new object[] { "商品名称,", "价格,", "销量,", "好评率,", "商品ID,", "店铺ID,", "店铺名称,", "店铺商品总数" }; writer.WriteLine(string.Concat(objArray1)); writer.Close(); int num = 0; while (num < totalPage) { if (cancellationTokenSource.IsCancellationRequested) break; currentPage = num + 1; Invoke(new Action(() => { this.statusBar.lblTaskStatus.Text = "正在采集类目:" + cateName + ",当前采集到第 " + (num + 1) + " 页"; this.statusBar.pgbTaskProgress.Value = (num + 1); this.statusBar.pgbTaskProgress.Text = (num + 1) + "/" + total; this.statusBar.pgbTaskProgress.Maximum = total; })); str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 正在采集类目:{cateName} 第{num + 1}页,成功采集{lstGoods.Count}条。"; object[] objArray2 = new object[] { str }; //base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray2); int num2 = num * 10; string str2 = ""; object[] objArray3 = new object[] { "https://aweme.snssdk.com/aweme/v2/commerce/favorite/feed?feature_version=new_operate_resource&tab_id=", cateIndex, "&entrances&cursor=", num2, "&request_tag_from=lynx&with_tabs=false&filters=2_3481030053603741229%2C2_3484212195024324357%2C1_6971768879194327844%2C2_3479930499084982767%2C2_3470779358002703750%2C1_6971770338224982791%2C2_3476553583086965127%2C2_3478793526744103385%2C1_6971747541322976035%2C2_3480664549739395118&page_num=2&feedbacks&page_name=order_homepage&only_entrance=false&os_api=22&device_type=TAS-AN00&ssmix=a&manifest_version_code=160002&dpi=240&uuid=865166029839340&app_name=aweme&version_name=16.0.1&ts=1623252303&cpu_support64=false&app_type=normal&appTheme=dark&ac=wifi&host_abi=armeabi-v7a&update_version_code=16019900&channel=aweGW&_rticket=1623252304036&device_platform=android&iid=1935603863652935&version_code=160001&cdid=61a31b75-b28a-448f-b688-2803bf62456b&openudid=2292f1eb6ba5f1c1&device_id=68214623104&resolution=1600*900&os_version=5.1.1&language=zh&device_brand=HUAWEI&aid=1128&minor_status=0&mcc_mnc=46000" }; string url = string.Concat(objArray3); object[] objArray4 = new object[0x13]; objArray4[0] = "https://aweme.snssdk.com/aweme/v2/commerce/favorite/feed?feature_version=new_operate_resource&tab_id="; objArray4[1] = cateIndex; objArray4[2] = "&render_purchase=0&entrances&cursor="; objArray4[3] = num2; objArray4[4] = "&request_tag_from=lynx&with_tabs=false&filters&page_num="; objArray4[5] = num + 1; objArray4[6] = "&feedbacks&page_name=order_homepage&only_entrance=false&os_api=22&device_type=TAS-AN00&ssmix=a&manifest_version_code=160002&dpi=240&uuid=8651"; objArray4[7] = DateTime.Now.ToString("ff"); objArray4[8] = "02983"; objArray4[9] = DateTime.Now.ToString("ss"); objArray4[10] = "40&app_name=aweme&version_name=16.0.1&ts="; objArray4[11] = ((DateTime.Now.ToUniversalTime().Ticks - 621355968000000000L) / 10000000L).ToString(); // Base.ConvertDateTimeToInt(); objArray4[12] = "&cpu_support64=false&app_type=normal&appTheme=dark&ac=wifi&host_abi=armeabi-v7a&update_version_code=16019900&channel=aweGW&_rticket=1623515082669&device_platform=android&iid=1935603863652935&version_code=160001&cdid=61a31b75-b28a-448f-b6"; objArray4[13] = DateTime.Now.ToString("ss"); objArray4[14] = "-2803bf62456b&openudid=22"; objArray4[15] = DateTime.Now.ToString("ss"); objArray4[16] = "f1eb6ba5f1c1&device_id=682"; objArray4[17] = DateTime.Now.ToString("ff"); objArray4[18] = "623104&resolution=1600*900&os_version=5.1.1&language=zh&device_brand=HUAWEI&aid=1128&minor_status=0&mcc_mnc=46000"; url = string.Concat(objArray4); if (string.IsNullOrWhiteSpace(dyCookies)) { dyCookies = "sid_guard=f7fbff5199b1bfe9de6eacb334368979|||"; } str2 = GetHtml.GetUtf8(url, AllBase.BaseClass.ChromeUserAgent, dyCookies); int num3 = 0; if (str2.IndexOf("用户未登录") != -1) { DateTime now = DateTime.Now; str = $"[{now.ToString("MM-dd HH:mm:ss")}] 请重新登入抖音."; object[] objArray7 = new object[] { str }; //base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray7); //this.int_6 = 0x1_869f; break; } while (true) { if (!(((str2.IndexOf("服务器打瞌睡了") != -1) || ((str2.IndexOf("你搜得太多啦") != -1) || (str2.Trim() == ""))) ? (num3 < 20) : false)) { char[] separator = new char[] { 'ㄨ' }; string[] strArray = str2.Replace("\"product\":", "ㄨ\"product\":").Split(separator); if (str2.Contains("\"item_cards\":")) { var jobj = Newtonsoft.Json.Linq.JObject.Parse(str2); var sObj = jobj["item_cards"]; var sList = new List(); foreach (var s in sObj) { sList.Add(JsonHelper.SerializeObject(s)); } strArray = sList.ToArray(); } int index = 0; while (true) { if (cancellationTokenSource.IsCancellationRequested) break; if (index >= strArray.Length) { Thread.Sleep(1000); num++; break; } try { string goodStr = strArray[index]; DouGoodsCollectInfo item = new DouGoodsCollectInfo(); item.Id = AllBase.BaseRegex.GetStringJson(ref goodStr, "product_id"); if (!string.IsNullOrEmpty(item.Id) && !lstGoods.Any(g => g.Id == item.Id)) { item.GoodsName = AllBase.BaseRegex.GetStringJson(ref goodStr, "title")?.Trim(); if (!string.IsNullOrEmpty(item.GoodsName)) { if (lstBanKeyText.Count == 0 || !lstBanKeyText.Any(t => item.GoodsName.Contains(t))) { // 销量数据 item.Sales = AllBase.BaseRegex.GetStringJson(ref goodStr, "sales"); if (string.IsNullOrEmpty(item.Sales)) item.Sales = "0"; // 销量符合筛选条件 if (Convert.ToInt32(item.Sales) >= Convert.ToInt32(this.minSaleCount) && Convert.ToInt32(item.Sales) <= Convert.ToInt32(this.maxSaleCount)) { string price = BaseRegex.GetStringJson(ref goodStr, "min_price"); if (string.IsNullOrEmpty(price)) price = "0"; item.Price = ((Convert.ToDouble(price) * 1.0) / 100.0).ToString("f2"); if (Convert.ToDouble(item.Price) >= Convert.ToDouble(this.minPrice) && Convert.ToDouble(item.Price) <= Convert.ToDouble(this.maxPrice)) { var img = BaseRegex.GetStringJson(ref goodStr, "url_list"); if (item.img == null) img = BaseRegex.GetMatchRegexString(ref goodStr, "\"cover\":[^\\[\\]]+?\"url_list\":\\[([^\\[\\]]+?)\\]"); if (!string.IsNullOrEmpty(img)) { item.img = img.Split(',')[0].Trim('"'); } // 获取商品评价 item.CommentsCount = this.GetRate(item.Id, ref item.ShopProductCount, ref item.CommentsRate); if (item.CommentsRate >= this.hplMin && item.CommentsRate <= hplMax) { //Application.DoEvents(); Thread.Sleep(1000); string content = HaoWuCollectBLL.getHtml( "https://ec.snssdk.com/product/fxgajaxstaticitem?b_type_new=0&device_id=0&is_outside=1&id=" + item.Id + "&preview=0", "", "utf8"); item.ShopId = BaseRegex.GetStringJson(ref content, "shop_id"); item.ShopName = BaseRegex.GetStringJson(ref content, "shop_name"); lstGoods.Add(item); this.ShowData(lstGoods.Count, item); if (this.isXqtj) { //this.method_3(spid); } if (((index % 2) == 0) && (index > 0)) Thread.Sleep(1000); } } } } } } } catch (Exception ex) { } index++; } break; } str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 服务器打瞌睡了,重试..."; object[] objArray5 = new object[] { str }; //base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray5); //Application.DoEvents(); Thread.Sleep(3000); num3++; } } } catch { } str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 采集结束,成功采集{lstGoods.Count}个."; object[] args = new object[] { str }; //base.BeginInvoke(new showMessageDelegate(this.showMsg), args); } private int GetRate(string productId, ref int productCount, ref double rate) { int num = 0; string url = "https://ec.snssdk.com/product/ajaxitem?b_type_new=0&device_id=0&is_outside=1&id=" + productId + "&abParams=0"; string content = HaoWuCollectBLL.getHtml_fxg_20201220(url, "", ""); int num2 = 0; string ip = ""; while ((num2 < 10) && (content.Trim().Length < 10)) { //Application.DoEvents(); // 刷新界面 Thread.Sleep(500); num2++; content = HaoWuCollectBLL.getHtml_fxg_20201220(url, "", ip); } num = BaseRegex.GetIntJson(ref content, "count"); productCount = BaseRegex.GetIntJson(ref content, "shop_product_count"); rate = (double)BaseRegex.GetDecimalJson(ref content, "value"); return num; } private void ShowData(int totalCount, DouGoodsCollectInfo goodsInfo) { try { StreamWriter writer = new StreamWriter(this.exportFileName, true, Encoding.GetEncoding("gb2312")); object[] objArray1 = new object[15]; objArray1[0] = goodsInfo.GoodsName; objArray1[1] = ","; objArray1[2] = goodsInfo.Price; objArray1[3] = ","; objArray1[4] = goodsInfo.Sales; objArray1[5] = ","; objArray1[6] = goodsInfo.CommentsRate; objArray1[7] = ","; objArray1[8] = "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsInfo.Id; objArray1[9] = ","; objArray1[10] = goodsInfo.ShopId; objArray1[11] = ","; objArray1[12] = goodsInfo.ShopName; objArray1[13] = ","; objArray1[14] = goodsInfo.ShopProductCount; writer.WriteLine(string.Concat(objArray1)); writer.Close(); } catch { } Invoke(new Action(() => { var index = this.dgvCollect.Rows.Add(false, goodsInfo.Id, goodsInfo.GoodsName, null, goodsInfo.Price, goodsInfo.Sales, goodsInfo.CommentsRate, goodsInfo.ShopId, goodsInfo.ShopName, goodsInfo.ShopProductCount, "删除"); if (this.isShowImg) { this.dgvCollect.Columns["colImage"].Visible = true; var row = dgvCollect.Rows[index]; row.Cells["colImage"].Value = this.GetImage2(goodsInfo.img, 60); row.Height = 70; } else this.dgvCollect.Columns["colImage"].Visible = false; })); } public Image GetImage2(string path, int size = 130) { try { return new Bitmap(Image.FromStream(WebRequest.Create(path).GetResponse().GetResponseStream()), size, size); } catch { return null; } } private void btnLoginDouyin_Click(object sender, EventArgs e) { DouUserLoginForm dForm = new DouUserLoginForm(); dForm.ShowDialog(); } private void btnCollect_Click(object sender, EventArgs e) { if (btnCollect.Text == "开始采集") { var cookie = GlobalConfig.DouUserCookies; var cate = this.cmbCategory.Text; var cid = this.cmbCategory.SelectedIndex.ToString(); var collectPage = this.txtCollectPage.Text.Trim().ParseToInt(); minSaleCount = this.txtSaleCountMin.Text.ParseToInt().ToString(); maxSaleCount = this.txtSaleCountMax.Text.ParseToInt().ToString(); minPrice = this.txtPriceMin.Text.ParseToDecimal().ToString(); maxPrice = this.txtPriceMax.Text.ParseToDecimal().ToString(); hplMin = this.txtRateMin.Text.ParseToDouble(); hplMax = this.txtRateMax.Text.ParseToDouble(); isShowImg = this.chkShowImage.Checked; this.dgvCollect.Rows.Clear(); this.lstProduct.Clear(); cancellationTokenSource = new CancellationTokenSource(); btnCollect.Text = "停止采集"; int n = 0; int total = collectPage; this.statusBar.lblTaskStatus.Text = "开始采集..."; this.statusBar.pgbTaskProgress.Value = n; this.statusBar.pgbTaskProgress.Text = n + "/" + total; this.statusBar.pgbTaskProgress.Maximum = total; Task.Run(() => { Collect(cookie, collectPage, cate, cid, ref lstProduct, ref n); Invoke(new Action(() => { btnCollect.Text = "开始采集"; this.statusBar.lblTaskStatus.Text = "采集结束,共采集到商品" + this.lstProduct.Count + "个"; this.statusBar.pgbTaskProgress.Value = n; this.statusBar.pgbTaskProgress.Text = n + "/" + total; this.statusBar.pgbTaskProgress.Maximum = total; })); }, cancellationTokenSource.Token); } else { cancellationTokenSource.Cancel(); } } private void btnExport_Click(object sender, EventArgs e) { try { SaveFileDialog saveFileDialog1 = new SaveFileDialog(); saveFileDialog1.Filter = "表格文件(*.csv)|*.csv"; saveFileDialog1.FileName = "好物采集" + DateTime.Now.ToString("yyyyMMddHHmmss") + ".csv"; saveFileDialog1.FilterIndex = 2; saveFileDialog1.RestoreDirectory = true; if (saveFileDialog1.ShowDialog() == DialogResult.OK) { StringBuilder sb = new StringBuilder(); sb.AppendLine("商品名称," + "价格," + "销量," + "好评率," + "商品ID," + "店铺ID," + "店铺名称," + "店铺商品总数"); foreach (var goodsInfo in lstProduct) { sb.AppendLine(goodsInfo.GoodsName + "," + goodsInfo.Price + "," + goodsInfo.Sales + "," + goodsInfo.CommentsRate + "," + "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsInfo.Id + "," + goodsInfo.ShopId + "," + goodsInfo.ShopName + "," + goodsInfo.ShopProductCount); } File.WriteAllText(saveFileDialog1.FileName, sb.ToString(), Encoding.Default); BaseForms.BaseMsgForm.ShowInfo("导出成功"); } } catch (Exception ex) { BaseForms.BaseMsgForm.ShowError(ex.Message); } } private void dgvShop_CellContentClick(object sender, DataGridViewCellEventArgs e) { if (e.RowIndex < 0 || e.ColumnIndex < 0) return; var goodsId = dgvCollect.Rows[e.RowIndex].Cells["colProductId"].Value.ToString(); var columnName = dgvCollect.Columns[e.ColumnIndex].Name; if (columnName == "colProductName") { string url = "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsId; GetHtml.OpenUrl(url); } } } }