438 lines
23 KiB
C#
438 lines
23 KiB
C#
using AllBase;
|
|
using AllBase.Helper;
|
|
using AllBase.Model.Result;
|
|
using TemuKJD.KJDForms.ToolForms;
|
|
using NaiBa.Util.Extension;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.ComponentModel;
|
|
using System.Data;
|
|
using System.Drawing;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using System.Windows.Forms;
|
|
using Work.BLL.Collect;
|
|
using TemuKJD.KJDForms.BaseForms;
|
|
|
|
namespace TemuKJD.KJDForms.CollectForms
|
|
{
|
|
public partial class HaoWuCollectForm : BaseForm
|
|
{
|
|
CancellationTokenSource cancellationTokenSource = null;
|
|
/// <summary>
|
|
/// 商品标题不包含的关键词列表
|
|
/// </summary>
|
|
private List<string> lstBanKeyText = new List<string>();
|
|
|
|
List<DouGoodsCollectInfo> lstProduct = new List<DouGoodsCollectInfo>();
|
|
|
|
/// <summary>
|
|
/// 商品过滤的最低销量
|
|
/// </summary>
|
|
private string minSaleCount = "0";
|
|
private string maxSaleCount = "99999999";
|
|
private string minPrice = "0";
|
|
private string maxPrice = "99999999";
|
|
/// <summary>
|
|
/// 好评率
|
|
/// </summary>
|
|
private double hplMin = -1;
|
|
private double hplMax = 100;
|
|
/// <summary>
|
|
/// 采集详情推荐中的商品
|
|
/// </summary>
|
|
private bool isXqtj = false;
|
|
/// <summary>
|
|
/// 是否显示图片
|
|
/// </summary>
|
|
private bool isShowImg = false;
|
|
private string exportFileName = "douyinhaowu_collect.csv";
|
|
private string[] cates = new string[] { "精选", "食品饮料", "生鲜", "女装", "鞋靴", "彩妆", "护肤", "珠宝文玩", "家电", "数码办公", "男装",
|
|
"家纺", "家装", "厨具", "图书", "配饰", "母婴", "运动", "百货", "洗护", "票务卡券", "车品", "绿植农资" };
|
|
|
|
public HaoWuCollectForm()
|
|
{
|
|
InitializeComponent();
|
|
}
|
|
|
|
private void HaoWuCollectForm_Load(object sender, EventArgs e)
|
|
{
|
|
foreach (var item in cates)
|
|
{
|
|
this.cmbCategory.Items.Add(item);
|
|
}
|
|
this.cmbCategory.SelectedIndex = 0;
|
|
this.splitContainer2.SplitterWidth = 1;
|
|
}
|
|
|
|
public void Collect(string dyCookies, int totalPage, string cateName, string cateIndex, ref List<DouGoodsCollectInfo> lstGoods, ref int currentPage)
|
|
{
|
|
int total = totalPage;
|
|
string str = "";
|
|
try
|
|
{
|
|
if (!Directory.Exists("采集导出"))
|
|
Directory.CreateDirectory("采集导出");
|
|
exportFileName = "采集导出\\抖音好物采集" + DateTime.Now.ToString("yyyyMMddHHmmss") + ".csv";
|
|
StreamWriter writer = new StreamWriter(this.exportFileName, true, Encoding.GetEncoding("gb2312"));
|
|
object[] objArray1 = new object[] { "商品名称,", "价格,", "销量,", "好评率,", "商品ID,", "店铺ID,", "店铺名称,", "店铺商品总数" };
|
|
writer.WriteLine(string.Concat(objArray1));
|
|
writer.Close();
|
|
|
|
int num = 0;
|
|
while (num < totalPage)
|
|
{
|
|
if (cancellationTokenSource.IsCancellationRequested)
|
|
break;
|
|
|
|
currentPage = num + 1;
|
|
Invoke(new Action(() =>
|
|
{
|
|
this.statusBar.lblTaskStatus.Text = "正在采集类目：" + cateName + "，当前采集到第 " + (num + 1) + " 页";
|
|
this.statusBar.pgbTaskProgress.Value = (num + 1);
|
|
this.statusBar.pgbTaskProgress.Text = (num + 1) + "/" + total;
|
|
this.statusBar.pgbTaskProgress.Maximum = total;
|
|
}));
|
|
|
|
str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 正在采集类目:{cateName} 第{num + 1}页，成功采集{lstGoods.Count}条。";
|
|
object[] objArray2 = new object[] { str };
|
|
//base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray2);
|
|
int num2 = num * 10;
|
|
string str2 = "";
|
|
object[] objArray3 = new object[] { "https://aweme.snssdk.com/aweme/v2/commerce/favorite/feed?feature_version=new_operate_resource&tab_id=", cateIndex, "&entrances&cursor=", num2, "&request_tag_from=lynx&with_tabs=false&filters=2_3481030053603741229%2C2_3484212195024324357%2C1_6971768879194327844%2C2_3479930499084982767%2C2_3470779358002703750%2C1_6971770338224982791%2C2_3476553583086965127%2C2_3478793526744103385%2C1_6971747541322976035%2C2_3480664549739395118&page_num=2&feedbacks&page_name=order_homepage&only_entrance=false&os_api=22&device_type=TAS-AN00&ssmix=a&manifest_version_code=160002&dpi=240&uuid=865166029839340&app_name=aweme&version_name=16.0.1&ts=1623252303&cpu_support64=false&app_type=normal&appTheme=dark&ac=wifi&host_abi=armeabi-v7a&update_version_code=16019900&channel=aweGW&_rticket=1623252304036&device_platform=android&iid=1935603863652935&version_code=160001&cdid=61a31b75-b28a-448f-b688-2803bf62456b&openudid=2292f1eb6ba5f1c1&device_id=68214623104&resolution=1600*900&os_version=5.1.1&language=zh&device_brand=HUAWEI&aid=1128&minor_status=0&mcc_mnc=46000" };
|
|
string url = string.Concat(objArray3);
|
|
object[] objArray4 = new object[0x13];
|
|
objArray4[0] = "https://aweme.snssdk.com/aweme/v2/commerce/favorite/feed?feature_version=new_operate_resource&tab_id=";
|
|
objArray4[1] = cateIndex;
|
|
objArray4[2] = "&render_purchase=0&entrances&cursor=";
|
|
objArray4[3] = num2;
|
|
objArray4[4] = "&request_tag_from=lynx&with_tabs=false&filters&page_num=";
|
|
objArray4[5] = num + 1;
|
|
objArray4[6] = "&feedbacks&page_name=order_homepage&only_entrance=false&os_api=22&device_type=TAS-AN00&ssmix=a&manifest_version_code=160002&dpi=240&uuid=8651";
|
|
objArray4[7] = DateTime.Now.ToString("ff");
|
|
objArray4[8] = "02983";
|
|
objArray4[9] = DateTime.Now.ToString("ss");
|
|
objArray4[10] = "40&app_name=aweme&version_name=16.0.1&ts=";
|
|
objArray4[11] = ((DateTime.Now.ToUniversalTime().Ticks - 621355968000000000L) / 10000000L).ToString(); // Base.ConvertDateTimeToInt();
|
|
objArray4[12] = "&cpu_support64=false&app_type=normal&appTheme=dark&ac=wifi&host_abi=armeabi-v7a&update_version_code=16019900&channel=aweGW&_rticket=1623515082669&device_platform=android&iid=1935603863652935&version_code=160001&cdid=61a31b75-b28a-448f-b6";
|
|
objArray4[13] = DateTime.Now.ToString("ss");
|
|
objArray4[14] = "-2803bf62456b&openudid=22";
|
|
objArray4[15] = DateTime.Now.ToString("ss");
|
|
objArray4[16] = "f1eb6ba5f1c1&device_id=682";
|
|
objArray4[17] = DateTime.Now.ToString("ff");
|
|
objArray4[18] = "623104&resolution=1600*900&os_version=5.1.1&language=zh&device_brand=HUAWEI&aid=1128&minor_status=0&mcc_mnc=46000";
|
|
url = string.Concat(objArray4);
|
|
if (string.IsNullOrWhiteSpace(dyCookies))
|
|
{
|
|
dyCookies = "sid_guard=f7fbff5199b1bfe9de6eacb334368979|||";
|
|
}
|
|
str2 = GetHtml.GetUtf8(url, AllBase.BaseClass.ChromeUserAgent, dyCookies);
|
|
int num3 = 0;
|
|
if (str2.IndexOf("用户未登录") != -1)
|
|
{
|
|
DateTime now = DateTime.Now;
|
|
str = $"[{now.ToString("MM-dd HH:mm:ss")}] 请重新登入抖音.";
|
|
object[] objArray7 = new object[] { str };
|
|
//base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray7);
|
|
//this.int_6 = 0x1_869f;
|
|
break;
|
|
}
|
|
while (true)
|
|
{
|
|
if (!(((str2.IndexOf("服务器打瞌睡了") != -1) || ((str2.IndexOf("你搜得太多啦") != -1) || (str2.Trim() == ""))) ? (num3 < 20) : false))
|
|
{
|
|
char[] separator = new char[] { 'ㄨ' };
|
|
string[] strArray = str2.Replace("\"product\":", "ㄨ\"product\":").Split(separator);
|
|
if (str2.Contains("\"item_cards\":"))
|
|
{
|
|
var jobj = Newtonsoft.Json.Linq.JObject.Parse(str2);
|
|
var sObj = jobj["item_cards"];
|
|
var sList = new List<string>();
|
|
foreach (var s in sObj)
|
|
{
|
|
sList.Add(JsonHelper.SerializeObject(s));
|
|
}
|
|
strArray = sList.ToArray();
|
|
}
|
|
int index = 0;
|
|
while (true)
|
|
{
|
|
if (cancellationTokenSource.IsCancellationRequested)
|
|
break;
|
|
|
|
if (index >= strArray.Length)
|
|
{
|
|
Thread.Sleep(1000);
|
|
num++;
|
|
break;
|
|
}
|
|
try
|
|
{
|
|
string goodStr = strArray[index];
|
|
DouGoodsCollectInfo item = new DouGoodsCollectInfo();
|
|
|
|
item.Id = AllBase.BaseRegex.GetStringJson(ref goodStr, "product_id");
|
|
if (!string.IsNullOrEmpty(item.Id) && !lstGoods.Any(g => g.Id == item.Id))
|
|
{
|
|
item.GoodsName = AllBase.BaseRegex.GetStringJson(ref goodStr, "title")?.Trim();
|
|
if (!string.IsNullOrEmpty(item.GoodsName))
|
|
{
|
|
if (lstBanKeyText.Count == 0 || !lstBanKeyText.Any(t => item.GoodsName.Contains(t)))
|
|
{
|
|
// 销量数据
|
|
item.Sales = AllBase.BaseRegex.GetStringJson(ref goodStr, "sales");
|
|
if (string.IsNullOrEmpty(item.Sales))
|
|
item.Sales = "0";
|
|
|
|
// 销量符合筛选条件
|
|
if (Convert.ToInt32(item.Sales) >= Convert.ToInt32(this.minSaleCount) && Convert.ToInt32(item.Sales) <= Convert.ToInt32(this.maxSaleCount))
|
|
{
|
|
string price = BaseRegex.GetStringJson(ref goodStr, "min_price");
|
|
if (string.IsNullOrEmpty(price))
|
|
price = "0";
|
|
item.Price = ((Convert.ToDouble(price) * 1.0) / 100.0).ToString("f2");
|
|
|
|
if (Convert.ToDouble(item.Price) >= Convert.ToDouble(this.minPrice) && Convert.ToDouble(item.Price) <= Convert.ToDouble(this.maxPrice))
|
|
{
|
|
var img = BaseRegex.GetStringJson(ref goodStr, "url_list");
|
|
if (item.img == null)
|
|
img = BaseRegex.GetMatchRegexString(ref goodStr, "\"cover\":[^\\[\\]]+?\"url_list\":\\[([^\\[\\]]+?)\\]");
|
|
if (!string.IsNullOrEmpty(img))
|
|
{
|
|
item.img = img.Split(',')[0].Trim('"');
|
|
}
|
|
|
|
// 获取商品评价
|
|
item.CommentsCount = this.GetRate(item.Id, ref item.ShopProductCount, ref item.CommentsRate);
|
|
if (item.CommentsRate >= this.hplMin && item.CommentsRate <= hplMax)
|
|
{
|
|
//Application.DoEvents();
|
|
Thread.Sleep(1000);
|
|
string content = HaoWuCollectBLL.getHtml(
|
|
"https://ec.snssdk.com/product/fxgajaxstaticitem?b_type_new=0&device_id=0&is_outside=1&id=" + item.Id + "&preview=0",
|
|
"", "utf8");
|
|
item.ShopId = BaseRegex.GetStringJson(ref content, "shop_id");
|
|
item.ShopName = BaseRegex.GetStringJson(ref content, "shop_name");
|
|
lstGoods.Add(item);
|
|
this.ShowData(lstGoods.Count, item);
|
|
if (this.isXqtj)
|
|
{
|
|
//this.method_3(spid);
|
|
}
|
|
if (((index % 2) == 0) && (index > 0))
|
|
Thread.Sleep(1000);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
}
|
|
index++;
|
|
}
|
|
break;
|
|
}
|
|
str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 服务器打瞌睡了，重试...";
|
|
object[] objArray5 = new object[] { str };
|
|
//base.BeginInvoke(new showMessageDelegate(this.showMsg), objArray5);
|
|
//Application.DoEvents();
|
|
Thread.Sleep(3000);
|
|
num3++;
|
|
}
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
}
|
|
str = $"[{DateTime.Now.ToString("MM-dd HH:mm:ss")}] 采集结束,成功采集{lstGoods.Count}个.";
|
|
object[] args = new object[] { str };
|
|
//base.BeginInvoke(new showMessageDelegate(this.showMsg), args);
|
|
}
|
|
|
|
private int GetRate(string productId, ref int productCount, ref double rate)
|
|
{
|
|
int num = 0;
|
|
string url = "https://ec.snssdk.com/product/ajaxitem?b_type_new=0&device_id=0&is_outside=1&id=" + productId + "&abParams=0";
|
|
string content = HaoWuCollectBLL.getHtml_fxg_20201220(url, "", "");
|
|
int num2 = 0;
|
|
string ip = "";
|
|
while ((num2 < 10) && (content.Trim().Length < 10))
|
|
{
|
|
//Application.DoEvents(); // 刷新界面
|
|
Thread.Sleep(500);
|
|
num2++;
|
|
content = HaoWuCollectBLL.getHtml_fxg_20201220(url, "", ip);
|
|
}
|
|
num = BaseRegex.GetIntJson(ref content, "count");
|
|
productCount = BaseRegex.GetIntJson(ref content, "shop_product_count");
|
|
rate = (double)BaseRegex.GetDecimalJson(ref content, "value");
|
|
|
|
return num;
|
|
}
|
|
|
|
private void ShowData(int totalCount, DouGoodsCollectInfo goodsInfo)
|
|
{
|
|
try
|
|
{
|
|
StreamWriter writer = new StreamWriter(this.exportFileName, true, Encoding.GetEncoding("gb2312"));
|
|
object[] objArray1 = new object[15];
|
|
objArray1[0] = goodsInfo.GoodsName;
|
|
objArray1[1] = ",";
|
|
objArray1[2] = goodsInfo.Price;
|
|
objArray1[3] = ",";
|
|
objArray1[4] = goodsInfo.Sales;
|
|
objArray1[5] = ",";
|
|
objArray1[6] = goodsInfo.CommentsRate;
|
|
objArray1[7] = ",";
|
|
objArray1[8] = "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsInfo.Id;
|
|
objArray1[9] = ",";
|
|
objArray1[10] = goodsInfo.ShopId;
|
|
objArray1[11] = ",";
|
|
objArray1[12] = goodsInfo.ShopName;
|
|
objArray1[13] = ",";
|
|
objArray1[14] = goodsInfo.ShopProductCount;
|
|
writer.WriteLine(string.Concat(objArray1));
|
|
writer.Close();
|
|
}
|
|
catch
|
|
{
|
|
}
|
|
Invoke(new Action(() =>
|
|
{
|
|
var index = this.dgvCollect.Rows.Add(false, goodsInfo.Id, goodsInfo.GoodsName, null, goodsInfo.Price, goodsInfo.Sales, goodsInfo.CommentsRate,
|
|
goodsInfo.ShopId, goodsInfo.ShopName, goodsInfo.ShopProductCount, "删除");
|
|
if (this.isShowImg)
|
|
{
|
|
this.dgvCollect.Columns["colImage"].Visible = true;
|
|
var row = dgvCollect.Rows[index];
|
|
row.Cells["colImage"].Value = this.GetImage2(goodsInfo.img, 60);
|
|
row.Height = 70;
|
|
}
|
|
else
|
|
this.dgvCollect.Columns["colImage"].Visible = false;
|
|
}));
|
|
|
|
}
|
|
|
|
public Image GetImage2(string path, int size = 130)
|
|
{
|
|
try
|
|
{
|
|
return new Bitmap(Image.FromStream(WebRequest.Create(path).GetResponse().GetResponseStream()), size, size);
|
|
}
|
|
catch
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private void btnLoginDouyin_Click(object sender, EventArgs e)
|
|
{
|
|
DouUserLoginForm dForm = new DouUserLoginForm();
|
|
dForm.ShowDialog();
|
|
}
|
|
|
|
private void btnCollect_Click(object sender, EventArgs e)
|
|
{
|
|
if (btnCollect.Text == "开始采集")
|
|
{
|
|
var cookie = GlobalConfig.DouUserCookies;
|
|
var cate = this.cmbCategory.Text;
|
|
var cid = this.cmbCategory.SelectedIndex.ToString();
|
|
var collectPage = this.txtCollectPage.Text.Trim().ParseToInt();
|
|
|
|
minSaleCount = this.txtSaleCountMin.Text.ParseToInt().ToString();
|
|
maxSaleCount = this.txtSaleCountMax.Text.ParseToInt().ToString();
|
|
minPrice = this.txtPriceMin.Text.ParseToDecimal().ToString();
|
|
maxPrice = this.txtPriceMax.Text.ParseToDecimal().ToString();
|
|
hplMin = this.txtRateMin.Text.ParseToDouble();
|
|
hplMax = this.txtRateMax.Text.ParseToDouble();
|
|
|
|
isShowImg = this.chkShowImage.Checked;
|
|
|
|
this.dgvCollect.Rows.Clear();
|
|
this.lstProduct.Clear();
|
|
|
|
cancellationTokenSource = new CancellationTokenSource();
|
|
btnCollect.Text = "停止采集";
|
|
|
|
int n = 0;
|
|
int total = collectPage;
|
|
this.statusBar.lblTaskStatus.Text = "开始采集...";
|
|
this.statusBar.pgbTaskProgress.Value = n;
|
|
this.statusBar.pgbTaskProgress.Text = n + "/" + total;
|
|
this.statusBar.pgbTaskProgress.Maximum = total;
|
|
|
|
Task.Run(() =>
|
|
{
|
|
Collect(cookie, collectPage, cate, cid, ref lstProduct, ref n);
|
|
Invoke(new Action(() =>
|
|
{
|
|
btnCollect.Text = "开始采集";
|
|
|
|
this.statusBar.lblTaskStatus.Text = "采集结束，共采集到商品" + this.lstProduct.Count + "个";
|
|
this.statusBar.pgbTaskProgress.Value = n;
|
|
this.statusBar.pgbTaskProgress.Text = n + "/" + total;
|
|
this.statusBar.pgbTaskProgress.Maximum = total;
|
|
}));
|
|
}, cancellationTokenSource.Token);
|
|
}
|
|
else
|
|
{
|
|
cancellationTokenSource.Cancel();
|
|
}
|
|
}
|
|
|
|
private void btnExport_Click(object sender, EventArgs e)
|
|
{
|
|
try
|
|
{
|
|
SaveFileDialog saveFileDialog1 = new SaveFileDialog();
|
|
saveFileDialog1.Filter = "表格文件(*.csv)|*.csv";
|
|
saveFileDialog1.FileName = "好物采集" + DateTime.Now.ToString("yyyyMMddHHmmss") + ".csv";
|
|
saveFileDialog1.FilterIndex = 2;
|
|
saveFileDialog1.RestoreDirectory = true;
|
|
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.AppendLine("商品名称," + "价格," + "销量," + "好评率," + "商品ID," + "店铺ID," + "店铺名称," + "店铺商品总数");
|
|
foreach (var goodsInfo in lstProduct)
|
|
{
|
|
sb.AppendLine(goodsInfo.GoodsName + "," + goodsInfo.Price + "," + goodsInfo.Sales + "," + goodsInfo.CommentsRate + ","
|
|
+ "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsInfo.Id + "," + goodsInfo.ShopId + ","
|
|
+ goodsInfo.ShopName + "," + goodsInfo.ShopProductCount);
|
|
}
|
|
File.WriteAllText(saveFileDialog1.FileName, sb.ToString(), Encoding.Default);
|
|
BaseForms.BaseMsgForm.ShowInfo("导出成功");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
BaseForms.BaseMsgForm.ShowError(ex.Message);
|
|
}
|
|
}
|
|
|
|
private void dgvShop_CellContentClick(object sender, DataGridViewCellEventArgs e)
|
|
{
|
|
if (e.RowIndex < 0 || e.ColumnIndex < 0)
|
|
return;
|
|
|
|
var goodsId = dgvCollect.Rows[e.RowIndex].Cells["colProductId"].Value.ToString();
|
|
var columnName = dgvCollect.Columns[e.ColumnIndex].Name;
|
|
if (columnName == "colProductName")
|
|
{
|
|
string url = "https://haohuo.jinritemai.com/views/product/item2?id=" + goodsId;
|
|
GetHtml.OpenUrl(url);
|
|
}
|
|
}
|
|
}
|
|
}
|