Files
KJD/TEMUKJD/ToolForms/Old/KeywordsForms/KeywordsCatchFrom.cs
T

640 lines
25 KiB
C#

using AllBase;
using CefSharp;
using CefSharp.WinForms;
using TemuKJD.Helper;
using TemuKJD.BaseForms;
using TemuKJD.KJDForms.ProductForms;
using NaiBa.CefHelper.Handler;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Work.BLL.Upload;
using Work.CollectClass;
namespace TemuKJD.KJDForms.ToolForms.KeywordsForms
{
public partial class KeywordsCatchFrom : LRBaseForm
{
public Fission fission = null;
private bool IsCf = false;
private string tempUrl = "";
public delegate void delegateView();
private List<string> titleList = new List<string>(); // 用于存储标题集合
public ChromiumWebBrowser cwbSearchTitle; // 用于根据关键词采集标题
private List<string> TemptitleList = new List<string>();//用于存储标题的临时集合
public KeywordsCatchFrom()
{
InitializeComponent();
}
private void KeywordsCatchFrom_Load(object sender, EventArgs e)
{
}
private void btnCollectZc_Click(object sender, EventArgs e)
{
if (txbSearchWord.Text.Trim().Length == 0)
{
BaseMsgForm.ShowInfo("请输入关键词!", "提示");
return;
}
titleList.Clear();
DisabledCaptureTitleZc();
//TemptitleList.Clear();
string targetUrl;
//根据文本框内容判断是网址还是关键词
if (txbSearchWord.Text.Trim().Contains("/"))
{
targetUrl = txbSearchWord.Text.Trim();
}
else
{
targetUrl = "https://s.taobao.com/search?initiative_id=tbindexz_20170306&ie=utf8&spm=a21bo.2017.201856-taobao-item.2&sourceId=tb.index&search_type=item&ssid=s5-e&commend=all&imgfile=&q=" + txbSearchWord.Text.Trim() + "&suggest=history_1&_input_charset=utf-8&wq=&suggest_query=&source=suggest";
if (priceCheckBox.Checked)
{
string minPrice = startPriceTextBox.Text;
string maxPrice = endPriceTextBox.Text;
targetUrl += "&filter=reserve_price%5B" + minPrice + "%2C" + maxPrice + "%5D";
}
}
if (cwbSearchTitle != null)
{
cwbSearchTitle.Load(targetUrl);
cwbSearchTitle.Visible = true;
return;
}
cwbSearchTitle = new ChromiumWebBrowser(targetUrl) { Dock = DockStyle.Fill, Visible = true };
splitContainer1.Panel2.Controls.Add(cwbSearchTitle);
cwbSearchTitle.BringToFront();
cwbSearchTitle.AddressChanged += CwbSearchTitle_AddressChanged; //url 地址有变化 则触发 add by tl 20190227
cwbSearchTitle.FrameLoadEnd += cwbSearchTitle_FrameLoadEnd;
cwbSearchTitle.LifeSpanHandler = new LifeSpanHandlerLocal(PopWindowType.SelfWindow);
cwbSearchTitle.KeyboardHandler = new KeyboardHandler();
}
/// <summary>
/// url 有变化时 则触发 add by tl 20190228
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void CwbSearchTitle_AddressChanged(object sender, AddressChangedEventArgs e)
{
Invoke(new delegateView(DisabledCaptureTitleZc));
if (!IsCf)
{
if (e.Address.Contains("https://s.taobao.com/search/_____tmd_____/verify/?nc_token="))
{
IsCf = true;
cwbSearchTitle.Load(tempUrl);
return;
}
}
//判断url是否有变化 有变化则需要load一下,这样获取网页源码才能有变化 add by tl 20190228
if (tempUrl != e.Address)
{
tempUrl = e.Address; //临时存储url
cwbSearchTitle.Load(e.Address);
}
//点击页面排序时 清空集合 add by tl 20190228
if (e.Address.Contains("default") || e.Address.Contains("sale-desc") || e.Address.Contains("credit-desc"))
{
if (e.Address.Contains("default&") || e.Address.Contains("sale-desc&") || e.Address.Contains("credit-desc&"))
{
//TempSort = "default&";
}
else
{
titleList.Clear();
TemptitleList.Clear();
return;
}
}
}
/// <summary>
/// 禁用抓取标题 add by tl 20190304
/// </summary>
public void DisabledCaptureTitleZc()
{
btnCaptureTitleZc.Enabled = false;
}
/// <summary>
/// 界面加载完成时 add by tl 20190228
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void cwbSearchTitle_FrameLoadEnd(object sender, FrameLoadEndEventArgs e)
{
//界面加载完成则获取标题 add by tl 20190228
new Thread(GetData).Start();
}
/// <summary>
/// 获取标题 add by tl 20190228
/// </summary>
private void GetData()
{
var task = cwbSearchTitle.GetSourceAsync();
task.Wait();
string html = task.Result;
string sContent = task.Result;
if (sContent.Length == 0) return;
string strReg = "g_page_config = [^<]+";
var titleCon = BaseRegex.GetMatchRegex(ref sContent, strReg);
if (titleCon.Count == 0)
{
return;
}
int titleLength = titleLengthTextBox.Text == "" ? 0 : Convert.ToInt32(titleLengthTextBox.Text);//去除标题小于这个长度的
string content = titleCon[0].Groups[0].Value;
string regTitle = "\"raw_title\":\"([\\s\\S]+?)\"";
string regPrice = "\"view_price\":\"([\\s\\S]+?)\"";
string regSales = "\"view_sales\":\"(\\d*)人付款\"";
var priceArr = BaseRegex.GetMatchRegex(ref content, regPrice);
var titleArr = BaseRegex.GetMatchRegex(ref content, regTitle);
var salesArr = BaseRegex.GetMatchRegex(ref content, regSales);
if (titleArr.Count > 0)
{
for (int i = 0; i < titleArr.Count; i++)
{
string titleItem = titleArr[i].Value;
if (titleLengthCheckBox.Checked)//判断标题长度够不够
{
int titleItemLength = titleItem.Length;
if (titleItemLength <= titleLength)
{
continue;
}
}
if (checkBox1.Checked && salesArr.Count == titleArr.Count)
{
string sales = salesArr[i].Value;
if (BaseRegex.GetStringJson(ref sales, "view_sales") == "0人付款")
continue;
}
string title = BaseRegex.GetStringJson(ref titleItem, "raw_title");
titleList.Add(title.Replace("\"", ""));
}
};
if (titleList.Count > 0)
{
Invoke(new delegateView(EnabledCaptureTitleZc));
}
//Invoke(new delegateView(ShowView));
//cwbSearchTitle.Visible = false;
}
/// <summary>
/// 显示抓取标题 add by tl 20190304
/// </summary>
public void EnabledCaptureTitleZc()
{
btnCaptureTitleZc.Enabled = true;
}
private void btnCaptureTitleZc_Click(object sender, EventArgs e)
{
//titleList.Clear();
descriptionTextBox.Text = "";
if (cwbSearchTitle == null) return;
//将数据显示到文本框中 add by tl 20190228
Invoke(new delegateView(ShowView));
cwbSearchTitle.Visible = false;//显示控件 add by tl 20190228
}
public void ShowView()
{
descriptionTextBox.Clear();
//if (TemptitleList.Count <= 0) return;
//获取标题临时变量赋值,同时去重 add by tl 20190228
titleList = titleList.Distinct().ToList();
for (int i = 0; i < titleList.Count; i++)
{
descriptionTextBox.AppendText(titleList[i] + "\r\n");
}
}
private void keyWordLengthButtonX_Click(object sender, EventArgs e)
{
Reload();
titleList = titleList.OrderByDescending(s => s.Length).ToList();
Invoke(new delegateView(ShowView));
}
private void Reload()
{
titleList.Clear();
string[] l = descriptionTextBox.Text.Split('\n');
for (int i = 0; i < l.Length; i++)
{
if (string.IsNullOrEmpty(l[i]))
continue;
l[i] = l[i].Trim();
titleList.Add(l[i]);
}
}
private void keyWordShortButtonX_Click(object sender, EventArgs e)
{
Reload();
titleList = titleList.OrderBy(s => s.Length).ToList();
Invoke(new delegateView(ShowView));
}
private void removeEnglishButtonX_Click(object sender, EventArgs e)
{
Reload();
new Thread(RemoveEnglish).Start();
}
public void RemoveEnglish()
{
for (int i = 0; i < titleList.Count; i++)
{
string pattern = @"[A-Za-z]{3,}";
var viewMatch = BaseRegex.GetMatchRegex(titleList[i], pattern);
if (viewMatch != null && viewMatch.Count > 0)
{
for (int j = 0; j < viewMatch.Count; j++)
{
string englishstr = viewMatch[j].Groups[0].Value;
titleList[i] = titleList[i].Replace(englishstr, "").Replace(" ", "");
}
}
}
Invoke(new delegateView(ShowView));
}
private void toSpecialButtonX_Click(object sender, EventArgs e)
{
Reload();
for (int i = 0; i < titleList.Count; i++)
{
titleList[i] = ReplaceStr(titleList[i]);
}
Invoke(new delegateView(ShowView));
}
public static string ReplaceStr(string str)
{
str = str.Replace(",", "");
str = str.Replace("'", "");
str = str.Replace(";", "");
str = str.Replace(":", "");
str = str.Replace("/", "");
str = str.Replace("?", "");
str = str.Replace("<", "");
str = str.Replace(">", "");
str = str.Replace(".", "");
str = str.Replace("#", "");
str = str.Replace("%", "");
str = str.Replace("%", "");
str = str.Replace("^", "");
str = str.Replace("//", "");
str = str.Replace("@", "");
str = str.Replace("(", "");
str = str.Replace(")", "");
str = str.Replace("*", "");
str = str.Replace("~", "");
str = str.Replace("`", "");
str = str.Replace("$", "");
str = str.Replace(" ", "");
return str;
}
private void IrregularitiesButtonX_Click(object sender, EventArgs e)
{
Reload();
for (int i = 0; i < titleList.Count; i++)
{
for (int j = 0; j < UploadProductBLL.FilterTitleWordList.Count; j++)
{
if (titleList[i].Contains(UploadProductBLL.FilterTitleWordList[j]))
titleList[i] = titleList[i].Replace(UploadProductBLL.FilterTitleWordList[j], "");
}
for (int j = 0; j < UploadProductBLL.FilterRemoveTitleWordList.Count; j++)
{
if (titleList[i].Contains(UploadProductBLL.FilterRemoveTitleWordList[j]))
titleList[i] = titleList[i].Replace(UploadProductBLL.FilterRemoveTitleWordList[j], "");
}
for (int j = 0; j < UploadProductBLL.FilterBrandWordList.Count; j++)
{
if (titleList[i].Contains(UploadProductBLL.FilterBrandWordList[j]))
titleList[i] = titleList[i].Replace(UploadProductBLL.FilterBrandWordList[j], "");
}
}
Invoke(new delegateView(ShowView));
}
private void remDataButtonX_Click(object sender, EventArgs e)
{
Reload();
for (int i = 0; i < titleList.Count; i++)
{
string pattern = @"[\d]+";
var viewMatch = BaseRegex.GetMatchRegex(titleList[i], pattern);
if (viewMatch != null && viewMatch.Count > 0)
{
for (int j = 0; j < viewMatch.Count; j++)
{
string englishstr = viewMatch[j].Groups[0].Value;
titleList[i] = titleList[i].Replace(englishstr, "").Replace(" ", "");
}
}
}
Invoke(new delegateView(ShowView));
}
private void toRepeatButtonX_Click(object sender, EventArgs e)
{
Reload();
titleList = titleList.Distinct().ToList();
Invoke(new delegateView(ShowView));
}
Dictionary<string, int> keys = new Dictionary<string, int>();
private void participleButtonX_Click(object sender, EventArgs e)
{
Reload();
keys.Clear();
dataGridViewX1.Rows.Clear();
for (int i = 0; i < titleList.Count; i++)
{
GetFenCi(titleList[i]);
}
var keylist = keys.ToList();
keylist = keylist.OrderByDescending(u => u.Value).ToList();
for (int i = 0; i < keylist.Count; i++)
{
dataGridViewX1.Rows.Add(dataGridViewX1.Rows.Count + 1, keylist[i].Key, keylist[i].Value, "删除");
}
}
public static Dictionary<string, int> Keywords = new Dictionary<string, int>();
public void GetFenCi(string title)
{
for (int i = 2; i < title.Length; i++)
{
for (int j = 0; j < title.Length - i; j++)
{
string key = title.Substring(j, i).Trim();
if (string.IsNullOrEmpty(key))
continue;
if (Keywords.ContainsKey(key))
{
if (!keys.ContainsKey(key))
{
keys.Add(key, 1);
}
else
{
keys[key] = keys[key] + 1;
}
}
}
}
}
private void wordReplaceButtonX_Click(object sender, EventArgs e)
{
string oldWord = this.oldWordTextBox.Text;
string newWord = this.newWordTextBox.Text;
if (string.IsNullOrEmpty(oldWord)) // 替换的字符不能为空 否则程序报错 add by tl20190301
{
BaseMsgForm.ShowInfo("需要替换的字符不能为空!");
return;
}
descriptionTextBox.Text = descriptionTextBox.Text.Replace(oldWord, newWord);
}
private void exportButtonX_Click(object sender, EventArgs e)
{
SaveFileDialog dialog = new SaveFileDialog();
dialog.Title = "Save file as...";
dialog.Filter = "Excel files (*.csv)|*.csv|All files (*.*)|*.*";
dialog.RestoreDirectory = true;
dialog.FileName = "淘宝关键词:" + DateTime.Now.ToString("yyyy-MM-dd") + ".csv";
if (dialog.ShowDialog() != DialogResult.OK)
return;
OutExcel.SaveFile(dialog.FileName, dataGridViewX1);
}
private void btnPicToMain_Click(object sender, EventArgs e)
{
if (cwbSearchTitle == null || !cwbSearchTitle.IsBrowserInitialized)
{
BaseMsgForm.ShowInfo("网页加载失败!");
return;
}
// 详情页转主图
if (cwbSearchTitle.Address.Contains("tmall.com/item.htm?") || cwbSearchTitle.Address.Contains("taobao.com/item.htm?"))
{
//9787030687623
var jsStr = "function getImgUrl() {" +
" var imgStr = '';" +
" var mainPicControls = document.getElementsByClassName('tb-thumb tb-clearfix');" +
" if (mainPicControls.length === 0) {" +
" var uls = document.getElementsByTagName('ul');" +
" for (i = 0; i < uls.length; i++) {" +
" if (uls[i].className.indexOf('PicGallery--thumbnails') >= 0) {" +
" imgStr = uls[i].innerHTML;" +
" break;" +
" }" +
" }" +
" }" +
" else {" +
" imgStr = mainPicControls[0].innerHTML;" +
" }" +
" return imgStr;" +
"} " +
"getImgUrl();";
var t1 = cwbSearchTitle.GetMainFrame().EvaluateScriptAsync(jsStr);
t1.Wait();
var html = t1.Result?.Result?.ToString();
if (!string.IsNullOrEmpty(html))
{
var res = BaseRegex.GetMatchRegex(ref html, " src=\"([\\s\\S]+?)\"");
var htmlTempArr = res.Cast<Match>().Select(m => m.ToString()).ToArray();
htmlTempArr = htmlTempArr.Where(s => !s.EndsWith("/s.gif")).ToArray();
var htmlList = new List<string>();
foreach (var str in htmlTempArr)
{
var strTemp = str.Trim().Replace("src=", "").Trim('\"').Replace(".jpg_230x230", "").Replace("_.webp", "");
if (strTemp.Contains(".jpg_"))
strTemp = strTemp.Substring(0, strTemp.IndexOf(".jpg_")) + ".jpg";
if (!strTemp.StartsWith("http"))
strTemp = "https://" + strTemp.Trim('/');
htmlList.Add(strTemp);
}
Task.Run(() =>
{
if (fission != null)
fission.MulitAddMainImageWithCatchWords(htmlList);
Invoke(new Action(() => { BaseMsgForm.ShowInfo("图片转主完成!"); }));
});
}
else
BaseMsgForm.ShowInfo("图片转主失败!");
return;
//var address = cwbSearchTitle.Address;
//var id = BaseRegex.GetMatchRegexString(ref address, "id=(\\d{5,15})");
//if (!string.IsNullOrEmpty(id))
//{
// Task.Run(() =>
// {
// ProductCollectBase pcb = new ProductCollectBase
// {
// from_url = "https://item.taobao.com/item.htm?id=" + id,
// from_id = Convert.ToInt64(id),
// };
// TaobaoMobileCollect tm = new TaobaoMobileCollect(pcb);
// tm.Get();
// if (tm.Product != null && tm.Product.Images?.Count > 0 == true)
// {
// var htmlList = new List<string>();
// foreach (var str in tm.Product.Images)
// {
// htmlList.Add(str.Replace(".jpg_230x230", "").Replace("_.webp", ""));
// }
// if (fission != null)
// {
// fission.MulitAddMainImageWithCatchWords(htmlList);
// Invoke(new Action(() => { BaseMsgForm.ShowInfo("图片转主完成!"); }));
// }
// }
// });
//}
}
// 搜索页转主图
else
{
var jsStr = "function getImgUrl() {" +
" var imgs = document.getElementsByClassName('J_ItemPic img'); " +
" var result = '';" +
" for(i = 0; i < imgs.length; i++) {" +
" result += imgs[i].src + ';';" +
" }" +
" return result;" +
"} getImgUrl();";
var t1 = cwbSearchTitle.GetMainFrame().EvaluateScriptAsync(jsStr);
t1.Wait();
var html = t1.Result?.Result?.ToString();
if (!string.IsNullOrEmpty(html))
{
var htmlTempArr = html.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
htmlTempArr = htmlTempArr.Where(s => !s.EndsWith("/s.gif")).ToArray();
var htmlList = new List<string>();
foreach (var str in htmlTempArr)
{
htmlList.Add(str.Replace(".jpg_230x230", "").Replace("_.webp", ""));
}
Task.Run(() =>
{
if (fission != null)
fission.MulitAddMainImageWithCatchWords(htmlList);
BaseMsgForm.ShowInfo("图片转主完成!");
});
}
else
BaseMsgForm.ShowInfo("图片转主失败!");
}
}
private void btnPicToContent_Click(object sender, EventArgs e)
{
//descV8-singleImage-image lazyload
var jsStr = "function getImgUrl() {" +
" var imgs = document.getElementsByClassName('lazyload'); " +
" var result = '';" +
" for(i = 0; i < imgs.length; i++) {" +
" result += imgs[i].src + ';';" +
" }" +
" return result;" +
"} getImgUrl();";
var t1 = cwbSearchTitle.GetMainFrame().EvaluateScriptAsync(jsStr);
t1.Wait();
var html = t1.Result?.Result?.ToString();
if (string.IsNullOrEmpty(html))
{
jsStr = "function getDescription() {" +
" var descriptions = document.getElementById('description'); " +
" var imgs = descriptions.getElementsByTagName('img'); " +
" var result = '';" +
" for(i = 0; i < imgs.length; i++) {" +
" result += imgs[i].src + ';';" +
" }" +
" return result;" +
"} getDescription();";
}
t1 = cwbSearchTitle.GetMainFrame().EvaluateScriptAsync(jsStr);
t1.Wait();
html = t1.Result?.Result?.ToString();
if (!string.IsNullOrEmpty(html))
{
var htmlTempArr = html.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
htmlTempArr = htmlTempArr.Where(s => !s.EndsWith("spaceball.gif") && !s.EndsWith("790-300.png")).ToArray();
var htmlList = new List<string>();
foreach (var str in htmlTempArr)
{
htmlList.Add(str.Replace(".jpg_230x230", "").Replace("_.webp", ""));
}
Task.Run(() =>
{
if (fission != null)
fission.MulitAddContentImageWithCatchWords(htmlList);
BaseMsgForm.ShowInfo("转内容图完成!");
});
}
else
BaseMsgForm.ShowInfo("转内容图失败!");
}
private void btnChangeFissionTitle_Click(object sender, EventArgs e)
{
if (fission != null)
fission.ChangeFissionTitle(descriptionTextBox.Text.Trim().Replace("\r\n", ";").Replace("\n", ";"));
BaseMsgForm.ShowInfo("替换完成!");
}
}
}