微信app采集
This commit is contained in:
1 parent
52220daa8b
commit
2f61dc1cdd
67 files changed
+6197
-1118
No files matched your search
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using KJD_Web.Upload.Prod;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Drawing;
|
||||
@@ -6,7 +7,10 @@ using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Cache;
|
||||
using System.Net.Security;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
@@ -72,6 +76,128 @@ namespace KJD_Web.Utils
|
||||
return decoder.GetString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// URL UTF-8转码
|
||||
/// </summary>
|
||||
/// <param name="IndexKey"></param>
|
||||
/// <returns></returns>
|
||||
public static string UrlEncodeUtf8(string IndexKey)
|
||||
{
|
||||
if (String.IsNullOrEmpty(IndexKey))
|
||||
return "";
|
||||
return HttpUtility.UrlEncode(IndexKey, Encoding.UTF8);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// post 数据
|
||||
/// </summary>
|
||||
/// <param name="url"></param>
|
||||
/// <param name="postData"></param>
|
||||
/// <returns></returns>
|
||||
public static string PostUtf8(string url, StringBuilder postData)
|
||||
{
|
||||
try
|
||||
{
|
||||
WebClientx wc = new WebClientx();
|
||||
byte[] sendData = Encoding.GetEncoding("utf-8").GetBytes(postData.ToString());
|
||||
wc.Headers.Add("ContentLength", sendData.Length.ToString());
|
||||
wc.Headers.Add("Content-Type", "application/x-www-form-urlencoded");
|
||||
wc.Proxy = null;
|
||||
return Encoding.UTF8.GetString(wc.UploadData(url, "POST", sendData));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 浏览器标识
|
||||
/// </summary>
|
||||
public static string UserAgent = @"Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+6.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/54.0.2840.99+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/31.0.1650.63+Safari/537.36
|
||||
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+Win64;+x64;+Trident/5.0)
|
||||
Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+6.3;+WOW64;+Trident/7.0;+rv:11.0)+like+Gecko
|
||||
Mozilla/5.0+(Windows+NT+10.0;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||||
Mozilla/5.0+(compatible;+MSIE+9.0;+Windows+NT+6.1;+WOW64;+Trident/5.0)
|
||||
Mozilla/5.0+(Windows+NT+6.3;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/57.0.2987.133+Safari/537.36
|
||||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1
|
||||
Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/45.0.2454.101+Safari/537.36
|
||||
Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2372.400 QQBrowser/9.5.10548.400";
|
||||
|
||||
/// <summary>
|
||||
/// post 数据
|
||||
/// </summary>
|
||||
/// <param name="url"></param>
|
||||
/// <param name="postData"></param>
|
||||
/// <returns></returns>
|
||||
public static string PostUtf8(string url, string postData, string contentType = "")
|
||||
{
|
||||
//return PostUtf8(url, Encoding.UTF8.GetBytes(postData));
|
||||
|
||||
bool isError = false;
|
||||
string[] l = UserAgent.Split('\n');
|
||||
reTry:
|
||||
try
|
||||
{
|
||||
string codeType = "utf-8";
|
||||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
|
||||
request.UserAgent = l[new Random(Guid.NewGuid().GetHashCode()).Next(l.Length)].Trim();
|
||||
byte[] byteData = Encoding.GetEncoding(codeType).GetBytes(postData);
|
||||
request.Method = "POST";
|
||||
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
||||
request.Timeout = 30 * 1000;
|
||||
request.ReadWriteTimeout = 30 * 1000;
|
||||
request.Proxy = null;
|
||||
request.ContentType = "application/x-www-form-urlencoded; charset=" + codeType;
|
||||
//if (BaseClass.SaLevel > 0)
|
||||
{
|
||||
//260306 采用GZip/deflate压缩
|
||||
//request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br, zstd");
|
||||
//request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
|
||||
}
|
||||
if (url.StartsWith("https://openapi.jinritemai.com/"))
|
||||
{
|
||||
request.Headers.Add("X-USE-BOE", "1");
|
||||
request.Headers.Add("Origin-From", "djt_prod");
|
||||
request.Headers.Add("djt_prod", "boe_time_sku");
|
||||
}
|
||||
if (!string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
request.ContentType = contentType;
|
||||
}
|
||||
request.ContentLength = byteData.Length;
|
||||
request.Referer = url;
|
||||
request.CookieContainer = new CookieContainer();
|
||||
request.AllowAutoRedirect = false;
|
||||
using (Stream requestStream = request.GetRequestStream())
|
||||
{
|
||||
requestStream.Write(byteData, 0, byteData.Length);
|
||||
}
|
||||
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
|
||||
|
||||
Stream receiveStream = response.GetResponseStream();
|
||||
StreamReader readStream = new StreamReader(receiveStream, Encoding.GetEncoding(codeType));
|
||||
|
||||
return readStream.ReadToEnd().Trim();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
//{
|
||||
// isError = true;
|
||||
// goto reTry;
|
||||
//}
|
||||
Thread.Sleep(50);
|
||||
return "";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static int HexToInt(char h)
|
||||
{
|
||||
if ((h >= '0') && (h <= '9'))
|
||||
@@ -150,4 +276,141 @@ namespace KJD_Web.Utils
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 下载器
|
||||
/// </summary>
|
||||
public class WebClientx : WebClient
|
||||
{
|
||||
/// <summary>
|
||||
/// 默认超时多少秒
|
||||
/// </summary>
|
||||
public int Timeout = 100;
|
||||
/// <summary>
|
||||
/// 允许自动重定向
|
||||
/// </summary>
|
||||
public bool AllowAutoRedirect = true;
|
||||
public bool IsDelete = false;
|
||||
|
||||
private string OutIP = "";
|
||||
/// <summary>
|
||||
/// 证书列表
|
||||
/// </summary>
|
||||
public static List<Certificate> X509List = new List<Certificate>();
|
||||
/// <summary>
|
||||
/// 绑定IP
|
||||
/// </summary>
|
||||
/// <param name="outIp"></param>
|
||||
public WebClientx(string outIp)
|
||||
{
|
||||
OutIP = outIp;
|
||||
CachePolicy = new RequestCachePolicy(RequestCacheLevel.NoCacheNoStore);
|
||||
}
|
||||
|
||||
public WebClientx()
|
||||
{
|
||||
}
|
||||
/// <summary>
|
||||
/// 重写
|
||||
/// </summary>
|
||||
/// <param name="address"></param>
|
||||
/// <returns></returns>
|
||||
protected override WebRequest GetWebRequest(Uri address)
|
||||
{
|
||||
string s = address.ToString();
|
||||
if (string.IsNullOrEmpty(s) || s.Length < 7 || !(s.Substring(0, 7).ToLower() == "http://" || s.Substring(0, 8).ToLower() == "https://"))
|
||||
return null;
|
||||
var request = (HttpWebRequest)base.GetWebRequest(address);
|
||||
request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
|
||||
request.Timeout = 1000 * Timeout;
|
||||
request.ReadWriteTimeout = 500 * Timeout;
|
||||
request.AllowAutoRedirect = AllowAutoRedirect;
|
||||
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(ValidateServerCertificate);
|
||||
if (IsDelete)
|
||||
request.Method = "DELETE";
|
||||
if (!string.IsNullOrEmpty(OutIP))
|
||||
{
|
||||
request.ServicePoint.BindIPEndPointDelegate = (servicePoint, remoteEndPoint, retryCount) =>
|
||||
{
|
||||
servicePoint.MaxIdleTime = 20;
|
||||
return new IPEndPoint(IPAddress.Parse(OutIP), 0);
|
||||
};
|
||||
}
|
||||
if (request.RequestUri.Scheme == "https") //挂接验证服务端证书的回调
|
||||
{
|
||||
X509Certificate2 x509c = null;
|
||||
for (int i = 0; i < X509List.Count; i++)
|
||||
{
|
||||
if (X509List[i].Domain.ToLower() == address.Host.ToLower())
|
||||
{
|
||||
x509c = X509List[i].X509;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (x509c == null)
|
||||
{
|
||||
ServicePointManager.ServerCertificateValidationCallback =
|
||||
new RemoteCertificateValidationCallback(RemoteCertificateValidationCallback);
|
||||
try
|
||||
{
|
||||
X509Store _store = new X509Store(StoreName.My, StoreLocation.LocalMachine);
|
||||
_store.Open(OpenFlags.ReadWrite);
|
||||
string findValue = Dns.GetHostName(); //获取本地主机名称作为证书查找的参数
|
||||
X509Certificate2Collection _certsCollection =
|
||||
_store.Certificates.Find(X509FindType.FindByIssuerName, findValue, false);
|
||||
if (_certsCollection.Count > 0)
|
||||
{
|
||||
x509c = _certsCollection[0];
|
||||
request.ClientCertificates.Add(x509c);
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
|
||||
}
|
||||
}
|
||||
return request;
|
||||
}
|
||||
/// <summary>
|
||||
/// 获取证书
|
||||
/// </summary>
|
||||
/// <param name="sender"></param>
|
||||
/// <param name="certificate"></param>
|
||||
/// <param name="chain"></param>
|
||||
/// <param name="sslPolicyErrors"></param>
|
||||
/// <returns></returns>
|
||||
public bool RemoteCertificateValidationCallback(Object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
public static bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
//if (sslPolicyErrors == SslPolicyErrors.None) {
|
||||
// return true;
|
||||
//}
|
||||
|
||||
//Console.WriteLine("Certificate error: {0}", sslPolicyErrors);
|
||||
|
||||
//// Do not allow this client to communicate with unauthenticated servers.
|
||||
//return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 网站证书
|
||||
/// </summary>
|
||||
public class Certificate
|
||||
{
|
||||
/// <summary>
|
||||
/// 域名
|
||||
/// </summary>
|
||||
public string Domain { get; set; }
|
||||
/// <summary>
|
||||
/// 证书
|
||||
/// </summary>
|
||||
public X509Certificate2 X509 { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
@@ -31,7 +32,12 @@ namespace KJD_Web.Utils
|
||||
}
|
||||
}
|
||||
|
||||
public static string SerializerObjectToString(object obj, JsonSerializerSettings jsonsetting = null)
|
||||
public static string SerializeObject(object obj)
|
||||
{
|
||||
return SerializeObjectToString(obj);
|
||||
}
|
||||
|
||||
public static string SerializeObjectToString(object obj, JsonSerializerSettings jsonsetting = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -47,5 +53,78 @@ namespace KJD_Web.Utils
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static T DeserializeJsonToObjectNullable<T>(string json) where T : class
|
||||
{
|
||||
try
|
||||
{
|
||||
JsonSerializer serializer = new JsonSerializer();
|
||||
StringReader sr = new StringReader(json);
|
||||
object o = serializer.Deserialize(new JsonTextReader(sr), typeof(T));
|
||||
T t = o as T;
|
||||
return t;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static JObject JObjectParse(string json)
|
||||
{
|
||||
try
|
||||
{
|
||||
JObject obj = JObject.Parse(json);
|
||||
return obj;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 解析JSON数组生成对象实体集合
|
||||
/// </summary>
|
||||
/// <typeparam name="T">对象类型</typeparam>
|
||||
/// <param name="json">json数组字符串(eg.[{"ID":"112","Name":"石子儿"}])</param>
|
||||
/// <returns>对象实体集合</returns>
|
||||
public static List<T> DeserializeJsonToList<T>(string json) where T : class
|
||||
{
|
||||
JsonSerializer serializer = new JsonSerializer();
|
||||
StringReader sr = new StringReader(json);
|
||||
object o = serializer.Deserialize(new JsonTextReader(sr), typeof(List<T>));
|
||||
List<T> list = o as List<T>;
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<T> DeserializeJsonToListNullable<T>(string json) where T : class
|
||||
{
|
||||
try
|
||||
{
|
||||
JsonSerializer serializer = new JsonSerializer();
|
||||
StringReader sr = new StringReader(json);
|
||||
object o = serializer.Deserialize(new JsonTextReader(sr), typeof(List<T>));
|
||||
List<T> list = o as List<T>;
|
||||
return list;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 反序列化JSON到给定的匿名对象.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">匿名对象类型</typeparam>
|
||||
/// <param name="json">json字符串</param>
|
||||
/// <param name="anonymousTypeObject">匿名对象</param>
|
||||
/// <returns>匿名对象</returns>
|
||||
public static T DeserializeAnonymousType<T>(string json, T anonymousTypeObject)
|
||||
{
|
||||
T t = JsonConvert.DeserializeAnonymousType(json, anonymousTypeObject);
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -664,5 +664,100 @@ namespace KJD_Web.Utils
|
||||
|
||||
return htmlStr.Trim();
|
||||
}
|
||||
|
||||
public static string GetOneJsonObjStrJson(ref string html, string reg)
|
||||
{
|
||||
return GetJsonObjStrJson(ref html, reg).FirstOrDefault();
|
||||
}
|
||||
|
||||
public static List<string> GetJsonObjStrJson(ref string html, string reg, int level = 0)
|
||||
{
|
||||
List<string> rstLst = new List<string>();
|
||||
if (html == null)
|
||||
return rstLst;
|
||||
|
||||
string htmlTmp = html.Replace("\\{", "\\这里是正花括号占位符").Replace("\\}", "\\这里是反花括号占位符").Replace("\"" + reg + "\": {", "\"" + reg + "\":{");
|
||||
|
||||
retry:
|
||||
string lvStr = "";
|
||||
if (level == 1)
|
||||
{
|
||||
lvStr += "\"";
|
||||
}
|
||||
var index1 = htmlTmp.IndexOf("\"" + reg + "\":" + lvStr + "{");
|
||||
if (index1 < 0)
|
||||
return rstLst;
|
||||
|
||||
htmlTmp = htmlTmp.Substring(index1 + ("\"" + reg + "\":" + lvStr + "{").Length - 1);
|
||||
// 括号计数
|
||||
int braceNum = 0;
|
||||
//string rst = "{";
|
||||
//while (htmlTmp.Length > 0)
|
||||
for (int i = 0; i < htmlTmp.Length; i++)
|
||||
{
|
||||
//rst += htmlTmp.Substring(0, 1);
|
||||
//if (htmlTmp.StartsWith("{"))
|
||||
// braceNum++;
|
||||
//else if (htmlTmp.StartsWith("}"))
|
||||
// braceNum--;
|
||||
if (htmlTmp[i] == '{')
|
||||
braceNum++;
|
||||
else if (htmlTmp[i] == '}')
|
||||
braceNum--;
|
||||
//htmlTmp = htmlTmp.Substring(1);
|
||||
if (braceNum == 0)
|
||||
{
|
||||
rstLst.Add(htmlTmp.Substring(0, i + 1).Replace("\\这里是正花括号占位符", "\\{").Replace("\\这里是反花括号占位符", "\\}"));
|
||||
htmlTmp = htmlTmp.Substring(i + 1);
|
||||
//rstLst.Add(rst.Replace("\\这里是正花括号占位符", "\\{").Replace("\\这里是反花括号占位符", "\\}"));
|
||||
break;
|
||||
}
|
||||
|
||||
// 循环完了还没结果返回空列表
|
||||
if (i == htmlTmp.Length - 1)
|
||||
return rstLst;
|
||||
}
|
||||
if (htmlTmp.Length > 0)
|
||||
goto retry;
|
||||
|
||||
return rstLst;
|
||||
}
|
||||
|
||||
public static List<string> GetJsonArrStrJson(ref string html, string reg)
|
||||
{
|
||||
List<string> rstLst = new List<string>();
|
||||
if (html == null)
|
||||
return rstLst;
|
||||
|
||||
string htmlTmp = html.Replace("\\[", "\\这里是正方括号占位符").Replace("\\]", "\\这里是反方括号占位符").Replace("\"" + reg + "\": [", "\"" + reg + "\":[");
|
||||
|
||||
retry:
|
||||
var index1 = htmlTmp.IndexOf("\"" + reg + "\":[");
|
||||
if (index1 < 0)
|
||||
return rstLst;
|
||||
|
||||
htmlTmp = htmlTmp.Substring(index1 + ("\"" + reg + "\":[").Length);
|
||||
// 括号计数
|
||||
int braceNum = 1;
|
||||
string rst = "[";
|
||||
while (htmlTmp.Length > 0)
|
||||
{
|
||||
rst += htmlTmp.Substring(0, 1);
|
||||
if (htmlTmp.StartsWith("["))
|
||||
braceNum++;
|
||||
else if (htmlTmp.StartsWith("]"))
|
||||
braceNum--;
|
||||
htmlTmp = htmlTmp.Substring(1);
|
||||
if (braceNum == 0)
|
||||
{
|
||||
rstLst.Add(rst.Replace("\\这里是正方括号占位符", "\\[").Replace("\\这里是反方括号占位符", "\\]"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (htmlTmp.Length > 0)
|
||||
goto retry;
|
||||
|
||||
return rstLst;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user