1109 lines
40 KiB
C#
1109 lines
40 KiB
C#
using AllBase.Extension;
|
||
using NaiBa.Util.Extension;
|
||
using Newtonsoft.Json;
|
||
using Newtonsoft.Json.Linq;
|
||
using System;
|
||
using System.Collections;
|
||
using System.Collections.Generic;
|
||
using System.ComponentModel;
|
||
using System.Diagnostics;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Linq;
|
||
using System.Net;
|
||
using System.Net.Security;
|
||
using System.Security.Cryptography.X509Certificates;
|
||
using System.Text;
|
||
using System.Text.RegularExpressions;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
|
||
namespace AllBase.Helper
|
||
{
|
||
public class HttpHelper
|
||
{
|
||
private const int RequestTimeout = 60 * 1000;
|
||
private const int DownloadTimeout = 300 * 1000;
|
||
|
||
static HttpHelper()
|
||
{
|
||
InitHttpsCertificate();
|
||
}
|
||
|
||
public static string UrlDecode(string url)
|
||
{
|
||
if (string.IsNullOrEmpty(url))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
else
|
||
{
|
||
return HttpUtility.UrlDecode(url);
|
||
}
|
||
}
|
||
|
||
public static string UrlDecode(string url, Encoding encoding)
|
||
{
|
||
if (string.IsNullOrEmpty(url))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
else
|
||
{
|
||
return System.Web.HttpUtility.UrlDecode(url, encoding);
|
||
}
|
||
}
|
||
|
||
public static string UrlEncode(string url, bool isUppercase = false)
|
||
{
|
||
if (string.IsNullOrEmpty(url))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
else
|
||
{
|
||
string encodeUrl = HttpUtility.UrlEncode(url, Encoding.UTF8);
|
||
if (isUppercase)
|
||
{
|
||
Regex reg = new Regex(@"%[a-f0-9]{2}");
|
||
encodeUrl = reg.Replace(encodeUrl, m => m.Value.ToUpperInvariant());
|
||
}
|
||
return encodeUrl;
|
||
}
|
||
}
|
||
|
||
public static string UrlEncode(string url, Encoding encoding)
|
||
{
|
||
if (string.IsNullOrEmpty(url))
|
||
{
|
||
return string.Empty;
|
||
}
|
||
else
|
||
{
|
||
string encodeUrl = HttpUtility.UrlEncode(url, encoding);
|
||
return encodeUrl;
|
||
}
|
||
}
|
||
|
||
public static string UnicodeToASCII(string unicode)
|
||
{
|
||
string result = string.Empty;
|
||
if (!string.IsNullOrEmpty(unicode))
|
||
{
|
||
Encoding utf8 = Encoding.UTF8;
|
||
Encoding ascii = Encoding.ASCII;
|
||
|
||
string input = unicode;
|
||
result = ascii.GetString(Encoding.Convert(utf8, ascii, utf8.GetBytes(input)));
|
||
result = result.Trim('?');
|
||
}
|
||
return result;
|
||
}
|
||
|
||
public static string UnicodeToString(string html)
|
||
{
|
||
return new Regex(@"\\u([0-9A-F]{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled).Replace(html, x => string.Empty + Convert.ToChar(Convert.ToUInt16(x.Result("$1"), 16)));
|
||
}
|
||
|
||
public static string StringToUnicode(string html)
|
||
{
|
||
byte[] bytes = Encoding.Unicode.GetBytes(html);
|
||
StringBuilder builder = new StringBuilder();
|
||
for (int i = 0; i < bytes.Length; i += 2)
|
||
{
|
||
builder.AppendFormat("\\u{0}{1}", bytes[i + 1].ToString("x").PadLeft(2, '0'), bytes[i].ToString("x").PadLeft(2, '0'));
|
||
}
|
||
return builder.ToString();
|
||
}
|
||
|
||
public static string FilterCookie(string url, string cookieListJson)
|
||
{
|
||
var uri = new Uri(url);
|
||
var domainList = new List<string>();
|
||
string domain = uri.Host;
|
||
int index = 0;
|
||
while (true)
|
||
{
|
||
domainList.Add(domain);
|
||
index = domain.IndexOf('.', 0);
|
||
if (index > 0)
|
||
{
|
||
domain = domain.Substring(index + 1);
|
||
}
|
||
else
|
||
{
|
||
break;
|
||
}
|
||
if (domain == "com" || domain == "cn" || domain == "com.cn")
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
string cookie = string.Empty;
|
||
JArray jArr = JArray.Parse(cookieListJson);
|
||
foreach (var item in jArr)
|
||
{
|
||
string name = item["Name"].ParseToString();
|
||
string value = item["Value"].ParseToString();
|
||
domain = item["Domain"].ParseToString();
|
||
if (!domainList.Contains(domain.Trim('.')))
|
||
{
|
||
continue;
|
||
}
|
||
cookie += $"{name}={value};";
|
||
}
|
||
return cookie;
|
||
}
|
||
|
||
public static string ParseCookieJsonToString(string cookieListJson)
|
||
{
|
||
string cookie = string.Empty;
|
||
JArray jArr = JArray.Parse(cookieListJson);
|
||
foreach (var item in jArr)
|
||
{
|
||
string name = item["Name"].ParseToString();
|
||
string value = item["Value"].ParseToString();
|
||
cookie += $"{name}={value};";
|
||
}
|
||
return cookie;
|
||
}
|
||
|
||
public static Dictionary<string, string> ParseCookieJsonToDictionary(string cookieListJson)
|
||
{
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
JArray jArr = JArray.Parse(cookieListJson);
|
||
foreach (var item in jArr)
|
||
{
|
||
string name = item["Name"].ParseToString();
|
||
string value = item["Value"].ParseToString();
|
||
if (!dict.ContainsKey(name))
|
||
{
|
||
dict.Add(name, value);
|
||
}
|
||
}
|
||
return dict;
|
||
}
|
||
|
||
public static Dictionary<string, string> ParseCookieStringToDictionary(string cookie)
|
||
{
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
var cookieArr = cookie.SplitToArray<string>(";");
|
||
foreach (string line in cookieArr)
|
||
{
|
||
int index = line.IndexOf('=');
|
||
if (index > 0)
|
||
{
|
||
string key = line.Substring(0, index).Trim();
|
||
string value = line.Substring(index + 1);
|
||
if (!dict.ContainsKey(key))
|
||
{
|
||
dict.Add(key, value);
|
||
}
|
||
}
|
||
}
|
||
return dict;
|
||
}
|
||
|
||
public static void AddHeader(HttpParam httpParam, string key, string value)
|
||
{
|
||
if (httpParam.Header == null)
|
||
{
|
||
httpParam.Header = new Dictionary<string, string>();
|
||
}
|
||
httpParam.Header.Remove(key);
|
||
httpParam.Header.Add(key, value);
|
||
}
|
||
|
||
public static string AddHttps(string url)
|
||
{
|
||
if (url.StartsWith("//"))
|
||
{
|
||
url = "https:" + url;
|
||
}
|
||
return url;
|
||
}
|
||
|
||
#region 是否是网址
|
||
public static bool IsUrl(string url)
|
||
{
|
||
url = url.ParseToString().ToLower();
|
||
if (url.StartsWith("http://") || url.StartsWith("https://"))
|
||
{
|
||
return true;
|
||
}
|
||
else
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
/// <summary>
|
||
/// 支持解析的url
|
||
/// https://shopee.co.th/--i.141607853.2809189585 or a=1&b=2
|
||
/// </summary>
|
||
/// <param name="url"></param>
|
||
/// <returns></returns>
|
||
public static Dictionary<string, string> ResolveUrlParamToDictionary(string url)
|
||
{
|
||
if (string.IsNullOrEmpty(url))
|
||
{
|
||
return new Dictionary<string, string>();
|
||
}
|
||
Dictionary<string, string> dict = new Dictionary<string, string>();
|
||
string partOne = string.Empty;
|
||
string partTwo = string.Empty;
|
||
|
||
int questionMark = url.IndexOf('?');
|
||
if (questionMark > 0)
|
||
{
|
||
partOne = url.Substring(0, questionMark);
|
||
partTwo = url.Substring(questionMark + 1);
|
||
}
|
||
else
|
||
{
|
||
int slashMark = url.IndexOf('/');
|
||
int firstEqualSign = url.IndexOf('=');
|
||
if (firstEqualSign == -1)
|
||
{
|
||
partOne = url;
|
||
}
|
||
else
|
||
{
|
||
if (firstEqualSign > slashMark)
|
||
{
|
||
partTwo = url;
|
||
}
|
||
else
|
||
{
|
||
if (slashMark > 0)
|
||
{
|
||
partOne = url.Substring(0, slashMark);
|
||
partTwo = url.Substring(slashMark + 1);
|
||
}
|
||
else
|
||
{
|
||
partOne = url;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
dict.Add("[url]", partOne);
|
||
if (!string.IsNullOrEmpty(partTwo))
|
||
{
|
||
string[] queryParams = partTwo.Split('&').OrderBy(p => p).ToArray();
|
||
foreach (string query in queryParams)
|
||
{
|
||
if (query.Contains("="))
|
||
{
|
||
string[] single = query.Split('=');
|
||
if (!dict.Keys.Contains(single[0]))
|
||
{
|
||
dict.Add(single[0], HttpHelper.UrlDecode(single[1]));
|
||
}
|
||
}
|
||
else
|
||
{
|
||
dict.Add(query, string.Empty);
|
||
}
|
||
}
|
||
}
|
||
return dict;
|
||
}
|
||
|
||
public static bool CheckNetworkAvailable()
|
||
{
|
||
return System.Net.NetworkInformation.NetworkInterface.GetIsNetworkAvailable();
|
||
}
|
||
|
||
#region Get请求
|
||
public static string Get(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
return ReadResponse(httpParam, (HttpWebResponse)request.GetResponse());
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
}
|
||
|
||
public static async Task<string> GetAsync(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
return await ReadResponseAsync(httpParam, (HttpWebResponse)request.GetResponse());
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
}
|
||
|
||
public static byte[] GetByte(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
if (string.IsNullOrEmpty(httpParam.Url))
|
||
{
|
||
return default(byte[]);
|
||
}
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
return ReadResponseByte(request);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
return default(byte[]);
|
||
}
|
||
|
||
public static async Task<byte[]> GetByteAsync(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
if (string.IsNullOrEmpty(httpParam.Url))
|
||
{
|
||
return default(byte[]);
|
||
}
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
return await ReadResponseByteAsync(request);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
return default(byte[]);
|
||
}
|
||
|
||
public static HttpResult GetHttpResult(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
HttpResult httpResult = new HttpResult();
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
ReadResponse(httpParam, httpResult, (HttpWebResponse)request.GetResponse());
|
||
UpdateCookie(httpParam, httpResult);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
return httpResult;
|
||
}
|
||
|
||
public static async Task<HttpResult> GetHttpResultAsync(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
HttpResult httpResult = new HttpResult();
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "GET";
|
||
SetHttpWebRequest(request, httpParam);
|
||
await ReadResponseAsync(httpParam, httpResult, (HttpWebResponse)request.GetResponse());
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
return httpResult;
|
||
}
|
||
#endregion
|
||
|
||
#region Post请求
|
||
public static string Post(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
if (string.IsNullOrEmpty(httpParam.Method))
|
||
{
|
||
request.Method = "POST";
|
||
}
|
||
else
|
||
{
|
||
request.Method = httpParam.Method;
|
||
}
|
||
SetHttpWebRequest(request, httpParam);
|
||
SetPostData(request, httpParam.PostData);
|
||
return ReadResponse(httpParam, (HttpWebResponse)request.GetResponse());
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
}
|
||
|
||
public static async Task<string> PostAsync(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "POST";
|
||
SetHttpWebRequest(request, httpParam);
|
||
SetPostData(request, httpParam.PostData);
|
||
return await ReadResponseAsync(httpParam, (HttpWebResponse)request.GetResponse());
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
}
|
||
|
||
public static byte[] PostByte(HttpParam httpParam, bool throwException = false)
|
||
{
|
||
if (string.IsNullOrEmpty(httpParam.Url))
|
||
{
|
||
return default(byte[]);
|
||
}
|
||
try
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "POST";
|
||
SetHttpWebRequest(request, httpParam);
|
||
SetPostData(request, httpParam.PostData);
|
||
return ReadResponseByte(request);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
return default(byte[]);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Post表单、文件或者二者都有
|
||
/// </summary>
|
||
/// <param name="httpParam"></param>
|
||
/// <param name="postDataList"></param>
|
||
/// <returns></returns>
|
||
public static string PostForm(HttpParam httpParam, List<PostDataParam> postDataList, bool throwException = false)
|
||
{
|
||
try
|
||
{
|
||
string html = string.Empty;
|
||
using (MemoryStream memoryStream = new MemoryStream())
|
||
{
|
||
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(httpParam.Url);
|
||
request.Method = "POST";
|
||
request.Headers.Remove("Cookie");
|
||
if (!string.IsNullOrEmpty(httpParam.Cookie))
|
||
{
|
||
request.Headers.Add("Cookie", httpParam.Cookie);
|
||
}
|
||
// 边界符
|
||
var boundary = "----WebKitFormBoundary" + DateTime.Now.Ticks.ToString("x");
|
||
// 边界符
|
||
var beginBoundary = Encoding.ASCII.GetBytes("--" + boundary + "\r\n");
|
||
// 最后的结束符
|
||
var endBoundary = Encoding.ASCII.GetBytes("--" + boundary + "--\r\n");
|
||
memoryStream.Write(beginBoundary, 0, beginBoundary.Length);
|
||
// 设置属性
|
||
request.ContentType = "multipart/form-data; boundary=" + boundary;
|
||
for (int i = 0; i < postDataList.Count; i++)
|
||
{
|
||
PostDataParam postData = postDataList[i];
|
||
string header = string.Empty;
|
||
byte[] headerBytes = null;
|
||
string contentType = "application/octet-stream";
|
||
string fileName = "blob";
|
||
if (!string.IsNullOrEmpty(postData.ContentType))
|
||
{
|
||
contentType = postData.ContentType;
|
||
}
|
||
if (!string.IsNullOrEmpty(postData.FileName))
|
||
{
|
||
fileName = postData.FileName;
|
||
}
|
||
switch (postData.Type)
|
||
{
|
||
case PostDataTypeEnum.String:
|
||
// 写入字符串的Key
|
||
var stringKeyHeader = "Content-Disposition: form-data; name=\"{0}\"" + "\r\n\r\n{1}\r\n";
|
||
header = string.Format(stringKeyHeader, postData.FieldName, postData.Value);
|
||
headerBytes = Encoding.UTF8.GetBytes(header);
|
||
memoryStream.Write(headerBytes, 0, headerBytes.Length);
|
||
break;
|
||
|
||
case PostDataTypeEnum.FileByteOrFilePath:
|
||
// 写入文件
|
||
string filePartHeader = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n" + "Content-Type: " + contentType + "\r\n\r\n";
|
||
header = string.Format(filePartHeader, postData.FieldName, fileName);
|
||
headerBytes = Encoding.UTF8.GetBytes(header);
|
||
memoryStream.Write(headerBytes, 0, headerBytes.Length);
|
||
if (postData.Value.GetType().IsArray)
|
||
{
|
||
headerBytes = postData.Value as byte[];
|
||
memoryStream.Write(headerBytes, 0, headerBytes.Length);
|
||
}
|
||
else
|
||
{
|
||
byte[] buffer = File.ReadAllBytes(postData.Value.ParseToString());
|
||
memoryStream.Write(buffer, 0, buffer.Length);
|
||
}
|
||
string end = "\r\n";
|
||
headerBytes = Encoding.UTF8.GetBytes(end);
|
||
memoryStream.Write(headerBytes, 0, headerBytes.Length);
|
||
break;
|
||
}
|
||
if (i != postDataList.Count - 1)
|
||
{
|
||
memoryStream.Write(beginBoundary, 0, beginBoundary.Length);
|
||
}
|
||
else
|
||
{
|
||
// 写入最后的结束边界符
|
||
memoryStream.Write(endBoundary, 0, endBoundary.Length);
|
||
}
|
||
}
|
||
request.ContentLength = memoryStream.Length;
|
||
using (Stream requestStream = request.GetRequestStream())
|
||
{
|
||
memoryStream.Position = 0;
|
||
byte[] tempBuffer = new byte[memoryStream.Length];
|
||
memoryStream.Read(tempBuffer, 0, tempBuffer.Length);
|
||
requestStream.Write(tempBuffer, 0, tempBuffer.Length);
|
||
|
||
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
||
{
|
||
using (Stream responseStream = response.GetResponseStream())
|
||
{
|
||
byte[] buffer = new byte[1024];
|
||
int read;
|
||
while ((read = responseStream.Read(buffer, 0, buffer.Length)) > 0)
|
||
{
|
||
html += Encoding.UTF8.GetString(buffer, 0, read);
|
||
}
|
||
}
|
||
response.Close();
|
||
}
|
||
}
|
||
}
|
||
return html;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
return GetHttpContentWhenException(httpParam, ex, throwException);
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
#region 下载文件
|
||
|
||
#endregion
|
||
|
||
public static void SetHttpWebRequest(HttpWebRequest request, HttpParam param)
|
||
{
|
||
if (param.Proxy != null)
|
||
{
|
||
request.Proxy = param.Proxy;
|
||
}
|
||
else
|
||
{
|
||
//request.Proxy = NaiBaGlobalConfig.GetDevelopProxy(param.Url);
|
||
}
|
||
if (!string.IsNullOrEmpty(param.UserAgent))
|
||
{
|
||
request.UserAgent = param.UserAgent;
|
||
}
|
||
else
|
||
{
|
||
request.UserAgent = GlobalConfig.UserAgent;
|
||
}
|
||
if (!string.IsNullOrEmpty(param.Accept))
|
||
{
|
||
request.Accept = param.Accept;
|
||
}
|
||
else
|
||
{
|
||
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
||
}
|
||
if (string.IsNullOrEmpty(param.AcceptLanguage))
|
||
{
|
||
request.Headers.Add("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
|
||
}
|
||
else
|
||
{
|
||
request.Headers.Add("Accept-Language", param.AcceptLanguage);
|
||
}
|
||
if (request.Method.ToUpper() != "GET")
|
||
{
|
||
if (!string.IsNullOrEmpty(param.ContentType))
|
||
{
|
||
request.ContentType = param.ContentType;
|
||
}
|
||
else
|
||
{
|
||
request.ContentType = "application/x-www-form-urlencoded;charset=UTF-8";
|
||
}
|
||
}
|
||
if (!string.IsNullOrEmpty(param.Referer))
|
||
{
|
||
request.Referer = param.Referer;
|
||
}
|
||
request.Headers.Remove("Cookie");
|
||
if (!string.IsNullOrEmpty(param.Cookie))
|
||
{
|
||
request.Headers.Add("Cookie", param.Cookie);
|
||
}
|
||
if (param.Header == null)
|
||
{
|
||
param.Header = new Dictionary<string, string>();
|
||
}
|
||
if (!param.Header.Where(p => p.Key.ToLower() == "Accept-Encoding".ToLower()).Any())
|
||
{
|
||
param.Header.Add("Accept-Encoding", "gzip, deflate");
|
||
}
|
||
if (param.Header != null)
|
||
{
|
||
foreach (KeyValuePair<string, string> header in param.Header)
|
||
{
|
||
request.Headers.Add(header.Key, header.Value);
|
||
}
|
||
}
|
||
if (param.Timeout > 0)
|
||
{
|
||
request.Timeout = param.Timeout * 1000;
|
||
}
|
||
else
|
||
{
|
||
request.Timeout = RequestTimeout;
|
||
}
|
||
}
|
||
|
||
public static void SetPostData(HttpWebRequest request, string postData)
|
||
{
|
||
request.ServicePoint.Expect100Continue = false;
|
||
byte[] postDataByte = Encoding.UTF8.GetBytes(postData ?? string.Empty);
|
||
request.ContentLength = postDataByte.Length;
|
||
request.AllowAutoRedirect = false;
|
||
using (Stream requestStream = request.GetRequestStream())
|
||
{
|
||
requestStream.Write(postDataByte, 0, postDataByte.Length);
|
||
}
|
||
}
|
||
|
||
private static Encoding GetResponseEncoding(HttpParam httpParam, HttpWebResponse response)
|
||
{
|
||
if (!string.IsNullOrEmpty(response.CharacterSet))
|
||
{
|
||
Encoding encoding = Encoding.GetEncoding(response.CharacterSet);
|
||
return encoding;
|
||
}
|
||
else
|
||
{
|
||
return httpParam.ResponseEncoding;
|
||
}
|
||
}
|
||
|
||
public static string ReadResponse(HttpParam httpParam, HttpWebResponse response)
|
||
{
|
||
using (Stream stream = DecompressStream(response))
|
||
{
|
||
using (StreamReader streamReader = new StreamReader(stream, GetResponseEncoding(httpParam, response)))
|
||
{
|
||
return streamReader.ReadToEnd();
|
||
}
|
||
}
|
||
}
|
||
|
||
public static void ReadResponse(HttpParam httpParam, HttpResult httpResult, HttpWebResponse response)
|
||
{
|
||
httpResult.CookieCollection = GetCookieFromResponse(string.Empty, response);
|
||
using (Stream stream = DecompressStream(response))
|
||
{
|
||
using (StreamReader streamReader = new StreamReader(stream, GetResponseEncoding(httpParam, response)))
|
||
{
|
||
httpResult.Html = streamReader.ReadToEnd();
|
||
}
|
||
}
|
||
}
|
||
|
||
public static async Task<string> ReadResponseAsync(HttpParam httpParam, HttpWebResponse response)
|
||
{
|
||
using (Stream stream = DecompressStream(response))
|
||
{
|
||
using (StreamReader streamReader = new StreamReader(stream, GetResponseEncoding(httpParam, response)))
|
||
{
|
||
return await streamReader.ReadToEndAsync();
|
||
}
|
||
}
|
||
}
|
||
|
||
public static async Task ReadResponseAsync(HttpParam httpParam, HttpResult httpResult, HttpWebResponse response)
|
||
{
|
||
httpResult.CookieCollection = GetCookieFromResponse(string.Empty, response);
|
||
using (Stream stream = DecompressStream(response))
|
||
{
|
||
using (StreamReader streamReader = new StreamReader(stream, GetResponseEncoding(httpParam, response)))
|
||
{
|
||
httpResult.Html = await streamReader.ReadToEndAsync();
|
||
}
|
||
}
|
||
}
|
||
|
||
public static Stream DecompressStream(HttpWebResponse response)
|
||
{
|
||
Stream responseStream = response.GetResponseStream();
|
||
Stream stream = null;
|
||
if (response.ContentEncoding.ToLower().Contains("gzip"))
|
||
{
|
||
stream = new GZipStream(responseStream, CompressionMode.Decompress);
|
||
}
|
||
else if (response.ContentEncoding.ToLower().Contains("deflate"))
|
||
{
|
||
stream = new DeflateStream(responseStream, CompressionMode.Decompress);
|
||
}
|
||
if (stream == null)
|
||
{
|
||
return responseStream;
|
||
}
|
||
else
|
||
{
|
||
return stream;
|
||
}
|
||
}
|
||
|
||
public static byte[] ReadResponseByte(HttpWebRequest request)
|
||
{
|
||
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
|
||
{
|
||
using (BinaryReader binaryReader = new BinaryReader(response.GetResponseStream()))
|
||
{
|
||
using (MemoryStream ms = new MemoryStream())
|
||
{
|
||
byte[] buffer = binaryReader.ReadBytes(1024);
|
||
while (buffer.Length > 0)
|
||
{
|
||
ms.Write(buffer, 0, buffer.Length);
|
||
buffer = binaryReader.ReadBytes(1024);
|
||
}
|
||
byte[] fileByte = new byte[ms.Length];
|
||
ms.Position = 0;
|
||
ms.Read(fileByte, 0, fileByte.Length);
|
||
return fileByte;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
public static async Task<byte[]> ReadResponseByteAsync(HttpWebRequest request)
|
||
{
|
||
using (HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync())
|
||
{
|
||
using (BinaryReader binaryReader = new BinaryReader(response.GetResponseStream()))
|
||
{
|
||
using (MemoryStream ms = new MemoryStream())
|
||
{
|
||
byte[] buffer = binaryReader.ReadBytes(1024);
|
||
while (buffer.Length > 0)
|
||
{
|
||
ms.Write(buffer, 0, buffer.Length);
|
||
buffer = binaryReader.ReadBytes(1024);
|
||
}
|
||
byte[] fileByte = new byte[ms.Length];
|
||
ms.Position = 0;
|
||
ms.Read(fileByte, 0, fileByte.Length);
|
||
return fileByte;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
#region 请求异常处理
|
||
private static string GetHttpContentWhenException(HttpParam httpParam, Exception ex, bool throwException = false)
|
||
{
|
||
TestProxyLog(httpParam, ex);
|
||
LogHelper.WriteWithTime(ex);
|
||
if (throwException)
|
||
{
|
||
throw ex;
|
||
}
|
||
else
|
||
{
|
||
WebException webException = ex as WebException;
|
||
if (webException != null && webException.Response != null)
|
||
{
|
||
return ReadResponse(httpParam, (HttpWebResponse)webException.Response);
|
||
}
|
||
}
|
||
return string.Empty;
|
||
}
|
||
|
||
private static void TestProxyLog(HttpParam httpParam, Exception ex)
|
||
{
|
||
if (httpParam.Proxy != null)
|
||
{
|
||
Debug.WriteLine("代理ip：" + httpParam.Proxy?.Address.ParseToString() + " 请求Url:" + httpParam.Url + " 错误信息：" + ex.GetOriginalException().Message);
|
||
}
|
||
else
|
||
{
|
||
DebugHelper.WriteLine("请求Url:" + httpParam.Url + " 错误信息：" + ex.GetOriginalException().Message);
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
#region 解析Response Cookie
|
||
private static CookieCollection GetCookieFromResponse(string host, HttpWebResponse response)
|
||
{
|
||
string setCookie = string.Empty;
|
||
if (response.Headers.AllKeys.Contains("Set-Cookie"))
|
||
{
|
||
setCookie = response.Headers["Set-Cookie"];
|
||
}
|
||
if (response.Headers.AllKeys.Contains("set-cookie"))
|
||
{
|
||
setCookie = response.Headers["set-cookie"];
|
||
}
|
||
ArrayList al = new ArrayList();
|
||
CookieCollection cc = new CookieCollection();
|
||
if (!string.IsNullOrEmpty(setCookie))
|
||
{
|
||
al = ConvertCookieToArrayList(setCookie);
|
||
cc = ConvertCookieArraysToCookieCollection(al, host);
|
||
}
|
||
return cc;
|
||
}
|
||
private static ArrayList ConvertCookieToArrayList(string setCookie)
|
||
{
|
||
setCookie = setCookie.Replace("\r", "");
|
||
setCookie = setCookie.Replace("\n", "");
|
||
string[] strCookTemp = setCookie.Split(',');
|
||
ArrayList al = new ArrayList();
|
||
int i = 0;
|
||
int n = strCookTemp.Length;
|
||
while (i < n)
|
||
{
|
||
if (strCookTemp[i].IndexOf("expires=", StringComparison.OrdinalIgnoreCase) > 0)
|
||
{
|
||
al.Add(strCookTemp[i] + "," + strCookTemp[i + 1]);
|
||
i = i + 1;
|
||
}
|
||
else
|
||
{
|
||
al.Add(strCookTemp[i]);
|
||
}
|
||
i = i + 1;
|
||
}
|
||
return al;
|
||
}
|
||
private static CookieCollection ConvertCookieArraysToCookieCollection(ArrayList al, string host)
|
||
{
|
||
CookieCollection cc = new CookieCollection();
|
||
|
||
int alcount = al.Count;
|
||
string strEachCook;
|
||
string[] strEachCookParts;
|
||
for (int i = 0; i < alcount; i++)
|
||
{
|
||
strEachCook = al[i].ToString();
|
||
strEachCookParts = strEachCook.Split(';');
|
||
int intEachCookPartsCount = strEachCookParts.Length;
|
||
string strCNameAndCValue = string.Empty;
|
||
string strPNameAndPValue = string.Empty;
|
||
string strDNameAndDValue = string.Empty;
|
||
string[] NameValuePairTemp;
|
||
Cookie cookTemp = new Cookie();
|
||
|
||
for (int j = 0; j < intEachCookPartsCount; j++)
|
||
{
|
||
if (j == 0)
|
||
{
|
||
strCNameAndCValue = strEachCookParts[j];
|
||
if (strCNameAndCValue != string.Empty)
|
||
{
|
||
int firstEqual = strCNameAndCValue.IndexOf("=");
|
||
string firstName = strCNameAndCValue.Substring(0, firstEqual);
|
||
string allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1));
|
||
cookTemp.Name = firstName;
|
||
cookTemp.Value = allValue;
|
||
}
|
||
continue;
|
||
}
|
||
if (strEachCookParts[j].IndexOf("path", StringComparison.OrdinalIgnoreCase) >= 0)
|
||
{
|
||
strPNameAndPValue = strEachCookParts[j];
|
||
if (strPNameAndPValue != string.Empty)
|
||
{
|
||
NameValuePairTemp = strPNameAndPValue.Split('=');
|
||
if (NameValuePairTemp[1] != string.Empty)
|
||
{
|
||
cookTemp.Path = NameValuePairTemp[1];
|
||
}
|
||
else
|
||
{
|
||
cookTemp.Path = "/";
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (strEachCookParts[j].IndexOf("domain", StringComparison.OrdinalIgnoreCase) >= 0)
|
||
{
|
||
strPNameAndPValue = strEachCookParts[j];
|
||
if (strPNameAndPValue != string.Empty)
|
||
{
|
||
NameValuePairTemp = strPNameAndPValue.Split('=');
|
||
|
||
if (NameValuePairTemp[1] != string.Empty)
|
||
{
|
||
cookTemp.Domain = NameValuePairTemp[1];
|
||
}
|
||
else
|
||
{
|
||
cookTemp.Domain = host;
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (cookTemp.Path == string.Empty)
|
||
{
|
||
cookTemp.Path = "/";
|
||
}
|
||
if (cookTemp.Domain == string.Empty)
|
||
{
|
||
cookTemp.Domain = host;
|
||
}
|
||
cc.Add(cookTemp);
|
||
}
|
||
return cc;
|
||
}
|
||
private static void UpdateCookie(HttpParam httpParam, HttpResult httpResult)
|
||
{
|
||
List<string> rawCookieList = new List<string>();
|
||
if (!string.IsNullOrEmpty(httpParam.Cookie))
|
||
{
|
||
rawCookieList = httpParam.Cookie.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToList();
|
||
}
|
||
foreach (Cookie cookie in httpResult.CookieCollection)
|
||
{
|
||
string newCookie = cookie.Name + "=" + cookie.Value;
|
||
if (rawCookieList.Where(p => p.StartsWith(cookie.Name + "=")).Any())
|
||
{
|
||
rawCookieList.RemoveAll(p => p.StartsWith(cookie.Name + "="));
|
||
rawCookieList.Add(newCookie);
|
||
}
|
||
else
|
||
{
|
||
rawCookieList.Add(newCookie);
|
||
}
|
||
}
|
||
if (rawCookieList.Any())
|
||
{
|
||
httpParam.Cookie = string.Join(";", rawCookieList);
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
#region 设置Https
|
||
public static void InitHttpsCertificate()
|
||
{
|
||
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
||
ServicePointManager.ServerCertificateValidationCallback += ValidateRemoteCertificate;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Certificate validation callback.
|
||
/// </summary>
|
||
private static bool ValidateRemoteCertificate(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors error)
|
||
{
|
||
// If the certificate is a valid, signed certificate, return true.
|
||
if (error == SslPolicyErrors.None)
|
||
{
|
||
return true;
|
||
}
|
||
Console.WriteLine("X509Certificate [{0}] Policy Error: '{1}'", cert.Subject, error.ToString());
|
||
return false;
|
||
}
|
||
#endregion
|
||
}
|
||
|
||
public class HttpParam
|
||
{
|
||
public HttpParam()
|
||
{
|
||
ResponseEncoding = Encoding.UTF8;
|
||
}
|
||
public string Url { get; set; }
|
||
public string Method { get; set; }
|
||
public string PostData { get; set; }
|
||
public string ContentType { get; set; }
|
||
public string Cookie { get; set; }
|
||
public string Referer { get; set; }
|
||
public string UserAgent { get; set; }
|
||
public UserAgentTypeEnum UserAgentType { get; set; }
|
||
public string AcceptLanguage { get; set; }
|
||
public string Accept { get; set; }
|
||
[JsonIgnore]
|
||
public WebProxy Proxy { get; set; }
|
||
/// <summary>
|
||
/// 秒
|
||
/// </summary>
|
||
public int Timeout { get; set; }
|
||
public Dictionary<string, string> Header { get; set; }
|
||
public int ProxySleep { get; set; }
|
||
public int TryTimes { get; set; }
|
||
public int ActualTryTimes { get; set; }
|
||
[JsonIgnore]
|
||
public CancellationTokenSource CancellationTokenSource { get; set; }
|
||
[JsonIgnore]
|
||
public Encoding ResponseEncoding { get; set; }
|
||
public HttpResultTypeEnum ResultType { get; set; }
|
||
}
|
||
|
||
public class PostDataParam
|
||
{
|
||
public PostDataTypeEnum Type { get; set; }
|
||
/// <summary>
|
||
/// 字段名
|
||
/// </summary>
|
||
public string FieldName { get; set; }
|
||
/// <summary>
|
||
/// 文件名
|
||
/// </summary>
|
||
public string FileName { get; set; }
|
||
/// <summary>
|
||
/// 可以是 string 或者 byte[]
|
||
/// </summary>
|
||
public object Value { get; set; }
|
||
public string ContentType { get; set; }
|
||
}
|
||
|
||
public class HttpResult
|
||
{
|
||
public CookieCollection CookieCollection { get; set; }
|
||
public string Html { get; set; }
|
||
public byte[] Byte { get; set; }
|
||
}
|
||
|
||
public enum PostDataTypeEnum
|
||
{
|
||
[Description("字符串")]
|
||
String,
|
||
|
||
[Description("文件流或者文件路径")]
|
||
FileByteOrFilePath
|
||
}
|
||
|
||
public enum HttpResultTypeEnum
|
||
{
|
||
String, //表示只返回字符串
|
||
Byte //表示返回字符串和字节流
|
||
}
|
||
}
|