C# .NET 利用 HttpWebRequest 模拟浏览器 http、https 请求。

/// <summary>
/// Http 请求
/// </summary>
/// <param name="url">url地址</param>
/// <param name="data">请求参数(可选参数)</param>
/// <param name="method">请求方法(可选参数,默认:GET)</param>
/// <param name="timeout">连接超时(可选参数,默认: 100,000 毫秒(100 秒))</param>
/// <param name="encoding">编码(可选参数,默认:utf-8)</param>
/// <param name="contentType">Content-type HTTP 标头(可选参数)</param>
/// <param name="userAgent">模拟浏览器UA(可选参数)</param>
/// <param name="headers">HTTP 标头(可选参数)</param>
/// <returns>远程服务器返回的数据</returns>
public static string HttpRequest(string url, string data = null, string method = "GET", int? timeout = null, Encoding encoding = null, string contentType = "application/x-www-form-urlencoded", string userAgent = null, Dictionary<string, string> headers = null)
{
	encoding = encoding ?? Encoding.UTF8;
	var myWebRequest = (HttpWebRequest)WebRequest.Create(url);
	myWebRequest.ContentType = contentType;
	myWebRequest.Method = method;
	if (!string.IsNullOrWhiteSpace(userAgent))
	{
		myWebRequest.UserAgent = userAgent;
	}
	if (timeout.HasValue)
	{
		myWebRequest.Timeout = timeout.Value;
	}
	if (myWebRequest.RequestUri.Scheme == "https")
	{
		myWebRequest.ProtocolVersion = HttpVersion.Version11;
		myWebRequest.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CertificateValidationCallback);
		// 这里设置了协议类型。
		ServicePointManager.SecurityProtocol = (SecurityProtocolType)3072;// SecurityProtocolType.Tls1.2; 
		myWebRequest.KeepAlive = false;
		ServicePointManager.CheckCertificateRevocationList = true;
		ServicePointManager.DefaultConnectionLimit = 100;
		ServicePointManager.Expect100Continue = false;
	}
	if (headers != null && headers.Count > 0)
	{
		foreach (KeyValuePair<string, string> item in headers)
		{
			myWebRequest.Headers[item.Key] = item.Value;
		}
	}
	try
	{
		if (!string.IsNullOrWhiteSpace(data))
		{
			using (Stream stream = myWebRequest.GetRequestStream())
			{
				var bytes = encoding.GetBytes(data);
				stream.Write(bytes, 0, bytes.Length);
				stream.Flush();
			}
		}
		using (HttpWebResponse myWebResponse = (HttpWebResponse)myWebRequest.GetResponse())
		{
			if (myWebResponse.StatusCode == HttpStatusCode.OK)
			{
				using (Stream receiveStream = myWebResponse.GetResponseStream())
				{
					using (StreamReader readStream = new StreamReader(receiveStream, Encoding.UTF8))
					{
						return readStream.ReadToEnd();
					}
				}
			}
			else
			{
				throw new WebException(myWebResponse.StatusDescription);
			}
		}
	}
	catch (Exception ex)
	{
		throw ex;
	}
	finally
	{
		myWebRequest.Abort();
	}
}
/// <summary>
/// (SSL) 证书验证回调
/// </summary>
/// <param name="sender">一个对象,它包含此验证的状态信息。</param>
/// <param name="certificate">用于对远程方进行身份验证的证书。</param>
/// <param name="chain">与远程证书关联的证书颁发机构链。</param>
/// <param name="sslPolicyErrors">与远程证书关联的一个或多个错误。</param>
/// <returns>是否接受指定证书进行身份验证</returns>
private static bool CertificateValidationCallback(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
{
	if (sslPolicyErrors == SslPolicyErrors.None)
		return true;

	Console.WriteLine("Certificate error: {0}", sslPolicyErrors);
	return false;
}

IP地址转数字

public long IPToNumber(string ip)
{
	string[] arrayIP = ip.Split('.');
	long sip1 = long.Parse(arrayIP[0]);
	long sip2 = long.Parse(arrayIP[1]);
	long sip3 = long.Parse(arrayIP[2]);
	long sip4 = long.Parse(arrayIP[3]);
	long ipNum = sip1 * 256 * 256 * 256 + sip2 * 256 * 256 + sip3 * 256 + sip4;
	return ipNum;
}

数字转IP地址

public string NumberToIP(long ipNum)
{
	StringBuilder builder = new StringBuilder();
	builder.Append((ipNum >> 24) & 0xFF).Append(".");
	builder.Append((ipNum >> 16) & 0xFF).Append(".");
	builder.Append((ipNum >> 8) & 0xFF).Append(".");
	builder.Append(ipNum & 0xFF);
	return builder.ToString();
}

.NET 版 Chromium WebBrowser 动态设置 http, https, socks5 代理方法。

private void InitWebBrowser(string url)
{
	ChromiumWebBrowser browser = new ChromiumWebBrowser(url);
	browser.Dock = DockStyle.Fill;
	//绑定 WebBrowser 初始化完成事件
	browser.IsBrowserInitializedChanged += Browser_IsBrowserInitializedChanged;
	//添加到WinForm中
	this.Controls.Add(browser);
	browser.BringToFront();
}

private void Browser_IsBrowserInitializedChanged(object sender, IsBrowserInitializedChangedEventArgs e)
{
	if(e.IsBrowserInitialized)
	{
		//只能在WebBrowser UI呈现后获取 Request 上下文
		Cef.UIThreadTaskFactory.StartNew(delegate
		{
			//获取 Request 上下文
			var rc = browser.GetBrowser().GetHost().RequestContext;
			var dict = new Dictionary<string, object>();
			dict.Add("mode", "fixed_servers");
			dict.Add("server", "127.0.0.1:1080");
			string error;
			//设置代理
			rc.SetPreference("proxy", dict, out error);
			//如果 error 不为空则表示设置失败。
			if(!string.IsNullOrWhiteSpace(error))
			{
				MessageBox.Show(error, "Tip", MessageBoxButtons.OK, MessageBoxIcon.Error);
			}
		});
	}
}

不要在 Cef.Initialize 时设置 CefCommandLineArgs.Add("proxy-server", "127.0.0.1:1080");,否则修改代理设置会失败。

利用 shell32 API清除IE缓存。好处是调用系统原生清理,保证效果,不好的就是会弹出窗口,影响用户体验。

public class ShellUtil
{
	[DllImport("shell32.dll")]
	public static extern IntPtr ShellExecute(IntPtr hwnd, string lpOperation, string lpFile, string lpParameters, string lpDirectory, ShowCommands nShowCmd);
}

public enum ShowCommands : int
{
	SW_HIDE = 0,
	SW_SHOWNORMAL = 1,
	SW_NORMAL = 1,
	SW_SHOWMINIMIZED = 2,
	SW_SHOWMAXIMIZED = 3,
	SW_MAXIMIZE = 3,
	SW_SHOWNOACTIVATE = 4,
	SW_SHOW = 5,
	SW_MINIMIZE = 6,
	SW_SHOWMINNOACTIVE = 7,
	SW_SHOWNA = 8,
	SW_RESTORE = 9,
	SW_SHOWDEFAULT = 10,
	SW_FORCEMINIMIZE = 11,
	SW_MAX = 11
}

1、 清除IE临时文件

ShellExecute(IntPtr.Zero, "open", "rundll32.exe", " InetCpl.cpl,ClearMyTracksByProcess 255", "", ShowCommands.SW_HIDE);

2、清除IE Cookies

ShellUtil.ShellExecute(IntPtr.Zero, "open", "rundll32.exe", " InetCpl.cpl,ClearMyTracksByProcess 2", "", ShowCommands.SW_HIDE);

3、清除IE History

ShellUtil.ShellExecute(IntPtr.Zero, "open", "rundll32.exe", " InetCpl.cpl,ClearMyTracksByProcess 1", "", ShowCommands.SW_HIDE);

本页中摘录的中英文对照地名,是没有按照地名的中文拼音拼写的地名。这类地名主要分布在中国的少数民族自治区。

内蒙古自治区
Inner Mongolia
呼和浩特市
Hohhot
鄂尔多斯市
Ordos
呼伦贝尔市
Hulunbuir
巴彦淖尔市
Bayannur
乌兰察布市
Ulanqab
兴安盟
Hinggan
锡林郭勒盟
Xilingol
阿拉善盟
Alxa
 
 
四川省
 
甘孜
Garze
阿坝
Ngawa
 
 
西藏自治区
Tibet
拉萨市
Lhasa
昌都地区
Qamdo
山南地区
Lhoka
日喀则地区
Xigaze
那曲地区
Nagqu
阿里地区
Ngari
林芝地区
Nyingchi
 
 
新疆维吾尔自治区
Xinjiang
乌鲁木齐市
Urumqi
克拉玛依市
Karamay
吐鲁番地区
Turpan
哈密地区
Kumul
昌吉回族自治州
Changji
博尔塔拉蒙古自治州
Bortala
巴音郭楞蒙古自治州
Bayingolin
阿克苏地区
Aksu
克孜勒苏柯尔克孜自治州
Kizilsu
喀什地区
Kashgar
和田地区
Hotan
伊犁哈萨克自治州
Ili
塔城地区
Tarbagatay
阿勒泰地区
Altay
石河子市
Shihezi
阿拉尔市
Aral
图木舒克市
Tumushuke
五家渠市
Wujiaqu
 
 
香港
Hongkong
澳门
Macco
 
 
台湾
Taiwan
新竹
Hsinchu
桃园
Taoyuan
基隆
Keelung
嘉义
Chiayi
台南
Tainan
台中
Taichung

using System;
using System.Data.SqlTypes;
using System.IO;
using System.Xml;
using System.Xml.Serialization;

namespace utility
{
    /// <summary>
    /// Xml序列化与反序列化
    /// </summary>
    public class XmlUtil
    {
        /// <summary>
        /// 反序列化
        /// </summary>
        /// <param name="xml">XML字符串</param>
        /// <returns></returns>
        public static T Deserialize<T>(string xml) where T : new()
        {
            try
            {
                using (Stream sr = new FileStream(xml, FileMode.Open))
                {
                    XmlSerializer xmldes = new XmlSerializer(typeof(T));
                    return (T)xmldes.Deserialize(sr);
                }
            }
            catch
            {
                return default(T);
            }
        }
		
        /// <summary>
        /// 序列化
        /// </summary>
        /// <param name="obj">对象</param>
        /// <returns></returns>
        public static string Serializer<T>(T obj)
        {
            using (MemoryStream Stream = new MemoryStream())
            {
                XmlSerializer xml = new XmlSerializer(typeof(T));
                try
                {
                    //序列化对象
                    xml.Serialize(Stream, obj);
                }
                catch (InvalidOperationException)
                {
                    throw;
                }
                Stream.Position = 0;
                using (StreamReader sr = new StreamReader(Stream))
                {
                    return sr.ReadToEnd();
                }
            }
        }
    }
}

Nginx 利用 IP2Locaion 模块实现地区负载均衡和IP定位。

由于GeoIP2的效率实在是“惊人”,和那“高到爆”的识别精度,真是让人想爆粗口。经过权衡,将IP定位模块更换为 IP2Locaion。

如果还没安装 IP2Location C Library,请先移步到 https://www.24kplus.com/linux/871.html 安装 IP2Location C Library

下载 Nginx IP2Locaion 模块

git clone https://github.com/ip2location/ip2location-nginx.git

查看当前 Nginx 版本信息

nginx -V

可以看到 Nginx 的版本为1.16.0 和 configure 参数,把 configure 参数拷贝保存下来,后面需要用到

到官网 https://nginx.org/en/download.html 找到对应的版本源码下载并解压。本站以1.16.0为例:

下载并解压

wget https://nginx.org/download/nginx-1.16.0.tar.gz
tar -zxf nginx-1.16.0.tar.gz
cd nginx-1.16.0

备用下载地址: https://down.24kplus.com/linux/nginx-1.16.0.tar.gz

生成新的 nginx

把刚刚复制的 configure 参数粘贴到 ./configure 后面, 在结尾处加入 -–add-module=../ip2location-nginx

./configure \
--prefix=/etc/nginx \
.....
# 在结尾处加入一行
-–add-module=../ip2location-nginx
# 如果想编译为动态模块,则添加
--add-dynamic-module=../ip2location-nginx
make

编译好之后不要安装, 停止 nginx 服务

systemctl stop nginx

复制编译好的新 nginx 文件拷贝到sbin下

cp /usr/sbin/nginx /usr/sbin/nginx.bak
cp objs/nginx /usr/sbin

编辑 nginx.conf 文件,在 http {} 中加入以下代码:

http {
……
#doc https://github.com/ip2location/ip2location-nginx
# on 为启用,off 为禁用
ip2location on;
#/usr/share/IP2Location/IP2LOCATION-LITE-DB3.BIN 替换成你的 IP2Locaion 数据路径。
ip2location_database /usr/share/IP2Location/IP2LOCATION-LITE-DB3.BIN;
# 可选参数 ip2location_access_type file_io|shared_memory|cache_memory
# 默认为 shared_memory
# 建议不要选择 file_io, 否则可能会严重拖慢响应速度。
ip2location_access_type shared_memory
……
}

官方提供免费版BIN文件下载: https://lite.ip2location.com/ip2location-lite ,根据自己的需要下载对应版本。

编辑 fastcgi_params 文件,在结尾加入以下几行代码(可选):

#IP2Location, with ip2location on;
fastcgi_param  IP_COUNTRY_CODE		$ip2location_country_short;
fastcgi_param  IP_COUNTRY_NAME		$ip2location_country_long;
fastcgi_param  IP_REGION_NAME		$ip2location_region;
fastcgi_param  IP_CITY_NAME  		$ip2location_city;

更多参数查考官方文档:https://github.com/ip2location/ip2location-nginx

启动 nginx 服务

systemctl start nginx

在 phpinfo 信息中可以看到:

当然,也可以在 nginx 中直接使用 $ip2location_country_short,$ip2location_region等变量来实现地区负载均衡。

IP2Location C Library

1、下载并解压

wget -O IP2Location-C-Library-8.0.8.tar.gz https://codeload.github.com/chrislim2888/IP2Location-C-Library/tar.gz/8.0.8
tar -zxf IP2Location-C-Library-8.0.8.tar.gz
cd IP2Location-C-Library-8.0.8

备用下载地址:https://down.24kplus.com/linux/IP2Location-C-Library-8.0.8.tar.gz

2、编译安装

autoreconf -i -v --force
./configure --prefix=/usr
make
make install
cd data
perl ip-country.pl

如果出现错误

configure.ac:42: error: possibly undefined macro: AC_PROG_LIBTOOL
If this token and others are legitimate, please use m4_pattern_allow.
See the Autoconf documentation.

执行:

yum install libtool libsysfs

3、测试(可选)

cd test
./test-IP2Location

IP2Location API version: 8.0.8 (80008)
IP2Location IPv4 Testing passed.
IP2Location IPv6 Testing passed.