织梦学校网站,怎么建设一个漫画网站,福田欧曼汽车官网,免费学校网站系统起因
抓取某个HTTPS网站的时候 开启charles代理能够抓取成功,关闭被风控 通过检测,怀疑可能是tls的时候有区别
尝试
golang的http中,Transport.TLSClientConfig是可以自定义设置的 但起初通过随意设置并不能绕过风控
困难
使用golang的http客户端,修改DialTLSContext函数的…起因
抓取某个HTTPS网站的时候 开启charles代理能够抓取成功,关闭被风控 通过检测,怀疑可能是tls的时候有区别
尝试
golang的http中,Transport.TLSClientConfig是可以自定义设置的 但起初通过随意设置并不能绕过风控
困难
使用golang的http客户端,修改DialTLSContext函数的方式是可以实绕过风控,但使用proxy的时候,代码会使用pconn.addTLS(ctx, cm.tlsHost(), trace) 重新以普通方式进行握手,导致JA3修改失败因为golang强关联,第三方库并不能完美的集成到现有代码中,都需要重构代码某些网站会对于新建链接进行ClientSession检测,因此需要 KeepAliveClientSessionCache,这样通过复用连接减少风控概率
最终实现
只需要拿到合法的参数,并且配置到TLSClientConfig里即可使用github.com/refraction-networking/utls中的UTLSIdToSpec拿到CipherSuites并传入
package main
import (bytescrypto/tlstlsx github.com/refraction-networking/utlsnet/http
)
func main() {c, _ : tlsx.UTLSIdToSpec(tlsx.HelloRandomized)a : http.Client{Transport: http.Transport{DisableKeepAlives: false,Proxy: proxy,TLSClientConfig: tls.Config{InsecureSkipVerify: true,MinVersion: c.TLSVersMin,MaxVersion: c.TLSVersMax,CipherSuites: c.CipherSuites,ClientSessionCache: tls.NewLRUClientSessionCache(32),},},}aw, bw : a.Get(https://tls.browserleaks.com/json)defer aw.Body.Close()var buf bytes.Bufferaw.Write(buf)println(string(buf.String()), bw)
}参考文章
https://github.com/baixudong007/gospiderhttps://juejin.cn/post/7073264626506399751 用Go构建你专属的JA3指纹https://blog.csdn.net/qq523176585/article/details/127116542 好库推荐|强烈推荐支持Ja3指纹修改的golang请求库https://github.com/wangluozhe/requestshttps://github.com/refraction-networking/utlshttp://www.ctfiot.com/64337.html 如何绕过 JA3 指纹校验https://www.coder.work/article/7192419 http - 发送请求时如何使用uTLS连接https://segmentfault.com/a/1190000041699815/en Build your own JA3 fingerprint with Gohttps://zhuanlan.zhihu.com/p/601474166 curl_cffi: 支持原生模拟浏览器 TLS/JA3 指纹的 Python 库https://tools.scrapfly.io/api/fp/ja3
历史编写的代码
这些代码都不太好用
package mainimport (contextfmttls github.com/refraction-networking/utlsxtls github.com/refraction-networking/utlsnetnet/http
)func GetTransport(helloID *xtls.ClientHelloID) *http.Transport {if helloID nil {helloID xtls.HelloChrome_83}transport : http.DefaultTransport.(*http.Transport).Clone()transport.DialTLSContext func(ctx context.Context, network, addr string) (_ net.Conn, err error) {dialer : net.Dialer{}con, err : dialer.DialContext(ctx, network, addr)if err ! nil {return nil, err}// 根据地址获取host信息host, _, err : net.SplitHostPort(addr)if err ! nil {return nil, err}c : transport.TLSClientConfig// 并且不验证host信息xtlsConf : xtls.Config{ServerName: host,//Renegotiation: xtls.RenegotiateNever,ClientSessionCache: xtls.NewLRUClientSessionCache(32),NextProtos: []string{h2, http/1.1},Rand: c.Rand,Time: c.Time,VerifyPeerCertificate: c.VerifyPeerCertificate,RootCAs: c.RootCAs,ClientCAs: c.ClientCAs,InsecureSkipVerify: c.InsecureSkipVerify,CipherSuites: c.CipherSuites,PreferServerCipherSuites: c.PreferServerCipherSuites,SessionTicketsDisabled: c.SessionTicketsDisabled,SessionTicketKey: c.SessionTicketKey,MinVersion: c.MinVersion,MaxVersion: c.MaxVersion,DynamicRecordSizingDisabled: c.DynamicRecordSizingDisabled,KeyLogWriter: c.KeyLogWriter,}// 构建tls.UConnxtlsConn : xtls.UClient(con, xtlsConf, *helloID)// 握手err xtlsConn.HandshakeContext(ctx)if err ! nil {return nil, err}fmt.Println(当前请求使用协议, xtlsConn.HandshakeState.ServerHello.AlpnProtocol)return xtlsConn, err}//transport.DialContext transport.DialTLSContexttransport.DisableKeepAlives truereturn transport
}
func CreateHTTPClient() *http.Transport {return http.Transport{DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {//initialize the tcp connectiontcpConn, err : (net.Dialer{}).DialContext(ctx, network, addr)if err ! nil {return nil, err}host, _, err : net.SplitHostPort(addr)if err ! nil {return nil, err}//initialize the conifg for tlsconfig : tls.Config{ServerName: host, //set the server name with the provided addrClientSessionCache: xtls.NewLRUClientSessionCache(0),}//initialize a tls connection with the underlying tcop connection and configtlsConn : tls.UClient(tcpConn, config, tls.HelloRandomized)//start the tls handshake between serverserr tlsConn.Handshake()if err ! nil {return nil, fmt.Errorf(uTlsConn.Handshake() error: %w, err)}return tlsConn, nil},ForceAttemptHTTP2: false,}}
