364 lines
7.8 KiB
Go
364 lines
7.8 KiB
Go
// bhttp Support for Http custom parameters and proxy
|
||
package bhttp
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"mime/multipart"
|
||
"net/http"
|
||
"net/url"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// BhttpService bhttp service
|
||
type BhttpService struct {
|
||
Client *bhttpClient
|
||
}
|
||
|
||
// NewBhttpService 创建bhttp service
|
||
func NewBhttpService(opts ...Option) (*BhttpService, error) {
|
||
client, err := NewBhttpClient(opts...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
return &BhttpService{
|
||
Client: client,
|
||
}, nil
|
||
}
|
||
|
||
type bhttpClient struct {
|
||
client *http.Client
|
||
httpRequest *http.Request
|
||
header *http.Header
|
||
values *url.Values
|
||
params map[string]string
|
||
postBody []byte
|
||
retry int32
|
||
checkStatusOk bool
|
||
proxyURL string
|
||
rateTimeLeft int //下次可执行剩余的时间毫秒
|
||
}
|
||
|
||
type options struct {
|
||
timeout int32
|
||
proxyURL string
|
||
retry int32
|
||
checkStatusOk bool
|
||
disableKeepAlives bool
|
||
}
|
||
type Option func(*options)
|
||
|
||
func WithTimeout(b int32) Option {
|
||
return func(c *options) {
|
||
c.timeout = b
|
||
}
|
||
}
|
||
func WithProxy(b string) Option {
|
||
return func(c *options) {
|
||
c.proxyURL = b
|
||
}
|
||
}
|
||
|
||
func WithRetry(b int32) Option {
|
||
return func(c *options) {
|
||
c.retry = b
|
||
}
|
||
}
|
||
|
||
func WithCheckStatusOk(b bool) Option {
|
||
return func(c *options) {
|
||
c.checkStatusOk = b
|
||
}
|
||
}
|
||
|
||
func WithDisableKeepAlives(b bool) Option {
|
||
return func(c *options) {
|
||
c.disableKeepAlives = b
|
||
}
|
||
}
|
||
|
||
func NewBhttpClient(opts ...Option) (*bhttpClient, error) {
|
||
proxyFunc := http.ProxyFromEnvironment
|
||
opt := options{
|
||
timeout: 60,
|
||
proxyURL: "",
|
||
retry: 0,
|
||
checkStatusOk: false,
|
||
disableKeepAlives: false,
|
||
}
|
||
for _, o := range opts {
|
||
o(&opt)
|
||
}
|
||
|
||
if opt.proxyURL != "" {
|
||
proxy, err := url.Parse(opt.proxyURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
proxyFunc = http.ProxyURL(proxy)
|
||
}
|
||
|
||
return &bhttpClient{
|
||
client: &http.Client{
|
||
Transport: &http.Transport{
|
||
Proxy: proxyFunc,
|
||
DisableKeepAlives: opt.disableKeepAlives,
|
||
},
|
||
Timeout: time.Duration(opt.timeout) * time.Second,
|
||
},
|
||
header: &http.Header{},
|
||
values: &url.Values{},
|
||
params: make(map[string]string),
|
||
retry: opt.retry,
|
||
checkStatusOk: opt.checkStatusOk,
|
||
proxyURL: opt.proxyURL,
|
||
}, nil
|
||
}
|
||
|
||
func (b *bhttpClient) SetParam(key, value string) *bhttpClient {
|
||
b.values.Add(key, value)
|
||
oParams := b.params
|
||
oParams[key] = value
|
||
b.params = oParams
|
||
|
||
return b
|
||
}
|
||
|
||
func (b *bhttpClient) SetBody(body []byte) *bhttpClient {
|
||
b.postBody = body
|
||
return b
|
||
}
|
||
|
||
func (b *bhttpClient) SetHeader(key, value string) *bhttpClient {
|
||
b.header.Add(key, value)
|
||
return b
|
||
}
|
||
|
||
// GetRedirectedURL 获取重定向链接
|
||
func (b *bhttpClient) GetRedirectedURL(url string) (string, error) {
|
||
var (
|
||
err error
|
||
)
|
||
|
||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
b.httpRequest = req
|
||
b.httpRequest.Header = *b.header
|
||
|
||
resp, err := b.client.Do(b.httpRequest)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
defer resp.Body.Close()
|
||
|
||
redirectedURL := resp.Request.URL.String()
|
||
return redirectedURL, nil
|
||
}
|
||
|
||
func (b *bhttpClient) DoGet(reqUrl string) ([]byte, error) {
|
||
requestURI, err := url.ParseRequestURI(reqUrl)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
reader := b.values.Encode()
|
||
if reader != "" {
|
||
if requestURI.RawQuery == "" {
|
||
requestURI.RawQuery = reader
|
||
} else {
|
||
requestURI.RawQuery = fmt.Sprintf("%s&%s", requestURI.RawQuery, reader)
|
||
}
|
||
// fmt.Println("test_debug", requestURI.String())
|
||
}
|
||
|
||
return b.doRequest(http.MethodGet, requestURI.String(), "")
|
||
}
|
||
|
||
func (b *bhttpClient) DoPost(url string) ([]byte, error) {
|
||
reader := b.values.Encode()
|
||
if len(b.postBody) > 0 {
|
||
reader = string(b.postBody)
|
||
}
|
||
return b.doRequest(http.MethodPost, url, reader)
|
||
}
|
||
|
||
// DoPatch is a patch method
|
||
func (b *bhttpClient) DoPatch(url string) ([]byte, error) {
|
||
reader := string(b.postBody)
|
||
return b.doRequest(http.MethodPatch, url, reader)
|
||
}
|
||
|
||
func (b *bhttpClient) DoPut(url string) ([]byte, error) {
|
||
reader := string(b.postBody)
|
||
return b.doRequest(http.MethodPut, url, reader)
|
||
}
|
||
|
||
func (b *bhttpClient) doRequest(method, url, reader string) ([]byte, error) {
|
||
payload := bytes.NewBuffer([]byte(reader))
|
||
//如果content_type是 multipart/form-data
|
||
cType := b.header.Get("Content-Type")
|
||
if strings.Contains(cType, "form-data") {
|
||
payload = &bytes.Buffer{}
|
||
bodyWriter := multipart.NewWriter(payload)
|
||
for k, v := range b.params {
|
||
bodyWriter.WriteField(k, v)
|
||
}
|
||
bodyWriter.Close()
|
||
b.header.Set("Content-Type", bodyWriter.FormDataContentType())
|
||
}
|
||
switch method {
|
||
case http.MethodGet:
|
||
req, err := http.NewRequest(method, url, nil)
|
||
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
b.httpRequest = req
|
||
case http.MethodPost, http.MethodPut:
|
||
req, err := http.NewRequest(method, url, payload)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
b.httpRequest = req
|
||
case http.MethodPatch:
|
||
fmt.Println("url: ", url)
|
||
fmt.Println("payload: ", reader)
|
||
payload1 := strings.NewReader(`{"status": "FINALIZED"}`)
|
||
req, err := http.NewRequest(method, url, payload1)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// 设置查询参数
|
||
// q := req.URL.Query()
|
||
// for k, v := range b.params {
|
||
// q.Add(k, v)
|
||
// }
|
||
// req.URL.RawQuery = q.Encode()
|
||
b.httpRequest = req
|
||
|
||
default:
|
||
return nil, errors.New("method not support")
|
||
}
|
||
|
||
b.httpRequest.Header = *b.header
|
||
|
||
var (
|
||
res *http.Response
|
||
err error
|
||
)
|
||
retry := b.retry + 1
|
||
for i := 1; i <= int(retry); i++ {
|
||
body, werr := b.client.Do(b.httpRequest)
|
||
err = werr
|
||
if err == nil {
|
||
res = body
|
||
break
|
||
}
|
||
fmt.Printf("bhttp-临时超时: %d , err: %s\n, proxy: %s, header-auth: %s \n", i, err.Error(), b.proxyURL, b.header.Get("Authorization"))
|
||
time.Sleep(time.Duration(i) * time.Second)
|
||
}
|
||
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
defer res.Body.Close()
|
||
|
||
rateTimeLeft := res.Header.Get("rate-limit-msec-left")
|
||
if rateTimeLeft != "" {
|
||
rateTimeLeftInt, _ := strconv.Atoi(rateTimeLeft)
|
||
if rateTimeLeftInt > 0 {
|
||
b.rateTimeLeft = rateTimeLeftInt
|
||
}
|
||
|
||
}
|
||
|
||
//ioutil.ReadAll(res.Body) //io/ioutil" has been deprecated since Go 1.19
|
||
newReader := bufio.NewReader(res.Body)
|
||
body := make([]byte, 0)
|
||
for {
|
||
data, err := newReader.ReadBytes('\n')
|
||
if err == io.EOF {
|
||
body = append(body, data...)
|
||
break
|
||
}
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
body = append(body, data...)
|
||
}
|
||
|
||
if b.checkStatusOk && res.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("status error code: %d, res: %s", res.StatusCode, string(body))
|
||
}
|
||
|
||
return body, nil
|
||
}
|
||
|
||
// GetRateTimeLeft get the reader of the request body
|
||
func (b *bhttpClient) GetRateTimeLeft() int {
|
||
return b.rateTimeLeft
|
||
}
|
||
|
||
// get the reader of the request body
|
||
func (b *bhttpClient) GetReader() string {
|
||
return b.values.Encode()
|
||
}
|
||
|
||
// GetRedirectedURL 获取重定向url
|
||
func GetRedirectedURL(initialURL string) (string, error) {
|
||
// 创建一个自定义的HTTP客户端
|
||
client := &http.Client{
|
||
// 自定义CheckRedirect函数，遇到重定向时获取Location头并返回
|
||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||
if len(via) > 0 {
|
||
// 返回重定向后的URL
|
||
return http.ErrUseLastResponse
|
||
}
|
||
return nil
|
||
},
|
||
}
|
||
|
||
// 发送GET请求
|
||
resp, err := client.Get(initialURL)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
// 检查响应状态码是否为302
|
||
if resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusMovedPermanently {
|
||
// 获取Location头中的重定向URL
|
||
redirectURL, err := resp.Location()
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return redirectURL.String(), nil
|
||
}
|
||
|
||
return "", fmt.Errorf("no redirect found")
|
||
|
||
}
|
||
|
||
func ParsedUrlWithRetry(link string) (*url.URL, error) {
|
||
for i := 0; i <= 5; i++ {
|
||
parsedURL, err := url.Parse(link)
|
||
if err == nil {
|
||
return parsedURL, nil
|
||
}
|
||
time.Sleep(1 * time.Second)
|
||
}
|
||
|
||
return nil, fmt.Errorf("failed to parsed url: %s", link)
|
||
|
||
}
|