// bhttp Support for Http custom parameters and proxy package bhttp import ( "bufio" "bytes" "errors" "fmt" "io" "mime/multipart" "net/http" "net/url" "strconv" "strings" "time" ) // BhttpService bhttp service type BhttpService struct { Client *bhttpClient } // NewBhttpService 创建bhttp service func NewBhttpService(opts ...Option) (*BhttpService, error) { client, err := NewBhttpClient(opts...) if err != nil { return nil, err } return &BhttpService{ Client: client, }, nil } type bhttpClient struct { client *http.Client httpRequest *http.Request header *http.Header values *url.Values params map[string]string postBody []byte retry int32 checkStatusOk bool proxyURL string rateTimeLeft int //下次可执行剩余的时间毫秒 } type options struct { timeout int32 proxyURL string retry int32 checkStatusOk bool disableKeepAlives bool } type Option func(*options) func WithTimeout(b int32) Option { return func(c *options) { c.timeout = b } } func WithProxy(b string) Option { return func(c *options) { c.proxyURL = b } } func WithRetry(b int32) Option { return func(c *options) { c.retry = b } } func WithCheckStatusOk(b bool) Option { return func(c *options) { c.checkStatusOk = b } } func WithDisableKeepAlives(b bool) Option { return func(c *options) { c.disableKeepAlives = b } } func NewBhttpClient(opts ...Option) (*bhttpClient, error) { proxyFunc := http.ProxyFromEnvironment opt := options{ timeout: 60, proxyURL: "", retry: 0, checkStatusOk: false, disableKeepAlives: false, } for _, o := range opts { o(&opt) } if opt.proxyURL != "" { proxy, err := url.Parse(opt.proxyURL) if err != nil { return nil, err } proxyFunc = http.ProxyURL(proxy) } return &bhttpClient{ client: &http.Client{ Transport: &http.Transport{ Proxy: proxyFunc, DisableKeepAlives: opt.disableKeepAlives, }, Timeout: time.Duration(opt.timeout) * time.Second, }, header: &http.Header{}, values: &url.Values{}, params: make(map[string]string), retry: opt.retry, checkStatusOk: opt.checkStatusOk, proxyURL: opt.proxyURL, }, nil } func (b *bhttpClient) SetParam(key, value string) *bhttpClient { b.values.Add(key, value) oParams := b.params oParams[key] = value b.params = oParams return b } func (b *bhttpClient) SetBody(body []byte) *bhttpClient { b.postBody = body return b } func (b *bhttpClient) SetHeader(key, value string) *bhttpClient { b.header.Add(key, value) return b } // GetRedirectedURL 获取重定向链接 func (b *bhttpClient) GetRedirectedURL(url string) (string, error) { var ( err error ) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return "", err } b.httpRequest = req b.httpRequest.Header = *b.header resp, err := b.client.Do(b.httpRequest) if err != nil { return "", err } defer resp.Body.Close() redirectedURL := resp.Request.URL.String() return redirectedURL, nil } func (b *bhttpClient) DoGet(reqUrl string) ([]byte, error) { requestURI, err := url.ParseRequestURI(reqUrl) if err != nil { return nil, err } reader := b.values.Encode() if reader != "" { if requestURI.RawQuery == "" { requestURI.RawQuery = reader } else { requestURI.RawQuery = fmt.Sprintf("%s&%s", requestURI.RawQuery, reader) } // fmt.Println("test_debug", requestURI.String()) } return b.doRequest(http.MethodGet, requestURI.String(), "") } func (b *bhttpClient) DoPost(url string) ([]byte, error) { reader := b.values.Encode() if len(b.postBody) > 0 { reader = string(b.postBody) } return b.doRequest(http.MethodPost, url, reader) } // DoPatch is a patch method func (b *bhttpClient) DoPatch(url string) ([]byte, error) { reader := string(b.postBody) return b.doRequest(http.MethodPatch, url, reader) } func (b *bhttpClient) DoPut(url string) ([]byte, error) { reader := string(b.postBody) return b.doRequest(http.MethodPut, url, reader) } func (b *bhttpClient) doRequest(method, url, reader string) ([]byte, error) { payload := bytes.NewBuffer([]byte(reader)) //如果content_type是 multipart/form-data cType := b.header.Get("Content-Type") if strings.Contains(cType, "form-data") { payload = &bytes.Buffer{} bodyWriter := multipart.NewWriter(payload) for k, v := range b.params { bodyWriter.WriteField(k, v) } bodyWriter.Close() b.header.Set("Content-Type", bodyWriter.FormDataContentType()) } switch method { case http.MethodGet: req, err := http.NewRequest(method, url, nil) if err != nil { return nil, err } b.httpRequest = req case http.MethodPost, http.MethodPut: req, err := http.NewRequest(method, url, payload) if err != nil { return nil, err } b.httpRequest = req case http.MethodPatch: fmt.Println("url: ", url) fmt.Println("payload: ", reader) payload1 := strings.NewReader(`{"status": "FINALIZED"}`) req, err := http.NewRequest(method, url, payload1) if err != nil { return nil, err } // 设置查询参数 // q := req.URL.Query() // for k, v := range b.params { // q.Add(k, v) // } // req.URL.RawQuery = q.Encode() b.httpRequest = req default: return nil, errors.New("method not support") } b.httpRequest.Header = *b.header var ( res *http.Response err error ) retry := b.retry + 1 for i := 1; i <= int(retry); i++ { body, werr := b.client.Do(b.httpRequest) err = werr if err == nil { res = body break } fmt.Printf("bhttp-临时超时: %d , err: %s\n, proxy: %s, header-auth: %s \n", i, err.Error(), b.proxyURL, b.header.Get("Authorization")) time.Sleep(time.Duration(i) * time.Second) } if err != nil { return nil, err } defer res.Body.Close() rateTimeLeft := res.Header.Get("rate-limit-msec-left") if rateTimeLeft != "" { rateTimeLeftInt, _ := strconv.Atoi(rateTimeLeft) if rateTimeLeftInt > 0 { b.rateTimeLeft = rateTimeLeftInt } } //ioutil.ReadAll(res.Body) //io/ioutil" has been deprecated since Go 1.19 newReader := bufio.NewReader(res.Body) body := make([]byte, 0) for { data, err := newReader.ReadBytes('\n') if err == io.EOF { body = append(body, data...) break } if err != nil { return nil, err } body = append(body, data...) } if b.checkStatusOk && res.StatusCode != http.StatusOK { return nil, fmt.Errorf("status error code: %d, res: %s", res.StatusCode, string(body)) } return body, nil } // GetRateTimeLeft get the reader of the request body func (b *bhttpClient) GetRateTimeLeft() int { return b.rateTimeLeft } // get the reader of the request body func (b *bhttpClient) GetReader() string { return b.values.Encode() } // GetRedirectedURL 获取重定向url func GetRedirectedURL(initialURL string) (string, error) { // 创建一个自定义的HTTP客户端 client := &http.Client{ // 自定义CheckRedirect函数,遇到重定向时获取Location头并返回 CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) > 0 { // 返回重定向后的URL return http.ErrUseLastResponse } return nil }, } // 发送GET请求 resp, err := client.Get(initialURL) if err != nil { return "", err } defer resp.Body.Close() // 检查响应状态码是否为302 if resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusMovedPermanently { // 获取Location头中的重定向URL redirectURL, err := resp.Location() if err != nil { return "", err } return redirectURL.String(), nil } return "", fmt.Errorf("no redirect found") } func ParsedUrlWithRetry(link string) (*url.URL, error) { for i := 0; i <= 5; i++ { parsedURL, err := url.Parse(link) if err == nil { return parsedURL, nil } time.Sleep(1 * time.Second) } return nil, fmt.Errorf("failed to parsed url: %s", link) }