bvbej/base-golang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 26 Wiki Activity

first commit

Browse Source
This commit is contained in:
bvbej
2024-07-23 10:23:43 +08:00
commit 7b4c2521a3
126 changed files with 15931 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
pkg/httpclient/alarm.go Normal file
View File

@ -0,0 +1,29 @@
package httpclient
import (
"bufio"
"bytes"
"go.uber.org/zap"
)
// Verify parse the body and verify that it is correct
type AlarmVerify func(body []byte) (shouldAlarm bool)
type AlarmObject interface {
Send(subject, body string) error
}
func onFailedAlarm(title string, raw []byte, logger *zap.Logger, alarmObject AlarmObject) {
buf := bytes.NewBuffer(nil)
scanner := bufio.NewScanner(bytes.NewReader(raw))
for scanner.Scan() {
buf.WriteString(scanner.Text())
buf.WriteString(" <br/>")
}
if err := alarmObject.Send(title, buf.String()); err != nil && logger != nil {
logger.Error("calls failed alarm err", zap.Error(err))
}
}

395
pkg/httpclient/client.go Normal file
View File

@ -0,0 +1,395 @@
package httpclient
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/spf13/cast"
"net/http"
httpURL "net/url"
"time"
"git.bvbej.com/bvbej/base-golang/pkg/trace"
)
const (
// DefaultTTL 一次http请求最长执行1分钟
DefaultTTL = time.Minute
)
// Get 请求
func Get(url string, form httpURL.Values, options ...Option) (body []byte, err error) {
return withoutBody(http.MethodGet, url, form, options...)
}
// Delete delete 请求
func Delete(url string, form httpURL.Values, options ...Option) (body []byte, err error) {
return withoutBody(http.MethodDelete, url, form, options...)
}
func withoutBody(method, url string, form httpURL.Values, options ...Option) (body []byte, err error) {
if url == "" {
return nil, errors.New("url required")
}
if len(form) > 0 {
if url, err = addFormValuesIntoURL(url, form); err != nil {
return
}
}
ts := time.Now()
opt := getOption()
defer func() {
if opt.trace != nil {
opt.dialog.Success = err == nil
opt.dialog.CostSeconds = time.Since(ts).Seconds()
opt.trace.AppendDialog(opt.dialog)
}
releaseOption(opt)
}()
for _, f := range options {
f(opt)
}
opt.header["Content-Type"] = []string{"application/x-www-form-urlencoded; charset=utf-8"}
if opt.trace != nil {
opt.header[trace.Header] = []string{opt.trace.ID()}
}
ttl := opt.ttl
if ttl <= 0 {
ttl = DefaultTTL
}
ctx, cancel := context.WithTimeout(context.Background(), ttl)
defer cancel()
if opt.dialog != nil {
decodedURL, _ := httpURL.QueryUnescape(url)
opt.dialog.Request = &trace.Request{
TTL: ttl.String(),
Method: method,
DecodedURL: decodedURL,
Header: opt.header,
}
}
retryTimes := opt.retryTimes
if retryTimes <= 0 {
retryTimes = DefaultRetryTimes
}
retryDelay := opt.retryDelay
if retryDelay <= 0 {
retryDelay = DefaultRetryDelay
}
var httpCode int
defer func() {
if opt.alarmObject == nil {
return
}
if opt.alarmVerify != nil && !opt.alarmVerify(body) && err == nil {
return
}
info := &struct {
TraceID string `json:"trace_id"`
Request struct {
Method string `json:"method"`
URL string `json:"url"`
} `json:"request"`
Response struct {
HTTPCode int `json:"http_code"`
Body string `json:"body"`
} `json:"response"`
Error string `json:"error"`
}{}
if opt.trace != nil {
info.TraceID = opt.trace.ID()
}
info.Request.Method = method
info.Request.URL = url
info.Response.HTTPCode = httpCode
info.Response.Body = string(body)
info.Error = ""
if err != nil {
info.Error = fmt.Sprintf("%+v", err)
}
raw, _ := json.MarshalIndent(info, "", " ")
onFailedAlarm(opt.alarmTitle, raw, opt.logger, opt.alarmObject)
}()
for k := 0; k < retryTimes; k++ {
body, httpCode, err = doHTTP(ctx, method, url, nil, opt)
if shouldRetry(ctx, httpCode) || (opt.retryVerify != nil && opt.retryVerify(body)) {
time.Sleep(retryDelay)
continue
}
return
}
return
}
// PostForm post form 请求
func PostForm(url string, form httpURL.Values, options ...Option) (body []byte, err error) {
return withFormBody(http.MethodPost, url, form, options...)
}
// PostJSON post json 请求
func PostJSON(url string, raw json.RawMessage, options ...Option) (body []byte, err error) {
return withJSONBody(http.MethodPost, url, raw, options...)
}
// PutForm put form 请求
func PutForm(url string, form httpURL.Values, options ...Option) (body []byte, err error) {
return withFormBody(http.MethodPut, url, form, options...)
}
// PutJSON put json 请求
func PutJSON(url string, raw json.RawMessage, options ...Option) (body []byte, err error) {
return withJSONBody(http.MethodPut, url, raw, options...)
}
// PatchFrom patch form 请求
func PatchFrom(url string, form httpURL.Values, options ...Option) (body []byte, err error) {
return withFormBody(http.MethodPatch, url, form, options...)
}
// PatchJSON patch json 请求
func PatchJSON(url string, raw json.RawMessage, options ...Option) (body []byte, err error) {
return withJSONBody(http.MethodPatch, url, raw, options...)
}
func withFormBody(method, url string, form httpURL.Values, options ...Option) (body []byte, err error) {
if url == "" {
return nil, errors.New("url required")
}
if len(form) == 0 {
return nil, errors.New("form required")
}
ts := time.Now()
opt := getOption()
defer func() {
if opt.trace != nil {
opt.dialog.Success = err == nil
opt.dialog.CostSeconds = time.Since(ts).Seconds()
opt.trace.AppendDialog(opt.dialog)
}
releaseOption(opt)
}()
for _, f := range options {
f(opt)
}
opt.header["Content-Type"] = []string{"application/x-www-form-urlencoded; charset=utf-8"}
if opt.trace != nil {
opt.header[trace.Header] = []string{opt.trace.ID()}
}
ttl := opt.ttl
if ttl <= 0 {
ttl = DefaultTTL
}
ctx, cancel := context.WithTimeout(context.Background(), ttl)
defer cancel()
formValue := form.Encode()
if opt.dialog != nil {
decodedURL, _ := httpURL.QueryUnescape(url)
opt.dialog.Request = &trace.Request{
TTL: ttl.String(),
Method: method,
DecodedURL: decodedURL,
Header: opt.header,
Body: formValue,
}
}
retryTimes := opt.retryTimes
if retryTimes <= 0 {
retryTimes = DefaultRetryTimes
}
retryDelay := opt.retryDelay
if retryDelay <= 0 {
retryDelay = DefaultRetryDelay
}
var httpCode int
defer func() {
if opt.alarmObject == nil {
return
}
if opt.alarmVerify != nil && !opt.alarmVerify(body) && err == nil {
return
}
info := &struct {
TraceID string `json:"trace_id"`
Request struct {
Method string `json:"method"`
URL string `json:"url"`
} `json:"request"`
Response struct {
HTTPCode int `json:"http_code"`
Body string `json:"body"`
} `json:"response"`
Error string `json:"error"`
}{}
if opt.trace != nil {
info.TraceID = opt.trace.ID()
}
info.Request.Method = method
info.Request.URL = url
info.Response.HTTPCode = httpCode
info.Response.Body = string(body)
info.Error = ""
if err != nil {
info.Error = fmt.Sprintf("%+v", err)
}
raw, _ := json.MarshalIndent(info, "", " ")
onFailedAlarm(opt.alarmTitle, raw, opt.logger, opt.alarmObject)
}()
for k := 0; k < retryTimes; k++ {
body, httpCode, err = doHTTP(ctx, method, url, []byte(formValue), opt)
if shouldRetry(ctx, httpCode) || (opt.retryVerify != nil && opt.retryVerify(body)) {
time.Sleep(retryDelay)
continue
}
return
}
return
}
func withJSONBody(method, url string, raw json.RawMessage, options ...Option) (body []byte, err error) {
if url == "" {
return nil, errors.New("url required")
}
if len(raw) == 0 {
return nil, errors.New("raw required")
}
ts := time.Now()
opt := getOption()
defer func() {
if opt.trace != nil {
opt.dialog.Success = err == nil
opt.dialog.CostSeconds = time.Since(ts).Seconds()
opt.trace.AppendDialog(opt.dialog)
}
releaseOption(opt)
}()
for _, f := range options {
f(opt)
}
opt.header["Content-Type"] = []string{"application/json; charset=utf-8"}
if opt.trace != nil {
opt.header[trace.Header] = []string{opt.trace.ID()}
}
ttl := opt.ttl
if ttl <= 0 {
ttl = DefaultTTL
}
ctx, cancel := context.WithTimeout(context.Background(), ttl)
defer cancel()
if opt.dialog != nil {
decodedURL, _ := httpURL.QueryUnescape(url)
opt.dialog.Request = &trace.Request{
TTL: ttl.String(),
Method: method,
DecodedURL: decodedURL,
Header: opt.header,
Body: cast.ToString(raw),
}
}
retryTimes := opt.retryTimes
if retryTimes <= 0 {
retryTimes = DefaultRetryTimes
}
retryDelay := opt.retryDelay
if retryDelay <= 0 {
retryDelay = DefaultRetryDelay
}
var httpCode int
defer func() {
if opt.alarmObject == nil {
return
}
if opt.alarmVerify != nil && !opt.alarmVerify(body) && err == nil {
return
}
info := &struct {
TraceID string `json:"trace_id"`
Request struct {
Method string `json:"method"`
URL string `json:"url"`
} `json:"request"`
Response struct {
HTTPCode int `json:"http_code"`
Body string `json:"body"`
} `json:"response"`
Error string `json:"error"`
}{}
if opt.trace != nil {
info.TraceID = opt.trace.ID()
}
info.Request.Method = method
info.Request.URL = url
info.Response.HTTPCode = httpCode
info.Response.Body = string(body)
info.Error = ""
if err != nil {
info.Error = fmt.Sprintf("%+v", err)
}
raw, _ := json.MarshalIndent(info, "", " ")
onFailedAlarm(opt.alarmTitle, raw, opt.logger, opt.alarmObject)
}()
for k := 0; k < retryTimes; k++ {
body, httpCode, err = doHTTP(ctx, method, url, raw, opt)
if shouldRetry(ctx, httpCode) || (opt.retryVerify != nil && opt.retryVerify(body)) {
time.Sleep(retryDelay)
continue
}
return
}
return
}

46
pkg/httpclient/error.go Normal file
View File

@ -0,0 +1,46 @@
package httpclient
var _ ReplyErr = (*replyErr)(nil)
// ReplyErr 错误响应，当 resp.StatusCode != http.StatusOK 时用来包装返回的 httpcode 和 body 。
type ReplyErr interface {
error
StatusCode() int
Body() []byte
}
type replyErr struct {
err error
statusCode int
body []byte
}
func (r *replyErr) Error() string {
return r.err.Error()
}
func (r *replyErr) StatusCode() int {
return r.statusCode
}
func (r *replyErr) Body() []byte {
return r.body
}
func newReplyErr(statusCode int, body []byte, err error) ReplyErr {
return &replyErr{
statusCode: statusCode,
body: body,
err: err,
}
}
// ToReplyErr 尝试将 err 转换为 ReplyErr
func ToReplyErr(err error) (ReplyErr, bool) {
if err == nil {
return nil, false
}
e, ok := err.(ReplyErr)
return e, ok
}

138
pkg/httpclient/option.go Normal file
View File

@ -0,0 +1,138 @@
package httpclient
import (
"sync"
"time"
"git.bvbej.com/bvbej/base-golang/pkg/trace"
"go.uber.org/zap"
)
var (
cache = &sync.Pool{
New: func() any {
return &option{
header: make(map[string][]string),
}
},
}
)
// Mock 定义接口Mock数据
type Mock func() (body []byte)
// Option 自定义设置http请求
type Option func(*option)
type basicAuth struct {
username string
password string
}
type option struct {
ttl time.Duration
basicAuth *basicAuth
header map[string][]string
trace *trace.Trace
dialog *trace.Dialog
logger *zap.Logger
retryTimes int
retryDelay time.Duration
retryVerify RetryVerify
alarmTitle string
alarmObject AlarmObject
alarmVerify AlarmVerify
mock Mock
}
func (o *option) reset() {
o.ttl = 0
o.basicAuth = nil
o.header = make(map[string][]string)
o.trace = nil
o.dialog = nil
o.logger = nil
o.retryTimes = 0
o.retryDelay = 0
o.retryVerify = nil
o.alarmTitle = ""
o.alarmObject = nil
o.alarmVerify = nil
o.mock = nil
}
func getOption() *option {
return cache.Get().(*option)
}
func releaseOption(opt *option) {
opt.reset()
cache.Put(opt)
}
// WithTTL 本次http请求最长执行时间
func WithTTL(ttl time.Duration) Option {
return func(opt *option) {
opt.ttl = ttl
}
}
// WithHeader 设置http header，可以调用多次设置多对key-value
func WithHeader(key, value string) Option {
return func(opt *option) {
opt.header[key] = []string{value}
}
}
// WithBasicAuth 设置基础认证权限
func WithBasicAuth(username, password string) Option {
return func(opt *option) {
opt.basicAuth = &basicAuth{
username: username,
password: password,
}
}
}
// WithTrace 设置trace信息
func WithTrace(t trace.T) Option {
return func(opt *option) {
if t != nil {
opt.trace = t.(*trace.Trace)
opt.dialog = new(trace.Dialog)
}
}
}
// WithLogger 设置logger以便打印关键日志
func WithLogger(logger *zap.Logger) Option {
return func(opt *option) {
opt.logger = logger
}
}
// WithMock 设置 mock 数据
func WithMock(m Mock) Option {
return func(opt *option) {
opt.mock = m
}
}
// WithOnFailedAlarm 设置告警通知
func WithOnFailedAlarm(alarmTitle string, alarmObject AlarmObject, alarmVerify AlarmVerify) Option {
return func(opt *option) {
opt.alarmTitle = alarmTitle
opt.alarmObject = alarmObject
opt.alarmVerify = alarmVerify
}
}
// WithOnFailedRetry 设置失败重试
func WithOnFailedRetry(retryTimes int, retryDelay time.Duration, retryVerify RetryVerify) Option {
return func(opt *option) {
opt.retryTimes = retryTimes
opt.retryDelay = retryDelay
opt.retryVerify = retryVerify
}
}

44
pkg/httpclient/retry.go Normal file
View File

@ -0,0 +1,44 @@
package httpclient
import (
"context"
"net/http"
"time"
)
const (
// DefaultRetryTimes 如果请求失败，最多重试3次
DefaultRetryTimes = 3
// DefaultRetryDelay 在重试前，延迟等待100毫秒
DefaultRetryDelay = time.Millisecond * 100
)
// Verify parse the body and verify that it is correct
type RetryVerify func(body []byte) (shouldRetry bool)
func shouldRetry(ctx context.Context, httpCode int) bool {
select {
case <-ctx.Done():
return false
default:
}
switch httpCode {
case
_StatusReadRespErr,
_StatusDoReqErr,
http.StatusRequestTimeout,
http.StatusLocked,
http.StatusTooEarly,
http.StatusTooManyRequests,
http.StatusServiceUnavailable,
http.StatusGatewayTimeout:
return true
default:
return false
}
}

147
pkg/httpclient/util.go Normal file
View File

@ -0,0 +1,147 @@
package httpclient
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
"git.bvbej.com/bvbej/base-golang/pkg/trace"
"go.uber.org/zap"
)
const (
// _StatusReadRespErr read resp body err, should re-call doHTTP again.
_StatusReadRespErr = -204
// _StatusDoReqErr do req err, should re-call doHTTP again.
_StatusDoReqErr = -500
)
var defaultClient = &http.Client{
Transport: &http.Transport{
DisableKeepAlives: true,
DisableCompression: true,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
MaxIdleConns: 100,
MaxConnsPerHost: 100,
MaxIdleConnsPerHost: 100,
},
}
func doHTTP(ctx context.Context, method, url string, payload []byte, opt *option) ([]byte, int, error) {
ts := time.Now()
if mock := opt.mock; mock != nil {
if opt.dialog != nil {
opt.dialog.AppendResponse(&trace.Response{
HttpCode: http.StatusOK,
HttpCodeMsg: http.StatusText(http.StatusOK),
Body: string(mock()),
CostSeconds: time.Since(ts).Seconds(),
})
}
return mock(), http.StatusOK, nil
}
req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(payload))
if err != nil {
return nil, -1, errors.Join(err, fmt.Errorf("new request [%s %s] err", method, url))
}
for key, value := range opt.header {
req.Header.Set(key, value[0])
}
if opt.basicAuth != nil {
req.SetBasicAuth(opt.basicAuth.username, opt.basicAuth.password)
}
resp, err := defaultClient.Do(req)
if err != nil {
err = errors.Join(err, fmt.Errorf("do request [%s %s] err", method, url))
if opt.dialog != nil {
opt.dialog.AppendResponse(&trace.Response{
Body: err.Error(),
CostSeconds: time.Since(ts).Seconds(),
})
}
if opt.logger != nil {
opt.logger.Warn("doHTTP got err", zap.Error(err))
}
return nil, _StatusDoReqErr, err
}
defer func() {
_ = resp.Body.Close()
}()
body, err := io.ReadAll(resp.Body)
if err != nil {
err = errors.Join(err, fmt.Errorf("read resp body from [%s %s] err", method, url))
if opt.dialog != nil {
opt.dialog.AppendResponse(&trace.Response{
Body: err.Error(),
CostSeconds: time.Since(ts).Seconds(),
})
}
if opt.logger != nil {
opt.logger.Warn("doHTTP got err", zap.Error(err))
}
return nil, _StatusReadRespErr, err
}
defer func() {
if opt.dialog != nil {
opt.dialog.AppendResponse(&trace.Response{
Header: resp.Header,
HttpCode: resp.StatusCode,
HttpCodeMsg: resp.Status,
Body: string(body), // unsafe
CostSeconds: time.Since(ts).Seconds(),
})
}
}()
if resp.StatusCode != http.StatusOK {
return nil, resp.StatusCode, newReplyErr(
resp.StatusCode,
body,
fmt.Errorf("do [%s %s] return code: %d message: %s", method, url, resp.StatusCode, string(body)),
)
}
return body, http.StatusOK, nil
}
// addFormValuesIntoURL append url.Values into url string
func addFormValuesIntoURL(rawURL string, form url.Values) (string, error) {
if rawURL == "" {
return "", errors.New("rawURL required")
}
if len(form) == 0 {
return "", errors.New("form required")
}
target, err := url.Parse(rawURL)
if err != nil {
return "", errors.Join(err, fmt.Errorf("parse rawURL `%s` err", rawURL))
}
urlValues := target.Query()
for key, values := range form {
for _, value := range values {
urlValues.Add(key, value)
}
}
target.RawQuery = urlValues.Encode()
return target.String(), nil
}