Merge pull request '[🚀] 64KB' (#7) from dev into main

Reviewed-on: #7

go.mod

@@ -1,4 +1,4 @@
-module git.bvbej.com/bvbej/base-golang
+module gitea.bvbej.com/bvbej/base-golang
 
 go 1.22.5
 
@@ -17,7 +17,7 @@ require (
 	github.com/panjf2000/ants/v2 v2.10.0
 	github.com/prometheus/client_golang v1.19.1
 	github.com/qiniu/go-sdk/v7 v7.21.1
-	github.com/redis/go-redis/v9 v9.6.0
+	github.com/redis/go-redis/v9 v9.6.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/cors v1.11.0
 	github.com/rs/cors/wrapper/gin v0.0.0-20240515105523-1562b1715b35
@@ -26,7 +26,7 @@ require (
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/buntdb v1.3.1
-	github.com/tidwall/gjson v1.17.1
+	github.com/tidwall/gjson v1.17.3
 	github.com/tus/tusd v1.13.0
 	github.com/xuri/excelize/v2 v2.8.1
 	go.mongodb.org/mongo-driver v1.16.0

go.sum

@@ -1246,8 +1246,8 @@ github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdk
 github.com/qiniu/go-sdk/v7 v7.21.1 h1:D/IjVOlg5pTw0jeDjqTo6H5QM73Obb1AYfPOHmIFN+Q=
 github.com/qiniu/go-sdk/v7 v7.21.1/go.mod h1:8EM2awITynlem2VML2dXGHkMYP2UyECsGLOdp6yMpco=
 github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
-github.com/redis/go-redis/v9 v9.6.0 h1:NLck+Rab3AOTHw21CGRpvQpgTrAU4sgdCswqGtlhGRA=
-github.com/redis/go-redis/v9 v9.6.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
+github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
@@ -1332,8 +1332,8 @@ github.com/tidwall/btree v1.4.2/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYms
 github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o=
 github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
-github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94=
+github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
 github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
 github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=

pkg/android_binary/apk/apk.go

@@ -4,7 +4,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/android_binary"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/android_binary"
 	"image"
 	"io"
 	"os"

pkg/android_binary/apk/apkxml.go

@@ -1,6 +1,6 @@
 package apk
 
-import "git.bvbej.com/bvbej/base-golang/pkg/android_binary"
+import "gitea.bvbej.com/bvbej/base-golang/pkg/android_binary"
 
 // Instrumentation is an application instrumentation code.
 type Instrumentation struct {

pkg/ants/pool.go

@@ -3,7 +3,7 @@ package ants
 import (
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/ticker"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/ticker"
 	"github.com/panjf2000/ants/v2"
 	"go.uber.org/zap"
 	"runtime/debug"

pkg/apollo/config.go

@@ -3,7 +3,7 @@ package apollo
 import (
 	"fmt"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/env"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/env"
 	"github.com/apolloconfig/agollo/v4"
 	"github.com/apolloconfig/agollo/v4/component/log"
 	apolloConfig "github.com/apolloconfig/agollo/v4/env/config"

pkg/cache/redis.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/time_parse"
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/time_parse"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 	"github.com/redis/go-redis/v9"
 )
 

pkg/captcha/base64.go

@@ -2,7 +2,7 @@ package captcha
 
 import (
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/cache"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/cache"
 	"github.com/mojocn/base64Captcha"
 	"go.uber.org/zap"
 	"strings"

pkg/database/mysql.go

@@ -7,8 +7,8 @@ import (
 	"os"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/time_parse"
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/time_parse"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"

pkg/database/trace.go

@@ -1,6 +1,6 @@
 package database
 
-import "git.bvbej.com/bvbej/base-golang/pkg/trace"
+import "gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 
 type Trace = trace.T
 

pkg/downloader/base/constants.go

@@ -0,0 +1,23 @@
+package base
+
+type Status int
+
+const (
+	DownloadStatusReady Status = iota
+	DownloadStatusStart
+	DownloadStatusPause
+	DownloadStatusError
+	DownloadStatusDone
+)
+
+const (
+	HttpCodeOK             = 200
+	HttpCodePartialContent = 206
+
+	HttpHeaderRange              = "Range"
+	HttpHeaderContentLength      = "Content-Length"
+	HttpHeaderContentRange       = "Content-Range"
+	HttpHeaderContentDisposition = "Content-Disposition"
+
+	HttpHeaderRangeFormat = "bytes=%d-%d"
+)

pkg/downloader/base/model.go

@@ -0,0 +1,37 @@
+package base
+
+// Request 下载请求
+type Request struct {
+	// 下载链接
+	URL string
+	// 附加信息
+	Extra any
+}
+
+// Resource 资源信息
+type Resource struct {
+	Req *Request
+	// 资源总大小
+	TotalSize int64
+	// 是否支持断点下载
+	Range bool
+	// 资源所包含的文件列表
+	Files []*FileInfo
+}
+
+// FileInfo 文件信息
+type FileInfo struct {
+	Name string
+	Path string
+	Size int64
+}
+
+// Options 下载选项
+type Options struct {
+	// 保存文件名
+	Name string
+	// 保存目录
+	Path string
+	// 并发连接数
+	Connections int
+}

pkg/downloader/controller/controller.go

@@ -0,0 +1,172 @@
+package controller
+
+import (
+	"golang.org/x/net/proxy"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+)
+
+type Controller interface {
+	Touch(name string, size int64) (file *os.File, err error)
+	Open(name string) (file *os.File, err error)
+	Write(name string, offset int64, buf []byte) (int, error)
+	Close(name string) error
+	ContextDialer() (proxy.Dialer, error)
+	ContextCookie() http.CookieJar
+	ContextTimeout() time.Duration
+	ContextProxy() func(*http.Request) (*url.URL, error)
+}
+
+type Option func(*option)
+
+type option struct {
+	CookieJar http.CookieJar
+	Timeout   time.Duration
+	Dialer    proxy.Dialer
+	Proxy     func(*http.Request) (*url.URL, error)
+}
+
+func WithCookie(cookieJar http.CookieJar) Option {
+	return func(opt *option) {
+		opt.CookieJar = cookieJar
+	}
+}
+
+func WithTimeout(timeout time.Duration) Option {
+	return func(opt *option) {
+		opt.Timeout = timeout
+	}
+}
+
+func WithDialer(dialer proxy.Dialer) Option {
+	return func(opt *option) {
+		opt.Dialer = dialer
+	}
+}
+
+func WithProxy(fn func(*http.Request) (*url.URL, error)) Option {
+	return func(opt *option) {
+		opt.Proxy = fn
+	}
+}
+
+type DefaultController struct {
+	*option
+	Files map[string]*os.File
+}
+
+func NewController(options ...Option) *DefaultController {
+	opt := new(option)
+	for _, f := range options {
+		f(opt)
+	}
+	if opt.Timeout == 0 {
+		opt.Timeout = time.Second * 30
+	}
+	if opt.Dialer == nil {
+		opt.Dialer = proxy.FromEnvironment()
+	}
+	return &DefaultController{
+		Files:  make(map[string]*os.File),
+		option: opt,
+	}
+}
+
+func (c *DefaultController) Touch(name string, size int64) (file *os.File, err error) {
+	file, err = os.Create(name)
+	if size > 0 {
+		err = os.Truncate(name, size)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if err == nil {
+		c.Files[name] = file
+	}
+	return
+}
+
+func (c *DefaultController) Open(name string) (file *os.File, err error) {
+	file, err = os.OpenFile(name, os.O_RDWR, os.ModePerm)
+	if err == nil {
+		c.Files[name] = file
+	}
+	return
+}
+
+func (c *DefaultController) Write(name string, offset int64, buf []byte) (int, error) {
+	return c.Files[name].WriteAt(buf, offset)
+}
+
+func (c *DefaultController) Close(name string) error {
+	err := c.Files[name].Close()
+	delete(c.Files, name)
+	return err
+}
+
+func (c *DefaultController) ContextDialer() (proxy.Dialer, error) {
+	return &DialerWarp{dialer: c.Dialer}, nil
+}
+
+func (c *DefaultController) ContextCookie() http.CookieJar {
+	return c.CookieJar
+}
+
+func (c *DefaultController) ContextTimeout() time.Duration {
+	return c.Timeout
+}
+
+func (c *DefaultController) ContextProxy() func(*http.Request) (*url.URL, error) {
+	return c.Proxy
+}
+
+type DialerWarp struct {
+	dialer proxy.Dialer
+}
+
+type ConnWarp struct {
+	conn net.Conn
+}
+
+func (c *ConnWarp) Read(b []byte) (n int, err error) {
+	return c.conn.Read(b)
+}
+
+func (c *ConnWarp) Write(b []byte) (n int, err error) {
+	return c.conn.Write(b)
+}
+
+func (c *ConnWarp) Close() error {
+	return c.conn.Close()
+}
+
+func (c *ConnWarp) LocalAddr() net.Addr {
+	return c.conn.LocalAddr()
+}
+
+func (c *ConnWarp) RemoteAddr() net.Addr {
+	return c.conn.RemoteAddr()
+}
+
+func (c *ConnWarp) SetDeadline(t time.Time) error {
+	return c.conn.SetDeadline(t)
+}
+
+func (c *ConnWarp) SetReadDeadline(t time.Time) error {
+	return c.conn.SetReadDeadline(t)
+}
+
+func (c *ConnWarp) SetWriteDeadline(t time.Time) error {
+	return c.conn.SetWriteDeadline(t)
+}
+
+func (d *DialerWarp) Dial(network, addr string) (c net.Conn, err error) {
+	conn, err := d.dialer.Dial(network, addr)
+	if err != nil {
+		return nil, err
+	}
+	return &ConnWarp{conn: conn}, nil
+}

pkg/downloader/downloader.go

@@ -2,11 +2,11 @@ package downloader
 
 import (
 	"errors"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/base"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/controller"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/fetcher"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/protocol/http"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/util"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/base"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/controller"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/fetcher"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/protocol/http"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/util"
 	"github.com/google/uuid"
 	"net/url"
 	"strings"

pkg/downloader/downloader_test.go

@@ -2,9 +2,9 @@ package downloader
 
 import (
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/base"
-	"git.bvbej.com/bvbej/base-golang/pkg/downloader/controller"
-	"git.bvbej.com/bvbej/base-golang/tool"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/base"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/controller"
+	"gitea.bvbej.com/bvbej/base-golang/tool"
 	"golang.org/x/net/proxy"
 	"net/http"
 	"net/url"

pkg/downloader/fetcher/fetcher.go

@@ -0,0 +1,52 @@
+package fetcher
+
+import (
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/base"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/controller"
+)
+
+// Fetcher 对应协议的下载支持
+type Fetcher interface {
+	// Setup 设置文件相关信息
+	Setup(ctl controller.Controller)
+	// Resolve 解析请求
+	Resolve(req *base.Request) (res *base.Resource, err error)
+	// Create 创建任务
+	Create(res *base.Resource, opts *base.Options) (err error)
+	// Start 开始
+	Start() (err error)
+	// Pause 暂停
+	Pause() (err error)
+	// Continue 继续
+	Continue() (err error)
+	// Progress 获取任务各个文件下载进度
+	Progress() Progress
+	// Wait 该方法会一直阻塞，直到任务下载结束
+	Wait() (err error)
+}
+
+type DefaultFetcher struct {
+	Ctl    controller.Controller
+	DoneCh chan error
+}
+
+func (f *DefaultFetcher) Setup(ctl controller.Controller) {
+	f.Ctl = ctl
+	f.DoneCh = make(chan error, 1)
+}
+
+func (f *DefaultFetcher) Wait() (err error) {
+	return <-f.DoneCh
+}
+
+// Progress 获取任务中各个文件的已下载字节数
+type Progress []int64
+
+// TotalDownloaded 获取任务总下载字节数
+func (p Progress) TotalDownloaded() int64 {
+	total := int64(0)
+	for _, d := range p {
+		total += d
+	}
+	return total
+}

pkg/downloader/protocol/http/fetcher.go

@@ -0,0 +1,409 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/base"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/downloader/fetcher"
+	"golang.org/x/sync/errgroup"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RequestError struct {
+	Code int
+	Msg  string
+}
+
+func NewRequestError(code int, msg string) *RequestError {
+	return &RequestError{Code: code, Msg: msg}
+}
+
+func (re *RequestError) Error() string {
+	return fmt.Sprintf("http request fail,code:%d", re.Code)
+}
+
+type Fetcher struct {
+	*fetcher.DefaultFetcher
+
+	res     *base.Resource
+	opts    *base.Options
+	status  base.Status
+	clients []*http.Response
+	chunks  []*Chunk
+
+	ctx     context.Context
+	cancel  context.CancelFunc
+	pauseCh chan any
+}
+
+func NewFetcher() *Fetcher {
+	return &Fetcher{
+		DefaultFetcher: new(fetcher.DefaultFetcher),
+		pauseCh:        make(chan any),
+	}
+}
+
+var protocols = []string{"HTTP", "HTTPS"}
+
+func FetcherBuilder() ([]string, func() fetcher.Fetcher) {
+	return protocols, func() fetcher.Fetcher {
+		return NewFetcher()
+	}
+}
+
+func (f *Fetcher) Resolve(req *base.Request) (*base.Resource, error) {
+	httpReq, err := f.buildRequest(nil, req)
+	if err != nil {
+		return nil, err
+	}
+	client, err := f.buildClient()
+	if err != nil {
+		return nil, err
+	}
+	// 只访问一个字节，测试资源是否支持Range请求
+	httpReq.Header.Set(base.HttpHeaderRange, fmt.Sprintf(base.HttpHeaderRangeFormat, 0, 0))
+	httpResp, err := client.Do(httpReq)
+	if err != nil {
+		return nil, err
+	}
+	// 拿到响应头就关闭，不用加defer
+	_ = httpResp.Body.Close()
+	res := &base.Resource{
+		Req:   req,
+		Range: false,
+		Files: []*base.FileInfo{},
+	}
+	if base.HttpCodePartialContent == httpResp.StatusCode {
+		// 返回206响应码表示支持断点下载
+		res.Range = true
+		// 解析资源大小: bytes 0-1000/1001 => 1001
+		contentTotal := path.Base(httpResp.Header.Get(base.HttpHeaderContentRange))
+		if contentTotal != "" {
+			parse, err := strconv.ParseInt(contentTotal, 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			res.TotalSize = parse
+		}
+	} else if base.HttpCodeOK == httpResp.StatusCode {
+		// 返回200响应码，不支持断点下载，通过Content-Length头获取文件大小，获取不到的话可能是chunked编码
+		contentLength := httpResp.Header.Get(base.HttpHeaderContentLength)
+		if contentLength != "" {
+			parse, err := strconv.ParseInt(contentLength, 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			res.TotalSize = parse
+		}
+	} else {
+		return nil, NewRequestError(httpResp.StatusCode, httpResp.Status)
+	}
+	file := &base.FileInfo{
+		Size: res.TotalSize,
+	}
+	contentDisposition := httpResp.Header.Get(base.HttpHeaderContentDisposition)
+	if contentDisposition != "" {
+		_, params, _ := mime.ParseMediaType(contentDisposition)
+		filename := params["filename"]
+		if filename != "" {
+			file.Name = filename
+		}
+	}
+	// Get file filename by URL
+	if file.Name == "" && strings.Count(req.URL, "/") > 2 {
+		file.Name = filepath.Base(req.URL)
+	}
+	// unknown file filename
+	if file.Name == "" {
+		file.Name = "unknown"
+	}
+	res.Files = append(res.Files, file)
+	return res, nil
+}
+
+func (f *Fetcher) Create(res *base.Resource, opts *base.Options) error {
+	f.res = res
+	f.opts = opts
+	f.status = base.DownloadStatusReady
+	return nil
+}
+
+func (f *Fetcher) Start() (err error) {
+	// 创建文件
+	name := f.filename()
+	_, err = f.Ctl.Touch(name, f.res.TotalSize)
+	if err != nil {
+		return err
+	}
+	f.status = base.DownloadStatusStart
+	if f.res.Range {
+		// 每个连接平均需要下载的分块大小
+		chunkSize := f.res.TotalSize / int64(f.opts.Connections)
+		f.chunks = make([]*Chunk, f.opts.Connections)
+		f.clients = make([]*http.Response, f.opts.Connections)
+		for i := 0; i < f.opts.Connections; i++ {
+			var (
+				begin = chunkSize * int64(i)
+				end   int64
+			)
+			if i == f.opts.Connections-1 {
+				// 最后一个分块需要保证把文件下载完
+				end = f.res.TotalSize - 1
+			} else {
+				end = begin + chunkSize - 1
+			}
+			chunk := NewChunk(begin, end)
+			f.chunks[i] = chunk
+		}
+	} else {
+		// 只支持单连接下载
+		f.chunks = make([]*Chunk, 1)
+		f.clients = make([]*http.Response, 1)
+		f.chunks[0] = NewChunk(0, 0)
+	}
+	f.fetch()
+	return
+}
+
+func (f *Fetcher) Pause() (err error) {
+	if base.DownloadStatusStart != f.status {
+		return
+	}
+	f.status = base.DownloadStatusPause
+	f.cancel()
+	<-f.pauseCh
+	return
+}
+
+func (f *Fetcher) Continue() (err error) {
+	if base.DownloadStatusStart == f.status || base.DownloadStatusDone == f.status {
+		return
+	}
+	f.status = base.DownloadStatusStart
+	var name = f.filename()
+	_, err = f.Ctl.Open(name)
+	if err != nil {
+		return err
+	}
+	f.fetch()
+	return
+}
+
+func (f *Fetcher) Progress() fetcher.Progress {
+	p := make(fetcher.Progress, 0)
+	if len(f.chunks) > 0 {
+		total := int64(0)
+		for _, chunk := range f.chunks {
+			total += chunk.Downloaded
+		}
+		p = append(p, total)
+	}
+	return p
+}
+
+func (f *Fetcher) filename() string {
+	// 创建文件
+	var filename = f.opts.Name
+	if filename == "" {
+		filename = f.res.Files[0].Name
+	}
+	return filepath.Join(f.opts.Path, filename)
+}
+
+func (f *Fetcher) fetch() {
+	f.ctx, f.cancel = context.WithCancel(context.Background())
+	eg, _ := errgroup.WithContext(f.ctx)
+	for i := 0; i < f.opts.Connections; i++ {
+		eg.Go(func() error {
+			return f.fetchChunk(i)
+		})
+	}
+
+	go func() {
+		err := eg.Wait()
+		// 下载停止，关闭文件句柄
+		_ = f.Ctl.Close(f.filename())
+		if f.status == base.DownloadStatusPause {
+			f.pauseCh <- nil
+		} else {
+			if err != nil {
+				f.status = base.DownloadStatusError
+			} else {
+				f.status = base.DownloadStatusDone
+			}
+			f.DoneCh <- err
+		}
+	}()
+}
+
+func (f *Fetcher) fetchChunk(index int) (err error) {
+	filename := f.filename()
+	chunk := f.chunks[index]
+
+	httpReq, err := f.buildRequest(f.ctx, f.res.Req)
+	if err != nil {
+		return err
+	}
+
+	client, err := f.buildClient()
+	if err != nil {
+		return err
+	}
+
+	var buf = make([]byte, 64*1024) // 64KB
+
+	// 重试10次
+	for i := 0; i < 10; i++ {
+		// 如果下载完成直接返回
+		if chunk.Status == base.DownloadStatusDone {
+			return
+		}
+		// 如果已暂停直接跳出
+		if f.status == base.DownloadStatusPause {
+			break
+		}
+		var (
+			resp  *http.Response
+			retry bool
+		)
+		if f.res.Range {
+			httpReq.Header.Set(base.HttpHeaderRange,
+				fmt.Sprintf(base.HttpHeaderRangeFormat, chunk.Begin+chunk.Downloaded, chunk.End))
+		} else {
+			chunk.Downloaded = 0
+		}
+		err = func() error {
+			resp, err = client.Do(httpReq)
+			if err != nil {
+				return err
+			}
+			f.clients[index] = resp
+			if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
+				err = NewRequestError(resp.StatusCode, resp.Status)
+				return err
+			}
+			return nil
+		}()
+		if err != nil {
+			//请求失败3s后重试
+			time.Sleep(time.Second * 3)
+			continue
+		}
+
+		// 请求成功就重置错误次数，连续失败10次才终止
+		i = 0
+
+		retry, err = func() (bool, error) {
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+			var n int
+			for {
+				n, err = resp.Body.Read(buf)
+				if n > 0 {
+					_, err = f.Ctl.Write(filename, chunk.Begin+chunk.Downloaded, buf[:n])
+					if err != nil {
+						return false, err
+					}
+					chunk.Downloaded += int64(n)
+				}
+				if err != nil {
+					if err != io.EOF {
+						return true, err
+					}
+					break
+				}
+			}
+			return false, nil
+		}()
+		if !retry {
+			// 下载成功，跳出重试
+			break
+		}
+	}
+
+	if f.status == base.DownloadStatusPause {
+		chunk.Status = base.DownloadStatusPause
+	} else if chunk.Downloaded >= chunk.End-chunk.Begin+1 {
+		chunk.Status = base.DownloadStatusDone
+	} else {
+		if err != nil {
+			chunk.Status = base.DownloadStatusError
+		} else {
+			chunk.Status = base.DownloadStatusDone
+		}
+	}
+	return
+}
+
+func (f *Fetcher) buildClient() (*http.Client, error) {
+	dialer, err := f.Ctl.ContextDialer()
+	if err != nil {
+		return nil, err
+	}
+	transport := &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return dialer.Dial(network, addr)
+		},
+	}
+	if f.Ctl.ContextProxy() != nil {
+		transport.Proxy = f.Ctl.ContextProxy()
+	}
+	return &http.Client{
+		Jar:       f.Ctl.ContextCookie(),
+		Timeout:   f.Ctl.ContextTimeout(),
+		Transport: transport,
+	}, nil
+}
+
+func (f *Fetcher) buildRequest(ctx context.Context, req *base.Request) (httpReq *http.Request, err error) {
+	reqUrl, err := url.Parse(req.URL)
+	if err != nil {
+		return
+	}
+
+	var (
+		method string
+		body   io.Reader
+	)
+	headers := make(map[string][]string)
+	if req.Extra == nil {
+		method = http.MethodGet
+	} else {
+		extra := req.Extra.(Extra)
+		if extra.Method != "" {
+			method = extra.Method
+		} else {
+			method = http.MethodGet
+		}
+		if len(extra.Header) > 0 {
+			for k, v := range extra.Header {
+				headers[k] = []string{v}
+			}
+		}
+		if extra.Body != "" {
+			body = io.NopCloser(bytes.NewBufferString(extra.Body))
+		}
+	}
+
+	if ctx != nil {
+		httpReq, err = http.NewRequestWithContext(ctx, method, reqUrl.String(), body)
+	} else {
+		httpReq, err = http.NewRequest(method, reqUrl.String(), body)
+	}
+	if err != nil {
+		return
+	}
+	httpReq.Header = headers
+	return httpReq, nil
+}

pkg/downloader/protocol/http/model.go

@@ -0,0 +1,24 @@
+package http
+
+import "gitea.bvbej.com/bvbej/base-golang/pkg/downloader/base"
+
+type Chunk struct {
+	Status     base.Status
+	Begin      int64
+	End        int64
+	Downloaded int64
+}
+
+func NewChunk(begin int64, end int64) *Chunk {
+	return &Chunk{
+		Status: base.DownloadStatusReady,
+		Begin:  begin,
+		End:    end,
+	}
+}
+
+type Extra struct {
+	Method string
+	Header map[string]string
+	Body   string
+}

pkg/downloader/util/timer.go

@@ -0,0 +1,25 @@
+package util
+
+import "time"
+
+// Timer 计时器
+type Timer struct {
+	t    int64
+	used int64
+}
+
+func (t *Timer) Start() {
+	t.t = time.Now().UnixNano()
+}
+
+func (t *Timer) Pause() {
+	t.used += time.Now().UnixNano() - t.t
+}
+
+func (t *Timer) Continue() {
+	t.t = time.Now().UnixNano()
+}
+
+func (t *Timer) Used() int64 {
+	return (time.Now().UnixNano() - t.t) + t.used
+}

pkg/httpclient/client.go

@@ -10,7 +10,7 @@ import (
 	httpURL "net/url"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 )
 
 const (

pkg/httpclient/option.go

@@ -4,7 +4,7 @@ import (
 	"sync"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 
 	"go.uber.org/zap"
 )

pkg/httpclient/util.go

@@ -11,7 +11,7 @@ import (
 	"net/url"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 	"go.uber.org/zap"
 )
 

pkg/limiter/rate.go

@@ -1,7 +1,7 @@
 package limiter
 
 import (
-	"git.bvbej.com/bvbej/base-golang/pkg/ticker"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/ticker"
 	"golang.org/x/time/rate"
 	"sync"
 	"time"

pkg/mux/context.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"sync"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/errno"
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/errno"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"

pkg/mux/core.go

@@ -8,12 +8,12 @@ import (
 	"runtime/debug"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/color"
-	"git.bvbej.com/bvbej/base-golang/pkg/env"
-	"git.bvbej.com/bvbej/base-golang/pkg/errno"
-	"git.bvbej.com/bvbej/base-golang/pkg/limiter"
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
-	"git.bvbej.com/bvbej/base-golang/pkg/validator"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/color"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/env"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/errno"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/limiter"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/validator"
 	"github.com/gin-contrib/pprof"
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"

pkg/p/print.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/trace"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/trace"
 )
 
 type Option func(*option)

pkg/qiniu/storage.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/md5"
-	"git.bvbej.com/bvbej/base-golang/tool"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/md5"
+	"gitea.bvbej.com/bvbej/base-golang/tool"
 	"github.com/qiniu/go-sdk/v7/auth/qbox"
 	"github.com/qiniu/go-sdk/v7/storage"
 	"github.com/tidwall/gjson"
@@ -57,12 +57,10 @@ type PutRet struct {
 }
 
 func New(accessKey, secretKey, bucket, domain, securityKey string) QiNiu {
-	region, _ := storage.GetRegion(accessKey, bucket)
 	mac := qbox.NewMac(accessKey, secretKey)
 	conf := &storage.Config{
-		Region:        region, //空间所在的存储区域
 		UseHTTPS:      true,  //是否使用https域名
-		UseCdnDomains: true,   //上传是否使用CDN上传加速
+		UseCdnDomains: false, //上传是否使用CDN上传加速
 	}
 	return &qiNiu{
 		mac:            mac,
@@ -216,7 +214,7 @@ func (q *qiNiu) GetFileHash(path, qhash string) (hash string, err error) {
 	}
 
 	sign := q.TimestampSecuritySign(path, time.Second*5)
-	addr := fmt.Sprintf("https://cdn.mogume.com/%s?%s&qhash/%s", path, sign, qhash)
+	addr := fmt.Sprintf("%s/%s?%s&qhash/%s", strings.TrimRight(q.domain, "/"), path, sign, qhash)
 
 	resp, err := http.Get(addr)
 	if err != nil {

pkg/signature/signature_generate.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"strings"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/time_parse"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/time_parse"
 )
 
 // Generate

pkg/signature/signature_verify.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/time_parse"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/time_parse"
 )
 
 func (s *signature) Verify(authorization, date string, path string, method string, params url.Values) (ok bool, err error) {

pkg/upload/tus.go

@@ -5,9 +5,9 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/color"
-	"git.bvbej.com/bvbej/base-golang/pkg/ticker"
-	"git.bvbej.com/bvbej/base-golang/pkg/token"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/color"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/ticker"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/token"
 	"github.com/rs/cors"
 	"github.com/tus/tusd/pkg/filestore"
 	tus "github.com/tus/tusd/pkg/handler"

pkg/websocket/client/client.go

@@ -3,12 +3,12 @@ package client
 import (
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/ticker"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/client/service"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
-	_ "git.bvbej.com/bvbej/base-golang/pkg/websocket/codec/json"
-	_ "git.bvbej.com/bvbej/base-golang/pkg/websocket/codec/protobuf"
-	"git.bvbej.com/bvbej/base-golang/tool"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/ticker"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/client/service"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
+	_ "gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec/json"
+	_ "gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec/protobuf"
+	"gitea.bvbej.com/bvbej/base-golang/tool"
 	"github.com/gorilla/websocket"
 	"go.uber.org/zap"
 	"net/http"

pkg/websocket/client/service/receiver.go

@@ -2,8 +2,8 @@ package service
 
 import (
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/util"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/util"
 	"reflect"
 	"strings"
 )

pkg/websocket/codec/json/json.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
 )
 
 type jsonCodec struct{}

pkg/websocket/codec/protobuf/protobuf.go

@@ -4,8 +4,8 @@ import (
 	"errors"
 	"fmt"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec/protobuf/protocol"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec/protobuf/protocol"
 	"google.golang.org/protobuf/proto"
 )
 

pkg/websocket/peer/connect/acceptor.go

@@ -4,14 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/mux"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/mux"
 	"github.com/gin-gonic/gin"
 	"go.uber.org/zap"
 	"net/http"
 	"net/url"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 	"github.com/gorilla/websocket"
 )
 

pkg/websocket/peer/connect/connection.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 	"github.com/gorilla/websocket"
 )
 

pkg/websocket/service/component.go

@@ -3,7 +3,7 @@ package service
 import (
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 )
 
 type Component interface {

pkg/websocket/service/method.go

@@ -1,7 +1,7 @@
 package service
 
 import (
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 	"reflect"
 )
 

pkg/websocket/service/service.go

@@ -2,11 +2,11 @@ package service
 
 import (
 	"errors"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/util"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/util"
 	"reflect"
 	"strings"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 )
 
 type Handler struct {

pkg/websocket/service/service_manager.go

@@ -2,9 +2,9 @@ package service
 
 import (
 	"fmt"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
-	_ "git.bvbej.com/bvbej/base-golang/pkg/websocket/codec/json"
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec"
+	_ "gitea.bvbej.com/bvbej/base-golang/pkg/websocket/codec/json"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 	"go.uber.org/zap"
 )
 

pkg/websocket/service/session_callback.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"git.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/websocket/peer"
 )
 
 type callBackEntity struct{}

tool/helper.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"github.com/spf13/cast"
 	"io"
 	"math"
 	"math/rand"
@@ -33,7 +34,7 @@ func GetOrderNumber() string {
 // ByteFmt 格式化显示文件大小
 func ByteFmt(size int64) string {
 	var unitArr = []string{"B", "KB", "MB", "GB", "TB", "EB"}
-	if size == 0 {
+	if size <= 0 {
 		return "unknown"
 	}
 	fs := float64(size)
@@ -43,7 +44,7 @@ func ByteFmt(size int64) string {
 	if frac > 0 {
 		return fmt.Sprintf("%.1f%s", math.Floor(val*10)/10, unitArr[p])
 	} else {
-		return fmt.Sprintf("%d%s", int(val), unitArr[p])
+		return fmt.Sprintf("%d%s", cast.ToInt(val), unitArr[p])
 	}
 }
 

tool/http.go

@@ -2,10 +2,10 @@ package tool
 
 import (
 	"encoding/json"
-	"git.bvbej.com/bvbej/base-golang/pkg/aes"
-	"git.bvbej.com/bvbej/base-golang/pkg/hmac"
-	"git.bvbej.com/bvbej/base-golang/pkg/httpclient"
-	"git.bvbej.com/bvbej/base-golang/pkg/time_parse"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/aes"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/hmac"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/httpclient"
+	"gitea.bvbej.com/bvbej/base-golang/pkg/time_parse"
 	"github.com/tidwall/gjson"
 	netUrl "net/url"
 	"strconv"