bvbej/base-golang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

[🚀] 七牛云SDK大改 #18

Merged
bvbej merged 1 commits from dev into main 2024-09-29 10:26:43 +08:00
Conversation 0 Commits 1 Files Changed 1 +133 -138
1 changed files with 133 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 13c6b4345f - Show all commits

271
pkg/qiniu/storage.go
View File

@ -5,9 +5,14 @@ import (
"errors" "errors"
"fmt" "fmt"
"gitea.bvbej.com/bvbej/base-golang/pkg/md5" "gitea.bvbej.com/bvbej/base-golang/pkg/md5"
"gitea.bvbej.com/bvbej/base-golang/tool" "github.com/qiniu/go-sdk/v7/auth"
"github.com/qiniu/go-sdk/v7/auth/qbox" "github.com/qiniu/go-sdk/v7/cdn"
"github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/go-sdk/v7/storagev2/credentials"
"github.com/qiniu/go-sdk/v7/storagev2/downloader"
"github.com/qiniu/go-sdk/v7/storagev2/http_client"
"github.com/qiniu/go-sdk/v7/storagev2/objects"
"github.com/qiniu/go-sdk/v7/storagev2/uploader"
"github.com/qiniu/go-sdk/v7/storagev2/uptoken"
"github.com/tidwall/gjson" "github.com/tidwall/gjson"
"io" "io"
"net/http" "net/http"
@ -17,33 +22,37 @@ import (
"time" "time"
) )
type Qhash string
const (
QSha1 Qhash = "sha1"
QSha256 Qhash = "sha256"
)
var _ QiNiu = (*qiNiu)(nil) var _ QiNiu = (*qiNiu)(nil)
type QiNiu interface { type QiNiu interface {
i() SetDefaultUploadTokenTTL(duration time.Duration)
SetDefaultUploadTokenTTL(ttl uint64) GetUploadToken() (string, error)
GetCallbackUploadToken(ttl uint64, callbackURL string) string GetCallbackUploadToken(callbackURL string) (string, error)
GetUploadToken(ttl uint64) string TimestampSecuritySign(urlStr string, ttl time.Duration) (string, error)
GetPrivateURL(key string, ttl uint64) string UploadFile(key, localFile, callbackURL string) (*PutRet, error)
GetPrivateURL(key string, ttl time.Duration) (string, error)
VerifyCallback(req *http.Request) (bool, error) VerifyCallback(req *http.Request) (bool, error)
UploadFile(key, localFile string) (*PutRet, error) GetFileInfo(key string) (*objects.ObjectDetails, error)
ResumeUploadFile(key, localFile string) (*PutRet, error)
DelFile(key string) error DelFile(key string) error
TimestampSecuritySign(path string, ttl time.Duration) string ListFiles(keyPrefix, marker string, limit uint64) ([]objects.ObjectDetails, error)
GetFileInfo(key string) *storage.FileInfo GetFileHash(path string, qhash Qhash) (hash string, err error)
ListFiles(prefix, delimiter, marker string, limit int) (entries []storage.ListItem, commonPrefixes []string, nextMarker string, hasNext bool, err error)
GetFileHash(path, qhash string) (hash string, err error)
} }
type qiNiu struct { type qiNiu struct {
mac *qbox.Mac
bucketManager *storage.BucketManager
conf *storage.Config
bucket string bucket string
domain string domain string
securityKey string securityKey string
credentials *auth.Credentials
bucketManager *objects.Bucket
md5 md5.MD5 md5 md5.MD5
uploadTokenTTL uint64 uploadTokenTTL time.Duration
} }
type PutRet struct { type PutRet struct {
@ -57,163 +66,149 @@ type PutRet struct {
} }
func New(accessKey, secretKey, bucket, domain, securityKey string) QiNiu { func New(accessKey, secretKey, bucket, domain, securityKey string) QiNiu {
mac := qbox.NewMac(accessKey, secretKey) cred := credentials.NewCredentials(accessKey, secretKey)
conf := &storage.Config{ objectsManager := objects.NewObjectsManager(&objects.ObjectsManagerOptions{
UseHTTPS: true, //是否使用https域名 Options: http_client.Options{Credentials: cred},
UseCdnDomains: false, //上传是否使用CDN上传加速 })
}
return &qiNiu{ return &qiNiu{
mac: mac,
bucketManager: storage.NewBucketManager(mac, conf),
bucket: bucket, bucket: bucket,
domain: domain, domain: domain,
securityKey: securityKey, securityKey: securityKey,
conf: conf, credentials: cred,
bucketManager: objectsManager.Bucket(bucket),
md5: md5.New(), md5: md5.New(),
uploadTokenTTL: 3600, uploadTokenTTL: time.Second * 3600,
} }
} }
func (q *qiNiu) i() {} func (q *qiNiu) SetDefaultUploadTokenTTL(duration time.Duration) {
q.uploadTokenTTL = duration
func (q *qiNiu) SetDefaultUploadTokenTTL(ttl uint64) {
q.uploadTokenTTL = ttl
} }
func (q *qiNiu) GetUploadToken(ttl uint64) string { func (q *qiNiu) GetUploadToken() (string, error) {
putPolicy := storage.PutPolicy{ putPolicy, err := uptoken.NewPutPolicy(q.bucket, time.Now().Add(q.uploadTokenTTL))
Scope: q.bucket, if err != nil {
Expires: ttl, return "", err
} }
return putPolicy.UploadToken(q.mac) return uptoken.NewSigner(putPolicy, q.credentials).GetUpToken(context.Background())
} }
func (q *qiNiu) GetCallbackUploadToken(ttl uint64, callbackURL string) string { func (q *qiNiu) GetCallbackUploadToken(callbackURL string) (string, error) {
putPolicy := storage.PutPolicy{ putPolicy, err := uptoken.NewPutPolicy(q.bucket, time.Now().Add(q.uploadTokenTTL))
Scope: q.bucket, if err != nil {
CallbackURL: callbackURL, return "", err
CallbackBody: `{"key":"$(key)","hash":"$(etag)","fname":"$(fname)","fsize":"$(fsize)","ext":"$(ext)","unique":"$(x:unique)","user":"$(x:user)"}`,
CallbackBodyType: "application/json",
Expires: ttl,
} }
return putPolicy.UploadToken(q.mac) putPolicy.SetCallbackUrl(callbackURL).
SetCallbackBody(`{"key":"$(key)","hash":"$(etag)","fname":"$(fname)","fsize":"$(fsize)","ext":"$(ext)","unique":"$(x:unique)","user":"$(x:user)"}`).
SetCallbackBodyType("application/json")
return uptoken.NewSigner(putPolicy, q.credentials).GetUpToken(context.Background())
} }
func (q *qiNiu) GetPrivateURL(key string, ttl uint64) string { func (q *qiNiu) TimestampSecuritySign(urlStr string, ttl time.Duration) (string, error) {
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix() deadline := time.Now().Add(ttl).Unix()
return storage.MakePrivateURL(q.mac, q.domain, key, deadline) tUrl, err := cdn.CreateTimestampAntileechURL(urlStr, q.securityKey, deadline)
if err != nil {
return "", err
}
return tUrl, nil
}
func (q *qiNiu) UploadFile(key, localFile, callbackURL string) (*PutRet, error) {
putPolicy, err := uptoken.NewPutPolicy(q.bucket, time.Now().Add(q.uploadTokenTTL))
if err != nil {
return nil, err
}
putPolicy.SetCallbackUrl(callbackURL).
SetCallbackBody(`{"key":"$(key)","hash":"$(etag)","fname":"$(fname)","fsize":"$(fsize)","ext":"$(ext)","unique":"$(x:unique)","user":"$(x:user)"}`).
SetCallbackBodyType("application/json")
ret := &PutRet{}
filename := path.Base(key)
fileSuffix := path.Ext(key)
filePrefix := filename[0 : len(filename)-len(fileSuffix)]
uploadManager := uploader.NewUploadManager(&uploader.UploadManagerOptions{})
err = uploadManager.UploadFile(context.Background(), localFile, &uploader.ObjectOptions{
UpToken: uptoken.NewSigner(putPolicy, q.credentials),
ObjectName: &key,
CustomVars: map[string]string{
"unique": filePrefix,
"user": "api",
},
}, ret)
if err != nil {
return nil, err
}
return ret, nil
}
func (q *qiNiu) GetPrivateURL(key string, ttl time.Duration) (string, error) {
urlsProvider := downloader.SignURLsProvider(
downloader.NewStaticDomainBasedURLsProvider([]string{q.domain}),
downloader.NewCredentialsSigner(q.credentials),
&downloader.SignOptions{
TTL: ttl,
})
iter, err := urlsProvider.GetURLsIter(context.Background(), key, &downloader.GenerateOptions{BucketName: q.bucket})
if err != nil {
return "", err
}
res := url.URL{}
peek, err := iter.Peek(&res)
if err != nil {
return "", err
}
if peek {
return res.String(), nil
}
return "", errors.New("get private url false")
} }
func (q *qiNiu) VerifyCallback(req *http.Request) (bool, error) { func (q *qiNiu) VerifyCallback(req *http.Request) (bool, error) {
return q.mac.VerifyCallback(req) return q.credentials.VerifyCallback(req)
} }
func (q *qiNiu) UploadFile(key, localFile string) (*PutRet, error) { func (q *qiNiu) GetFileInfo(key string) (*objects.ObjectDetails, error) {
upToken := q.GetUploadToken(q.uploadTokenTTL) objectInfo, err := q.bucketManager.Object(key).Stat().Call(context.Background())
//构建表单上传的对象
formUploader := storage.NewFormUploader(q.conf)
//请求参数
filename := path.Base(key)
fileSuffix := path.Ext(key)
filePrefix := filename[0 : len(filename)-len(fileSuffix)]
putExtra := &storage.PutExtra{
Params: map[string]string{
"x:unique": filePrefix,
"x:user": "-",
},
}
//自定义返回body
ret := new(PutRet)
err := formUploader.PutFile(context.Background(), ret, upToken, key, localFile, putExtra)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return objectInfo, nil
return ret, nil
}
func (q *qiNiu) ResumeUploadFile(key, localFile string) (*PutRet, error) {
upToken := q.GetUploadToken(q.uploadTokenTTL)
//构建分片上传的对象
resumeUploader := storage.NewResumeUploaderV2(q.conf)
//请求参数
filename := path.Base(key)
fileSuffix := path.Ext(key)
filePrefix := filename[0 : len(filename)-len(fileSuffix)]
putExtra := &storage.RputV2Extra{
CustomVars: map[string]string{
"x:unique": filePrefix,
"x:user": "-",
},
}
//自定义返回body
ret := new(PutRet)
err := resumeUploader.PutFile(context.Background(), ret, upToken, key, localFile, putExtra)
if err != nil {
return nil, err
}
return ret, nil
} }
func (q *qiNiu) DelFile(key string) error { func (q *qiNiu) DelFile(key string) error {
err := q.bucketManager.Delete(q.bucket, key) err := q.bucketManager.Object(key).Delete().Call(context.Background())
if err != nil { if err != nil {
return err return err
} }
return nil return nil
} }
func (q *qiNiu) TimestampSecuritySign(path string, ttl time.Duration) string { func (q *qiNiu) ListFiles(keyPrefix, marker string, limit uint64) ([]objects.ObjectDetails, error) {
sep := "/" iter := q.bucketManager.List(context.Background(), &objects.ListObjectsOptions{
path = strings.Trim(path, sep) Limit: &limit,
splits := strings.Split(path, sep) Prefix: keyPrefix,
for i, split := range splits { Marker: marker,
splits[i] = url.QueryEscape(split) })
defer func() {
_ = iter.Close()
}()
objectInfos := make([]objects.ObjectDetails, 0)
var objectInfo objects.ObjectDetails
for iter.Next(&objectInfo) {
objectInfos = append(objectInfos, objectInfo)
} }
path = sep + strings.Join(splits, sep) if err := iter.Error(); err != nil {
return nil, err
unix := time.Now().Add(ttl).Unix() }
hex := fmt.Sprintf("%x", unix) return objectInfos, nil
encrypt := q.md5.Encrypt(q.securityKey + path + hex)
param := make(url.Values)
param.Set("sign", encrypt)
param.Set("t", hex)
return param.Encode()
} }
func (q *qiNiu) GetFileInfo(key string) *storage.FileInfo { func (q *qiNiu) GetFileHash(path string, qhash Qhash) (hash string, err error) {
fileInfo, sErr := q.bucketManager.Stat(q.bucket, key) sign, err := q.TimestampSecuritySign(path, time.Second*5)
if sErr != nil { if err != nil {
return nil return "", err
} }
return &fileInfo
}
func (q *qiNiu) ListFiles(prefix, delimiter, marker string, limit int) (entries []storage.ListItem,
commonPrefixes []string, nextMarker string, hasNext bool, err error) {
return q.bucketManager.ListFiles(q.bucket, prefix, delimiter, marker, limit)
}
func (q *qiNiu) GetFileHash(path, qhash string) (hash string, err error) {
if !tool.InArray(qhash, []string{"sha1", "md5", "sha256"}) {
return "", errors.New("qhash invalid")
}
sign := q.TimestampSecuritySign(path, time.Second*5)
addr := fmt.Sprintf("%s/%s?%s&qhash/%s", strings.TrimRight(q.domain, "/"), path, sign, qhash) addr := fmt.Sprintf("%s/%s?%s&qhash/%s", strings.TrimRight(q.domain, "/"), path, sign, qhash)
resp, err := http.Get(addr) resp, err := http.Get(addr)