//go:build !minio package storage import ( "context" "crypto/md5" "encoding/hex" "errors" "fmt" "io" "path/filepath" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" ) // Config S3配置 type Config struct { Endpoint string `yaml:"endpoint" json:"endpoint" mapstructure:"endpoint"` AccessKeyID string `yaml:"access_key_id" json:"access_key_id" mapstructure:"access_key_id"` SecretAccessKey string `yaml:"secret_access_key" json:"secret_access_key" mapstructure:"secret_access_key"` UseSSL bool `yaml:"use_ssl" json:"use_ssl" mapstructure:"use_ssl"` BucketName string `yaml:"bucket_name" json:"bucket_name" mapstructure:"bucket_name"` Region string `yaml:"region" json:"region" mapstructure:"region"` CDNDomain string `yaml:"cdn_domain" json:"cdn_domain" mapstructure:"cdn_domain"` PresignExpires time.Duration `yaml:"presign_expires" json:"presign_expires" mapstructure:"presign_expires"` } // Client S3客户端 type Client struct { client *s3.Client presignClient *s3.PresignClient config *Config } // NewClient 创建S3客户端 func NewClient(ctx context.Context, cfg *Config) (*Client, error) { // 设置默认值 if cfg.PresignExpires == 0 { cfg.PresignExpires = 15 * time.Minute } if cfg.UseSSL && cfg.Endpoint == "" { cfg.UseSSL = true // AWS默认使用SSL } // 构建AWS配置选项 var opts []func(*config.LoadOptions) error // 设置区域 if cfg.Region != "" { opts = append(opts, config.WithRegion(cfg.Region)) } else { opts = append(opts, config.WithRegion("us-east-1")) // 默认区域 cfg.Region = "us-east-1" } // 设置凭证 if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { opts = append(opts, config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), )) } awsCfg, err := config.LoadDefaultConfig(ctx, opts...) if err != nil { return nil, fmt.Errorf("加载AWS配置失败: %w", err) } // 创建S3客户端选项 s3Opts := []func(*s3.Options){ func(o *s3.Options) { // 自定义端点(用于兼容S3的对象存储) if cfg.Endpoint != "" { o.BaseEndpoint = aws.String(cfg.Endpoint) o.UsePathStyle = true // 使用路径风格访问,兼容MinIO } // SSL设置 if !cfg.UseSSL { o.EndpointOptions.DisableHTTPS = true } }, } s3Client := s3.NewFromConfig(awsCfg, s3Opts...) presignClient := s3.NewPresignClient(s3Client) c := &Client{ client: s3Client, presignClient: presignClient, config: cfg, } // 确保默认桶存在 if cfg.BucketName != "" { timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() exists, err := c.BucketExists(timeoutCtx, cfg.BucketName) if err != nil { return nil, fmt.Errorf("检查桶失败: %w", err) } if !exists { if err := c.CreateBucket(timeoutCtx, cfg.BucketName); err != nil { return nil, fmt.Errorf("创建桶失败: %w", err) } } } return c, nil } // UploadToken 上传凭证 type UploadToken struct { Key string `json:"key"` // 文件存储路径 UploadURL string `json:"upload_url"` // 预签名上传URL ExpiresAt time.Time `json:"expires_at"` // 过期时间 BucketName string `json:"bucket_name"` // 桶名称 AccessURL string `json:"access_url"` // 访问URL(可选) } // DownloadToken 下载凭证 type DownloadToken struct { Key string `json:"key"` // 文件key DownloadURL string `json:"download_url"` // 预签名下载URL ExpiresAt time.Time `json:"expires_at"` // 过期时间 Filename string `json:"filename"` // 文件名(可选) } // FileInfo 文件信息 type FileInfo struct { Key string `json:"key"` // 文件key Size int64 `json:"size"` // 文件大小 ETag string `json:"etag"` // ETag(MD5) ContentType string `json:"content_type"` // Content-Type LastModified time.Time `json:"last_modified"` // 最后修改时间 Metadata map[string]string `json:"metadata"` // 元数据 URL string `json:"url"` // 访问URL Exists bool `json:"exists"` // 是否存在 } // GenerateUploadToken 生成上传凭证 func (c *Client) GenerateUploadToken(ctx context.Context, key string, bucketName ...string) (*UploadToken, error) { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } // 生成预签名PUT URL presignedReq, err := c.presignClient.PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }, s3.WithPresignExpires(c.config.PresignExpires)) if err != nil { return nil, fmt.Errorf("生成上传凭证失败: %w", err) } token := &UploadToken{ Key: key, UploadURL: presignedReq.URL, ExpiresAt: time.Now().Add(c.config.PresignExpires), BucketName: bucket, } // 如果配置了CDN域名,生成访问URL if c.config.CDNDomain != "" { token.AccessURL = c.buildCDNURL(bucket, key) } return token, nil } // GenerateDownloadToken 生成下载凭证 func (c *Client) GenerateDownloadToken(ctx context.Context, key string, bucketName ...string) (*DownloadToken, error) { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } // 检查文件是否存在 exists, err := c.FileExists(ctx, key, bucket) if err != nil { return nil, err } if !exists { return nil, fmt.Errorf("文件不存在: %s", key) } // 生成预签名GET URL presignedReq, err := c.presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }, s3.WithPresignExpires(c.config.PresignExpires)) if err != nil { return nil, fmt.Errorf("生成下载凭证失败: %w", err) } token := &DownloadToken{ Key: key, DownloadURL: presignedReq.URL, ExpiresAt: time.Now().Add(c.config.PresignExpires), Filename: filepath.Base(key), } return token, nil } // VerifyFile 验证文件完整性 func (c *Client) VerifyFile(ctx context.Context, key string, expectedMD5 string, bucketName ...string) (*FileInfo, error) { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } // 获取文件信息 output, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { // 检查是否是 NoSuchKey 错误 if c.isNotFoundError(err) { return &FileInfo{ Key: key, Exists: false, }, nil } return nil, fmt.Errorf("获取文件信息失败: %w", err) } // 转换元数据 metadata := make(map[string]string) for k, v := range output.Metadata { metadata[k] = v } contentType := "" if output.ContentType != nil { contentType = *output.ContentType } fileInfo := &FileInfo{ Key: key, Size: aws.ToInt64(output.ContentLength), ETag: strings.Trim(aws.ToString(output.ETag), "\""), // 去除引号 ContentType: contentType, LastModified: aws.ToTime(output.LastModified), Metadata: metadata, Exists: true, URL: c.buildAccessURL(bucket, key), } // 如果提供了期望的MD5,进行验证 if expectedMD5 != "" { if !c.compareMD5(fileInfo.ETag, expectedMD5) { return fileInfo, fmt.Errorf("文件MD5不匹配,期望: %s, 实际: %s", expectedMD5, fileInfo.ETag) } } return fileInfo, nil } // CalculateFileMD5 计算文件MD5(从S3下载并计算) func (c *Client) CalculateFileMD5(ctx context.Context, key string, bucketName ...string) (string, error) { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } // 下载文件 output, err := c.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { return "", fmt.Errorf("下载文件失败: %w", err) } defer func() { _ = output.Body.Close() }() // 计算MD5 hash := md5.New() if _, err := io.Copy(hash, output.Body); err != nil { return "", fmt.Errorf("计算MD5失败: %w", err) } return hex.EncodeToString(hash.Sum(nil)), nil } // FileExists 检查文件是否存在 func (c *Client) FileExists(ctx context.Context, key string, bucketName ...string) (bool, error) { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } _, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { // 检查是否是 NoSuchKey 错误 if c.isNotFoundError(err) { return false, nil } return false, fmt.Errorf("检查文件失败: %w", err) } return true, nil } // DeleteFile 删除文件 func (c *Client) DeleteFile(ctx context.Context, key string, bucketName ...string) error { bucket := c.config.BucketName if len(bucketName) > 0 && bucketName[0] != "" { bucket = bucketName[0] } _, err := c.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { return fmt.Errorf("删除文件失败: %w", err) } return nil } // GetFileInfo 获取文件信息 func (c *Client) GetFileInfo(ctx context.Context, key string, bucketName ...string) (*FileInfo, error) { return c.VerifyFile(ctx, key, "", bucketName...) } // BucketExists 检查桶是否存在 func (c *Client) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.client.HeadBucket(ctx, &s3.HeadBucketInput{ Bucket: aws.String(bucketName), }) if err != nil { // 检查是否是 NotFound 错误 if c.isNotFoundError(err) { return false, nil } return false, fmt.Errorf("检查桶失败: %w", err) } return true, nil } // CreateBucket 创建桶 func (c *Client) CreateBucket(ctx context.Context, bucketName string) error { input := &s3.CreateBucketInput{ Bucket: aws.String(bucketName), } // 如果不是 us-east-1 区域,需要设置 LocationConstraint if c.config.Region != "" && c.config.Region != "us-east-1" { input.CreateBucketConfiguration = &types.CreateBucketConfiguration{ LocationConstraint: types.BucketLocationConstraint(c.config.Region), } } _, err := c.client.CreateBucket(ctx, input) if err != nil { return fmt.Errorf("创建桶失败: %w", err) } return nil } // SetBucketPublic 设置桶为公开访问 func (c *Client) SetBucketPublic(ctx context.Context, bucketName string) error { policy := fmt.Sprintf(`{ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::%s/*"] }] }`, bucketName) _, err := c.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ Bucket: aws.String(bucketName), Policy: aws.String(policy), }) if err != nil { return fmt.Errorf("设置桶策略失败: %w", err) } return nil } // buildAccessURL 构建访问URL func (c *Client) buildAccessURL(bucket, key string) string { if c.config.CDNDomain != "" { return c.buildCDNURL(bucket, key) } protocol := "http" if c.config.UseSSL { protocol = "https" } if c.config.Endpoint != "" { return fmt.Sprintf("%s://%s/%s/%s", protocol, c.config.Endpoint, bucket, key) } // AWS S3 默认URL格式 return fmt.Sprintf("%s://%s.s3.%s.amazonaws.com/%s", protocol, bucket, c.config.Region, key) } // buildCDNURL 构建CDN URL func (c *Client) buildCDNURL(bucket, key string) string { return fmt.Sprintf("%s/%s/%s", strings.TrimRight(c.config.CDNDomain, "/"), bucket, key) } // compareMD5 比较MD5 func (c *Client) compareMD5(etag, md5 string) bool { etag = strings.ToLower(strings.Trim(etag, "\"")) md5 = strings.ToLower(strings.Trim(md5, "\"")) return etag == md5 } // isNotFoundError 检查是否为NotFound错误 func (c *Client) isNotFoundError(err error) bool { if err == nil { return false } var apiErr smithy.APIError if errors.As(err, &apiErr) { code := apiErr.ErrorCode() return code == "NotFound" || code == "NoSuchKey" || code == "NoSuchBucket" } return false }