SCHISANDRA/schisandra-album-cloud-microservices
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ optimized image list interface

Browse Source
This commit is contained in:
landaiqing
2025-02-17 11:21:38 +08:00
parent ab4e9c4d59
commit b196e50aee
72 changed files with 1676 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/auth/api/auth.api
View File

@@ -520,6 +520,7 @@ type (
Width float64 `json:"width"`
Height float64 `json:"height"`
CreatedAt string `json:"created_at"`
Thumbnail string `json:"thumbnail"`
}
AllImageDetail {
Date string `json:"date"`
@@ -588,6 +589,10 @@ type (
ThingDetailListResponse {
records []AllImageDetail `json:"records"`
}
// 单张图片请求参数
SingleImageRequest {
ID int64 `json:"id"`
}
)
// 文件上传
@@ -670,5 +675,9 @@ service auth {
// 获取事物详情列表
@handler queryThingDetailList
post /image/thing/detail/list (ThingDetailListRequest) returns (ThingDetailListResponse)
// 获取单张图片连接
@handler getImageUrl
post /image/url/single (SingleImageRequest) returns (string)
}

3
app/auth/api/auth.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/zeromicro/go-zero/core/conf"
"github.com/zeromicro/go-zero/rest"
"schisandra-album-cloud-microservices/app/auth/api/internal/mq"
"schisandra-album-cloud-microservices/common/idgenerator"
"schisandra-album-cloud-microservices/common/middleware"
@@ -31,6 +32,8 @@ func main() {
server.Use(middleware.I18nMiddleware)
ctx := svc.NewServiceContext(c)
handler.RegisterHandlers(server, ctx)
// start image process consumer
go mq.NewImageProcessConsumer(ctx)
// initialize id generator
idgenerator.NewIDGenerator(0)
fmt.Printf("Starting server at %s:%d...\n", c.Host, c.Port)

7
app/auth/api/etc/auth.yaml
View File

@@ -171,4 +171,9 @@ Minio:
# Minio 访问密钥
SecretAccessKey: XEHkwExqQdAlEPfpRk36xpc0Sie8hZkcmlhXQJXw
# Minio 使用SSL
UseSSL: false
UseSSL: false
#NSQ配置
NSQ:
# NSQD地址
NSQDHost: 1.95.0.111:4150
LookUpdHost: 1.95.0.111:4161

4
app/auth/api/internal/config/config.go
View File

@@ -73,4 +73,8 @@ type Config struct {
SecretAccessKey string
UseSSL bool
}
NSQ struct {
NSQDHost string
LookUpdHost string
}
}

5
app/auth/api/internal/handler/routes.go
View File

@@ -268,6 +268,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
Path: "/image/thing/list",
Handler: storage.QueryThingImageListHandler(serverCtx),
},
{
Method: http.MethodPost,
Path: "/image/url/single",
Handler: storage.GetImageUrlHandler(serverCtx),
},
{
Method: http.MethodPost,
Path: "/uploads",

29
app/auth/api/internal/handler/storage/get_image_url_handler.go Normal file
View File

@@ -0,0 +1,29 @@
package storage
import (
"net/http"
"github.com/zeromicro/go-zero/rest/httpx"
"schisandra-album-cloud-microservices/app/auth/api/internal/logic/storage"
"schisandra-album-cloud-microservices/app/auth/api/internal/svc"
"schisandra-album-cloud-microservices/app/auth/api/internal/types"
"schisandra-album-cloud-microservices/common/xhttp"
)
func GetImageUrlHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req types.SingleImageRequest
if err := httpx.Parse(r, &req); err != nil {
xhttp.JsonBaseResponseCtx(r.Context(), w, err)
return
}
l := storage.NewGetImageUrlLogic(r.Context(), svcCtx)
resp, err := l.GetImageUrl(&req)
if err != nil {
xhttp.JsonBaseResponseCtx(r.Context(), w, err)
} else {
xhttp.JsonBaseResponseCtx(r.Context(), w, resp)
}
}
}

60
app/auth/api/internal/logic/storage/get_album_detail_logic.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"github.com/redis/go-redis/v9"
"math/rand"
"net/url"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/app/auth/model/mysql/query"
"schisandra-album-cloud-microservices/common/constant"
@@ -56,18 +57,28 @@ func (l *GetAlbumDetailLogic) GetAlbumDetail(req *types.AlbumDetailListRequest)
return nil, errors.New("get cached image list failed")
}
// 缓存未命中，从数据库中查询
storageInfo := l.svcCtx.DB.ScaStorageInfo
storageThumb := l.svcCtx.DB.ScaStorageThumb
// 数据库查询文件信息列表
var storageInfoQuery query.IScaStorageInfoDo
var storageInfoList []types.FileInfoResult
storageInfoQuery = storageInfo.Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.AlbumID.Eq(req.ID)).
storageInfoQuery = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageThumb.ThumbPath,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).
Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.AlbumID.Eq(req.ID)).
Order(storageInfo.CreatedAt.Desc())
storageInfoList, err := storageInfoQuery.Find()
err = storageInfoQuery.Scan(&storageInfoList)
if err != nil {
return nil, err
}
@@ -75,17 +86,17 @@ func (l *GetAlbumDetailLogic) GetAlbumDetail(req *types.AlbumDetailListRequest)
return &types.AlbumDetailListResponse{}, nil
}
// 加载用户oss配置信息
cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
if err != nil {
return nil, err
}
service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
if err != nil {
return nil, errors.New("get storage failed")
}
//// 加载用户oss配置信息
//cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
//ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
//if err != nil {
// return nil, err
//}
//
//service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
//if err != nil {
// return nil, errors.New("get storage failed")
//}
// 按日期进行分组
var wg sync.WaitGroup
@@ -93,11 +104,12 @@ func (l *GetAlbumDetailLogic) GetAlbumDetail(req *types.AlbumDetailListRequest)
for _, dbFileInfo := range storageInfoList {
wg.Add(1)
go func(dbFileInfo *model.ScaStorageInfo) {
go func(dbFileInfo *types.FileInfoResult) {
defer wg.Done()
weekday := WeekdayMap[dbFileInfo.CreatedAt.Weekday()]
date := dbFileInfo.CreatedAt.Format("2006年1月2日 星期" + weekday)
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, dbFileInfo.Path, time.Hour*24*7)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, dbFileInfo.ThumbPath, time.Hour*24*7, reqParams)
if err != nil {
logx.Error(err)
return
@@ -109,15 +121,15 @@ func (l *GetAlbumDetailLogic) GetAlbumDetail(req *types.AlbumDetailListRequest)
images = append(images, types.ImageMeta{
ID: dbFileInfo.ID,
FileName: dbFileInfo.FileName,
URL: url,
Width: dbFileInfo.Width,
Height: dbFileInfo.Height,
URL: presignedUrl.String(),
Width: dbFileInfo.ThumbW,
Height: dbFileInfo.ThumbH,
CreatedAt: dbFileInfo.CreatedAt.Format("2006-01-02 15:04:05"),
})
// 重新存储更新后的图像列表
groupedImages.Store(date, images)
}(dbFileInfo)
}(&dbFileInfo)
}
wg.Wait()
var imageList []types.AllImageDetail

36
app/auth/api/internal/logic/storage/get_face_detail_list_logic.go
View File

@@ -7,8 +7,8 @@ import (
"fmt"
"github.com/redis/go-redis/v9"
"math/rand"
"net/url"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/app/auth/model/mysql/query"
"schisandra-album-cloud-microservices/common/constant"
"schisandra-album-cloud-microservices/common/encrypt"
storageConfig "schisandra-album-cloud-microservices/common/storage/config"
@@ -58,16 +58,24 @@ func (l *GetFaceDetailListLogic) GetFaceDetailList(req *types.FaceDetailListRequ
// 缓存未命中，从数据库中查询
storageInfo := l.svcCtx.DB.ScaStorageInfo
// 数据库查询文件信息列表
var storageInfoQuery query.IScaStorageInfoDo
storageThumb := l.svcCtx.DB.ScaStorageThumb
var storageInfoList []types.FileInfoResult
storageInfoQuery = storageInfo.Where(
err = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageInfo.Path,
storageThumb.ThumbPath,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.FaceID.Eq(req.FaceID)).
Order(storageInfo.CreatedAt.Desc())
storageInfoList, err := storageInfoQuery.Find()
Order(storageInfo.CreatedAt.Desc()).Scan(&storageInfoList)
if err != nil {
return nil, err
}
@@ -93,15 +101,22 @@ func (l *GetFaceDetailListLogic) GetFaceDetailList(req *types.FaceDetailListRequ
for _, dbFileInfo := range storageInfoList {
wg.Add(1)
go func(dbFileInfo *model.ScaStorageInfo) {
go func(dbFileInfo *types.FileInfoResult) {
defer wg.Done()
weekday := WeekdayMap[dbFileInfo.CreatedAt.Weekday()]
date := dbFileInfo.CreatedAt.Format("2006年1月2日 星期" + weekday)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, dbFileInfo.ThumbPath, time.Hour*24*7, reqParams)
if err != nil {
logx.Error(err)
return
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, dbFileInfo.Path, time.Hour*24*7)
if err != nil {
logx.Error(err)
return
}
// 使用 Load 或 Store 确保原子操作
value, _ := groupedImages.LoadOrStore(date, []types.ImageMeta{})
images := value.([]types.ImageMeta)
@@ -109,15 +124,16 @@ func (l *GetFaceDetailListLogic) GetFaceDetailList(req *types.FaceDetailListRequ
images = append(images, types.ImageMeta{
ID: dbFileInfo.ID,
FileName: dbFileInfo.FileName,
Thumbnail: presignedUrl.String(),
URL: url,
Width: dbFileInfo.Width,
Height: dbFileInfo.Height,
Width: dbFileInfo.ThumbW,
Height: dbFileInfo.ThumbH,
CreatedAt: dbFileInfo.CreatedAt.Format("2006-01-02 15:04:05"),
})
// 重新存储更新后的图像列表
groupedImages.Store(date, images)
}(dbFileInfo)
}(&dbFileInfo)
}
wg.Wait()
var imageList []types.AllImageDetail

148
app/auth/api/internal/logic/storage/get_image_url_logic.go Normal file
View File

@@ -0,0 +1,148 @@
package storage
import (
"context"
"encoding/json"
"errors"
"github.com/redis/go-redis/v9"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/common/constant"
"schisandra-album-cloud-microservices/common/encrypt"
storageConfig "schisandra-album-cloud-microservices/common/storage/config"
"strconv"
"time"
"schisandra-album-cloud-microservices/app/auth/api/internal/svc"
"schisandra-album-cloud-microservices/app/auth/api/internal/types"
"github.com/zeromicro/go-zero/core/logx"
)
type GetImageUrlLogic struct {
logx.Logger
ctx context.Context
svcCtx *svc.ServiceContext
}
func NewGetImageUrlLogic(ctx context.Context, svcCtx *svc.ServiceContext) *GetImageUrlLogic {
return &GetImageUrlLogic{
Logger: logx.WithContext(ctx),
ctx: ctx,
svcCtx: svcCtx,
}
}
func (l *GetImageUrlLogic) GetImageUrl(req *types.SingleImageRequest) (resp string, err error) {
uid, ok := l.ctx.Value("user_id").(string)
if !ok {
return "", errors.New("user_id not found")
}
// 从redis 获取url
cacheKey := constant.ImageSinglePrefix + uid + ":" + strconv.FormatInt(req.ID, 10)
cacheUrl, err := l.svcCtx.RedisClient.Get(l.ctx, cacheKey).Result()
if err != nil && !errors.Is(err, redis.Nil) {
logx.Info(err)
return "", errors.New("get image url failed")
}
if cacheUrl != "" {
return cacheUrl, nil
}
storageInfo := l.svcCtx.DB.ScaStorageInfo
result, err := storageInfo.Select(
storageInfo.ID,
storageInfo.Provider,
storageInfo.Bucket,
storageInfo.Path).
Where(storageInfo.ID.Eq(req.ID), storageInfo.UserID.Eq(uid)).
First()
if err != nil {
logx.Info(err)
return "", errors.New("get storage info failed")
}
if result == nil {
return "", errors.New("get storage info failed")
}
// 加载用户oss配置信息
cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + result.Provider
ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, result.Provider)
if err != nil {
return "", err
}
service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
if err != nil {
return "", errors.New("get storage failed")
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, result.Path, 7*24*time.Hour)
if err != nil {
return "", errors.New("get presigned url failed")
}
// 缓存url
err = l.svcCtx.RedisClient.Set(l.ctx, cacheKey, url, 7*24*time.Hour).Err()
if err != nil {
logx.Info(err)
}
return url, nil
}
// 提取解密操作为函数
func (l *GetImageUrlLogic) decryptConfig(config *model.ScaStorageConfig) (*storageConfig.StorageConfig, error) {
accessKey, err := encrypt.Decrypt(config.AccessKey, l.svcCtx.Config.Encrypt.Key)
if err != nil {
return nil, errors.New("decrypt access key failed")
}
secretKey, err := encrypt.Decrypt(config.SecretKey, l.svcCtx.Config.Encrypt.Key)
if err != nil {
return nil, errors.New("decrypt secret key failed")
}
return &storageConfig.StorageConfig{
Provider: config.Provider,
Endpoint: config.Endpoint,
AccessKey: accessKey,
SecretKey: secretKey,
BucketName: config.Bucket,
Region: config.Region,
}, nil
}
// 从缓存或数据库中获取 OSS 配置
func (l *GetImageUrlLogic) getOssConfigFromCacheOrDb(cacheKey, uid, provider string) (*storageConfig.StorageConfig, error) {
result, err := l.svcCtx.RedisClient.Get(l.ctx, cacheKey).Result()
if err != nil && !errors.Is(err, redis.Nil) {
return nil, errors.New("get oss config failed")
}
var ossConfig *storageConfig.StorageConfig
if result != "" {
var redisOssConfig model.ScaStorageConfig
if err = json.Unmarshal([]byte(result), &redisOssConfig); err != nil {
return nil, errors.New("unmarshal oss config failed")
}
return l.decryptConfig(&redisOssConfig)
}
// 缓存未命中，从数据库中加载
scaOssConfig := l.svcCtx.DB.ScaStorageConfig
dbOssConfig, err := scaOssConfig.Where(scaOssConfig.UserID.Eq(uid), scaOssConfig.Provider.Eq(provider)).First()
if err != nil {
return nil, err
}
// 缓存数据库配置
ossConfig, err = l.decryptConfig(dbOssConfig)
if err != nil {
return nil, err
}
marshalData, err := json.Marshal(dbOssConfig)
if err != nil {
return nil, errors.New("marshal oss config failed")
}
err = l.svcCtx.RedisClient.Set(l.ctx, cacheKey, marshalData, 0).Err()
if err != nil {
return nil, errors.New("set oss config failed")
}
return ossConfig, nil
}

68
app/auth/api/internal/logic/storage/query_all_image_list_logic.go
View File

@@ -8,10 +8,10 @@ import (
"github.com/redis/go-redis/v9"
"github.com/zeromicro/go-zero/core/logx"
"math/rand"
"net/url"
"schisandra-album-cloud-microservices/app/auth/api/internal/svc"
"schisandra-album-cloud-microservices/app/auth/api/internal/types"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/app/auth/model/mysql/query"
"schisandra-album-cloud-microservices/common/constant"
"schisandra-album-cloud-microservices/common/encrypt"
storageConfig "schisandra-album-cloud-microservices/common/storage/config"
@@ -66,25 +66,44 @@ func (l *QueryAllImageListLogic) QueryAllImageList(req *types.AllImageListReques
// 缓存未命中，从数据库中查询
storageInfo := l.svcCtx.DB.ScaStorageInfo
// 数据库查询文件信息列表
var storageInfoQuery query.IScaStorageInfoDo
storageThumb := l.svcCtx.DB.ScaStorageThumb
var storageInfoList []types.FileInfoResult
if req.Sort {
storageInfoQuery = storageInfo.Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Type.Eq(req.Type),
storageInfo.AlbumID.IsNull()).
Order(storageInfo.CreatedAt.Desc())
err = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageInfo.Path,
storageThumb.ThumbPath,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).
Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Type.Eq(req.Type),
storageInfo.AlbumID.IsNull()).
Order(storageInfo.CreatedAt.Desc()).Scan(&storageInfoList)
} else {
storageInfoQuery = storageInfo.Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Type.Eq(req.Type)).
Order(storageInfo.CreatedAt.Desc())
err = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageThumb.ThumbPath,
storageInfo.Path,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).
Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Type.Eq(req.Type)).
Order(storageInfo.CreatedAt.Desc()).Scan(&storageInfoList)
}
storageInfoList, err := storageInfoQuery.Find()
if err != nil {
return nil, err
}
@@ -110,10 +129,16 @@ func (l *QueryAllImageListLogic) QueryAllImageList(req *types.AllImageListReques
for _, dbFileInfo := range storageInfoList {
wg.Add(1)
go func(dbFileInfo *model.ScaStorageInfo) {
go func(dbFileInfo *types.FileInfoResult) {
defer wg.Done()
weekday := WeekdayMap[dbFileInfo.CreatedAt.Weekday()]
date := dbFileInfo.CreatedAt.Format("2006年1月2日 星期" + weekday)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, dbFileInfo.ThumbPath, time.Hour*24*7, reqParams)
if err != nil {
logx.Error(err)
return
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, dbFileInfo.Path, time.Hour*24*7)
if err != nil {
logx.Error(err)
@@ -126,15 +151,16 @@ func (l *QueryAllImageListLogic) QueryAllImageList(req *types.AllImageListReques
images = append(images, types.ImageMeta{
ID: dbFileInfo.ID,
FileName: dbFileInfo.FileName,
Thumbnail: presignedUrl.String(),
URL: url,
Width: dbFileInfo.Width,
Height: dbFileInfo.Height,
Width: dbFileInfo.ThumbW,
Height: dbFileInfo.ThumbH,
CreatedAt: dbFileInfo.CreatedAt.Format("2006-01-02 15:04:05"),
})
// 重新存储更新后的图像列表
groupedImages.Store(date, images)
}(dbFileInfo)
}(&dbFileInfo)
}
wg.Wait()
var imageList []types.AllImageDetail

32
app/auth/api/internal/logic/storage/query_location_detail_list_logic.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"github.com/redis/go-redis/v9"
"math/rand"
"net/url"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/app/auth/model/mysql/query"
"schisandra-album-cloud-microservices/common/constant"
@@ -56,18 +57,28 @@ func (l *QueryLocationDetailListLogic) QueryLocationDetailList(req *types.Locati
return nil, errors.New("get cached image list failed")
}
// 缓存未命中，从数据库中查询
storageInfo := l.svcCtx.DB.ScaStorageInfo
storageThumb := l.svcCtx.DB.ScaStorageThumb
// 数据库查询文件信息列表
var storageInfoQuery query.IScaStorageInfoDo
var storageInfoList []types.FileInfoResult
storageInfoQuery = storageInfo.Where(
storageInfoQuery = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageThumb.ThumbPath,
storageInfo.Path,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.LocationID.Eq(req.ID)).
Order(storageInfo.CreatedAt.Desc())
storageInfoList, err := storageInfoQuery.Find()
err = storageInfoQuery.Scan(&storageInfoList)
if err != nil {
return nil, err
}
@@ -93,10 +104,16 @@ func (l *QueryLocationDetailListLogic) QueryLocationDetailList(req *types.Locati
for _, dbFileInfo := range storageInfoList {
wg.Add(1)
go func(dbFileInfo *model.ScaStorageInfo) {
go func(dbFileInfo *types.FileInfoResult) {
defer wg.Done()
weekday := WeekdayMap[dbFileInfo.CreatedAt.Weekday()]
date := dbFileInfo.CreatedAt.Format("2006年1月2日 星期" + weekday)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, dbFileInfo.ThumbPath, time.Hour*24*7, reqParams)
if err != nil {
logx.Error(err)
return
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, dbFileInfo.Path, time.Hour*24*7)
if err != nil {
logx.Error(err)
@@ -109,15 +126,16 @@ func (l *QueryLocationDetailListLogic) QueryLocationDetailList(req *types.Locati
images = append(images, types.ImageMeta{
ID: dbFileInfo.ID,
FileName: dbFileInfo.FileName,
Thumbnail: presignedUrl.String(),
URL: url,
Width: dbFileInfo.Width,
Height: dbFileInfo.Height,
Width: dbFileInfo.ThumbW,
Height: dbFileInfo.ThumbH,
CreatedAt: dbFileInfo.CreatedAt.Format("2006-01-02 15:04:05"),
})
// 重新存储更新后的图像列表
groupedImages.Store(date, images)
}(dbFileInfo)
}(&dbFileInfo)
}
wg.Wait()
var imageList []types.AllImageDetail

26
app/auth/api/internal/logic/storage/query_location_image_list_logic.go
View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"github.com/redis/go-redis/v9"
"net/url"
"schisandra-album-cloud-microservices/app/auth/api/internal/svc"
"schisandra-album-cloud-microservices/app/auth/api/internal/types"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
@@ -53,16 +54,16 @@ func (l *QueryLocationImageListLogic) QueryLocationImageList(req *types.Location
}
// 加载用户oss配置信息
cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
if err != nil {
return nil, err
}
service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
if err != nil {
return nil, errors.New("get storage failed")
}
//cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
//ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
//if err != nil {
// return nil, err
//}
//
//service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
//if err != nil {
// return nil, errors.New("get storage failed")
//}
locationMap := make(map[string][]types.LocationMeta)
for _, loc := range locations {
@@ -77,7 +78,8 @@ func (l *QueryLocationImageListLogic) QueryLocationImageList(req *types.Location
if city == "" {
city = loc.Country
}
url, err := service.PresignedURL(l.ctx, req.Bucket, loc.CoverImage, 7*24*time.Hour)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, loc.CoverImage, 7*24*time.Hour, reqParams)
if err != nil {
return nil, errors.New("get presigned url failed")
}
@@ -85,7 +87,7 @@ func (l *QueryLocationImageListLogic) QueryLocationImageList(req *types.Location
ID: loc.ID,
City: city,
Total: loc.Total,
CoverImage: url,
CoverImage: presignedUrl.String(),
}
locationMap[locationKey] = append(locationMap[locationKey], locationMeta)
}

53
app/auth/api/internal/logic/storage/query_recent_image_list_logic.go
View File

@@ -62,37 +62,34 @@ func (l *QueryRecentImageListLogic) QueryRecentImageList() (resp *types.RecentLi
if cmd == nil {
continue
}
val, ok := cmd.(string)
if !ok {
logx.Error("invalid value type")
return nil, errors.New("invalid value type")
}
var imageMeta types.ImageMeta
err = json.Unmarshal([]byte(val), &imageMeta)
if err != nil {
logx.Error(err)
return nil, errors.New("unmarshal recent file list failed")
}
parse, err := time.Parse("2006-01-02 15:04:05", imageMeta.CreatedAt)
if err != nil {
logx.Error(err)
return nil, errors.New("parse recent file list failed")
}
date := parse.Format("2006年1月2日 星期" + WeekdayMap[parse.Weekday()])
// 使用LoadOrStore来检查并存储或者追加
wg.Add(1)
go func(cmd interface{}) {
go func(date string, imageMeta types.ImageMeta) {
defer wg.Done()
val, ok := cmd.(string)
if !ok {
logx.Error("invalid value type")
return
value, loaded := groupedImages.LoadOrStore(date, []types.ImageMeta{imageMeta})
if loaded {
images := value.([]types.ImageMeta)
images = append(images, imageMeta)
groupedImages.Store(date, images)
}
var imageMeta types.ImageMeta
err = json.Unmarshal([]byte(val), &imageMeta)
if err != nil {
logx.Error(err)
return
}
parse, err := time.Parse("2006-01-02 15:04:05", imageMeta.CreatedAt)
if err != nil {
logx.Error(err)
return
}
date := parse.Format("2006年1月2日 星期" + WeekdayMap[parse.Weekday()])
groupedImages.Range(func(key, value interface{}) bool {
if key == date {
images := value.([]types.ImageMeta)
images = append(images, imageMeta)
groupedImages.Store(date, images)
return false
}
return true
})
groupedImages.Store(date, []types.ImageMeta{imageMeta})
}(cmd)
}(date, imageMeta)
}
wg.Wait()

41
app/auth/api/internal/logic/storage/query_thing_detail_list_logic.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"github.com/redis/go-redis/v9"
"math/rand"
"net/url"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/app/auth/model/mysql/query"
"schisandra-album-cloud-microservices/common/constant"
@@ -56,18 +57,29 @@ func (l *QueryThingDetailListLogic) QueryThingDetailList(req *types.ThingDetailL
return nil, errors.New("get cached image list failed")
}
// 缓存未命中，从数据库中查询
storageInfo := l.svcCtx.DB.ScaStorageInfo
storageThumb := l.svcCtx.DB.ScaStorageThumb
// 数据库查询文件信息列表
var storageInfoQuery query.IScaStorageInfoDo
var storageInfoList []types.FileInfoResult
storageInfoQuery = storageInfo.Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Tag.Eq(req.TagName)).
storageInfoQuery = storageInfo.Select(
storageInfo.ID,
storageInfo.FileName,
storageInfo.CreatedAt,
storageThumb.ThumbPath,
storageInfo.Path,
storageThumb.ThumbW,
storageThumb.ThumbH,
storageThumb.ThumbSize).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).
Where(
storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
storageInfo.Tag.Eq(req.TagName)).
Order(storageInfo.CreatedAt.Desc())
storageInfoList, err := storageInfoQuery.Find()
err = storageInfoQuery.Scan(&storageInfoList)
if err != nil {
return nil, err
}
@@ -93,10 +105,16 @@ func (l *QueryThingDetailListLogic) QueryThingDetailList(req *types.ThingDetailL
for _, dbFileInfo := range storageInfoList {
wg.Add(1)
go func(dbFileInfo *model.ScaStorageInfo) {
go func(dbFileInfo *types.FileInfoResult) {
defer wg.Done()
weekday := WeekdayMap[dbFileInfo.CreatedAt.Weekday()]
date := dbFileInfo.CreatedAt.Format("2006年1月2日 星期" + weekday)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, dbFileInfo.ThumbPath, time.Hour*24*7, reqParams)
if err != nil {
logx.Error(err)
return
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, dbFileInfo.Path, time.Hour*24*7)
if err != nil {
logx.Error(err)
@@ -109,15 +127,16 @@ func (l *QueryThingDetailListLogic) QueryThingDetailList(req *types.ThingDetailL
images = append(images, types.ImageMeta{
ID: dbFileInfo.ID,
FileName: dbFileInfo.FileName,
Thumbnail: presignedUrl.String(),
URL: url,
Width: dbFileInfo.Width,
Height: dbFileInfo.Height,
Width: dbFileInfo.ThumbW,
Height: dbFileInfo.ThumbH,
CreatedAt: dbFileInfo.CreatedAt.Format("2006-01-02 15:04:05"),
})
// 重新存储更新后的图像列表
groupedImages.Store(date, images)
}(dbFileInfo)
}(&dbFileInfo)
}
wg.Wait()
var imageList []types.AllImageDetail

43
app/auth/api/internal/logic/storage/query_thing_image_list_logic.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"github.com/redis/go-redis/v9"
"net/url"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/common/constant"
"schisandra-album-cloud-microservices/common/encrypt"
@@ -38,12 +39,15 @@ func (l *QueryThingImageListLogic) QueryThingImageList(req *types.ThingListReque
return nil, errors.New("user_id not found")
}
storageInfo := l.svcCtx.DB.ScaStorageInfo
storageInfos, err := storageInfo.Select(
storageThumb := l.svcCtx.DB.ScaStorageThumb
var thingList []types.ThingImageList
err = storageInfo.Select(
storageInfo.ID,
storageInfo.Category,
storageInfo.Tag,
storageInfo.Path,
storageThumb.ThumbPath,
storageInfo.CreatedAt).
LeftJoin(storageThumb, storageInfo.ThumbID.EqCol(storageThumb.ID)).
Where(storageInfo.UserID.Eq(uid),
storageInfo.Provider.Eq(req.Provider),
storageInfo.Bucket.Eq(req.Bucket),
@@ -52,28 +56,28 @@ func (l *QueryThingImageListLogic) QueryThingImageList(req *types.ThingListReque
storageInfo.Category.Length().Gt(0),
storageInfo.Tag.Length().Gte(0)).
Order(storageInfo.CreatedAt.Desc()).
Find()
Scan(&thingList)
if err != nil {
return nil, err
}
// 加载用户oss配置信息
cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
if err != nil {
return nil, err
}
service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
if err != nil {
return nil, errors.New("get storage failed")
}
//cacheOssConfigKey := constant.UserOssConfigPrefix + uid + ":" + req.Provider
//ossConfig, err := l.getOssConfigFromCacheOrDb(cacheOssConfigKey, uid, req.Provider)
//if err != nil {
// return nil, err
//}
//
//service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
//if err != nil {
// return nil, errors.New("get storage failed")
//}
categoryMap := sync.Map{}
tagCountMap := sync.Map{}
tagCoverMap := sync.Map{} // 用于存储每个 Tag 的封面图片路径
for _, info := range storageInfos {
for _, info := range thingList {
tagKey := info.Category + "::" + info.Tag
if _, exists := tagCountMap.Load(tagKey); !exists {
tagCountMap.Store(tagKey, int64(0))
@@ -90,14 +94,15 @@ func (l *QueryThingImageListLogic) QueryThingImageList(req *types.ThingListReque
// 为每个 Tag 存储封面图片路径
if _, exists := tagCoverMap.Load(tagKey); !exists {
// 使用服务生成预签名 URL
coverImageURL, err := service.PresignedURL(l.ctx, req.Bucket, info.Path, 7*24*time.Hour)
reqParams := make(url.Values)
presignedUrl, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, info.ThumbPath, time.Hour*24*7, reqParams)
if err == nil {
tagCoverMap.Store(tagKey, coverImageURL)
tagCoverMap.Store(tagKey, presignedUrl.String())
}
}
}
var thingListData []types.ThingListData
var thingListResponse []types.ThingListData
categoryMap.Range(func(category, tagData interface{}) bool {
var metas []types.ThingMeta
tagData.(*sync.Map).Range(func(tag, item interface{}) bool {
@@ -113,14 +118,14 @@ func (l *QueryThingImageListLogic) QueryThingImageList(req *types.ThingListReque
metas = append(metas, meta)
return true
})
thingListData = append(thingListData, types.ThingListData{
thingListResponse = append(thingListResponse, types.ThingListData{
Category: category.(string),
List: metas,
})
return true
})
return &types.ThingListResponse{Records: thingListData}, nil
return &types.ThingListResponse{Records: thingListResponse}, nil
}
// 提取解密操作为函数

288
app/auth/api/internal/logic/storage/upload_file_logic.go
View File

@@ -1,17 +1,23 @@
package storage
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/ccpwcn/kgo"
"github.com/minio/minio-go/v7"
"github.com/redis/go-redis/v9"
"github.com/zeromicro/go-zero/core/logx"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
"gorm.io/gorm"
"io"
"mime/multipart"
"net/http"
"net/url"
"path"
"path/filepath"
"schisandra-album-cloud-microservices/app/aisvc/rpc/pb"
@@ -56,72 +62,159 @@ func (l *UploadFileLogic) UploadFile(r *http.Request) (resp string, err error) {
if err != nil {
return "", err
}
defer file.Close()
defer func(file multipart.File) {
_ = file.Close()
}(file)
// 解析 AI 识别结果
result, err := l.parseAIRecognitionResult(r)
if err != nil {
return "", err
}
var faceId int64 = 0
bytes, err := io.ReadAll(file)
data, err := io.ReadAll(file)
if err != nil {
return "", err
}
// 人脸识别
if result.FileType == "image/png" || result.FileType == "image/jpeg" {
face, err := l.svcCtx.AiSvcRpc.FaceRecognition(l.ctx, &pb.FaceRecognitionRequest{Face: bytes, UserId: uid})
if err != nil {
return "", err
// 解析上传的缩略图
thumbnail, _, err := l.getUploadedThumbnail(r)
if err != nil {
return "", err
}
defer thumbnail.Close()
// 解析图片信息识别结果
result, err := l.parseImageInfoResult(r)
if err != nil {
return "", err
}
// 使用 `errgroup.Group` 处理并发任务
var (
faceId int64
filePath string
minioFilePath string
presignedURL string
)
g, ctx := errgroup.WithContext(context.Background())
// 创建信号量，限制最大并发上传数（比如最多同时 5 个任务）
sem := semaphore.NewWeighted(5)
// 进行人脸识别
g.Go(func() error {
if result.FileType == "image/png" || result.FileType == "image/jpeg" {
face, err := l.svcCtx.AiSvcRpc.FaceRecognition(l.ctx, &pb.FaceRecognitionRequest{
Face: data,
UserId: uid,
})
if err != nil {
return err
}
if face != nil {
faceId = face.GetFaceId()
}
}
if face != nil {
faceId = face.GetFaceId()
}
}
// 根据 GPS 信息获取地理位置信息
country, province, city, err := l.getGeoLocation(result.Latitude, result.Longitude)
if err != nil {
return "", err
}
return nil
})
// 上传文件到 OSS
// 重新设置文件指针到文件开头
if _, err = file.Seek(0, 0); err != nil {
return "", err
}
bucket, provider, filePath, url, err := l.uploadFileToOSS(uid, header, file, result)
if err != nil {
g.Go(func() error {
if err := sem.Acquire(ctx, 1); err != nil {
return err
}
defer sem.Release(1)
// 重新创建 `multipart.File` 兼容的 `Reader`
fileReader := struct {
*bytes.Reader
io.Closer
}{
Reader: bytes.NewReader(data),
Closer: io.NopCloser(nil),
}
fileUrl, err := l.uploadFileToOSS(uid, header, fileReader, result)
if err != nil {
return err
}
filePath = fileUrl
return nil
})
// 上传缩略图到 MinIO
g.Go(func() error {
if err := sem.Acquire(ctx, 1); err != nil {
return err
}
defer sem.Release(1)
path, url, err := l.uploadFileToMinio(uid, header, thumbnail, result)
if err != nil {
return err
}
minioFilePath = path
presignedURL = url
return nil
})
// 等待所有 goroutine 执行完毕
if err = g.Wait(); err != nil {
return "", err
}
// 将地址信息保存到数据库
locationId, err := l.saveFileLocationInfoToDB(uid, result.Provider, result.Bucket, result.Latitude, result.Longitude, country, province, city, filePath)
fileUploadMessage := &types.FileUploadMessage{
UID: uid,
Data: result,
FaceID: faceId,
FileHeader: header,
FilePath: filePath,
PresignedURL: presignedURL,
ThumbPath: minioFilePath,
}
// 转换为 JSON
messageData, err := json.Marshal(fileUploadMessage)
if err != nil {
return "", err
}
err = l.svcCtx.NSQProducer.Publish(constant.MQTopicImageProcess, messageData)
if err != nil {
return "", errors.New("publish message failed")
}
// 将 EXIF 和文件信息存入数据库
id, err := l.saveFileInfoToDB(uid, bucket, provider, header, result, locationId, faceId, filePath)
if err != nil {
return "", err
}
// 删除缓存
l.afterImageUpload(uid, provider, bucket)
// ------------------------------------------------------------------------
// redis 保存最近7天上传的文件列表
err = l.saveRecentFileList(uid, url, id, result, header.Filename)
if err != nil {
return "", err
}
//// 根据 GPS 信息获取地理位置信息
//country, province, city, err := l.getGeoLocation(result.Latitude, result.Longitude)
//if err != nil {
// return "", err
//}
//// 将地址信息保存到数据库
//locationId, err := l.saveFileLocationInfoToDB(uid, result.Provider, result.Bucket, result.Latitude, result.Longitude, country, province, city, filePath)
//if err != nil {
// return "", err
//}
//
//// 将 EXIF 和文件信息存入数据库
//id, err := l.saveFileInfoToDB(uid, bucket, provider, header, result, locationId, faceId, filePath)
//if err != nil {
// return "", err
//}
//// 删除缓存
//l.afterImageUpload(uid, provider, bucket)
//
//// redis 保存最近7天上传的文件列表
//err = l.saveRecentFileList(uid, url, id, result, header.Filename)
//if err != nil {
// return "", err
//}
return "success", nil
}
// 将 multipart.File 转为 Base64 字符串
func (l *UploadFileLogic) fileToBase64(file multipart.File) (string, error) {
// 读取文件内容
fileBytes, err := io.ReadAll(file)
if err != nil {
return "", err
}
// 将文件内容转为 Base64 编码
return base64.StdEncoding.EncodeToString(fileBytes), nil
}
// 获取用户 ID
func (l *UploadFileLogic) getUserID() (string, error) {
uid, ok := l.ctx.Value("user_id").(string)
@@ -150,8 +243,17 @@ func (l *UploadFileLogic) getUploadedFile(r *http.Request) (multipart.File, *mul
return file, header, nil
}
// 解析 AI 识别结果
func (l *UploadFileLogic) parseAIRecognitionResult(r *http.Request) (types.File, error) {
// 解析上传的文件
func (l *UploadFileLogic) getUploadedThumbnail(r *http.Request) (multipart.File, *multipart.FileHeader, error) {
file, header, err := r.FormFile("thumbnail")
if err != nil {
return nil, nil, errors.New("file not found")
}
return file, header, nil
}
// 解析图片信息结果
func (l *UploadFileLogic) parseImageInfoResult(r *http.Request) (types.File, error) {
formValue := r.PostFormValue("data")
var result types.File
if err := json.Unmarshal([]byte(formValue), &result); err != nil {
@@ -160,47 +262,11 @@ func (l *UploadFileLogic) parseAIRecognitionResult(r *http.Request) (types.File,
return result, nil
}
// 提取拍摄时间
func (l *UploadFileLogic) extractOriginalDateTime(exif map[string]interface{}) (string, error) {
if dateTimeOriginal, ok := exif["DateTimeOriginal"].(string); ok {
parsedTime, err := time.Parse(time.RFC3339, dateTimeOriginal)
if err == nil {
return parsedTime.Format("2006-01-02 15:04:05"), nil
}
}
return "", nil
}
// 根据 GPS 信息获取地理位置信息
func (l *UploadFileLogic) getGeoLocation(latitude, longitude float64) (string, string, string, error) {
if latitude == 0.000000 || longitude == 0.000000 {
return "", "", "", nil
}
//request := gao_map.ReGeoRequest{Location: fmt.Sprintf("%f,%f", latitude, longitude)}
//
//location, err := l.svcCtx.GaoMap.Location.ReGeo(&request)
//if err != nil {
// return nil, errors.New("regeo failed")
//}
//
//addressInfo := map[string]string{}
//if location.ReGeoCode.AddressComponent.Country != "" {
// addressInfo["country"] = location.ReGeoCode.AddressComponent.Country
//}
//if location.ReGeoCode.AddressComponent.Province != "" {
// addressInfo["province"] = location.ReGeoCode.AddressComponent.Province
//}
//if location.ReGeoCode.AddressComponent.City != "" {
// addressInfo["city"] = location.ReGeoCode.AddressComponent.City.(string)
//}
//if location.ReGeoCode.AddressComponent.District != "" {
// addressInfo["district"] = location.ReGeoCode.AddressComponent.District.(string)
//}
//if location.ReGeoCode.AddressComponent.Township != "" {
// addressInfo["township"] = location.ReGeoCode.AddressComponent.Township
//}
country, province, city, err := geo_json.GetAddress(latitude, longitude, l.svcCtx.GeoRegionData)
if err != nil {
return "", "", "", errors.New("get geo location failed")
@@ -210,20 +276,21 @@ func (l *UploadFileLogic) getGeoLocation(latitude, longitude float64) (string, s
}
// 上传文件到 OSS
func (l *UploadFileLogic) uploadFileToOSS(uid string, header *multipart.FileHeader, file multipart.File, result types.File) (string, string, string, string, error) {
func (l *UploadFileLogic) uploadFileToOSS(uid string, header *multipart.FileHeader, file multipart.File, result types.File) (string, error) {
cacheKey := constant.UserOssConfigPrefix + uid + ":" + result.Provider
ossConfig, err := l.getOssConfigFromCacheOrDb(cacheKey, uid, result.Provider)
if err != nil {
return "", "", "", "", errors.New("get oss config failed")
return "", errors.New("get oss config failed")
}
service, err := l.svcCtx.StorageManager.GetStorage(uid, ossConfig)
if err != nil {
return "", "", "", "", errors.New("get storage failed")
return "", errors.New("get storage failed")
}
objectKey := path.Join(
uid,
time.Now().Format("2006/01"), // 按年/月划分目录
l.classifyFile(result.FileType, result.IsScreenshot),
fmt.Sprintf("%s_%s%s", strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename)), kgo.SimpleUuid(), filepath.Ext(header.Filename)),
)
@@ -231,13 +298,50 @@ func (l *UploadFileLogic) uploadFileToOSS(uid string, header *multipart.FileHead
"Content-Type": header.Header.Get("Content-Type"),
})
if err != nil {
return "", "", "", "", errors.New("upload file failed")
return "", errors.New("upload file failed")
}
url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, objectKey, time.Hour*24*7)
//url, err := service.PresignedURL(l.ctx, ossConfig.BucketName, objectKey, time.Hour*24*7)
//if err != nil {
// return "", "", errors.New("presigned url failed")
//}
return objectKey, nil
}
func (l *UploadFileLogic) uploadFileToMinio(uid string, header *multipart.FileHeader, file multipart.File, result types.File) (string, string, error) {
objectKey := path.Join(
uid,
time.Now().Format("2006/01"), // 按年/月划分目录
l.classifyFile(result.FileType, result.IsScreenshot),
fmt.Sprintf("%s_%s%s", strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename)), kgo.SimpleUuid(), filepath.Ext(header.Filename)),
)
exists, err := l.svcCtx.MinioClient.BucketExists(l.ctx, constant.ThumbnailBucketName)
if err != nil || !exists {
err = l.svcCtx.MinioClient.MakeBucket(l.ctx, constant.ThumbnailBucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
if err != nil {
logx.Errorf("Failed to create MinIO bucket: %v", err)
return "", "", err
}
}
// 上传到MinIO
_, err = l.svcCtx.MinioClient.PutObject(
l.ctx,
constant.ThumbnailBucketName,
objectKey,
file,
int64(result.ThumbSize),
minio.PutObjectOptions{
ContentType: result.FileType,
},
)
if err != nil {
return "", "", "", "", errors.New("presigned url failed")
return "", "", err
}
return ossConfig.BucketName, ossConfig.Provider, objectKey, url, nil
reqParams := make(url.Values)
presignedURL, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.ThumbnailBucketName, objectKey, time.Hour*24*7, reqParams)
if err != nil {
return "", "", err
}
return objectKey, presignedURL.String(), nil
}
func (l *UploadFileLogic) saveFileLocationInfoToDB(uid string, provider string, bucket string, latitude float64, longitude float64, country string, province string, city string, filePath string) (int64, error) {

305
app/auth/api/internal/mq/image_process_consumer.go Normal file
View File

@@ -0,0 +1,305 @@
package mq
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nsqio/go-nsq"
"github.com/redis/go-redis/v9"
"github.com/zeromicro/go-zero/core/logx"
"gorm.io/gorm"
"mime/multipart"
"schisandra-album-cloud-microservices/app/auth/api/internal/svc"
"schisandra-album-cloud-microservices/app/auth/api/internal/types"
"schisandra-album-cloud-microservices/app/auth/model/mysql/model"
"schisandra-album-cloud-microservices/common/constant"
"schisandra-album-cloud-microservices/common/encrypt"
"schisandra-album-cloud-microservices/common/geo_json"
"schisandra-album-cloud-microservices/common/nsqx"
"schisandra-album-cloud-microservices/common/storage/config"
"strconv"
"time"
)
type NsqImageProcessConsumer struct {
svcCtx *svc.ServiceContext
ctx context.Context
}
func NewImageProcessConsumer(svcCtx *svc.ServiceContext) {
consumer := nsqx.NewNSQConsumer(constant.MQTopicImageProcess)
consumer.AddHandler(&NsqImageProcessConsumer{
svcCtx: svcCtx,
ctx: context.Background(),
})
err := consumer.ConnectToNSQD(svcCtx.Config.NSQ.NSQDHost)
if err != nil {
panic(err)
}
}
func (c *NsqImageProcessConsumer) HandleMessage(msg *nsq.Message) error {
if len(msg.Body) == 0 {
return errors.New("empty message body")
}
var message types.FileUploadMessage
err := json.Unmarshal(msg.Body, &message)
if err != nil {
return err
}
// 根据 GPS 信息获取地理位置信息
country, province, city, err := c.getGeoLocation(message.Data.Latitude, message.Data.Longitude)
if err != nil {
return err
}
// 将地址信息保存到数据库
locationId, err := c.saveFileLocationInfoToDB(message.UID, message.Data.Provider, message.Data.Bucket, message.Data.Latitude, message.Data.Longitude, country, province, city, message.ThumbPath)
if err != nil {
return err
}
thumbnailId, err := c.saveFileThumbnailInfoToDB(message.UID, message.ThumbPath, message.Data.ThumbW, message.Data.ThumbH, message.Data.ThumbSize)
// 将文件信息存入数据库
id, err := c.saveFileInfoToDB(message.UID, message.Data.Bucket, message.Data.Provider, message.FileHeader, message.Data, locationId, message.FaceID, message.FilePath, thumbnailId)
if err != nil {
return err
}
// 删除缓存
c.afterImageUpload(message.UID, message.Data.Provider, message.Data.Bucket)
// redis 保存最近7天上传的文件列表
err = c.saveRecentFileList(message.UID, message.PresignedURL, id, message.Data, message.FileHeader.Filename)
if err != nil {
return err
}
return nil
}
// 根据 GPS 信息获取地理位置信息
func (c *NsqImageProcessConsumer) getGeoLocation(latitude, longitude float64) (string, string, string, error) {
if latitude == 0.000000 || longitude == 0.000000 {
return "", "", "", nil
}
country, province, city, err := geo_json.GetAddress(latitude, longitude, c.svcCtx.GeoRegionData)
if err != nil {
return "", "", "", errors.New("get geo location failed")
}
return country, province, city, nil
}
// 从缓存或数据库中获取 OSS 配置
func (c *NsqImageProcessConsumer) getOssConfigFromCacheOrDb(cacheKey, uid, provider string) (*config.StorageConfig, error) {
result, err := c.svcCtx.RedisClient.Get(c.ctx, cacheKey).Result()
if err != nil && !errors.Is(err, redis.Nil) {
return nil, errors.New("get oss config failed")
}
var ossConfig *config.StorageConfig
if result != "" {
var redisOssConfig model.ScaStorageConfig
if err = json.Unmarshal([]byte(result), &redisOssConfig); err != nil {
return nil, errors.New("unmarshal oss config failed")
}
return c.decryptConfig(&redisOssConfig)
}
// 缓存未命中，从数据库中加载
scaOssConfig := c.svcCtx.DB.ScaStorageConfig
dbOssConfig, err := scaOssConfig.Where(scaOssConfig.UserID.Eq(uid), scaOssConfig.Provider.Eq(provider)).First()
if err != nil {
return nil, err
}
// 缓存数据库配置
ossConfig, err = c.decryptConfig(dbOssConfig)
if err != nil {
return nil, err
}
marshalData, err := json.Marshal(dbOssConfig)
if err != nil {
return nil, errors.New("marshal oss config failed")
}
err = c.svcCtx.RedisClient.Set(c.ctx, cacheKey, marshalData, 0).Err()
if err != nil {
return nil, errors.New("set oss config failed")
}
return ossConfig, nil
}
func (c *NsqImageProcessConsumer) classifyFile(mimeType string, isScreenshot bool) string {
// 使用map存储MIME类型及其对应的分类
typeMap := map[string]string{
"image/jpeg": "image",
"image/png": "image",
"image/gif": "gif",
"image/bmp": "image",
"image/tiff": "image",
"image/webp": "image",
"video/mp4": "video",
"video/avi": "video",
"video/mpeg": "video",
"video/quicktime": "video",
"video/x-msvideo": "video",
"video/x-flv": "video",
"video/x-matroska": "video",
}
// 根据MIME类型从map中获取分类
if classification, exists := typeMap[mimeType]; exists {
return classification
}
// 如果isScreenshot为true，则返回"screenshot"
if isScreenshot {
return "screenshot"
}
return "unknown"
}
// 保存最近7天上传的文件列表
func (c *NsqImageProcessConsumer) saveRecentFileList(uid, url string, id int64, result types.File, filename string) error {
redisKey := constant.ImageRecentPrefix + uid + ":" + strconv.FormatInt(id, 10)
imageMeta := types.ImageMeta{
ID: id,
URL: url,
FileName: filename,
Width: result.Width,
Height: result.Height,
CreatedAt: time.Now().Format("2006-01-02 15:04:05"),
}
marshal, err := json.Marshal(imageMeta)
if err != nil {
logx.Error(err)
return errors.New("marshal image meta failed")
}
err = c.svcCtx.RedisClient.Set(c.ctx, redisKey, marshal, time.Hour*24*7).Err()
if err != nil {
logx.Error(err)
return errors.New("save recent file list failed")
}
return nil
}
// 提取解密操作为函数
func (c *NsqImageProcessConsumer) decryptConfig(dbConfig *model.ScaStorageConfig) (*config.StorageConfig, error) {
accessKey, err := encrypt.Decrypt(dbConfig.AccessKey, c.svcCtx.Config.Encrypt.Key)
if err != nil {
return nil, errors.New("decrypt access key failed")
}
secretKey, err := encrypt.Decrypt(dbConfig.SecretKey, c.svcCtx.Config.Encrypt.Key)
if err != nil {
return nil, errors.New("decrypt secret key failed")
}
return &config.StorageConfig{
Provider: dbConfig.Provider,
Endpoint: dbConfig.Endpoint,
AccessKey: accessKey,
SecretKey: secretKey,
BucketName: dbConfig.Bucket,
Region: dbConfig.Region,
}, nil
}
func (c *NsqImageProcessConsumer) saveFileLocationInfoToDB(uid string, provider string, bucket string, latitude float64, longitude float64, country string, province string, city string, filePath string) (int64, error) {
if latitude == 0.000000 || longitude == 0.000000 {
return 0, nil
}
locationDB := c.svcCtx.DB.ScaStorageLocation
storageLocations, err := locationDB.Where(locationDB.UserID.Eq(uid), locationDB.Province.Eq(province), locationDB.City.Eq(city)).First()
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return 0, err
}
if storageLocations == nil {
locationInfo := model.ScaStorageLocation{
Provider: provider,
Bucket: bucket,
UserID: uid,
Country: country,
City: city,
Province: province,
Latitude: fmt.Sprintf("%f", latitude),
Longitude: fmt.Sprintf("%f", longitude),
Total: 1,
CoverImage: filePath,
}
err = locationDB.Create(&locationInfo)
if err != nil {
return 0, err
}
return locationInfo.ID, nil
} else {
info, err := locationDB.Where(locationDB.ID.Eq(storageLocations.ID), locationDB.UserID.Eq(uid)).UpdateColumnSimple(locationDB.Total.Add(1), locationDB.CoverImage.Value(filePath))
if err != nil {
return 0, err
}
if info.RowsAffected == 0 {
return 0, errors.New("update location failed")
}
return storageLocations.ID, nil
}
}
func (c *NsqImageProcessConsumer) saveFileThumbnailInfoToDB(uid string, filePath string, width, height float64, size float64) (int64, error) {
storageThumb := c.svcCtx.DB.ScaStorageThumb
storageThumbInfo := &model.ScaStorageThumb{
UserID: uid,
ThumbPath: filePath,
ThumbW: width,
ThumbH: height,
ThumbSize: size,
}
err := storageThumb.Create(storageThumbInfo)
if err != nil {
logx.Error(err)
return 0, errors.New("create storage thumb failed")
}
return storageThumbInfo.ID, nil
}
// 将 EXIF 和文件信息存入数据库
func (c *NsqImageProcessConsumer) saveFileInfoToDB(uid, bucket, provider string, header *multipart.FileHeader, result types.File, locationId, faceId int64, filePath string, thumbnailId int64) (int64, error) {
typeName := c.classifyFile(result.FileType, result.IsScreenshot)
scaStorageInfo := &model.ScaStorageInfo{
UserID: uid,
Provider: provider,
Bucket: bucket,
FileName: header.Filename,
FileSize: strconv.FormatInt(header.Size, 10),
FileType: result.FileType,
Path: filePath,
Landscape: result.Landscape,
Tag: result.TagName,
IsAnime: strconv.FormatBool(result.IsAnime),
Category: result.TopCategory,
LocationID: locationId,
FaceID: faceId,
Type: typeName,
Width: result.Width,
Height: result.Height,
ThumbID: thumbnailId,
}
err := c.svcCtx.DB.ScaStorageInfo.Create(scaStorageInfo)
if err != nil {
return 0, errors.New("create storage info failed")
}
return scaStorageInfo.ID, nil
}
// 在UploadImageLogic或其他需要使缓存失效的逻辑中添加：
func (c *NsqImageProcessConsumer) afterImageUpload(uid, provider, bucket string) {
for _, sort := range []bool{true, false} {
key := fmt.Sprintf("%s%s:%s:%s:%v", constant.ImageListPrefix, uid, provider, bucket, sort)
if err := c.svcCtx.RedisClient.Del(c.ctx, key).Err(); err != nil {
logx.Errorf("删除缓存键 %s 失败: %v", key, err)
}
}
}

7
app/auth/api/internal/svc/service_context.go
View File

@@ -5,6 +5,7 @@ import (
"github.com/casbin/casbin/v2"
"github.com/lionsoul2014/ip2region/binding/golang/xdb"
"github.com/minio/minio-go/v7"
"github.com/nsqio/go-nsq"
"github.com/redis/go-redis/v9"
"github.com/wenlng/go-captcha/v2/rotate"
"github.com/wenlng/go-captcha/v2/slide"
@@ -21,6 +22,7 @@ import (
"schisandra-album-cloud-microservices/common/geo_json"
"schisandra-album-cloud-microservices/common/ip2region"
"schisandra-album-cloud-microservices/common/miniox"
"schisandra-album-cloud-microservices/common/nsqx"
"schisandra-album-cloud-microservices/common/redisx"
"schisandra-album-cloud-microservices/common/sensitivex"
"schisandra-album-cloud-microservices/common/storage"
@@ -45,13 +47,14 @@ type ServiceContext struct {
StorageManager *manager.Manager
MinioClient *minio.Client
GeoRegionData *geo_json.RegionData
NSQProducer *nsq.Producer
}
func NewServiceContext(c config.Config) *ServiceContext {
redisClient := redisx.NewRedis(c.Redis.Host, c.Redis.Pass, c.Redis.DB)
db, queryDB := mysql.NewMySQL(c.Mysql.DataSource, c.Mysql.MaxOpenConn, c.Mysql.MaxIdleConn, redisClient)
casbinEnforcer := casbinx.NewCasbin(db)
return &ServiceContext{
serviceContext := &ServiceContext{
Config: c,
SecurityHeadersMiddleware: middleware.NewSecurityHeadersMiddleware().Handle,
CasbinVerifyMiddleware: middleware.NewCasbinVerifyMiddleware(casbinEnforcer).Handle,
@@ -68,5 +71,7 @@ func NewServiceContext(c config.Config) *ServiceContext {
AiSvcRpc: aiservice.NewAiService(zrpc.MustNewClient(c.AiSvcRpc)),
MinioClient: miniox.NewMinio(c.Minio.Endpoint, c.Minio.AccessKeyID, c.Minio.SecretAccessKey, c.Minio.UseSSL),
GeoRegionData: geo_json.NewGeoJSON(),
NSQProducer: nsqx.NewNsqProducer(c.NSQ.NSQDHost),
}
return serviceContext
}

0
app/auth/api/internal/types/comment.go → app/auth/api/internal/types/comment_types.go
View File

17
app/auth/api/internal/types/file.go
View File

@@ -1,17 +0,0 @@
package types
// File represents a file uploaded by the user.
type File struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
FileType string `json:"fileType"`
IsAnime bool `json:"isAnime"`
TagName string `json:"tagName"`
Landscape string `json:"landscape"`
TopCategory string `json:"topCategory"`
IsScreenshot bool `json:"isScreenshot"`
Width float64 `json:"width"`
Height float64 `json:"height"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
}

56
app/auth/api/internal/types/file_types.go Normal file
View File

@@ -0,0 +1,56 @@
package types
import (
"mime/multipart"
"time"
)
// File represents a file uploaded by the user.
type File struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
FileType string `json:"fileType"`
IsAnime bool `json:"isAnime"`
TagName string `json:"tagName"`
Landscape string `json:"landscape"`
TopCategory string `json:"topCategory"`
IsScreenshot bool `json:"isScreenshot"`
Width float64 `json:"width"`
Height float64 `json:"height"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
ThumbW float64 `json:"thumb_w"`
ThumbH float64 `json:"thumb_h"`
ThumbSize float64 `json:"thumb_size"`
}
// FileUploadMessage represents a message sent to the user after a file upload.
type FileUploadMessage struct {
FaceID int64 `json:"face_id"`
FileHeader *multipart.FileHeader `json:"fileHeader"`
Data File `json:"data"`
UID string `json:"uid"`
FilePath string `json:"filePath"`
PresignedURL string `json:"presignedURL"`
ThumbPath string `json:"thumbPath"`
}
type FileInfoResult struct {
ID int64 `json:"id"`
FileName string `json:"file_name"`
ThumbPath string `json:"thumb_path"`
ThumbW float64 `json:"thumb_w"`
ThumbH float64 `json:"thumb_h"`
ThumbSize float64 `json:"thumb_size"`
CreatedAt time.Time `json:"created_at"`
Path string `json:"path"`
}
type ThingImageList struct {
ID int64 `json:"id"`
Category string `json:"category"`
Tag string `json:"tag"`
CreatedAt time.Time `json:"created_at"`
ThumbPath string `json:"thumb_path"`
Path string `json:"path"`
}

0
app/auth/api/internal/types/token.go → app/auth/api/internal/types/token_types.go
View File

5
app/auth/api/internal/types/types.go
View File

@@ -178,6 +178,7 @@ type ImageMeta struct {
Width float64 `json:"width"`
Height float64 `json:"height"`
CreatedAt string `json:"created_at"`
Thumbnail string `json:"thumbnail"`
}
type LocationDetailListRequest struct {
@@ -305,6 +306,10 @@ type RotateCaptchaResponse struct {
Thumb string `json:"thumb"`
}
type SingleImageRequest struct {
ID int64 `json:"id"`
}
type SlideCaptchaResponse struct {
Key string `json:"key"`
Image string `json:"image"`