SCHISANDRA/schisandra-album-cloud-microservices
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

♻️ use minio instead of mongodb

Browse Source
This commit is contained in:
landaiqing
2025-02-05 18:08:29 +08:00
parent a3d4f2c8d1
commit d2b0d7b42e
53 changed files with 2446 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/aisvc/rpc/internal/config/config.go
View File

@@ -15,4 +15,10 @@ type Config struct {
Pass string
DB int
}
Minio struct {
Endpoint string
AccessKeyID string
SecretAccessKey string
UseSSL bool
}
}

91
app/aisvc/rpc/internal/logic/aiservice/face_recognition_logic.go
View File

@@ -7,12 +7,12 @@ import (
"fmt"
"github.com/Kagami/go-face"
"github.com/ccpwcn/kgo"
"github.com/minio/minio-go/v7"
"github.com/zeromicro/go-zero/core/logx"
"image"
"image/jpeg"
_ "image/png"
"os"
"path/filepath"
"path"
"schisandra-album-cloud-microservices/app/aisvc/model/mysql/model"
"schisandra-album-cloud-microservices/app/aisvc/rpc/internal/svc"
"schisandra-album-cloud-microservices/app/aisvc/rpc/pb"
@@ -60,7 +60,7 @@ func (l *FaceRecognitionLogic) FaceRecognition(in *pb.FaceRecognitionRequest) (*
return nil, nil
}
hashKey := fmt.Sprintf("user:%s:faces", in.GetUserId())
hashKey := constant.FaceVectorPrefix + in.GetUserId()
// 从 Redis 加载人脸数据
samples, ids, err := l.loadFacesFromRedisHash(hashKey)
if err != nil {
@@ -89,10 +89,10 @@ func (l *FaceRecognitionLogic) FaceRecognition(in *pb.FaceRecognitionRequest) (*
l.svcCtx.FaceRecognizer.SetSamples(samples, ids)
// 人脸分类
classify := l.svcCtx.FaceRecognizer.ClassifyThreshold(faceFeatures.Descriptor, 0.6)
if classify >= 0 && classify < len(ids) {
classify := l.svcCtx.FaceRecognizer.ClassifyThreshold(faceFeatures.Descriptor, 0.3)
if classify > 0 {
return &pb.FaceRecognitionResponse{
FaceId: int64(ids[classify]),
FaceId: int64(classify),
}, nil
}
@@ -131,7 +131,7 @@ func (l *FaceRecognitionLogic) saveNewFace(in *pb.FaceRecognitionRequest, faceFe
}
// 保存人脸图片到本地
faceImagePath, err := l.saveCroppedFaceToLocal(in.GetFace(), faceFeatures.Rectangle, "face_samples", in.GetUserId())
faceImagePath, err := l.saveCroppedFaceToLocal(in.GetFace(), faceFeatures.Rectangle, in.GetUserId())
if err != nil {
return nil, err
}
@@ -161,7 +161,7 @@ func (l *FaceRecognitionLogic) loadExistingFaces(userId string) ([]face.Descript
storageFace := l.svcCtx.DB.ScaStorageFace
existingFaces, err := storageFace.
Select(storageFace.FaceVector, storageFace.ID).
Where(storageFace.UserID.Eq(userId), storageFace.FaceType.Eq(constant.FaceTypeSample)).
Where(storageFace.UserID.Eq(userId)).
Find()
if err != nil {
return nil, nil, err
@@ -215,7 +215,6 @@ func (l *FaceRecognitionLogic) saveFaceToDatabase(userId string, descriptor face
storageFace := model.ScaStorageFace{
FaceVector: string(jsonBytes),
FaceImagePath: faceImagePath,
FaceType: constant.FaceTypeSample,
UserID: userId,
}
err = l.svcCtx.DB.ScaStorageFace.Create(&storageFace)
@@ -225,17 +224,12 @@ func (l *FaceRecognitionLogic) saveFaceToDatabase(userId string, descriptor face
return &storageFace, nil
}
func (l *FaceRecognitionLogic) saveCroppedFaceToLocal(faceImage []byte, rect image.Rectangle, baseSavePath string, userID string) (string, error) {
// 动态生成用户目录和时间分级目录
subDir := filepath.Join(baseSavePath, userID, time.Now().Format("2006/01")) // 格式：<baseSavePath>/<userID>/YYYY/MM
// 缓存目录检查，避免重复调用 os.MkdirAll
if !l.isDirectoryCached(subDir) {
if err := os.MkdirAll(subDir, os.ModePerm); err != nil {
return "", fmt.Errorf("failed to create directory: %w", err)
}
l.cacheDirectory(subDir) // 缓存已创建的目录路径
}
func (l *FaceRecognitionLogic) saveCroppedFaceToLocal(faceImage []byte, rect image.Rectangle, userID string) (string, error) {
objectKey := path.Join(
userID,
time.Now().Format("2006/01"), // 按年/月划分目录
fmt.Sprintf("%s_%s.jpg", time.Now().Format("20060102150405"), kgo.SimpleUuid()),
)
// 解码图像
img, _, err := image.Decode(bytes.NewReader(faceImage))
@@ -258,42 +252,35 @@ func (l *FaceRecognitionLogic) saveCroppedFaceToLocal(faceImage []byte, rect ima
SubImage(r image.Rectangle) image.Image
}).SubImage(extendedRect)
// 生成唯一文件名（时间戳 + UUID）
fileName := fmt.Sprintf("%s_%s.jpg", time.Now().Format("20060102_150405"), kgo.SimpleUuid())
outputPath := filepath.Join(subDir, fileName)
// 写入文件
if err = l.writeImageToFile(outputPath, croppedImage); err != nil {
return "", err
// 将图像编码为JPEG字节流
var buf bytes.Buffer
if err = jpeg.Encode(&buf, croppedImage, nil); err != nil {
return "", fmt.Errorf("failed to encode image to JPEG: %w", err)
}
exists, err := l.svcCtx.MinioClient.BucketExists(l.ctx, constant.FaceBucketName)
if err != nil || !exists {
err = l.svcCtx.MinioClient.MakeBucket(l.ctx, constant.FaceBucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true})
if err != nil {
logx.Errorf("Failed to create MinIO bucket: %v", err)
return "", err
}
}
return outputPath, nil
}
// 判断目录是否已缓存
func (l *FaceRecognitionLogic) isDirectoryCached(dir string) bool {
_, exists := l.directoryCache.Load(dir)
return exists
}
// 缓存目录
func (l *FaceRecognitionLogic) cacheDirectory(dir string) {
l.directoryCache.Store(dir, struct{}{})
}
// 将图像写入文件
func (l *FaceRecognitionLogic) writeImageToFile(path string, img image.Image) error {
file, err := os.Create(path)
// 上传到MinIO
_, err = l.svcCtx.MinioClient.PutObject(
l.ctx,
constant.FaceBucketName,
objectKey,
bytes.NewReader(buf.Bytes()),
int64(buf.Len()),
minio.PutObjectOptions{
ContentType: "image/jpeg",
},
)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
return "", fmt.Errorf("failed to upload image to MinIO: %w", err)
}
defer func(file *os.File) {
_ = file.Close()
}(file)
if err = jpeg.Encode(file, img, nil); err != nil {
return fmt.Errorf("failed to encode and save image: %w", err)
}
return nil
return objectKey, nil
}
// 从 Redis 的 Hash 中加载人脸数据

40
app/aisvc/rpc/internal/logic/aiservice/modify_face_name_logic.go Normal file
View File

@@ -0,0 +1,40 @@
package aiservicelogic
import (
"context"
"errors"
"schisandra-album-cloud-microservices/app/aisvc/rpc/internal/svc"
"schisandra-album-cloud-microservices/app/aisvc/rpc/pb"
"github.com/zeromicro/go-zero/core/logx"
)
type ModifyFaceNameLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewModifyFaceNameLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ModifyFaceNameLogic {
return &ModifyFaceNameLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
func (l *ModifyFaceNameLogic) ModifyFaceName(in *pb.ModifyFaceNameRequest) (*pb.ModifyFaceNameResponse, error) {
storageFace := l.svcCtx.DB.ScaStorageFace
affected, err := storageFace.Where(storageFace.ID.Eq(in.GetFaceId()), storageFace.UserID.Eq(in.GetUserId())).Update(storageFace.FaceName, in.GetFaceName())
if err != nil {
return nil, err
}
if affected.RowsAffected == 0 {
return nil, errors.New("update failed, no rows affected")
}
return &pb.ModifyFaceNameResponse{
FaceId: in.GetFaceId(),
FaceName: in.GetFaceName(),
}, nil
}

43
app/aisvc/rpc/internal/logic/aiservice/modify_face_type_logic.go Normal file
View File

@@ -0,0 +1,43 @@
package aiservicelogic
import (
"context"
"errors"
"schisandra-album-cloud-microservices/app/aisvc/rpc/internal/svc"
"schisandra-album-cloud-microservices/app/aisvc/rpc/pb"
"github.com/zeromicro/go-zero/core/logx"
)
type ModifyFaceTypeLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
}
func NewModifyFaceTypeLogic(ctx context.Context, svcCtx *svc.ServiceContext) *ModifyFaceTypeLogic {
return &ModifyFaceTypeLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
}
}
// ModifyFaceType
func (l *ModifyFaceTypeLogic) ModifyFaceType(in *pb.ModifyFaceTypeRequest) (*pb.ModifyFaceTypeResponse, error) {
storageFace := l.svcCtx.DB.ScaStorageFace
faceIds := in.GetFaceId()
info, err := storageFace.Where(storageFace.ID.In(faceIds...), storageFace.UserID.Eq(in.GetUserId())).Update(storageFace.FaceType, in.GetType())
if err != nil {
return nil, err
}
if info.RowsAffected == 0 {
return &pb.ModifyFaceTypeResponse{
Result: "fail",
}, errors.New("face not found")
}
return &pb.ModifyFaceTypeResponse{
Result: "success",
}, nil
}

102
app/aisvc/rpc/internal/logic/aiservice/query_face_library_logic.go Normal file
View File

@@ -0,0 +1,102 @@
package aiservicelogic
import (
"context"
"fmt"
"net/url"
"schisandra-album-cloud-microservices/app/aisvc/model/mysql/model"
"schisandra-album-cloud-microservices/common/constant"
"strconv"
"sync"
"time"
"schisandra-album-cloud-microservices/app/aisvc/rpc/internal/svc"
"schisandra-album-cloud-microservices/app/aisvc/rpc/pb"
"github.com/zeromicro/go-zero/core/logx"
)
type QueryFaceLibraryLogic struct {
ctx context.Context
svcCtx *svc.ServiceContext
logx.Logger
wg sync.WaitGroup
mu sync.Mutex
}
type FaceLibrary struct {
ID int64 `json:"id"`
FaceImage []byte `json:"face_image"`
FaceName string `json:"face_name"`
}
func NewQueryFaceLibraryLogic(ctx context.Context, svcCtx *svc.ServiceContext) *QueryFaceLibraryLogic {
return &QueryFaceLibraryLogic{
ctx: ctx,
svcCtx: svcCtx,
Logger: logx.WithContext(ctx),
wg: sync.WaitGroup{},
mu: sync.Mutex{},
}
}
// QueryFaceLibrary queries the face library
func (l *QueryFaceLibraryLogic) QueryFaceLibrary(in *pb.QueryFaceLibraryRequest) (*pb.QueryFaceLibraryResponse, error) {
if in.GetUserId() == "" {
return nil, fmt.Errorf("user ID is required")
}
storageFace := l.svcCtx.DB.ScaStorageFace
samples, err := storageFace.Select(
storageFace.ID,
storageFace.FaceVector,
storageFace.FaceImagePath,
storageFace.FaceName).
Where(storageFace.UserID.Eq(in.GetUserId()), storageFace.FaceType.Eq(in.GetType())).
Find()
if err != nil {
return nil, fmt.Errorf("failed to query face library: %v", err)
}
if len(samples) == 0 {
return nil, nil
}
faceLibrary := make([]*pb.FaceLibrary, len(samples))
for i, sample := range samples {
l.wg.Add(1)
go func(i int, sample *model.ScaStorageFace) {
defer l.wg.Done()
redisKey := constant.FaceSamplePrefix + in.GetUserId() + ":" + strconv.FormatInt(sample.ID, 10)
file, err := l.svcCtx.RedisClient.Get(l.ctx, redisKey).Result()
if err == nil {
l.mu.Lock()
faceLibrary[i] = &pb.FaceLibrary{
Id: sample.ID,
FaceName: sample.FaceName,
FaceImage: file,
}
l.mu.Unlock()
return
}
reqParams := make(url.Values)
presignedURL, err := l.svcCtx.MinioClient.PresignedGetObject(l.ctx, constant.FaceBucketName, sample.FaceImagePath, time.Hour*24, reqParams)
err = l.svcCtx.RedisClient.Set(l.ctx, redisKey, presignedURL.String(), time.Hour*24).Err()
if err != nil {
return
}
l.mu.Lock()
faceLibrary[i] = &pb.FaceLibrary{
Id: sample.ID,
FaceName: sample.FaceName,
FaceImage: presignedURL.String(),
}
l.mu.Unlock()
}(i, sample)
}
l.wg.Wait()
return &pb.QueryFaceLibraryResponse{
Faces: faceLibrary,
}, nil
}

18
app/aisvc/rpc/internal/server/aiservice/ai_service_server.go
View File

@@ -40,3 +40,21 @@ func (s *AiServiceServer) CaffeClassification(ctx context.Context, in *pb.CaffeC
l := aiservicelogic.NewCaffeClassificationLogic(ctx, s.svcCtx)
return l.CaffeClassification(in)
}
// QueryFaceLibrary
func (s *AiServiceServer) QueryFaceLibrary(ctx context.Context, in *pb.QueryFaceLibraryRequest) (*pb.QueryFaceLibraryResponse, error) {
l := aiservicelogic.NewQueryFaceLibraryLogic(ctx, s.svcCtx)
return l.QueryFaceLibrary(in)
}
// ModifyFaceName
func (s *AiServiceServer) ModifyFaceName(ctx context.Context, in *pb.ModifyFaceNameRequest) (*pb.ModifyFaceNameResponse, error) {
l := aiservicelogic.NewModifyFaceNameLogic(ctx, s.svcCtx)
return l.ModifyFaceName(in)
}
// ModifyFaceType
func (s *AiServiceServer) ModifyFaceType(ctx context.Context, in *pb.ModifyFaceTypeRequest) (*pb.ModifyFaceTypeResponse, error) {
l := aiservicelogic.NewModifyFaceTypeLogic(ctx, s.svcCtx)
return l.ModifyFaceType(in)
}

4
app/aisvc/rpc/internal/svc/service_context.go
View File

@@ -2,6 +2,7 @@ package svc
import (
"github.com/Kagami/go-face"
"github.com/minio/minio-go/v7"
"github.com/redis/go-redis/v9"
"gocv.io/x/gocv"
"schisandra-album-cloud-microservices/app/aisvc/model/mysql"
@@ -9,6 +10,7 @@ import (
"schisandra-album-cloud-microservices/app/aisvc/rpc/internal/config"
"schisandra-album-cloud-microservices/common/caffe_classifier"
"schisandra-album-cloud-microservices/common/face_recognizer"
"schisandra-album-cloud-microservices/common/miniox"
"schisandra-album-cloud-microservices/common/redisx"
"schisandra-album-cloud-microservices/common/tf_classifier"
)
@@ -22,6 +24,7 @@ type ServiceContext struct {
TfDesc []string
CaffeNet *gocv.Net
CaffeDesc []string
MinioClient *minio.Client
}
func NewServiceContext(c config.Config) *ServiceContext {
@@ -38,5 +41,6 @@ func NewServiceContext(c config.Config) *ServiceContext {
TfDesc: tfDesc,
CaffeNet: caffeClassifier,
CaffeDesc: caffeDesc,
MinioClient: miniox.NewMinio(c.Minio.Endpoint, c.Minio.AccessKeyID, c.Minio.SecretAccessKey, c.Minio.UseSSL),
}
}