414 lines
9.5 KiB
Go
414 lines
9.5 KiB
Go
package snapshotstore
|
||
|
||
import (
|
||
"archive/tar"
|
||
"bytes"
|
||
"compress/gzip"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
"voidraft/internal/syncer/backend"
|
||
"voidraft/internal/syncer/backend/snapshotstore/blob"
|
||
)
|
||
|
||
const (
|
||
defaultNamespace = "sync"
|
||
defaultHeadKey = "head.json"
|
||
bundleDirName = "bundles"
|
||
)
|
||
|
||
var stableBundleTime = time.Unix(0, 0).UTC()
|
||
|
||
// Config 描述 snapshot_store 后端配置。
|
||
type Config struct {
|
||
Store blob.Store
|
||
Namespace string
|
||
HeadKey string
|
||
}
|
||
|
||
type headDocument struct {
|
||
Revision string `json:"revision"`
|
||
BundleKey string `json:"bundle_key"`
|
||
UpdatedAt string `json:"updated_at"`
|
||
}
|
||
|
||
type headState struct {
|
||
Document headDocument
|
||
Info blob.ObjectInfo
|
||
}
|
||
|
||
// Backend 提供基于对象/文件存储的快照后端实现。
|
||
type Backend struct {
|
||
config Config
|
||
}
|
||
|
||
// New 创建新的 snapshot_store 后端。
|
||
func New(config Config) (*Backend, error) {
|
||
if config.Store == nil {
|
||
return nil, errors.New("snapshot store blob backend is required")
|
||
}
|
||
if strings.TrimSpace(config.Namespace) == "" {
|
||
config.Namespace = defaultNamespace
|
||
}
|
||
if strings.TrimSpace(config.HeadKey) == "" {
|
||
config.HeadKey = defaultHeadKey
|
||
}
|
||
return &Backend{config: config}, nil
|
||
}
|
||
|
||
// Verify 校验后端是否可读。
|
||
func (b *Backend) Verify(ctx context.Context) error {
|
||
_, _, err := b.readHead(ctx)
|
||
return err
|
||
}
|
||
|
||
// DownloadLatest 下载远端最新快照包并解压到目标目录。
|
||
func (b *Backend) DownloadLatest(ctx context.Context, dst string) (backend.RemoteState, error) {
|
||
head, exists, err := b.readHead(ctx)
|
||
if err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
if !exists {
|
||
return backend.RemoteState{}, nil
|
||
}
|
||
|
||
reader, _, err := b.config.Store.Get(ctx, head.Document.BundleKey)
|
||
if err != nil {
|
||
if errors.Is(err, blob.ErrObjectNotFound) {
|
||
return backend.RemoteState{}, nil
|
||
}
|
||
return backend.RemoteState{}, err
|
||
}
|
||
defer reader.Close()
|
||
|
||
if err := recreateDir(dst); err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
if err := extractBundle(reader, dst); err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
|
||
return backend.RemoteState{
|
||
Exists: true,
|
||
Revision: head.Document.Revision,
|
||
}, nil
|
||
}
|
||
|
||
// Upload 打包并发布本地快照目录。
|
||
func (b *Backend) Upload(ctx context.Context, src string, options backend.PublishOptions) (backend.RemoteState, error) {
|
||
currentHead, exists, err := b.readHead(ctx)
|
||
if err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
|
||
switch {
|
||
case options.ExpectedRevision != "" && !exists:
|
||
return backend.RemoteState{}, backend.ErrRevisionConflict
|
||
case options.ExpectedRevision != "" && currentHead.Document.Revision != options.ExpectedRevision:
|
||
return backend.RemoteState{}, backend.ErrRevisionConflict
|
||
}
|
||
|
||
bundlePath, revision, err := createBundle(src)
|
||
if err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
defer os.Remove(bundlePath)
|
||
|
||
if exists && currentHead.Document.Revision == revision {
|
||
return backend.RemoteState{
|
||
Exists: true,
|
||
Revision: revision,
|
||
}, nil
|
||
}
|
||
|
||
bundleKey := b.bundleKey(revision)
|
||
file, err := os.Open(bundlePath)
|
||
if err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
defer file.Close()
|
||
|
||
if _, err := b.config.Store.Put(ctx, bundleKey, file, blob.PutOptions{}); err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
|
||
nextHead := headDocument{
|
||
Revision: revision,
|
||
BundleKey: bundleKey,
|
||
UpdatedAt: time.Now().Format(time.RFC3339),
|
||
}
|
||
headPayload, err := json.MarshalIndent(nextHead, "", " ")
|
||
if err != nil {
|
||
return backend.RemoteState{}, err
|
||
}
|
||
headPayload = append(headPayload, '\n')
|
||
|
||
putOptions := blob.PutOptions{}
|
||
if exists {
|
||
putOptions.IfMatch = currentHead.Info.Revision
|
||
}
|
||
|
||
if _, err := b.config.Store.Put(ctx, b.headKey(), bytes.NewReader(headPayload), putOptions); err != nil {
|
||
if errors.Is(err, blob.ErrConditionNotMet) {
|
||
return backend.RemoteState{}, backend.ErrRevisionConflict
|
||
}
|
||
return backend.RemoteState{}, err
|
||
}
|
||
|
||
return backend.RemoteState{
|
||
Exists: true,
|
||
Revision: revision,
|
||
}, nil
|
||
}
|
||
|
||
// Close 关闭后端。
|
||
func (b *Backend) Close() error {
|
||
return nil
|
||
}
|
||
|
||
// readHead 读取远端 head 指针。
|
||
func (b *Backend) readHead(ctx context.Context) (headState, bool, error) {
|
||
reader, info, err := b.config.Store.Get(ctx, b.headKey())
|
||
if err != nil {
|
||
if errors.Is(err, blob.ErrObjectNotFound) {
|
||
return headState{}, false, nil
|
||
}
|
||
return headState{}, false, err
|
||
}
|
||
defer reader.Close()
|
||
|
||
data, err := io.ReadAll(reader)
|
||
if err != nil {
|
||
return headState{}, false, err
|
||
}
|
||
|
||
var document headDocument
|
||
if err := json.Unmarshal(data, &document); err != nil {
|
||
return headState{}, false, err
|
||
}
|
||
if document.Revision == "" || document.BundleKey == "" {
|
||
return headState{}, false, errors.New("snapshot store head is invalid")
|
||
}
|
||
|
||
return headState{
|
||
Document: document,
|
||
Info: info,
|
||
}, true, nil
|
||
}
|
||
|
||
// headKey 返回完整的 head 对象键。
|
||
func (b *Backend) headKey() string {
|
||
return path.Join(b.config.Namespace, b.config.HeadKey)
|
||
}
|
||
|
||
// bundleKey 返回 revision 对应的 bundle 键。
|
||
func (b *Backend) bundleKey(revision string) string {
|
||
return path.Join(b.config.Namespace, bundleDirName, revision+".tar.gz")
|
||
}
|
||
|
||
// createBundle 将目录稳定打包成 tar.gz，并返回文件路径与摘要。
|
||
func createBundle(root string) (string, string, error) {
|
||
tempFile, err := os.CreateTemp("", "voidraft-snapshot-*.tar.gz")
|
||
if err != nil {
|
||
return "", "", err
|
||
}
|
||
tempName := tempFile.Name()
|
||
|
||
hasher := sha256.New()
|
||
multiWriter := io.MultiWriter(tempFile, hasher)
|
||
|
||
gzipWriter := gzip.NewWriter(multiWriter)
|
||
gzipWriter.ModTime = stableBundleTime
|
||
gzipWriter.Name = ""
|
||
gzipWriter.Comment = ""
|
||
|
||
tarWriter := tar.NewWriter(gzipWriter)
|
||
|
||
writeErr := writeBundle(root, tarWriter)
|
||
closeErr := tarWriter.Close()
|
||
gzipCloseErr := gzipWriter.Close()
|
||
fileCloseErr := tempFile.Close()
|
||
if writeErr != nil {
|
||
_ = os.Remove(tempName)
|
||
return "", "", writeErr
|
||
}
|
||
if closeErr != nil {
|
||
_ = os.Remove(tempName)
|
||
return "", "", closeErr
|
||
}
|
||
if gzipCloseErr != nil {
|
||
_ = os.Remove(tempName)
|
||
return "", "", gzipCloseErr
|
||
}
|
||
if fileCloseErr != nil {
|
||
_ = os.Remove(tempName)
|
||
return "", "", fileCloseErr
|
||
}
|
||
|
||
revision := hex.EncodeToString(hasher.Sum(nil))
|
||
return tempName, revision, nil
|
||
}
|
||
|
||
// writeBundle 将目录内容按稳定顺序写入 tar。
|
||
func writeBundle(root string, writer *tar.Writer) error {
|
||
paths, err := collectPaths(root)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
for _, entryPath := range paths {
|
||
info, err := os.Lstat(entryPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
relativePath, err := filepath.Rel(root, entryPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
header, err := tar.FileInfoHeader(info, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
header.Name = filepath.ToSlash(relativePath)
|
||
header.ModTime = stableBundleTime
|
||
header.AccessTime = stableBundleTime
|
||
header.ChangeTime = stableBundleTime
|
||
header.Uid = 0
|
||
header.Gid = 0
|
||
header.Uname = ""
|
||
header.Gname = ""
|
||
|
||
if info.IsDir() && !strings.HasSuffix(header.Name, "/") {
|
||
header.Name += "/"
|
||
}
|
||
|
||
if err := writer.WriteHeader(header); err != nil {
|
||
return err
|
||
}
|
||
|
||
if info.IsDir() {
|
||
continue
|
||
}
|
||
|
||
file, err := os.Open(entryPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if _, err := io.Copy(writer, file); err != nil {
|
||
file.Close()
|
||
return err
|
||
}
|
||
if err := file.Close(); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// collectPaths 返回稳定排序后的目录项列表。
|
||
func collectPaths(root string) ([]string, error) {
|
||
entries := make([]string, 0)
|
||
if err := filepath.WalkDir(root, func(entryPath string, entry os.DirEntry, err error) error {
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if entryPath == root {
|
||
return nil
|
||
}
|
||
entries = append(entries, entryPath)
|
||
return nil
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
sort.Strings(entries)
|
||
return entries, nil
|
||
}
|
||
|
||
// extractBundle 将 tar.gz 包解压到目标目录。
|
||
func extractBundle(reader io.Reader, dst string) error {
|
||
gzipReader, err := gzip.NewReader(reader)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer gzipReader.Close()
|
||
|
||
tarReader := tar.NewReader(gzipReader)
|
||
for {
|
||
header, err := tarReader.Next()
|
||
if errors.Is(err, io.EOF) {
|
||
return nil
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
targetPath, err := resolveExtractPath(dst, header.Name)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
switch header.Typeflag {
|
||
case tar.TypeDir:
|
||
if err := os.MkdirAll(targetPath, 0755); err != nil {
|
||
return err
|
||
}
|
||
case tar.TypeReg:
|
||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||
return err
|
||
}
|
||
file, err := os.OpenFile(targetPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if _, err := io.Copy(file, tarReader); err != nil {
|
||
file.Close()
|
||
return err
|
||
}
|
||
if err := file.Close(); err != nil {
|
||
return err
|
||
}
|
||
default:
|
||
return fmt.Errorf("unsupported tar entry type: %d", header.Typeflag)
|
||
}
|
||
}
|
||
}
|
||
|
||
// recreateDir 清空并重建目录。
|
||
func recreateDir(dir string) error {
|
||
if err := os.RemoveAll(dir); err != nil {
|
||
return err
|
||
}
|
||
return os.MkdirAll(dir, 0755)
|
||
}
|
||
|
||
// resolveExtractPath 将归档路径安全映射到目标目录。
|
||
func resolveExtractPath(root string, name string) (string, error) {
|
||
clean := filepath.Clean(filepath.FromSlash(name))
|
||
if clean == "." {
|
||
return "", errors.New("invalid archive entry")
|
||
}
|
||
targetPath := filepath.Join(root, clean)
|
||
relativePath, err := filepath.Rel(root, targetPath)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if strings.HasPrefix(relativePath, "..") {
|
||
return "", errors.New("archive entry escapes target directory")
|
||
}
|
||
return targetPath, nil
|
||
}
|