82 lines
2.6 KiB
Go
82 lines
2.6 KiB
Go
package media
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
|
|
"git.mstar.dev/mstar/goutils/sliceutils"
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/rs/zerolog/log"
|
|
|
|
"git.mstar.dev/mstar/linstrom/config"
|
|
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
|
|
"git.mstar.dev/mstar/linstrom/storage-new/models"
|
|
)
|
|
|
|
// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between
|
|
// the db's metadata for the files and the actual files in s3.
|
|
// All files without matching metadata will be deleted. Same for all metadata without a matching file.
|
|
// No attempt at restoring a connection will be made
|
|
func (s *Server) ServiceEnsureFileSynchronisation() {
|
|
mm := dbgen.MediaMetadata
|
|
allFiles, err := mm.
|
|
Select(mm.ID, mm.OwnedById, mm.Name, mm.Location).
|
|
Where(mm.Location.NotLike("linstrom://%"), mm.Remote.Is(false)).
|
|
Find()
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("Failed to get a list of all known media")
|
|
return
|
|
}
|
|
foundInDb := []string{}
|
|
objectMissingInDb := []minio.ObjectInfo{}
|
|
// Go over all objects in the bucket. Note down if it has an entry in the db or not
|
|
for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{
|
|
Recursive: true,
|
|
}) {
|
|
log.Debug().Str("object-key", obj.Key).Msg("Checking object")
|
|
if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
|
|
return e.Location == obj.Key
|
|
}) {
|
|
foundInDb = append(foundInDb, obj.Key)
|
|
} else {
|
|
objectMissingInDb = append(objectMissingInDb, obj)
|
|
}
|
|
}
|
|
|
|
// Find every db entry not in the list of found objects
|
|
entryMissingAnObject := []string{}
|
|
for _, dbFile := range allFiles {
|
|
if !slices.ContainsFunc(foundInDb, func(e string) bool {
|
|
return dbFile.Location == e
|
|
}) {
|
|
entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
|
|
}
|
|
}
|
|
|
|
// For every object missing in the db, delete it
|
|
minioErrChan := s.client.RemoveObjects(
|
|
context.TODO(),
|
|
config.GlobalConfig.S3.BucketName,
|
|
sliceutils.ToChannel(objectMissingInDb),
|
|
minio.RemoveObjectsOptions{GovernanceBypass: true},
|
|
)
|
|
s3Errors := sliceutils.FromChannel(minioErrChan, 0)
|
|
s3Errors = sliceutils.Filter(
|
|
s3Errors,
|
|
func(t minio.RemoveObjectError) bool { return t.Err != nil },
|
|
)
|
|
for _, s3Err := range s3Errors {
|
|
log.Error().
|
|
Err(s3Err.Err).
|
|
Str("object-name", s3Err.ObjectName).
|
|
Msg("Failed to delete object missing in db")
|
|
}
|
|
// And perform a batch delete
|
|
_, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete()
|
|
if err != nil {
|
|
log.Error().
|
|
Err(err).
|
|
Strs("media-ids", entryMissingAnObject).
|
|
Msg("Failed to batch delete all media metadata without a matching object in s3")
|
|
}
|
|
}
|