package media import ( "context" "slices" "git.mstar.dev/mstar/goutils/sliceutils" "github.com/minio/minio-go/v7" "github.com/rs/zerolog/log" "git.mstar.dev/mstar/linstrom/config" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" "git.mstar.dev/mstar/linstrom/storage-new/models" ) // ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between // the db's metadata for the files and the actual files in s3. // All files without matching metadata will be deleted. Same for all metadata without a matching file. // No attempt at restoring a connection will be made func (s *Server) ServiceEnsureFileSynchronisation() { allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name). Find() if err != nil { log.Error().Err(err).Msg("Failed to get a list of all known media") return } foundInDb := []string{} objectMissingInDb := []minio.ObjectInfo{} // Go over all objects in the bucket. Note down if it has an entry in the db or not for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) { if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool { return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key }) { foundInDb = append(foundInDb, obj.Key) } else { objectMissingInDb = append(objectMissingInDb, obj) } } // Find every db entry not in the list of found objects entryMissingAnObject := []string{} for _, dbFile := range allFiles { if !slices.ContainsFunc(foundInDb, func(e string) bool { return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e }) { entryMissingAnObject = append(entryMissingAnObject, dbFile.ID) } } // For every object missing in the db, delete it minioErrChan := s.client.RemoveObjects( context.TODO(), config.GlobalConfig.S3.BucketName, sliceutils.ToChannel(objectMissingInDb), minio.RemoveObjectsOptions{GovernanceBypass: true}, ) s3Errors := sliceutils.FromChannel(minioErrChan, 0) s3Errors = sliceutils.Filter( s3Errors, func(t minio.RemoveObjectError) bool { return t.Err != nil }, ) for _, s3Err := range s3Errors { log.Error(). Err(s3Err.Err). Str("object-name", s3Err.ObjectName). Msg("Failed to delete object missing in db") } // And perform a batch delete _, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete() if err != nil { log.Error(). Err(err). Strs("media-ids", entryMissingAnObject). Msg("Failed to batch delete all media metadata without a matching object in s3") } }