Add media upload functionality

media/services.go

@@ -1 +1,76 @@
 package media
+
+import (
+	"context"
+	"slices"
+
+	"git.mstar.dev/mstar/goutils/sliceutils"
+	"github.com/minio/minio-go/v7"
+	"github.com/rs/zerolog/log"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"git.mstar.dev/mstar/linstrom/storage-new/models"
+)
+
+// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between
+// the db's metadata for the files and the actual files in s3.
+// All files without matching metadata will be deleted. Same for all metadata without a matching file.
+// No attempt at restoring a connection will be made
+func (s *Server) ServiceEnsureFileSynchronisation() {
+	allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name).
+		Find()
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to get a list of all known media")
+		return
+	}
+	foundInDb := []string{}
+	objectMissingInDb := []minio.ObjectInfo{}
+	// Go over all objects in the bucket. Note down if it has an entry in the db or not
+	for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) {
+		if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
+			return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key
+		}) {
+			foundInDb = append(foundInDb, obj.Key)
+		} else {
+			objectMissingInDb = append(objectMissingInDb, obj)
+		}
+	}
+
+	// Find every db entry not in the list of found objects
+	entryMissingAnObject := []string{}
+	for _, dbFile := range allFiles {
+		if !slices.ContainsFunc(foundInDb, func(e string) bool {
+			return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e
+		}) {
+			entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
+		}
+	}
+
+	// For every object missing in the db, delete it
+	minioErrChan := s.client.RemoveObjects(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		sliceutils.ToChannel(objectMissingInDb),
+		minio.RemoveObjectsOptions{GovernanceBypass: true},
+	)
+	s3Errors := sliceutils.FromChannel(minioErrChan, 0)
+	s3Errors = sliceutils.Filter(
+		s3Errors,
+		func(t minio.RemoveObjectError) bool { return t.Err != nil },
+	)
+	for _, s3Err := range s3Errors {
+		log.Error().
+			Err(s3Err.Err).
+			Str("object-name", s3Err.ObjectName).
+			Msg("Failed to delete object missing in db")
+	}
+	// And perform a batch delete
+	_, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete()
+	if err != nil {
+		log.Error().
+			Err(err).
+			Strs("media-ids", entryMissingAnObject).
+			Msg("Failed to batch delete all media metadata without a matching object in s3")
+	}
+}