Add media upload functionality

media/addFile.go

@@ -3,6 +3,9 @@ package media
 import (
 	"context"
 	"database/sql"
+	"git.mstar.dev/mstar/goutils/other"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"github.com/gabriel-vasile/mimetype"
 	"io"
 	"os"
 	"path"
@@ -27,18 +30,23 @@ func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error {
 	}
 	_ = file.Close()
 	if s.transcoderClient == nil {
-		return s.addFileAsIs(filename, userId, filePath)
+		return s.addFileAsIs(filename, userId, filePath, nil)
 	} else {
 		return s.addFileWithTranscoder(filename, userId, filePath)
 	}
 }
 
-func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
-	panic("not implemented")
-}
-
-func (s *Server) addFileAsIs(filename, userId, filepath string) error {
-	_, err := s.client.FPutObject(
+// adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict)
+// is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually
+func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error {
+	if mtype == nil {
+		mType, err := mimetype.DetectFile(filepath)
+		if err != nil {
+			return err
+		}
+		mtype = other.IntoPointer(mType.String())
+	}
+	s3Result, err := s.client.FPutObject(
 		context.TODO(),
 		config.GlobalConfig.S3.BucketName,
 		UsernameFilename(userId, filename),
@@ -52,7 +60,15 @@ func (s *Server) addFileAsIs(filename, userId, filepath string) error {
 		ID:        shared.NewId(),
 		OwnedById: sql.NullString{Valid: true, String: userId},
 		Remote:    false,
-		// Location: string, // TODO: Figure this out
+		Location:  s3Result.Location,
+		Type:      *mtype,
+		Name:      UsernameFilename(userId, filename),
+		AltText:   "",
+		Blurred:   false,
 	}
-	panic("not implemented")
+	err = dbgen.MediaMetadata.Create(&fileMetadata)
+	if err != nil {
+		return err
+	}
+	return nil
 }

media/services.go

@@ -1 +1,76 @@
 package media
+
+import (
+	"context"
+	"slices"
+
+	"git.mstar.dev/mstar/goutils/sliceutils"
+	"github.com/minio/minio-go/v7"
+	"github.com/rs/zerolog/log"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"git.mstar.dev/mstar/linstrom/storage-new/models"
+)
+
+// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between
+// the db's metadata for the files and the actual files in s3.
+// All files without matching metadata will be deleted. Same for all metadata without a matching file.
+// No attempt at restoring a connection will be made
+func (s *Server) ServiceEnsureFileSynchronisation() {
+	allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name).
+		Find()
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to get a list of all known media")
+		return
+	}
+	foundInDb := []string{}
+	objectMissingInDb := []minio.ObjectInfo{}
+	// Go over all objects in the bucket. Note down if it has an entry in the db or not
+	for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) {
+		if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
+			return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key
+		}) {
+			foundInDb = append(foundInDb, obj.Key)
+		} else {
+			objectMissingInDb = append(objectMissingInDb, obj)
+		}
+	}
+
+	// Find every db entry not in the list of found objects
+	entryMissingAnObject := []string{}
+	for _, dbFile := range allFiles {
+		if !slices.ContainsFunc(foundInDb, func(e string) bool {
+			return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e
+		}) {
+			entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
+		}
+	}
+
+	// For every object missing in the db, delete it
+	minioErrChan := s.client.RemoveObjects(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		sliceutils.ToChannel(objectMissingInDb),
+		minio.RemoveObjectsOptions{GovernanceBypass: true},
+	)
+	s3Errors := sliceutils.FromChannel(minioErrChan, 0)
+	s3Errors = sliceutils.Filter(
+		s3Errors,
+		func(t minio.RemoveObjectError) bool { return t.Err != nil },
+	)
+	for _, s3Err := range s3Errors {
+		log.Error().
+			Err(s3Err.Err).
+			Str("object-name", s3Err.ObjectName).
+			Msg("Failed to delete object missing in db")
+	}
+	// And perform a batch delete
+	_, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete()
+	if err != nil {
+		log.Error().
+			Err(err).
+			Strs("media-ids", entryMissingAnObject).
+			Msg("Failed to batch delete all media metadata without a matching object in s3")
+	}
+}

media/transcoder.go

@@ -1 +1,40 @@
 package media
+
+import (
+	"git.mstar.dev/mstar/linstrom/config"
+	"github.com/rs/zerolog/log"
+)
+
+// WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go
+// TODO: Maybe move to a separate repo outside of linstrom-transcoder
+
+type TranscodeArgs struct {
+	Secret   string
+	Filename string
+}
+
+type TranscodeReply struct {
+	Error             error
+	Mimetype          string
+	ThumbnailFilename *string
+	Filename          string
+}
+
+// addFileWithTranscoder will try to transcode the given file using the helper application.
+// If the transcode fails, it uploads the file as is
+func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
+	args := TranscodeArgs{
+		Secret:   config.GlobalConfig.Transcoder.Secret,
+		Filename: filepath,
+	}
+	reply := TranscodeReply{}
+	err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply)
+	if err != nil {
+		return err
+	}
+	if reply.Error != nil {
+		log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file")
+		return s.addFileAsIs(filename, userId, filepath, nil)
+	}
+	return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype)
+}