Add media upload functionality

go.mod

@@ -3,11 +3,12 @@ module git.mstar.dev/mstar/linstrom
 go 1.24.2
 
 require (
-	git.mstar.dev/mstar/goutils v1.14.2
+	git.mstar.dev/mstar/goutils v1.16.1
 	github.com/BurntSushi/toml v1.5.0
 	github.com/PeerDB-io/gluabit32 v1.0.2
 	github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9
 	github.com/cosmotek/loguago v1.0.0
+	github.com/gabriel-vasile/mimetype v1.4.9
 	github.com/go-acme/lego/v4 v4.23.1
 	github.com/go-webauthn/webauthn v0.12.3
 	github.com/google/uuid v1.6.0

go.sum

@@ -1,7 +1,7 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-git.mstar.dev/mstar/goutils v1.14.2 h1:2W9AtsAAGR9OeztPnyVCkxiuZDe7h1DlXzil35wU+vs=
-git.mstar.dev/mstar/goutils v1.14.2/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA=
+git.mstar.dev/mstar/goutils v1.16.1 h1:uVsT+a8Ad0DuYy7rnXAVZ5NjoE6AHit6DGxFn5XiSrU=
+git.mstar.dev/mstar/goutils v1.16.1/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA=
 github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/PeerDB-io/gluabit32 v1.0.2 h1:AGI1Z7dwDVotakpuOOuyTX4/QGi5HUYsipL/VfodmO4=
@@ -32,6 +32,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU=
 github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY=
+github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
 github.com/go-acme/lego/v4 v4.23.1 h1:lZ5fGtGESA2L9FB8dNTvrQUq3/X4QOb8ExkKyY7LSV4=
 github.com/go-acme/lego/v4 v4.23.1/go.mod h1:7UMVR7oQbIYw6V7mTgGwi4Er7B6Ww0c+c8feiBM0EgI=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=

media/addFile.go

@@ -3,6 +3,9 @@ package media
 import (
 	"context"
 	"database/sql"
+	"git.mstar.dev/mstar/goutils/other"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"github.com/gabriel-vasile/mimetype"
 	"io"
 	"os"
 	"path"
@@ -27,18 +30,23 @@ func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error {
 	}
 	_ = file.Close()
 	if s.transcoderClient == nil {
-		return s.addFileAsIs(filename, userId, filePath)
+		return s.addFileAsIs(filename, userId, filePath, nil)
 	} else {
 		return s.addFileWithTranscoder(filename, userId, filePath)
 	}
 }
 
-func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
-	panic("not implemented")
-}
-
-func (s *Server) addFileAsIs(filename, userId, filepath string) error {
-	_, err := s.client.FPutObject(
+// adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict)
+// is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually
+func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error {
+	if mtype == nil {
+		mType, err := mimetype.DetectFile(filepath)
+		if err != nil {
+			return err
+		}
+		mtype = other.IntoPointer(mType.String())
+	}
+	s3Result, err := s.client.FPutObject(
 		context.TODO(),
 		config.GlobalConfig.S3.BucketName,
 		UsernameFilename(userId, filename),
@@ -52,7 +60,15 @@ func (s *Server) addFileAsIs(filename, userId, filepath string) error {
 		ID:        shared.NewId(),
 		OwnedById: sql.NullString{Valid: true, String: userId},
 		Remote:    false,
-		// Location: string, // TODO: Figure this out
+		Location:  s3Result.Location,
+		Type:      *mtype,
+		Name:      UsernameFilename(userId, filename),
+		AltText:   "",
+		Blurred:   false,
 	}
-	panic("not implemented")
+	err = dbgen.MediaMetadata.Create(&fileMetadata)
+	if err != nil {
+		return err
+	}
+	return nil
 }

media/services.go

@@ -1 +1,76 @@
 package media
+
+import (
+	"context"
+	"slices"
+
+	"git.mstar.dev/mstar/goutils/sliceutils"
+	"github.com/minio/minio-go/v7"
+	"github.com/rs/zerolog/log"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"git.mstar.dev/mstar/linstrom/storage-new/models"
+)
+
+// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between
+// the db's metadata for the files and the actual files in s3.
+// All files without matching metadata will be deleted. Same for all metadata without a matching file.
+// No attempt at restoring a connection will be made
+func (s *Server) ServiceEnsureFileSynchronisation() {
+	allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name).
+		Find()
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to get a list of all known media")
+		return
+	}
+	foundInDb := []string{}
+	objectMissingInDb := []minio.ObjectInfo{}
+	// Go over all objects in the bucket. Note down if it has an entry in the db or not
+	for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) {
+		if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
+			return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key
+		}) {
+			foundInDb = append(foundInDb, obj.Key)
+		} else {
+			objectMissingInDb = append(objectMissingInDb, obj)
+		}
+	}
+
+	// Find every db entry not in the list of found objects
+	entryMissingAnObject := []string{}
+	for _, dbFile := range allFiles {
+		if !slices.ContainsFunc(foundInDb, func(e string) bool {
+			return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e
+		}) {
+			entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
+		}
+	}
+
+	// For every object missing in the db, delete it
+	minioErrChan := s.client.RemoveObjects(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		sliceutils.ToChannel(objectMissingInDb),
+		minio.RemoveObjectsOptions{GovernanceBypass: true},
+	)
+	s3Errors := sliceutils.FromChannel(minioErrChan, 0)
+	s3Errors = sliceutils.Filter(
+		s3Errors,
+		func(t minio.RemoveObjectError) bool { return t.Err != nil },
+	)
+	for _, s3Err := range s3Errors {
+		log.Error().
+			Err(s3Err.Err).
+			Str("object-name", s3Err.ObjectName).
+			Msg("Failed to delete object missing in db")
+	}
+	// And perform a batch delete
+	_, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete()
+	if err != nil {
+		log.Error().
+			Err(err).
+			Strs("media-ids", entryMissingAnObject).
+			Msg("Failed to batch delete all media metadata without a matching object in s3")
+	}
+}

media/transcoder.go

@@ -1 +1,40 @@
 package media
+
+import (
+	"git.mstar.dev/mstar/linstrom/config"
+	"github.com/rs/zerolog/log"
+)
+
+// WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go
+// TODO: Maybe move to a separate repo outside of linstrom-transcoder
+
+type TranscodeArgs struct {
+	Secret   string
+	Filename string
+}
+
+type TranscodeReply struct {
+	Error             error
+	Mimetype          string
+	ThumbnailFilename *string
+	Filename          string
+}
+
+// addFileWithTranscoder will try to transcode the given file using the helper application.
+// If the transcode fails, it uploads the file as is
+func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
+	args := TranscodeArgs{
+		Secret:   config.GlobalConfig.Transcoder.Secret,
+		Filename: filepath,
+	}
+	reply := TranscodeReply{}
+	err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply)
+	if err != nil {
+		return err
+	}
+	if reply.Error != nil {
+		log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file")
+		return s.addFileAsIs(filename, userId, filepath, nil)
+	}
+	return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype)
+}