From c813c4784afb10fd2d75e1b369d3969f755705ce Mon Sep 17 00:00:00 2001 From: mstar Date: Tue, 17 Jun 2025 16:48:24 +0200 Subject: [PATCH] Add media upload functionality --- go.mod | 3 +- go.sum | 6 ++-- media/addFile.go | 34 ++++++++++++++------ media/services.go | 75 +++++++++++++++++++++++++++++++++++++++++++++ media/transcoder.go | 39 +++++++++++++++++++++++ 5 files changed, 145 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 75ea6b3..e74b5e3 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module git.mstar.dev/mstar/linstrom go 1.24.2 require ( - git.mstar.dev/mstar/goutils v1.14.2 + git.mstar.dev/mstar/goutils v1.16.1 github.com/BurntSushi/toml v1.5.0 github.com/PeerDB-io/gluabit32 v1.0.2 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 github.com/cosmotek/loguago v1.0.0 + github.com/gabriel-vasile/mimetype v1.4.9 github.com/go-acme/lego/v4 v4.23.1 github.com/go-webauthn/webauthn v0.12.3 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 6a18a4d..10ef9e5 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -git.mstar.dev/mstar/goutils v1.14.2 h1:2W9AtsAAGR9OeztPnyVCkxiuZDe7h1DlXzil35wU+vs= -git.mstar.dev/mstar/goutils v1.14.2/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA= +git.mstar.dev/mstar/goutils v1.16.1 h1:uVsT+a8Ad0DuYy7rnXAVZ5NjoE6AHit6DGxFn5XiSrU= +git.mstar.dev/mstar/goutils v1.16.1/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/PeerDB-io/gluabit32 v1.0.2 h1:AGI1Z7dwDVotakpuOOuyTX4/QGi5HUYsipL/VfodmO4= @@ -32,6 +32,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= github.com/go-acme/lego/v4 v4.23.1 h1:lZ5fGtGESA2L9FB8dNTvrQUq3/X4QOb8ExkKyY7LSV4= github.com/go-acme/lego/v4 v4.23.1/go.mod h1:7UMVR7oQbIYw6V7mTgGwi4Er7B6Ww0c+c8feiBM0EgI= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= diff --git a/media/addFile.go b/media/addFile.go index 304f854..da4cfeb 100644 --- a/media/addFile.go +++ b/media/addFile.go @@ -3,6 +3,9 @@ package media import ( "context" "database/sql" + "git.mstar.dev/mstar/goutils/other" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "github.com/gabriel-vasile/mimetype" "io" "os" "path" @@ -27,18 +30,23 @@ func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error { } _ = file.Close() if s.transcoderClient == nil { - return s.addFileAsIs(filename, userId, filePath) + return s.addFileAsIs(filename, userId, filePath, nil) } else { return s.addFileWithTranscoder(filename, userId, filePath) } } -func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error { - panic("not implemented") -} - -func (s *Server) addFileAsIs(filename, userId, filepath string) error { - _, err := s.client.FPutObject( +// adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict) +// is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually +func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error { + if mtype == nil { + mType, err := mimetype.DetectFile(filepath) + if err != nil { + return err + } + mtype = other.IntoPointer(mType.String()) + } + s3Result, err := s.client.FPutObject( context.TODO(), config.GlobalConfig.S3.BucketName, UsernameFilename(userId, filename), @@ -52,7 +60,15 @@ func (s *Server) addFileAsIs(filename, userId, filepath string) error { ID: shared.NewId(), OwnedById: sql.NullString{Valid: true, String: userId}, Remote: false, - // Location: string, // TODO: Figure this out + Location: s3Result.Location, + Type: *mtype, + Name: UsernameFilename(userId, filename), + AltText: "", + Blurred: false, } - panic("not implemented") + err = dbgen.MediaMetadata.Create(&fileMetadata) + if err != nil { + return err + } + return nil } diff --git a/media/services.go b/media/services.go index 90dec23..d846549 100644 --- a/media/services.go +++ b/media/services.go @@ -1 +1,76 @@ package media + +import ( + "context" + "slices" + + "git.mstar.dev/mstar/goutils/sliceutils" + "github.com/minio/minio-go/v7" + "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between +// the db's metadata for the files and the actual files in s3. +// All files without matching metadata will be deleted. Same for all metadata without a matching file. +// No attempt at restoring a connection will be made +func (s *Server) ServiceEnsureFileSynchronisation() { + allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name). + Find() + if err != nil { + log.Error().Err(err).Msg("Failed to get a list of all known media") + return + } + foundInDb := []string{} + objectMissingInDb := []minio.ObjectInfo{} + // Go over all objects in the bucket. Note down if it has an entry in the db or not + for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) { + if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool { + return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key + }) { + foundInDb = append(foundInDb, obj.Key) + } else { + objectMissingInDb = append(objectMissingInDb, obj) + } + } + + // Find every db entry not in the list of found objects + entryMissingAnObject := []string{} + for _, dbFile := range allFiles { + if !slices.ContainsFunc(foundInDb, func(e string) bool { + return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e + }) { + entryMissingAnObject = append(entryMissingAnObject, dbFile.ID) + } + } + + // For every object missing in the db, delete it + minioErrChan := s.client.RemoveObjects( + context.TODO(), + config.GlobalConfig.S3.BucketName, + sliceutils.ToChannel(objectMissingInDb), + minio.RemoveObjectsOptions{GovernanceBypass: true}, + ) + s3Errors := sliceutils.FromChannel(minioErrChan, 0) + s3Errors = sliceutils.Filter( + s3Errors, + func(t minio.RemoveObjectError) bool { return t.Err != nil }, + ) + for _, s3Err := range s3Errors { + log.Error(). + Err(s3Err.Err). + Str("object-name", s3Err.ObjectName). + Msg("Failed to delete object missing in db") + } + // And perform a batch delete + _, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete() + if err != nil { + log.Error(). + Err(err). + Strs("media-ids", entryMissingAnObject). + Msg("Failed to batch delete all media metadata without a matching object in s3") + } +} diff --git a/media/transcoder.go b/media/transcoder.go index 90dec23..8bf1ecc 100644 --- a/media/transcoder.go +++ b/media/transcoder.go @@ -1 +1,40 @@ package media + +import ( + "git.mstar.dev/mstar/linstrom/config" + "github.com/rs/zerolog/log" +) + +// WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go +// TODO: Maybe move to a separate repo outside of linstrom-transcoder + +type TranscodeArgs struct { + Secret string + Filename string +} + +type TranscodeReply struct { + Error error + Mimetype string + ThumbnailFilename *string + Filename string +} + +// addFileWithTranscoder will try to transcode the given file using the helper application. +// If the transcode fails, it uploads the file as is +func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error { + args := TranscodeArgs{ + Secret: config.GlobalConfig.Transcoder.Secret, + Filename: filepath, + } + reply := TranscodeReply{} + err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply) + if err != nil { + return err + } + if reply.Error != nil { + log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file") + return s.addFileAsIs(filename, userId, filepath, nil) + } + return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype) +}