diff --git a/config/config.go b/config/config.go index ed8a575..c708080 100644 --- a/config/config.go +++ b/config/config.go @@ -75,10 +75,11 @@ type ConfigStorage struct { } type ConfigTranscoder struct { - SharedDirectory string `toml:"shared_directory"` - Secret string `toml:"secret"` - ServerAddress string `toml:"server_address"` - ServerPort int `toml:"server_port"` + SharedDirectory string `toml:"shared_directory"` + Secret string `toml:"secret"` + ServerAddress string `toml:"server_address"` + ServerPort int `toml:"server_port"` + IgnoreTranscoder bool `toml:"ignore_transcoder"` } type ConfigS3 struct { @@ -198,10 +199,11 @@ var defaultConfig Config = Config{ MaxReconnectAttempts: 3, }, Transcoder: ConfigTranscoder{ - SharedDirectory: "/tmp/linstrom-transcoder", - Secret: "The same secret as configured in the transcoder", - ServerAddress: "127.0.0.1", - ServerPort: 5594, + SharedDirectory: "/tmp/linstrom-transcoder", + Secret: "The same secret as configured in the transcoder", + ServerAddress: "127.0.0.1", + ServerPort: 5594, + IgnoreTranscoder: false, }, Mail: ConfigMail{ Host: "localhost", diff --git a/main.go b/main.go index 97bc573..78df4c9 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "embed" "flag" "fmt" - "git.mstar.dev/mstar/linstrom/media" "net/http" "os" "os/signal" @@ -19,6 +18,7 @@ import ( "gorm.io/gorm" "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/media" "git.mstar.dev/mstar/linstrom/shared" storagenew "git.mstar.dev/mstar/linstrom/storage-new" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" @@ -106,7 +106,11 @@ func newServer() { } log.Info().Msg("Connecting to s3 storage and transcoder") - _, err = media.NewServer() + mediaServer, err := media.NewServer() + if err != nil { + log.Fatal().Err(err).Msg("Failed to set up the media server") + } + media.GlobalServer = mediaServer debugShutdownChan := make(chan *sync.WaitGroup, 1) interuptChan := make(chan os.Signal, 1) diff --git a/media/addFile.go b/media/addFile.go index da4cfeb..3297203 100644 --- a/media/addFile.go +++ b/media/addFile.go @@ -3,72 +3,96 @@ package media import ( "context" "database/sql" - "git.mstar.dev/mstar/goutils/other" - "git.mstar.dev/mstar/linstrom/storage-new/dbgen" - "github.com/gabriel-vasile/mimetype" + "errors" "io" "os" "path" + "git.mstar.dev/mstar/goutils/other" + "github.com/gabriel-vasile/mimetype" "github.com/minio/minio-go/v7" + "github.com/rs/zerolog/log" "git.mstar.dev/mstar/linstrom/config" "git.mstar.dev/mstar/linstrom/shared" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" "git.mstar.dev/mstar/linstrom/storage-new/models" ) -func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error { +var ErrFileAlreadyExists = errors.New("a file with that name already exists") + +func (s *Server) AddFile( + fileReader io.Reader, + filename, userId string, + blurred bool, + altText string, +) (string, error) { + fileCount, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.OwnedById.Eq(sql.NullString{Valid: true, String: userId}), dbgen.MediaMetadata.Name.Eq(filename)). + Count() + if err != nil { + return "", err + } + if fileCount > 0 { + return "", ErrFileAlreadyExists + } transcoderInDir := config.GlobalConfig.Transcoder.InDir() filePath := path.Join(transcoderInDir, filename) file, err := os.Create(filePath) if err != nil { - return err + return "", err } if _, err = io.Copy(file, fileReader); err != nil { _ = file.Close() - return err + return "", err } _ = file.Close() if s.transcoderClient == nil { - return s.addFileAsIs(filename, userId, filePath, nil) + return s.addFileAsIs(filename, userId, filePath, nil, blurred, altText) } else { - return s.addFileWithTranscoder(filename, userId, filePath) + return s.addFileWithTranscoder(filename, userId, filePath, blurred, altText) } } // adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict) // is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually -func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error { +func (s *Server) addFileAsIs( + filename, userId, filepath string, + mtype *string, + blurred bool, + altText string, +) (string, error) { if mtype == nil { mType, err := mimetype.DetectFile(filepath) if err != nil { - return err + return "", err } mtype = other.IntoPointer(mType.String()) } + id := shared.NewId() s3Result, err := s.client.FPutObject( context.TODO(), config.GlobalConfig.S3.BucketName, - UsernameFilename(userId, filename), + id, filepath, minio.PutObjectOptions{}, ) if err != nil { - return err + return "", err } + log.Debug().Any("result", s3Result).Msg("Upload result") fileMetadata := models.MediaMetadata{ - ID: shared.NewId(), + ID: id, OwnedById: sql.NullString{Valid: true, String: userId}, Remote: false, - Location: s3Result.Location, + Location: s3Result.Key, Type: *mtype, - Name: UsernameFilename(userId, filename), - AltText: "", - Blurred: false, + Name: filename, + AltText: altText, + Blurred: blurred, } err = dbgen.MediaMetadata.Create(&fileMetadata) if err != nil { - return err + return "", err } - return nil + return id, nil } diff --git a/media/fileInfo.go b/media/fileInfo.go new file mode 100644 index 0000000..acbd8ec --- /dev/null +++ b/media/fileInfo.go @@ -0,0 +1,17 @@ +package media + +import ( + "database/sql" + + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +func (s *Server) FileExists(userid, filename string) (bool, error) { + mm := dbgen.MediaMetadata + c, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename)). + Count() + if err != nil { + return false, err + } + return c > 0, nil +} diff --git a/media/media.go b/media/media.go index e26775e..0e1fbfa 100644 --- a/media/media.go +++ b/media/media.go @@ -3,11 +3,11 @@ package media import ( "context" "errors" - "github.com/rs/zerolog/log" "net/rpc" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rs/zerolog/log" "git.mstar.dev/mstar/linstrom/config" ) @@ -21,6 +21,8 @@ var ( ErrNoBucketAccess = errors.New("can't access configured bucket") ) +var GlobalServer *Server + func NewServer() (*Server, error) { client, err := minio.New(config.GlobalConfig.S3.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4( @@ -51,6 +53,9 @@ func NewServer() (*Server, error) { } } + if config.GlobalConfig.Transcoder.IgnoreTranscoder { + return &Server{client: client, transcoderClient: nil}, nil + } transcoderClient, err := rpc.DialHTTP("tcp", config.GlobalConfig.Transcoder.Address()) if err != nil { log.Warn().Err(err). @@ -60,22 +65,3 @@ func NewServer() (*Server, error) { } return &Server{client: client, transcoderClient: transcoderClient}, nil } - -// UsernameFilename converts a userId and filename into a proper filepath for s3. -// Reason for this is that the userId for external users is a valid url which needs to be encoded -func UsernameFilename(userId, filename string) string { - return userId + "//" + filename -} - -func (s *Server) HasFileScoped(userId, filename string) (bool, error) { - info, err := s.client.StatObject( - context.Background(), - config.GlobalConfig.S3.BucketName, - UsernameFilename(userId, filename), - minio.GetObjectOptions{}, - ) - if err != nil { - return false, err - } - return info.IsDeleteMarker, nil -} diff --git a/media/readFile.go b/media/readFile.go new file mode 100644 index 0000000..9c61b2e --- /dev/null +++ b/media/readFile.go @@ -0,0 +1,56 @@ +package media + +import ( + "context" + "database/sql" + "errors" + "io" + + "github.com/minio/minio-go/v7" + "gorm.io/gorm" + + "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +var ErrFileNotFound = errors.New("file not found") + +func (s *Server) ReadFile(userid, filename string) (io.ReadCloser, error) { + mm := dbgen.MediaMetadata + metadata, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename), mm.Remote.Is(false)). + Select(mm.ID, mm.Location). + First() + switch err { + case gorm.ErrRecordNotFound: + return nil, ErrFileNotFound + case nil: + default: + return nil, err + } + return s.client.GetObject( + context.TODO(), + config.GlobalConfig.S3.BucketName, + metadata.Location, + minio.GetObjectOptions{}, + ) +} + +func (s *Server) ReadFileId(id string) (io.ReadCloser, error) { + mm := dbgen.MediaMetadata + metadata, err := mm.Where(mm.ID.Eq(id), mm.Remote.Is(false)). + Select(mm.Location). + First() + switch err { + case gorm.ErrRecordNotFound: + return nil, ErrFileNotFound + case nil: + default: + return nil, err + } + return s.client.GetObject( + context.TODO(), + config.GlobalConfig.S3.BucketName, + metadata.Location, + minio.GetObjectOptions{}, + ) +} diff --git a/media/removeFile.go b/media/removeFile.go new file mode 100644 index 0000000..8041f3e --- /dev/null +++ b/media/removeFile.go @@ -0,0 +1,5 @@ +package media + +func (s *Server) RemoveFile(userId, filename string) error { + panic("not implemented") +} diff --git a/media/services.go b/media/services.go index d846549..8a326c8 100644 --- a/media/services.go +++ b/media/services.go @@ -18,7 +18,10 @@ import ( // All files without matching metadata will be deleted. Same for all metadata without a matching file. // No attempt at restoring a connection will be made func (s *Server) ServiceEnsureFileSynchronisation() { - allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name). + mm := dbgen.MediaMetadata + allFiles, err := mm. + Select(mm.ID, mm.OwnedById, mm.Name, mm.Location). + Where(mm.Location.NotLike("linstrom://%"), mm.Remote.Is(false)). Find() if err != nil { log.Error().Err(err).Msg("Failed to get a list of all known media") @@ -27,9 +30,12 @@ func (s *Server) ServiceEnsureFileSynchronisation() { foundInDb := []string{} objectMissingInDb := []minio.ObjectInfo{} // Go over all objects in the bucket. Note down if it has an entry in the db or not - for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) { + for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{ + Recursive: true, + }) { + log.Debug().Str("object-key", obj.Key).Msg("Checking object") if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool { - return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key + return e.Location == obj.Key }) { foundInDb = append(foundInDb, obj.Key) } else { @@ -41,7 +47,7 @@ func (s *Server) ServiceEnsureFileSynchronisation() { entryMissingAnObject := []string{} for _, dbFile := range allFiles { if !slices.ContainsFunc(foundInDb, func(e string) bool { - return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e + return dbFile.Location == e }) { entryMissingAnObject = append(entryMissingAnObject, dbFile.ID) } diff --git a/media/transcoder.go b/media/transcoder.go index 8bf1ecc..b27794e 100644 --- a/media/transcoder.go +++ b/media/transcoder.go @@ -1,8 +1,9 @@ package media import ( - "git.mstar.dev/mstar/linstrom/config" "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/config" ) // WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go @@ -22,7 +23,11 @@ type TranscodeReply struct { // addFileWithTranscoder will try to transcode the given file using the helper application. // If the transcode fails, it uploads the file as is -func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error { +func (s *Server) addFileWithTranscoder( + filename, userId, filepath string, + blurred bool, + altText string, +) (string, error) { args := TranscodeArgs{ Secret: config.GlobalConfig.Transcoder.Secret, Filename: filepath, @@ -30,11 +35,11 @@ func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error reply := TranscodeReply{} err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply) if err != nil { - return err + return "", err } if reply.Error != nil { log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file") - return s.addFileAsIs(filename, userId, filepath, nil) + return s.addFileAsIs(filename, userId, filepath, nil, blurred, altText) } - return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype) + return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype, blurred, altText) } diff --git a/storage-new/models/MediaMetadata.go b/storage-new/models/MediaMetadata.go index 25f6082..2502060 100644 --- a/storage-new/models/MediaMetadata.go +++ b/storage-new/models/MediaMetadata.go @@ -22,7 +22,7 @@ type MediaMetadata struct { // OwnedBy User OwnedById sql.NullString // Account id this media belongs to Remote bool // whether the attachment is a remote one - // Where the media is stored. Url + // Where the media is stored. Url for remote files, the object name for local files Location string Type string // What media type this is following mime types, eg image/png // Name of the file diff --git a/web/debug/media.go b/web/debug/media.go new file mode 100644 index 0000000..bd0219f --- /dev/null +++ b/web/debug/media.go @@ -0,0 +1,72 @@ +package webdebug + +import ( + "database/sql" + "net/http" + + webutils "git.mstar.dev/mstar/goutils/http" + "git.mstar.dev/mstar/goutils/sliceutils" + "github.com/rs/zerolog/hlog" + + "git.mstar.dev/mstar/linstrom/media" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +func uploadMedia(w http.ResponseWriter, r *http.Request) { + log := hlog.FromRequest(r) + _ = r.ParseMultipartForm(10 << 20) // 10MB + + userId := r.FormValue("user-id") + blurred := r.FormValue("blurred") != "" + altText := r.FormValue("alt-text") + file, handler, err := r.FormFile("file") + if err != nil { + log.Warn().Err(err).Msg("Failed to get file from form") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusBadRequest) + return + } + defer func() { _ = file.Close() }() + _, err = media.GlobalServer.AddFile(file, handler.Filename, userId, blurred, altText) + if err != nil { + log.Error().Err(err).Msg("Failed to upload file to storage") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + } +} + +func forceMediaSync(w http.ResponseWriter, r *http.Request) { + go media.GlobalServer.ServiceEnsureFileSynchronisation() +} + +func getOwnedFiles(w http.ResponseWriter, r *http.Request) { + type File struct { + Name, Id, Mime string + } + type Outbound struct { + Files []File + } + log := hlog.FromRequest(r) + userId := r.FormValue("id") + + mm := dbgen.MediaMetadata + files, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userId})). + Select(mm.Name, mm.Type, mm.ID). + Find() + if err != nil { + log.Error().Err(err).Str("user-id", userId).Msg("Failed to get files of user") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + err = webutils.SendJson(w, Outbound{ + Files: sliceutils.Map(files, func(t *models.MediaMetadata) File { + return File{ + Name: t.Name, + Id: t.ID, + Mime: t.Type, + } + }), + }) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal response to json") + } +} diff --git a/web/debug/server.go b/web/debug/server.go index 99569ac..67197e4 100644 --- a/web/debug/server.go +++ b/web/debug/server.go @@ -37,6 +37,9 @@ func New(addr string) *Server { handler.HandleFunc("GET /replies-to/{id}", inReplyTo) handler.HandleFunc("POST /fetch", requestAs) handler.HandleFunc("POST /follow", requestFollow) + handler.HandleFunc("POST /upload-file", uploadMedia) + handler.HandleFunc("/force-media-sync", forceMediaSync) + handler.HandleFunc("GET /files-owned-by", getOwnedFiles) web := http.Server{ Addr: addr, Handler: webutils.ChainMiddlewares( diff --git a/web/public/media.go b/web/public/media.go new file mode 100644 index 0000000..e098855 --- /dev/null +++ b/web/public/media.go @@ -0,0 +1,44 @@ +package webpublic + +import ( + "io" + "net/http" + + webutils "git.mstar.dev/mstar/goutils/http" + "github.com/rs/zerolog/hlog" + + "git.mstar.dev/mstar/linstrom/media" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +func downloadMediaHander(w http.ResponseWriter, r *http.Request) { + mediaId := r.PathValue("id") + log := hlog.FromRequest(r) + mm := dbgen.MediaMetadata + fileReader, err := media.GlobalServer.ReadFileId(mediaId) + switch err { + case nil: + case media.ErrFileNotFound: + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusNotFound) + return + default: + log.Error().Err(err).Str("file-id", mediaId).Msg("Failed to get file") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + defer func() { _ = fileReader.Close() }() + meta, err := mm.Where(mm.ID.Eq(mediaId)).Select(mm.Type).First() + if err != nil { + log.Error(). + Err(err). + Str("file-id", mediaId). + Msg("Failed to get file metadata after already getting file") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + w.Header().Add("Content-Type", meta.Type) + _, err = io.Copy(w, fileReader) + if err != nil { + log.Error().Err(err).Msg("Failed to copy file to request") + } +} diff --git a/web/public/server.go b/web/public/server.go index 944f716..e3b2953 100644 --- a/web/public/server.go +++ b/web/public/server.go @@ -56,6 +56,7 @@ func New(addr string, duckFs fs.FS) *Server { handler.HandleFunc("GET /errors/{name}", errorTypeHandler) handler.HandleFunc("GET /default-image", buildServeDefaultImage(duckFs)) handler.HandleFunc("GET /default-image.webp", buildServeDefaultImage(duckFs)) + handler.HandleFunc("GET /media/{id}", downloadMediaHander) rootHandler := webutils.ChainMiddlewares( handler, webutils.BuildLoggingMiddleware(