Add more things for file handling

config/config.go

@@ -75,10 +75,11 @@ type ConfigStorage struct {
 }
 
 type ConfigTranscoder struct {
-	SharedDirectory string `toml:"shared_directory"`
+	SharedDirectory  string `toml:"shared_directory"`
-	Secret          string `toml:"secret"`
+	Secret           string `toml:"secret"`
-	ServerAddress   string `toml:"server_address"`
+	ServerAddress    string `toml:"server_address"`
-	ServerPort      int    `toml:"server_port"`
+	ServerPort       int    `toml:"server_port"`
+	IgnoreTranscoder bool   `toml:"ignore_transcoder"`
 }
 
 type ConfigS3 struct {
@@ -198,10 +199,11 @@ var defaultConfig Config = Config{
 		MaxReconnectAttempts: 3,
 	},
 	Transcoder: ConfigTranscoder{
-		SharedDirectory: "/tmp/linstrom-transcoder",
+		SharedDirectory:  "/tmp/linstrom-transcoder",
-		Secret:          "The same secret as configured in the transcoder",
+		Secret:           "The same secret as configured in the transcoder",
-		ServerAddress:   "127.0.0.1",
+		ServerAddress:    "127.0.0.1",
-		ServerPort:      5594,
+		ServerPort:       5594,
+		IgnoreTranscoder: false,
 	},
 	Mail: ConfigMail{
 		Host:                           "localhost",

main.go

@@ -5,7 +5,6 @@ import (
 	"embed"
 	"flag"
 	"fmt"
-	"git.mstar.dev/mstar/linstrom/media"
 	"net/http"
 	"os"
 	"os/signal"
@@ -19,6 +18,7 @@ import (
 	"gorm.io/gorm"
 
 	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/media"
 	"git.mstar.dev/mstar/linstrom/shared"
 	storagenew "git.mstar.dev/mstar/linstrom/storage-new"
 	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
@@ -106,7 +106,11 @@ func newServer() {
 	}
 
 	log.Info().Msg("Connecting to s3 storage and transcoder")
-	_, err = media.NewServer()
+	mediaServer, err := media.NewServer()
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to set up the media server")
+	}
+	media.GlobalServer = mediaServer
 
 	debugShutdownChan := make(chan *sync.WaitGroup, 1)
 	interuptChan := make(chan os.Signal, 1)

media/addFile.go

@@ -3,72 +3,96 @@ package media
 import (
 	"context"
 	"database/sql"
-	"git.mstar.dev/mstar/goutils/other"
+	"errors"
-	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
-	"github.com/gabriel-vasile/mimetype"
 	"io"
 	"os"
 	"path"
 
+	"git.mstar.dev/mstar/goutils/other"
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/minio/minio-go/v7"
+	"github.com/rs/zerolog/log"
 
 	"git.mstar.dev/mstar/linstrom/config"
 	"git.mstar.dev/mstar/linstrom/shared"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
 	"git.mstar.dev/mstar/linstrom/storage-new/models"
 )
 
-func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error {
+var ErrFileAlreadyExists = errors.New("a file with that name already exists")
+
+func (s *Server) AddFile(
+	fileReader io.Reader,
+	filename, userId string,
+	blurred bool,
+	altText string,
+) (string, error) {
+	fileCount, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.OwnedById.Eq(sql.NullString{Valid: true, String: userId}), dbgen.MediaMetadata.Name.Eq(filename)).
+		Count()
+	if err != nil {
+		return "", err
+	}
+	if fileCount > 0 {
+		return "", ErrFileAlreadyExists
+	}
 	transcoderInDir := config.GlobalConfig.Transcoder.InDir()
 	filePath := path.Join(transcoderInDir, filename)
 	file, err := os.Create(filePath)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err = io.Copy(file, fileReader); err != nil {
 		_ = file.Close()
-		return err
+		return "", err
 	}
 	_ = file.Close()
 	if s.transcoderClient == nil {
-		return s.addFileAsIs(filename, userId, filePath, nil)
+		return s.addFileAsIs(filename, userId, filePath, nil, blurred, altText)
 	} else {
-		return s.addFileWithTranscoder(filename, userId, filePath)
+		return s.addFileWithTranscoder(filename, userId, filePath, blurred, altText)
 	}
 }
 
 // adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict)
 // is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually
-func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error {
+func (s *Server) addFileAsIs(
+	filename, userId, filepath string,
+	mtype *string,
+	blurred bool,
+	altText string,
+) (string, error) {
 	if mtype == nil {
 		mType, err := mimetype.DetectFile(filepath)
 		if err != nil {
-			return err
+			return "", err
 		}
 		mtype = other.IntoPointer(mType.String())
 	}
+	id := shared.NewId()
 	s3Result, err := s.client.FPutObject(
 		context.TODO(),
 		config.GlobalConfig.S3.BucketName,
-		UsernameFilename(userId, filename),
+		id,
 		filepath,
 		minio.PutObjectOptions{},
 	)
 	if err != nil {
-		return err
+		return "", err
 	}
+	log.Debug().Any("result", s3Result).Msg("Upload result")
 	fileMetadata := models.MediaMetadata{
-		ID:        shared.NewId(),
+		ID:        id,
 		OwnedById: sql.NullString{Valid: true, String: userId},
 		Remote:    false,
-		Location:  s3Result.Location,
+		Location:  s3Result.Key,
 		Type:      *mtype,
-		Name:      UsernameFilename(userId, filename),
+		Name:      filename,
-		AltText:   "",
+		AltText:   altText,
-		Blurred:   false,
+		Blurred:   blurred,
 	}
 	err = dbgen.MediaMetadata.Create(&fileMetadata)
 	if err != nil {
-		return err
+		return "", err
 	}
-	return nil
+	return id, nil
 }

media/fileInfo.go

@@ -0,0 +1,17 @@
+package media
+
+import (
+	"database/sql"
+
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+)
+
+func (s *Server) FileExists(userid, filename string) (bool, error) {
+	mm := dbgen.MediaMetadata
+	c, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename)).
+		Count()
+	if err != nil {
+		return false, err
+	}
+	return c > 0, nil
+}

media/media.go

@@ -3,11 +3,11 @@ package media
 import (
 	"context"
 	"errors"
-	"github.com/rs/zerolog/log"
 	"net/rpc"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/rs/zerolog/log"
 
 	"git.mstar.dev/mstar/linstrom/config"
 )
@@ -21,6 +21,8 @@ var (
 	ErrNoBucketAccess = errors.New("can't access configured bucket")
 )
 
+var GlobalServer *Server
+
 func NewServer() (*Server, error) {
 	client, err := minio.New(config.GlobalConfig.S3.Endpoint, &minio.Options{
 		Creds: credentials.NewStaticV4(
@@ -51,6 +53,9 @@ func NewServer() (*Server, error) {
 		}
 	}
 
+	if config.GlobalConfig.Transcoder.IgnoreTranscoder {
+		return &Server{client: client, transcoderClient: nil}, nil
+	}
 	transcoderClient, err := rpc.DialHTTP("tcp", config.GlobalConfig.Transcoder.Address())
 	if err != nil {
 		log.Warn().Err(err).
@@ -60,22 +65,3 @@ func NewServer() (*Server, error) {
 	}
 	return &Server{client: client, transcoderClient: transcoderClient}, nil
 }
-
-// UsernameFilename converts a userId and filename into a proper filepath for s3.
-// Reason for this is that the userId for external users is a valid url which needs to be encoded
-func UsernameFilename(userId, filename string) string {
-	return userId + "//" + filename
-}
-
-func (s *Server) HasFileScoped(userId, filename string) (bool, error) {
-	info, err := s.client.StatObject(
-		context.Background(),
-		config.GlobalConfig.S3.BucketName,
-		UsernameFilename(userId, filename),
-		minio.GetObjectOptions{},
-	)
-	if err != nil {
-		return false, err
-	}
-	return info.IsDeleteMarker, nil
-}

media/readFile.go

@@ -0,0 +1,56 @@
+package media
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"io"
+
+	"github.com/minio/minio-go/v7"
+	"gorm.io/gorm"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+)
+
+var ErrFileNotFound = errors.New("file not found")
+
+func (s *Server) ReadFile(userid, filename string) (io.ReadCloser, error) {
+	mm := dbgen.MediaMetadata
+	metadata, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename), mm.Remote.Is(false)).
+		Select(mm.ID, mm.Location).
+		First()
+	switch err {
+	case gorm.ErrRecordNotFound:
+		return nil, ErrFileNotFound
+	case nil:
+	default:
+		return nil, err
+	}
+	return s.client.GetObject(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		metadata.Location,
+		minio.GetObjectOptions{},
+	)
+}
+
+func (s *Server) ReadFileId(id string) (io.ReadCloser, error) {
+	mm := dbgen.MediaMetadata
+	metadata, err := mm.Where(mm.ID.Eq(id), mm.Remote.Is(false)).
+		Select(mm.Location).
+		First()
+	switch err {
+	case gorm.ErrRecordNotFound:
+		return nil, ErrFileNotFound
+	case nil:
+	default:
+		return nil, err
+	}
+	return s.client.GetObject(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		metadata.Location,
+		minio.GetObjectOptions{},
+	)
+}

media/removeFile.go

@@ -0,0 +1,5 @@
+package media
+
+func (s *Server) RemoveFile(userId, filename string) error {
+	panic("not implemented")
+}

media/services.go

@@ -18,7 +18,10 @@ import (
 // All files without matching metadata will be deleted. Same for all metadata without a matching file.
 // No attempt at restoring a connection will be made
 func (s *Server) ServiceEnsureFileSynchronisation() {
-	allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name).
+	mm := dbgen.MediaMetadata
+	allFiles, err := mm.
+		Select(mm.ID, mm.OwnedById, mm.Name, mm.Location).
+		Where(mm.Location.NotLike("linstrom://%"), mm.Remote.Is(false)).
 		Find()
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to get a list of all known media")
@@ -27,9 +30,12 @@ func (s *Server) ServiceEnsureFileSynchronisation() {
 	foundInDb := []string{}
 	objectMissingInDb := []minio.ObjectInfo{}
 	// Go over all objects in the bucket. Note down if it has an entry in the db or not
-	for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) {
+	for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{
+		Recursive: true,
+	}) {
+		log.Debug().Str("object-key", obj.Key).Msg("Checking object")
 		if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
-			return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key
+			return e.Location == obj.Key
 		}) {
 			foundInDb = append(foundInDb, obj.Key)
 		} else {
@@ -41,7 +47,7 @@ func (s *Server) ServiceEnsureFileSynchronisation() {
 	entryMissingAnObject := []string{}
 	for _, dbFile := range allFiles {
 		if !slices.ContainsFunc(foundInDb, func(e string) bool {
-			return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e
+			return dbFile.Location == e
 		}) {
 			entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
 		}

media/transcoder.go

@@ -1,8 +1,9 @@
 package media
 
 import (
-	"git.mstar.dev/mstar/linstrom/config"
 	"github.com/rs/zerolog/log"
+
+	"git.mstar.dev/mstar/linstrom/config"
 )
 
 // WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go
@@ -22,7 +23,11 @@ type TranscodeReply struct {
 
 // addFileWithTranscoder will try to transcode the given file using the helper application.
 // If the transcode fails, it uploads the file as is
-func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
+func (s *Server) addFileWithTranscoder(
+	filename, userId, filepath string,
+	blurred bool,
+	altText string,
+) (string, error) {
 	args := TranscodeArgs{
 		Secret:   config.GlobalConfig.Transcoder.Secret,
 		Filename: filepath,
@@ -30,11 +35,11 @@ func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error
 	reply := TranscodeReply{}
 	err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if reply.Error != nil {
 		log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file")
-		return s.addFileAsIs(filename, userId, filepath, nil)
+		return s.addFileAsIs(filename, userId, filepath, nil, blurred, altText)
 	}
-	return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype)
+	return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype, blurred, altText)
 }

storage-new/models/MediaMetadata.go

@@ -22,7 +22,7 @@ type MediaMetadata struct {
 	// OwnedBy   User
 	OwnedById sql.NullString // Account id this media belongs to
 	Remote    bool           // whether the attachment is a remote one
-	// Where the media is stored. Url
+	// Where the media is stored. Url for remote files, the object name for local files
 	Location string
 	Type     string // What media type this is following mime types, eg image/png
 	// Name of the file

web/debug/media.go

@@ -0,0 +1,72 @@
+package webdebug
+
+import (
+	"database/sql"
+	"net/http"
+
+	webutils "git.mstar.dev/mstar/goutils/http"
+	"git.mstar.dev/mstar/goutils/sliceutils"
+	"github.com/rs/zerolog/hlog"
+
+	"git.mstar.dev/mstar/linstrom/media"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	"git.mstar.dev/mstar/linstrom/storage-new/models"
+)
+
+func uploadMedia(w http.ResponseWriter, r *http.Request) {
+	log := hlog.FromRequest(r)
+	_ = r.ParseMultipartForm(10 << 20) // 10MB
+
+	userId := r.FormValue("user-id")
+	blurred := r.FormValue("blurred") != ""
+	altText := r.FormValue("alt-text")
+	file, handler, err := r.FormFile("file")
+	if err != nil {
+		log.Warn().Err(err).Msg("Failed to get file from form")
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusBadRequest)
+		return
+	}
+	defer func() { _ = file.Close() }()
+	_, err = media.GlobalServer.AddFile(file, handler.Filename, userId, blurred, altText)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to upload file to storage")
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
+	}
+}
+
+func forceMediaSync(w http.ResponseWriter, r *http.Request) {
+	go media.GlobalServer.ServiceEnsureFileSynchronisation()
+}
+
+func getOwnedFiles(w http.ResponseWriter, r *http.Request) {
+	type File struct {
+		Name, Id, Mime string
+	}
+	type Outbound struct {
+		Files []File
+	}
+	log := hlog.FromRequest(r)
+	userId := r.FormValue("id")
+
+	mm := dbgen.MediaMetadata
+	files, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userId})).
+		Select(mm.Name, mm.Type, mm.ID).
+		Find()
+	if err != nil {
+		log.Error().Err(err).Str("user-id", userId).Msg("Failed to get files of user")
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
+		return
+	}
+	err = webutils.SendJson(w, Outbound{
+		Files: sliceutils.Map(files, func(t *models.MediaMetadata) File {
+			return File{
+				Name: t.Name,
+				Id:   t.ID,
+				Mime: t.Type,
+			}
+		}),
+	})
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to marshal response to json")
+	}
+}

web/debug/server.go

@@ -37,6 +37,9 @@ func New(addr string) *Server {
 	handler.HandleFunc("GET /replies-to/{id}", inReplyTo)
 	handler.HandleFunc("POST /fetch", requestAs)
 	handler.HandleFunc("POST /follow", requestFollow)
+	handler.HandleFunc("POST /upload-file", uploadMedia)
+	handler.HandleFunc("/force-media-sync", forceMediaSync)
+	handler.HandleFunc("GET /files-owned-by", getOwnedFiles)
 	web := http.Server{
 		Addr: addr,
 		Handler: webutils.ChainMiddlewares(

web/public/media.go

@@ -0,0 +1,44 @@
+package webpublic
+
+import (
+	"io"
+	"net/http"
+
+	webutils "git.mstar.dev/mstar/goutils/http"
+	"github.com/rs/zerolog/hlog"
+
+	"git.mstar.dev/mstar/linstrom/media"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+)
+
+func downloadMediaHander(w http.ResponseWriter, r *http.Request) {
+	mediaId := r.PathValue("id")
+	log := hlog.FromRequest(r)
+	mm := dbgen.MediaMetadata
+	fileReader, err := media.GlobalServer.ReadFileId(mediaId)
+	switch err {
+	case nil:
+	case media.ErrFileNotFound:
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusNotFound)
+		return
+	default:
+		log.Error().Err(err).Str("file-id", mediaId).Msg("Failed to get file")
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
+		return
+	}
+	defer func() { _ = fileReader.Close() }()
+	meta, err := mm.Where(mm.ID.Eq(mediaId)).Select(mm.Type).First()
+	if err != nil {
+		log.Error().
+			Err(err).
+			Str("file-id", mediaId).
+			Msg("Failed to get file metadata after already getting file")
+		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
+		return
+	}
+	w.Header().Add("Content-Type", meta.Type)
+	_, err = io.Copy(w, fileReader)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to copy file to request")
+	}
+}

web/public/server.go

@@ -56,6 +56,7 @@ func New(addr string, duckFs fs.FS) *Server {
 	handler.HandleFunc("GET /errors/{name}", errorTypeHandler)
 	handler.HandleFunc("GET /default-image", buildServeDefaultImage(duckFs))
 	handler.HandleFunc("GET /default-image.webp", buildServeDefaultImage(duckFs))
+	handler.HandleFunc("GET /media/{id}", downloadMediaHander)
 	rootHandler := webutils.ChainMiddlewares(
 		handler,
 		webutils.BuildLoggingMiddleware(