mstar/linstrom
1
0
Fork 0
Code Issues 27 Pull requests Projects Releases Packages 1 Wiki Activity Actions

Add more things for file handling

Browse source
This commit is contained in:
Melody Becker 2025-06-18 15:36:33 +02:00
parent c813c4784a
commit 1fcf47bffc
Signed by: mstar
SSH key fingerprint: SHA256:9VAo09aaVNTWKzPW7Hq2LW+ox9OdwmTSHRoD4mlz1yI
14 changed files with 284 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

62
media/addFile.go
View file

@ -3,72 +3,96 @@ package media
import (
"context"
"database/sql"
"git.mstar.dev/mstar/goutils/other"
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
"github.com/gabriel-vasile/mimetype"
"errors"
"io"
"os"
"path"
"git.mstar.dev/mstar/goutils/other"
"github.com/gabriel-vasile/mimetype"
"github.com/minio/minio-go/v7"
"github.com/rs/zerolog/log"
"git.mstar.dev/mstar/linstrom/config"
"git.mstar.dev/mstar/linstrom/shared"
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
"git.mstar.dev/mstar/linstrom/storage-new/models"
)
func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error {
var ErrFileAlreadyExists = errors.New("a file with that name already exists")
func (s *Server) AddFile(
fileReader io.Reader,
filename, userId string,
blurred bool,
altText string,
) (string, error) {
fileCount, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.OwnedById.Eq(sql.NullString{Valid: true, String: userId}), dbgen.MediaMetadata.Name.Eq(filename)).
Count()
if err != nil {
return "", err
}
if fileCount > 0 {
return "", ErrFileAlreadyExists
}
transcoderInDir := config.GlobalConfig.Transcoder.InDir()
filePath := path.Join(transcoderInDir, filename)
file, err := os.Create(filePath)
if err != nil {
return err
return "", err
}
if _, err = io.Copy(file, fileReader); err != nil {
_ = file.Close()
return err
return "", err
}
_ = file.Close()
if s.transcoderClient == nil {
return s.addFileAsIs(filename, userId, filePath, nil)
return s.addFileAsIs(filename, userId, filePath, nil, blurred, altText)
} else {
return s.addFileWithTranscoder(filename, userId, filePath)
return s.addFileWithTranscoder(filename, userId, filePath, blurred, altText)
}
}
// adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict)
// is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually
func (s *Server) addFileAsIs(filename, userId, filepath string, mtype *string) error {
func (s *Server) addFileAsIs(
filename, userId, filepath string,
mtype *string,
blurred bool,
altText string,
) (string, error) {
if mtype == nil {
mType, err := mimetype.DetectFile(filepath)
if err != nil {
return err
return "", err
}
mtype = other.IntoPointer(mType.String())
}
id := shared.NewId()
s3Result, err := s.client.FPutObject(
context.TODO(),
config.GlobalConfig.S3.BucketName,
UsernameFilename(userId, filename),
id,
filepath,
minio.PutObjectOptions{},
)
if err != nil {
return err
return "", err
}
log.Debug().Any("result", s3Result).Msg("Upload result")
fileMetadata := models.MediaMetadata{
ID: shared.NewId(),
ID: id,
OwnedById: sql.NullString{Valid: true, String: userId},
Remote: false,
Location: s3Result.Location,
Location: s3Result.Key,
Type: *mtype,
Name: UsernameFilename(userId, filename),
AltText: "",
Blurred: false,
Name: filename,
AltText: altText,
Blurred: blurred,
}
err = dbgen.MediaMetadata.Create(&fileMetadata)
if err != nil {
return err
return "", err
}
return nil
return id, nil
}

17
media/fileInfo.go Normal file
View file

@ -0,0 +1,17 @@
package media
import (
"database/sql"
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
)
func (s *Server) FileExists(userid, filename string) (bool, error) {
mm := dbgen.MediaMetadata
c, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename)).
Count()
if err != nil {
return false, err
}
return c > 0, nil
}

26
media/media.go
View file

@ -3,11 +3,11 @@ package media
import (
"context"
"errors"
"github.com/rs/zerolog/log"
"net/rpc"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/rs/zerolog/log"
"git.mstar.dev/mstar/linstrom/config"
)
@ -21,6 +21,8 @@ var (
ErrNoBucketAccess = errors.New("can't access configured bucket")
)
var GlobalServer *Server
func NewServer() (*Server, error) {
client, err := minio.New(config.GlobalConfig.S3.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(
@ -51,6 +53,9 @@ func NewServer() (*Server, error) {
}
}
if config.GlobalConfig.Transcoder.IgnoreTranscoder {
return &Server{client: client, transcoderClient: nil}, nil
}
transcoderClient, err := rpc.DialHTTP("tcp", config.GlobalConfig.Transcoder.Address())
if err != nil {
log.Warn().Err(err).
@ -60,22 +65,3 @@ func NewServer() (*Server, error) {
}
return &Server{client: client, transcoderClient: transcoderClient}, nil
}
// UsernameFilename converts a userId and filename into a proper filepath for s3.
// Reason for this is that the userId for external users is a valid url which needs to be encoded
func UsernameFilename(userId, filename string) string {
return userId + "//" + filename
}
func (s *Server) HasFileScoped(userId, filename string) (bool, error) {
info, err := s.client.StatObject(
context.Background(),
config.GlobalConfig.S3.BucketName,
UsernameFilename(userId, filename),
minio.GetObjectOptions{},
)
if err != nil {
return false, err
}
return info.IsDeleteMarker, nil
}

56
media/readFile.go Normal file
View file

@ -0,0 +1,56 @@
package media
import (
"context"
"database/sql"
"errors"
"io"
"github.com/minio/minio-go/v7"
"gorm.io/gorm"
"git.mstar.dev/mstar/linstrom/config"
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
)
var ErrFileNotFound = errors.New("file not found")
func (s *Server) ReadFile(userid, filename string) (io.ReadCloser, error) {
mm := dbgen.MediaMetadata
metadata, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename), mm.Remote.Is(false)).
Select(mm.ID, mm.Location).
First()
switch err {
case gorm.ErrRecordNotFound:
return nil, ErrFileNotFound
case nil:
default:
return nil, err
}
return s.client.GetObject(
context.TODO(),
config.GlobalConfig.S3.BucketName,
metadata.Location,
minio.GetObjectOptions{},
)
}
func (s *Server) ReadFileId(id string) (io.ReadCloser, error) {
mm := dbgen.MediaMetadata
metadata, err := mm.Where(mm.ID.Eq(id), mm.Remote.Is(false)).
Select(mm.Location).
First()
switch err {
case gorm.ErrRecordNotFound:
return nil, ErrFileNotFound
case nil:
default:
return nil, err
}
return s.client.GetObject(
context.TODO(),
config.GlobalConfig.S3.BucketName,
metadata.Location,
minio.GetObjectOptions{},
)
}

5
media/removeFile.go Normal file
View file

@ -0,0 +1,5 @@
package media
func (s *Server) RemoveFile(userId, filename string) error {
panic("not implemented")
}

14
media/services.go
View file

@ -18,7 +18,10 @@ import (
// All files without matching metadata will be deleted. Same for all metadata without a matching file.
// No attempt at restoring a connection will be made
func (s *Server) ServiceEnsureFileSynchronisation() {
allFiles, err := dbgen.MediaMetadata.Select(dbgen.MediaMetadata.ID, dbgen.MediaMetadata.OwnedById, dbgen.MediaMetadata.Name).
mm := dbgen.MediaMetadata
allFiles, err := mm.
Select(mm.ID, mm.OwnedById, mm.Name, mm.Location).
Where(mm.Location.NotLike("linstrom://%"), mm.Remote.Is(false)).
Find()
if err != nil {
log.Error().Err(err).Msg("Failed to get a list of all known media")
@ -27,9 +30,12 @@ func (s *Server) ServiceEnsureFileSynchronisation() {
foundInDb := []string{}
objectMissingInDb := []minio.ObjectInfo{}
// Go over all objects in the bucket. Note down if it has an entry in the db or not
for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{}) {
for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{
Recursive: true,
}) {
log.Debug().Str("object-key", obj.Key).Msg("Checking object")
if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool {
return UsernameFilename(e.OwnedById.String, e.Name) == obj.Key
return e.Location == obj.Key
}) {
foundInDb = append(foundInDb, obj.Key)
} else {
@ -41,7 +47,7 @@ func (s *Server) ServiceEnsureFileSynchronisation() {
entryMissingAnObject := []string{}
for _, dbFile := range allFiles {
if !slices.ContainsFunc(foundInDb, func(e string) bool {
return UsernameFilename(dbFile.OwnedById.String, dbFile.Name) == e
return dbFile.Location == e
}) {
entryMissingAnObject = append(entryMissingAnObject, dbFile.ID)
}

15
media/transcoder.go
View file

@ -1,8 +1,9 @@
package media
import (
"git.mstar.dev/mstar/linstrom/config"
"github.com/rs/zerolog/log"
"git.mstar.dev/mstar/linstrom/config"
)
// WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go
@ -22,7 +23,11 @@ type TranscodeReply struct {
// addFileWithTranscoder will try to transcode the given file using the helper application.
// If the transcode fails, it uploads the file as is
func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
func (s *Server) addFileWithTranscoder(
filename, userId, filepath string,
blurred bool,
altText string,
) (string, error) {
args := TranscodeArgs{
Secret: config.GlobalConfig.Transcoder.Secret,
Filename: filepath,
@ -30,11 +35,11 @@ func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error
reply := TranscodeReply{}
err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply)
if err != nil {
return err
return "", err
}
if reply.Error != nil {
log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file")
return s.addFileAsIs(filename, userId, filepath, nil)
return s.addFileAsIs(filename, userId, filepath, nil, blurred, altText)
}
return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype)
return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype, blurred, altText)
}