More work on file storage

config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"os"
+	"path"
 	"strings"
 
 	"git.mstar.dev/mstar/goutils/other"
@@ -73,6 +74,13 @@ type ConfigStorage struct {
 	MaxReconnectAttempts int `toml:"max_reconnect_attempts"`
 }
 
+type ConfigTranscoder struct {
+	SharedDirectory string `toml:"shared_directory"`
+	Secret          string `toml:"secret"`
+	ServerAddress   string `toml:"server_address"`
+	ServerPort      int    `toml:"server_port"`
+}
+
 type ConfigS3 struct {
 	KeyId      string `toml:"key_id"`
 	Secret     string `toml:"secret"`
@@ -143,6 +151,7 @@ type Config struct {
 	Admin        ConfigAdmin        `toml:"admin"`
 	Webauthn     ConfigWebAuthn     `toml:"webauthn"`
 	Storage      ConfigStorage      `toml:"storage"`
+	Transcoder   ConfigTranscoder   `toml:"transcoder"`
 	Mail         ConfigMail         `toml:"mail"`
 	Self         ConfigSelf         `toml:"self"`
 	S3           ConfigS3           `toml:"s3"`
@@ -185,13 +194,15 @@ var defaultConfig Config = Config{
 		Port:                 5432,
 		SslMode:              other.IntoPointer("disable"),
 		TimeZone:             other.IntoPointer("Europe/Berlin"),
-		RedisUrl:             nil,
-		MaxInMemoryCacheSize: 1e6, // 1 Megabyte
-		MaxInMemoryCacheTTL:  5,
-		MaxRedisCacheTTL:     nil,
 		EncryptionKey:        "Encryption key for sensitive information. DO NOT CHANGE THIS AFTER SETUP",
 		MaxReconnectAttempts: 3,
 	},
+	Transcoder: ConfigTranscoder{
+		SharedDirectory: "/tmp/linstrom-transcoder",
+		Secret:          "The same secret as configured in the transcoder",
+		ServerAddress:   "127.0.0.1",
+		ServerPort:      5594,
+	},
 	Mail: ConfigMail{
 		Host:                           "localhost",
 		Port:                           587,
@@ -275,6 +286,18 @@ func (sc *ConfigStorage) BuildPostgresDSN() string {
 	return dsn
 }
 
+func (tc *ConfigTranscoder) Address() string {
+	return fmt.Sprintf("%s:%d", tc.ServerAddress, tc.ServerPort)
+}
+
+func (tc *ConfigTranscoder) InDir() string {
+	return path.Join(tc.SharedDirectory, "input")
+}
+
+func (tc *ConfigTranscoder) OutDir() string {
+	return path.Join(tc.SharedDirectory, "output")
+}
+
 func WriteDefaultConfig(toFile string) error {
 	log.Trace().Caller().Send()
 	log.Info().Str("config-file", toFile).Msg("Writing default config to file")

main.go

@@ -5,6 +5,7 @@ import (
 	"embed"
 	"flag"
 	"fmt"
+	"git.mstar.dev/mstar/linstrom/media"
 	"net/http"
 	"os"
 	"os/signal"
@@ -103,6 +104,10 @@ func newServer() {
 	if err = storagenew.InsertUnknownActorPlaceholder(); err != nil {
 		log.Fatal().Err(err).Msg("Failed to insert self properly")
 	}
+
+	log.Info().Msg("Connecting to s3 storage and transcoder")
+	_, err = media.NewServer()
+
 	debugShutdownChan := make(chan *sync.WaitGroup, 1)
 	interuptChan := make(chan os.Signal, 1)
 	debugShutdownWaitgroup := sync.WaitGroup{}

media/addFile.go

@@ -0,0 +1,58 @@
+package media
+
+import (
+	"context"
+	"database/sql"
+	"io"
+	"os"
+	"path"
+
+	"github.com/minio/minio-go/v7"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/shared"
+	"git.mstar.dev/mstar/linstrom/storage-new/models"
+)
+
+func (s *Server) AddFile(fileReader io.Reader, filename, userId string) error {
+	transcoderInDir := config.GlobalConfig.Transcoder.InDir()
+	filePath := path.Join(transcoderInDir, filename)
+	file, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	if _, err = io.Copy(file, fileReader); err != nil {
+		_ = file.Close()
+		return err
+	}
+	_ = file.Close()
+	if s.transcoderClient == nil {
+		return s.addFileAsIs(filename, userId, filePath)
+	} else {
+		return s.addFileWithTranscoder(filename, userId, filePath)
+	}
+}
+
+func (s *Server) addFileWithTranscoder(filename, userId, filepath string) error {
+	panic("not implemented")
+}
+
+func (s *Server) addFileAsIs(filename, userId, filepath string) error {
+	_, err := s.client.FPutObject(
+		context.TODO(),
+		config.GlobalConfig.S3.BucketName,
+		UsernameFilename(userId, filename),
+		filepath,
+		minio.PutObjectOptions{},
+	)
+	if err != nil {
+		return err
+	}
+	fileMetadata := models.MediaMetadata{
+		ID:        shared.NewId(),
+		OwnedById: sql.NullString{Valid: true, String: userId},
+		Remote:    false,
+		// Location: string, // TODO: Figure this out
+	}
+	panic("not implemented")
+}

media/media.go

@@ -3,6 +3,8 @@ package media
 import (
 	"context"
 	"errors"
+	"github.com/rs/zerolog/log"
+	"net/rpc"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -11,17 +13,10 @@ import (
 )
 
 type Server struct {
-	client *minio.Client
+	client           *minio.Client
+	transcoderClient *rpc.Client
 }
 
-/*
-TODO: Figure out an api for a microservice for transcoding media, see https://tech.lgbt/@lina/114682780787448797
-- Read endpoint from config
-- Try to reach transcoder
-- If transcoder is alive, use it for transcoding
-- If not alive, store files as is
-*/
-
 var (
 	ErrNoBucketAccess = errors.New("can't access configured bucket")
 )
@@ -55,7 +50,15 @@ func NewServer() (*Server, error) {
 			return nil, ErrNoBucketAccess
 		}
 	}
-	return &Server{client: client}, nil
+
+	transcoderClient, err := rpc.DialHTTP("tcp", config.GlobalConfig.Transcoder.Address())
+	if err != nil {
+		log.Warn().Err(err).
+			Str("address", config.GlobalConfig.Transcoder.Address()).
+			Msg("failed to dial transcoder, various media related features won't be available")
+		transcoderClient = nil
+	}
+	return &Server{client: client, transcoderClient: transcoderClient}, nil
 }
 
 // UsernameFilename converts a userId and filename into a proper filepath for s3.

web/public/api/activitypub/inbox.go

@@ -213,7 +213,7 @@ func handleLike(w http.ResponseWriter, r *http.Request, object map[string]any) b
 		Reactor:   *liker,
 		ReactorId: liker.ID,
 		Emote:     nil,
-		EmoteId:   sql.NullInt64{Valid: false},
+		EmoteId:   sql.NullString{Valid: false},
 	}
 	tx := dbgen.Q.Begin()
 

web/public/api/activitypub/inboxUndo.go

@@ -113,8 +113,7 @@ func undoLike(w http.ResponseWriter, r *http.Request, object map[string]any, tar
 		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
 		return false
 	}
-	reactionId := uint(other.Must(strconv.ParseUint(act.ObjectId, 10, 64)))
-	reaction, err := dbgen.Reaction.Where(dbgen.Reaction.ID.Eq(reactionId)).First()
+	reaction, err := dbgen.Reaction.Where(dbgen.Reaction.ID.Eq(act.ObjectId)).First()
 	switch err {
 	case gorm.ErrRecordNotFound:
 		return true
@@ -140,7 +139,7 @@ func undoLike(w http.ResponseWriter, r *http.Request, object map[string]any, tar
 		_ = tx.Rollback()
 		log.Error().
 			Err(err).
-			Uint("reaction-id", reaction.ID).
+			Str("reaction-id", reaction.ID).
 			Msg("Failed to delete reaction on undo")
 		_ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError)
 		return false