diff --git a/activitypub/importNote.go b/activitypub/importNote.go index b451a1d..5f71bae 100644 --- a/activitypub/importNote.go +++ b/activitypub/importNote.go @@ -177,6 +177,12 @@ func importRemoteNoteRecursive( // In which case, you need to remove tags that don't exist anymore // and only create the ones not yet stored err = dbgen.NoteToPing.Save(pings...) + if err != nil { + return "", err + } err = dbgen.NoteTag.Save(hashtags...) + if err != nil { + return "", err + } return dbNote.ID, nil } diff --git a/activitypub/translators/announce.go b/activitypub/translators/announce.go new file mode 100644 index 0000000..07b28da --- /dev/null +++ b/activitypub/translators/announce.go @@ -0,0 +1,23 @@ +package translators + +import ( + "context" + "time" +) + +// Announce is boost + +type activityAnnounceOut struct { + Context any `json:"@context,omitempty"` + Id string + Type string // Always "Announce" + Actor string // The one doing the boost + Published time.Time + To []string + CC []string + Object string // Link to object being boosted +} + +func AnnounceFromStorage(ctx context.Context, id string) (*activityAnnounceOut, error) { + panic("not implemented") +} diff --git a/activitypub/translators/media.go b/activitypub/translators/media.go index 2ca177d..654f591 100644 --- a/activitypub/translators/media.go +++ b/activitypub/translators/media.go @@ -2,7 +2,9 @@ package translators import ( "context" + "strings" + "git.mstar.dev/mstar/linstrom/config" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" ) @@ -10,17 +12,39 @@ type Media struct { Type string `json:"type"` Url string `json:"url"` MediaType string `json:"mediaType"` + Name string `json:"name"` + Summary string `json:"summary"` + Sensitive bool `json:"sensitive"` } func MediaFromStorage(ctx context.Context, id string) (*Media, error) { - metadata, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.Eq(id)).First() + metadata, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.Eq(id), dbgen.MediaMetadata.Remote.Is(false)). + First() if err != nil { return nil, err } data := Media{ - Type: "Image", // FIXME: Change this to a sort of dynamic detection based on mimetype MediaType: metadata.Type, - Url: metadata.Location, + Url: config.GlobalConfig.General.GetFullPublicUrl() + "/media/" + id, + Name: metadata.AltText, + Summary: metadata.AltText, + Sensitive: metadata.Blurred, + } + switch strings.SplitN(metadata.Type, "/", 2)[0] { + case "audio": + data.Type = "Audio" + case "application": + data.Type = "Document" + case "image": + data.Type = "Image" + case "video": + data.Type = "Video" + case "text": + data.Type = "Document" + case "font": + data.Type = "Document" + default: + data.Type = "Document" } return &data, nil } diff --git a/activitypub/translators/user.go b/activitypub/translators/user.go index 4083c7e..a240a77 100644 --- a/activitypub/translators/user.go +++ b/activitypub/translators/user.go @@ -51,6 +51,9 @@ func UserFromStorage(ctx context.Context, id string) (*User, error) { Preload(dbgen.User.Icon).Preload(dbgen.User.Banner). Preload(dbgen.User.BeingTypes). First() + if err != nil { + return nil, err + } err = storage.EnsureLocalUserIdHasLinks(id) if err != nil { diff --git a/auth-new/passkey.go b/auth-new/passkey.go index 28b4fbc..4da055a 100644 --- a/auth-new/passkey.go +++ b/auth-new/passkey.go @@ -15,6 +15,13 @@ import ( "git.mstar.dev/mstar/linstrom/storage-new/models" ) +// Helper for discovering a user during a dicovering login (user not known yet). +// Not thread safe and should only be used once. +// FoundUser will be set after a successful call to userWithPasskeyDiscoverer +type passkeyDiscoverer struct { + FoundUser *models.User +} + // TODO: Check if passkey encryption is viable // Check if encryption for passkey info data is viable to implement // and if we should do it. @@ -25,7 +32,7 @@ import ( // Start the login process via passkey for a given username. // Returns the credential options the passkey needs to sign -func (a *Authenticator) StartPasskeyLogin( +func (a *Authenticator) StartPasskeyLoginWithUsername( username string, ) (*protocol.CredentialAssertion, string, error) { if ok, err := a.canUsernameLogin(username); !ok { @@ -58,7 +65,7 @@ func (a *Authenticator) StartPasskeyLogin( // Complete a passkey login request // Takes the username logging in as well as the raw request containing the passkey response -func (a *Authenticator) CompletePasskeyLogin( +func (a *Authenticator) CompletePasskeyLoginWithUsername( username string, sessionId string, response *http.Request, @@ -132,6 +139,82 @@ func (a *Authenticator) CompletePasskeyLogin( return dbAccessToken.Token, nil } +// Start the login process via passkey for an unknown username. +// Returns the credential options the passkey needs to sign. +// The relevant user will be discovered during the completion stage +func (a *Authenticator) StartPasskeyLoginDiscovery() (*protocol.CredentialAssertion, string, error) { + panic("not implemented") +} + +// Complete a passkey login request for an unknown user. +func (a *Authenticator) CompletePasskeyLoginDiscovery( + sessionId string, + response *http.Request, +) (accessToken string, err error) { + // Get user in question + // Get latest login token data + loginToken, err := dbgen.LoginProcessToken.Where(dbgen.LoginProcessToken.Token.Eq(sessionId)). + First() + if err != nil { + return "", other.Error( + "auth", + "failed to get user's login token for passkey login completion", + err, + ) + } + // Check if that token has expired + if loginToken.ExpiresAt.Before(time.Now()) { + return "", ErrProcessTimeout + } + var pkeySession webauthn.SessionData + err = json.Unmarshal([]byte(loginToken.Name), &pkeySession) + if err != nil { + return "", other.Error("auth", "failed to unmarshal passkey session for user", err) + } + discoverer := a.getPasskeyDiscoverer() + // Hand data to webauthn for completion + newSession, err := a.webauthn.FinishDiscoverableLogin( + discoverer.userWithPasskeyDiscoverer, + pkeySession, + response, + ) + if err != nil { + return "", other.Error("auth", "passkey completion failed", err) + } + // TODO: Utilise clone warning + // newSession.Authenticator.CloneWarning + + jsonSessionId, err := json.Marshal(newSession.ID) + if err != nil { + return "", other.Error("auth", "failed to marshal session", err) + } + jsonSession, err := json.Marshal(newSession.ID) + if err != nil { + return "", other.Error("auth", "failed to marshal session", err) + } + // Update credentials + // WARN: I am not sure if this will work + // Using the ID of the passkey session *should* be unique enough to identify the correct one + // Of course, even then, there's still the problem of matching as + // I can't yet guarantee that the parsed json content for the ID would be the same + _, err = dbgen.UserAuthMethod.Where(dbgen.UserAuthMethod.Token.Like("%"+string(jsonSessionId)+"%")). + Update(dbgen.UserAuthMethod.Token, jsonSession) + if err != nil { + return "", other.Error("auth", "failed to update credentials", err) + } + dbAccessToken := models.AccessToken{ + User: *discoverer.FoundUser, + UserId: discoverer.FoundUser.ID, + ExpiresAt: calcAccessExpirationTimestamp(), + } + err = dbgen.AccessToken.Omit(dbgen.AccessToken.Token).Create(&dbAccessToken) + if err != nil { + return "", other.Error("auth", "failed to generate access token", err) + } + + return dbAccessToken.Token, nil +} + // Start the process of registrating a passkey to an account func (a *Authenticator) StartPasskeyRegistration( username string, @@ -219,3 +302,22 @@ func (a *Authenticator) CompletePasskeyRegistration( } return nil } + +// Get a new passkey discoverer. +func (a *Authenticator) getPasskeyDiscoverer() *passkeyDiscoverer { + // nothing special yet, might use session id for further verification + return &passkeyDiscoverer{} +} + +// userWithPasskeyDiscoverer implements webauthn.DiscoverableUserHandler +// for the use during a discovering login process +func (d *passkeyDiscoverer) userWithPasskeyDiscoverer( + rawID, userHandle []byte, +) (user webauthn.User, err error) { + dbUser, err := dbgen.User.Where(dbgen.User.PasskeyId.Eq(userHandle)).First() + if err != nil { + return nil, err + } + d.FoundUser = dbUser + return &fakeUser{dbUser}, nil +} diff --git a/cmd/model-gen/main.go b/cmd/model-gen/main.go index 8481ef5..f4ab1ce 100644 --- a/cmd/model-gen/main.go +++ b/cmd/model-gen/main.go @@ -54,6 +54,7 @@ func main() { g.ApplyInterface(func(models.IAccessToken) {}, models.AccessToken{}) g.ApplyInterface(func(models.INote) {}, models.Note{}) g.ApplyInterface(func(models.IUserToUserRelation) {}, models.UserToUserRelation{}) + g.ApplyInterface(func(models.IFailedOutboundRequest) {}, models.FailedOutboundRequest{}) log.Info().Msg("Extra features applied, starting generation") g.Execute() diff --git a/cmd/sample-id-gen/main.go b/cmd/sample-id-gen/main.go new file mode 100644 index 0000000..5d76e96 --- /dev/null +++ b/cmd/sample-id-gen/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/google/uuid" + "github.com/nrednav/cuid2" + "github.com/rs/xid" +) + +var flagGenerator = flag.String( + "generator", + "xid", + "Which generator to showcase. Options: xid, cuid, uuid. Defaults to xid", +) + +func main() { + flag.Parse() + generator := "xid" + switch *flagGenerator { + case "uuid": + generator = "uuid" + case "cuid": + generator = "cuid" + } + fmt.Printf("Generator used: %s\n", generator) + var gen func() string + switch generator { + case "xid": + gen = func() string { return xid.New().String() } + case "uuid": + gen = uuid.NewString + case "cuid": + gen = cuid2.Generate + } + fmt.Println("Generating 10 ids") + for range 10 { + fmt.Println(gen()) + } +} diff --git a/config/config.go b/config/config.go index 6f0aa08..c708080 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "os" + "path" "strings" "git.mstar.dev/mstar/goutils/other" @@ -73,6 +74,14 @@ type ConfigStorage struct { MaxReconnectAttempts int `toml:"max_reconnect_attempts"` } +type ConfigTranscoder struct { + SharedDirectory string `toml:"shared_directory"` + Secret string `toml:"secret"` + ServerAddress string `toml:"server_address"` + ServerPort int `toml:"server_port"` + IgnoreTranscoder bool `toml:"ignore_transcoder"` +} + type ConfigS3 struct { KeyId string `toml:"key_id"` Secret string `toml:"secret"` @@ -143,6 +152,7 @@ type Config struct { Admin ConfigAdmin `toml:"admin"` Webauthn ConfigWebAuthn `toml:"webauthn"` Storage ConfigStorage `toml:"storage"` + Transcoder ConfigTranscoder `toml:"transcoder"` Mail ConfigMail `toml:"mail"` Self ConfigSelf `toml:"self"` S3 ConfigS3 `toml:"s3"` @@ -185,13 +195,16 @@ var defaultConfig Config = Config{ Port: 5432, SslMode: other.IntoPointer("disable"), TimeZone: other.IntoPointer("Europe/Berlin"), - RedisUrl: nil, - MaxInMemoryCacheSize: 1e6, // 1 Megabyte - MaxInMemoryCacheTTL: 5, - MaxRedisCacheTTL: nil, EncryptionKey: "Encryption key for sensitive information. DO NOT CHANGE THIS AFTER SETUP", MaxReconnectAttempts: 3, }, + Transcoder: ConfigTranscoder{ + SharedDirectory: "/tmp/linstrom-transcoder", + Secret: "The same secret as configured in the transcoder", + ServerAddress: "127.0.0.1", + ServerPort: 5594, + IgnoreTranscoder: false, + }, Mail: ConfigMail{ Host: "localhost", Port: 587, @@ -275,6 +288,18 @@ func (sc *ConfigStorage) BuildPostgresDSN() string { return dsn } +func (tc *ConfigTranscoder) Address() string { + return fmt.Sprintf("%s:%d", tc.ServerAddress, tc.ServerPort) +} + +func (tc *ConfigTranscoder) InDir() string { + return path.Join(tc.SharedDirectory, "input") +} + +func (tc *ConfigTranscoder) OutDir() string { + return path.Join(tc.SharedDirectory, "output") +} + func WriteDefaultConfig(toFile string) error { log.Trace().Caller().Send() log.Info().Str("config-file", toFile).Msg("Writing default config to file") diff --git a/devserver/.awsrc b/devserver/.awsrc deleted file mode 100644 index 8f9ac69..0000000 --- a/devserver/.awsrc +++ /dev/null @@ -1,4 +0,0 @@ -export AWS_SECRET_ACCESS_KEY=b6d1c8ec97052b8a40ae953de34f336170d85554fbe7875acce0ff51464724ee -export AWS_ACCESS_KEY_ID=GK458f9d7315fc6c9686c41045 -export AWS_DEFAULT_REGION="garage" -export AWS_ENDPOINT_URL="http://localhost:3900" diff --git a/go.mod b/go.mod index 75ea6b3..e8546a8 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module git.mstar.dev/mstar/linstrom go 1.24.2 require ( - git.mstar.dev/mstar/goutils v1.14.2 + golang.org/x/sync v0.15.0 + git.mstar.dev/mstar/goutils v1.16.1 github.com/BurntSushi/toml v1.5.0 github.com/PeerDB-io/gluabit32 v1.0.2 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 github.com/cosmotek/loguago v1.0.0 + github.com/gabriel-vasile/mimetype v1.4.9 github.com/go-acme/lego/v4 v4.23.1 github.com/go-webauthn/webauthn v0.12.3 github.com/google/uuid v1.6.0 @@ -79,7 +81,6 @@ require ( go.uber.org/mock v0.5.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.32.0 // indirect gorm.io/datatypes v1.2.5 // indirect diff --git a/go.sum b/go.sum index 6a18a4d..b4be8f2 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -git.mstar.dev/mstar/goutils v1.14.2 h1:2W9AtsAAGR9OeztPnyVCkxiuZDe7h1DlXzil35wU+vs= -git.mstar.dev/mstar/goutils v1.14.2/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA= +git.mstar.dev/mstar/goutils v1.16.1 h1:uVsT+a8Ad0DuYy7rnXAVZ5NjoE6AHit6DGxFn5XiSrU= +git.mstar.dev/mstar/goutils v1.16.1/go.mod h1:juxY0eZEMnA95fedRp2LVXvUBgEjz66nE8SEdGKcxMA= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/PeerDB-io/gluabit32 v1.0.2 h1:AGI1Z7dwDVotakpuOOuyTX4/QGi5HUYsipL/VfodmO4= @@ -32,6 +32,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= github.com/go-acme/lego/v4 v4.23.1 h1:lZ5fGtGESA2L9FB8dNTvrQUq3/X4QOb8ExkKyY7LSV4= github.com/go-acme/lego/v4 v4.23.1/go.mod h1:7UMVR7oQbIYw6V7mTgGwi4Er7B6Ww0c+c8feiBM0EgI= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -191,6 +193,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/main.go b/main.go index 45a54b6..78df4c9 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "gorm.io/gorm" "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/media" "git.mstar.dev/mstar/linstrom/shared" storagenew "git.mstar.dev/mstar/linstrom/storage-new" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" @@ -103,6 +104,14 @@ func newServer() { if err = storagenew.InsertUnknownActorPlaceholder(); err != nil { log.Fatal().Err(err).Msg("Failed to insert self properly") } + + log.Info().Msg("Connecting to s3 storage and transcoder") + mediaServer, err := media.NewServer() + if err != nil { + log.Fatal().Err(err).Msg("Failed to set up the media server") + } + media.GlobalServer = mediaServer + debugShutdownChan := make(chan *sync.WaitGroup, 1) interuptChan := make(chan os.Signal, 1) debugShutdownWaitgroup := sync.WaitGroup{} diff --git a/media/addFile.go b/media/addFile.go new file mode 100644 index 0000000..3297203 --- /dev/null +++ b/media/addFile.go @@ -0,0 +1,98 @@ +package media + +import ( + "context" + "database/sql" + "errors" + "io" + "os" + "path" + + "git.mstar.dev/mstar/goutils/other" + "github.com/gabriel-vasile/mimetype" + "github.com/minio/minio-go/v7" + "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/shared" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +var ErrFileAlreadyExists = errors.New("a file with that name already exists") + +func (s *Server) AddFile( + fileReader io.Reader, + filename, userId string, + blurred bool, + altText string, +) (string, error) { + fileCount, err := dbgen.MediaMetadata.Where(dbgen.MediaMetadata.OwnedById.Eq(sql.NullString{Valid: true, String: userId}), dbgen.MediaMetadata.Name.Eq(filename)). + Count() + if err != nil { + return "", err + } + if fileCount > 0 { + return "", ErrFileAlreadyExists + } + transcoderInDir := config.GlobalConfig.Transcoder.InDir() + filePath := path.Join(transcoderInDir, filename) + file, err := os.Create(filePath) + if err != nil { + return "", err + } + if _, err = io.Copy(file, fileReader); err != nil { + _ = file.Close() + return "", err + } + _ = file.Close() + if s.transcoderClient == nil { + return s.addFileAsIs(filename, userId, filePath, nil, blurred, altText) + } else { + return s.addFileWithTranscoder(filename, userId, filePath, blurred, altText) + } +} + +// adFileAsIs uploads the given file. If mtype (short for mimetype, shortened because of module naming conflict) +// is not nil, use that as the file's mimetype. Otherwise, the mimetype will be detected manually +func (s *Server) addFileAsIs( + filename, userId, filepath string, + mtype *string, + blurred bool, + altText string, +) (string, error) { + if mtype == nil { + mType, err := mimetype.DetectFile(filepath) + if err != nil { + return "", err + } + mtype = other.IntoPointer(mType.String()) + } + id := shared.NewId() + s3Result, err := s.client.FPutObject( + context.TODO(), + config.GlobalConfig.S3.BucketName, + id, + filepath, + minio.PutObjectOptions{}, + ) + if err != nil { + return "", err + } + log.Debug().Any("result", s3Result).Msg("Upload result") + fileMetadata := models.MediaMetadata{ + ID: id, + OwnedById: sql.NullString{Valid: true, String: userId}, + Remote: false, + Location: s3Result.Key, + Type: *mtype, + Name: filename, + AltText: altText, + Blurred: blurred, + } + err = dbgen.MediaMetadata.Create(&fileMetadata) + if err != nil { + return "", err + } + return id, nil +} diff --git a/media/fileInfo.go b/media/fileInfo.go new file mode 100644 index 0000000..acbd8ec --- /dev/null +++ b/media/fileInfo.go @@ -0,0 +1,17 @@ +package media + +import ( + "database/sql" + + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +func (s *Server) FileExists(userid, filename string) (bool, error) { + mm := dbgen.MediaMetadata + c, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename)). + Count() + if err != nil { + return false, err + } + return c > 0, nil +} diff --git a/media/media.go b/media/media.go index 3ee981b..0e1fbfa 100644 --- a/media/media.go +++ b/media/media.go @@ -3,29 +3,26 @@ package media import ( "context" "errors" + "net/rpc" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rs/zerolog/log" "git.mstar.dev/mstar/linstrom/config" ) type Server struct { - client *minio.Client + client *minio.Client + transcoderClient *rpc.Client } -/* -TODO: Figure out an api for a microservice for transcoding media, see https://tech.lgbt/@lina/114682780787448797 -- Read endpoint from config -- Try to reach transcoder -- If transcoder is alive, use it for transcoding -- If not alive, store files as is -*/ - var ( ErrNoBucketAccess = errors.New("can't access configured bucket") ) +var GlobalServer *Server + func NewServer() (*Server, error) { client, err := minio.New(config.GlobalConfig.S3.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4( @@ -55,24 +52,16 @@ func NewServer() (*Server, error) { return nil, ErrNoBucketAccess } } - return &Server{client: client}, nil -} -// UsernameFilename converts a userId and filename into a proper filepath for s3. -// Reason for this is that the userId for external users is a valid url which needs to be encoded -func UsernameFilename(userId, filename string) string { - return userId + "//" + filename -} - -func (s *Server) HasFileScoped(userId, filename string) (bool, error) { - info, err := s.client.StatObject( - context.Background(), - config.GlobalConfig.S3.BucketName, - UsernameFilename(userId, filename), - minio.GetObjectOptions{}, - ) - if err != nil { - return false, err + if config.GlobalConfig.Transcoder.IgnoreTranscoder { + return &Server{client: client, transcoderClient: nil}, nil } - return info.IsDeleteMarker, nil + transcoderClient, err := rpc.DialHTTP("tcp", config.GlobalConfig.Transcoder.Address()) + if err != nil { + log.Warn().Err(err). + Str("address", config.GlobalConfig.Transcoder.Address()). + Msg("failed to dial transcoder, various media related features won't be available") + transcoderClient = nil + } + return &Server{client: client, transcoderClient: transcoderClient}, nil } diff --git a/media/readFile.go b/media/readFile.go new file mode 100644 index 0000000..9c61b2e --- /dev/null +++ b/media/readFile.go @@ -0,0 +1,56 @@ +package media + +import ( + "context" + "database/sql" + "errors" + "io" + + "github.com/minio/minio-go/v7" + "gorm.io/gorm" + + "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +var ErrFileNotFound = errors.New("file not found") + +func (s *Server) ReadFile(userid, filename string) (io.ReadCloser, error) { + mm := dbgen.MediaMetadata + metadata, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userid}), mm.Name.Eq(filename), mm.Remote.Is(false)). + Select(mm.ID, mm.Location). + First() + switch err { + case gorm.ErrRecordNotFound: + return nil, ErrFileNotFound + case nil: + default: + return nil, err + } + return s.client.GetObject( + context.TODO(), + config.GlobalConfig.S3.BucketName, + metadata.Location, + minio.GetObjectOptions{}, + ) +} + +func (s *Server) ReadFileId(id string) (io.ReadCloser, error) { + mm := dbgen.MediaMetadata + metadata, err := mm.Where(mm.ID.Eq(id), mm.Remote.Is(false)). + Select(mm.Location). + First() + switch err { + case gorm.ErrRecordNotFound: + return nil, ErrFileNotFound + case nil: + default: + return nil, err + } + return s.client.GetObject( + context.TODO(), + config.GlobalConfig.S3.BucketName, + metadata.Location, + minio.GetObjectOptions{}, + ) +} diff --git a/media/removeFile.go b/media/removeFile.go new file mode 100644 index 0000000..8041f3e --- /dev/null +++ b/media/removeFile.go @@ -0,0 +1,5 @@ +package media + +func (s *Server) RemoveFile(userId, filename string) error { + panic("not implemented") +} diff --git a/media/services.go b/media/services.go new file mode 100644 index 0000000..8a326c8 --- /dev/null +++ b/media/services.go @@ -0,0 +1,82 @@ +package media + +import ( + "context" + "slices" + + "git.mstar.dev/mstar/goutils/sliceutils" + "github.com/minio/minio-go/v7" + "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/config" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +// ServiceEnsureFileSynchronisation is a service function for ensuring data synchronicity between +// the db's metadata for the files and the actual files in s3. +// All files without matching metadata will be deleted. Same for all metadata without a matching file. +// No attempt at restoring a connection will be made +func (s *Server) ServiceEnsureFileSynchronisation() { + mm := dbgen.MediaMetadata + allFiles, err := mm. + Select(mm.ID, mm.OwnedById, mm.Name, mm.Location). + Where(mm.Location.NotLike("linstrom://%"), mm.Remote.Is(false)). + Find() + if err != nil { + log.Error().Err(err).Msg("Failed to get a list of all known media") + return + } + foundInDb := []string{} + objectMissingInDb := []minio.ObjectInfo{} + // Go over all objects in the bucket. Note down if it has an entry in the db or not + for obj := range s.client.ListObjects(context.TODO(), config.GlobalConfig.S3.BucketName, minio.ListObjectsOptions{ + Recursive: true, + }) { + log.Debug().Str("object-key", obj.Key).Msg("Checking object") + if slices.ContainsFunc(allFiles, func(e *models.MediaMetadata) bool { + return e.Location == obj.Key + }) { + foundInDb = append(foundInDb, obj.Key) + } else { + objectMissingInDb = append(objectMissingInDb, obj) + } + } + + // Find every db entry not in the list of found objects + entryMissingAnObject := []string{} + for _, dbFile := range allFiles { + if !slices.ContainsFunc(foundInDb, func(e string) bool { + return dbFile.Location == e + }) { + entryMissingAnObject = append(entryMissingAnObject, dbFile.ID) + } + } + + // For every object missing in the db, delete it + minioErrChan := s.client.RemoveObjects( + context.TODO(), + config.GlobalConfig.S3.BucketName, + sliceutils.ToChannel(objectMissingInDb), + minio.RemoveObjectsOptions{GovernanceBypass: true}, + ) + s3Errors := sliceutils.FromChannel(minioErrChan, 0) + s3Errors = sliceutils.Filter( + s3Errors, + func(t minio.RemoveObjectError) bool { return t.Err != nil }, + ) + for _, s3Err := range s3Errors { + log.Error(). + Err(s3Err.Err). + Str("object-name", s3Err.ObjectName). + Msg("Failed to delete object missing in db") + } + // And perform a batch delete + _, err = dbgen.MediaMetadata.Where(dbgen.MediaMetadata.ID.In(entryMissingAnObject...)).Delete() + if err != nil { + log.Error(). + Err(err). + Strs("media-ids", entryMissingAnObject). + Msg("Failed to batch delete all media metadata without a matching object in s3") + } +} diff --git a/media/transcoder.go b/media/transcoder.go new file mode 100644 index 0000000..b27794e --- /dev/null +++ b/media/transcoder.go @@ -0,0 +1,45 @@ +package media + +import ( + "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/config" +) + +// WARN: These types need to always be in sync with linstrom-transcoder/transcode/transcoder.go +// TODO: Maybe move to a separate repo outside of linstrom-transcoder + +type TranscodeArgs struct { + Secret string + Filename string +} + +type TranscodeReply struct { + Error error + Mimetype string + ThumbnailFilename *string + Filename string +} + +// addFileWithTranscoder will try to transcode the given file using the helper application. +// If the transcode fails, it uploads the file as is +func (s *Server) addFileWithTranscoder( + filename, userId, filepath string, + blurred bool, + altText string, +) (string, error) { + args := TranscodeArgs{ + Secret: config.GlobalConfig.Transcoder.Secret, + Filename: filepath, + } + reply := TranscodeReply{} + err := s.transcoderClient.Call("Transcoder.Transcode", &args, &reply) + if err != nil { + return "", err + } + if reply.Error != nil { + log.Warn().Err(reply.Error).Msg("Transcoder failed, uploading raw file") + return s.addFileAsIs(filename, userId, filepath, nil, blurred, altText) + } + return s.addFileAsIs(filename, userId, reply.Filename, &reply.Mimetype, blurred, altText) +} diff --git a/storage-new/cleaners/killDeadServers.go b/storage-new/cleaners/killDeadServers.go new file mode 100644 index 0000000..764ad48 --- /dev/null +++ b/storage-new/cleaners/killDeadServers.go @@ -0,0 +1,30 @@ +package cleaners + +import ( + "time" + + "github.com/rs/zerolog/log" + + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +const maxServerAge = time.Hour * 24 * 30 // One month + +func init() { + cleanerBuilders = append(cleanerBuilders, buildKillDeadServers) +} + +// Marks all servers where the last interaction time is older than maxServerAge +func tickKillDeadServers(now time.Time) { + _, err := dbgen.RemoteServer.Where(dbgen.RemoteServer.LastInteraction.Lt(now.Add(-maxServerAge)), dbgen.RemoteServer.IsSelf.Is(false)). + UpdateColumn(dbgen.RemoteServer.IsDead, true) + if err != nil { + log.Error(). + Err(err). + Msg("Failed to mark servers without interaction for over a 30 days as dead") + } +} + +func buildKillDeadServers() (onTick func(time.Time), name string, tickSpeed time.Duration) { + return tickKillDeadServers, "kill-dead-servers", time.Hour +} diff --git a/storage-new/cleaners/retryFailedRequests.go b/storage-new/cleaners/retryFailedRequests.go new file mode 100644 index 0000000..536af70 --- /dev/null +++ b/storage-new/cleaners/retryFailedRequests.go @@ -0,0 +1,113 @@ +package cleaners + +import ( + "net/http" + "strings" + "sync" + "time" + + "github.com/rs/zerolog/log" + "golang.org/x/sync/errgroup" + "gorm.io/gen" + + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" + webshared "git.mstar.dev/mstar/linstrom/web/shared" +) + +const maxFailedRequestsBeforeDeath = 10 + +var ( + reqErrStrUnknownHost = "no such host" +) + +func init() { + cleanerBuilders = append(cleanerBuilders, buildRetryRequests) +} + +func TickRetryRequests(now time.Time) { + batchResults := []*models.FailedOutboundRequest{} + fo := dbgen.FailedOutboundRequest + idsToDelete := []uint64{} + var idsDeleteLock sync.Mutex + err := fo.Preload(fo.TargetServer, fo.ActingUser). + // Join with server data to exclude dead servers + LeftJoin(dbgen.RemoteServer, dbgen.RemoteServer.ID.EqCol(fo.TargetServerId)). + Where(dbgen.RemoteServer.IsDead.Is(false)). + Order(fo.Id.Asc()). + FindInBatches(&batchResults, 50, func(tx gen.Dao, batch int) error { + var g errgroup.Group + for _, failedRequest := range batchResults { + g.Go(func() (err error) { + defer func() { + failedRequest.NrOfAttempts += 1 + err = dbgen.FailedOutboundRequest.Save(failedRequest) + if err != nil { + return + } + if failedRequest.NrOfAttempts >= maxFailedRequestsBeforeDeath { + _, err = dbgen.RemoteServer.Where(dbgen.RemoteServer.ID.Eq(failedRequest.TargetServerId)). + UpdateColumn(dbgen.RemoteServer.IsDead, true) + } + }() + var res *http.Response + res, _, err = webshared.RequestSigned( + "POST", + failedRequest.Target, + failedRequest.RawData, + failedRequest.ActingUser, + ) + if err != nil { + failedRequest.NrOfAttempts += 1 + errString := err.Error() + // FIXME: Use the actual error types instead of error string + // Using substring matching is awful and probably not reliable. Using error type is likely more reliable + if strings.Contains(errString, reqErrStrUnknownHost) { + failedRequest.LastFailureReason = string( + models.RequestFailureRequestError, + ) + } else { + failedRequest.LastFailureReason = string(models.RequestFailureRequestError) + } + return + } + if res.StatusCode < 400 { + idsDeleteLock.Lock() + idsToDelete = append(idsToDelete, failedRequest.Id) + idsDeleteLock.Unlock() + // Defer func will always add one (to make the expected failure case easier) + // Sub one here to prevent a potential server kill if it was at maxFailedRequestsBeforeDeath-1 failed requests before + failedRequest.NrOfAttempts -= 1 + return nil + } + + switch res.StatusCode { + case http.StatusInternalServerError: + failedRequest.LastFailureReason = string(models.RequestFailureInternalError) + case http.StatusForbidden: + failedRequest.LastFailureReason = string(models.RequestFailureRejected) + case http.StatusTooManyRequests: + // TODO: Check Timeout headers and write apropriate message + } + + return nil + }) + } + return nil + }) + if err != nil { + log.Error().Err(err).Msg("Failed to batch-process all failed outbound requests") + } + _, err = fo.Where(fo.Id.In(idsToDelete...)).Delete() + if err != nil { + log.Error().Err(err).Msg("Failed to batch-delete all successful retries") + } + err = dbgen.FailedOutboundRequest.KillServers(maxFailedRequestsBeforeDeath) + if err != nil { + log.Error().Err(err).Msg("Failed to kill all servers with too many failed requests") + } +} + +func buildRetryRequests() (onTick func(time.Time), name string, tickSpeed time.Duration) { + return TickRetryRequests, "retry-requests", time.Hour +} diff --git a/storage-new/dbgen/failed_outbound_requests.gen.go b/storage-new/dbgen/failed_outbound_requests.gen.go index 45b4405..01741f7 100644 --- a/storage-new/dbgen/failed_outbound_requests.gen.go +++ b/storage-new/dbgen/failed_outbound_requests.gen.go @@ -7,6 +7,7 @@ package dbgen import ( "context" "database/sql" + "strings" "git.mstar.dev/mstar/linstrom/storage-new/models" "gorm.io/gorm" @@ -607,6 +608,34 @@ type IFailedOutboundRequestDo interface { Returning(value interface{}, columns ...string) IFailedOutboundRequestDo UnderlyingDB() *gorm.DB schema.Tabler + + KillServers(maxAttempts uint32) (err error) +} + +// KillServers marks all servers where a request has more than X failed attempts as dead +// +// WITH servers_to_kill AS ( +// +// SELECT target_server_id +// FROM failed_outbound_requests +// WHERE nr_of_attempts > @maxAttempts +// +// ) +// UPDATE remote_servers +// SET is_dead = true +// WHERE id IN (SELECT target_server_id FROM servers_to_kill) +func (f failedOutboundRequestDo) KillServers(maxAttempts uint32) (err error) { + var params []interface{} + + var generateSQL strings.Builder + params = append(params, maxAttempts) + generateSQL.WriteString("WITH servers_to_kill AS ( SELECT target_server_id FROM failed_outbound_requests WHERE nr_of_attempts > ? ) UPDATE remote_servers SET is_dead = true WHERE id IN (SELECT target_server_id FROM servers_to_kill) ") + + var executeSQL *gorm.DB + executeSQL = f.UnderlyingDB().Exec(generateSQL.String(), params...) // ignore_security_alert + err = executeSQL.Error + + return } func (f failedOutboundRequestDo) Debug() IFailedOutboundRequestDo { diff --git a/storage-new/dbgen/roles.gen.go b/storage-new/dbgen/roles.gen.go index 635807b..5786e5c 100644 --- a/storage-new/dbgen/roles.gen.go +++ b/storage-new/dbgen/roles.gen.go @@ -55,6 +55,8 @@ func newRole(db *gorm.DB, opts ...gen.DOOption) role { _role.CanMentionOthers = field.NewBool(tableName, "can_mention_others") _role.HasMentionCountLimit = field.NewBool(tableName, "has_mention_count_limit") _role.MentionLimit = field.NewUint32(tableName, "mention_limit") + _role.MaxIndividualFileSize = field.NewUint64(tableName, "max_individual_file_size") + _role.MaxTotalFileSize = field.NewUint64(tableName, "max_total_file_size") _role.AutoNsfwMedia = field.NewBool(tableName, "auto_nsfw_media") _role.AutoCwPosts = field.NewBool(tableName, "auto_cw_posts") _role.AutoCwPostsText = field.NewString(tableName, "auto_cw_posts_text") @@ -118,6 +120,8 @@ type role struct { CanMentionOthers field.Bool HasMentionCountLimit field.Bool MentionLimit field.Uint32 + MaxIndividualFileSize field.Uint64 + MaxTotalFileSize field.Uint64 AutoNsfwMedia field.Bool AutoCwPosts field.Bool AutoCwPostsText field.String @@ -187,6 +191,8 @@ func (r *role) updateTableName(table string) *role { r.CanMentionOthers = field.NewBool(table, "can_mention_others") r.HasMentionCountLimit = field.NewBool(table, "has_mention_count_limit") r.MentionLimit = field.NewUint32(table, "mention_limit") + r.MaxIndividualFileSize = field.NewUint64(table, "max_individual_file_size") + r.MaxTotalFileSize = field.NewUint64(table, "max_total_file_size") r.AutoNsfwMedia = field.NewBool(table, "auto_nsfw_media") r.AutoCwPosts = field.NewBool(table, "auto_cw_posts") r.AutoCwPostsText = field.NewString(table, "auto_cw_posts_text") @@ -228,7 +234,7 @@ func (r *role) GetFieldByName(fieldName string) (field.OrderExpr, bool) { } func (r *role) fillFieldMap() { - r.fieldMap = make(map[string]field.Expr, 53) + r.fieldMap = make(map[string]field.Expr, 55) r.fieldMap["id"] = r.ID r.fieldMap["created_at"] = r.CreatedAt r.fieldMap["updated_at"] = r.UpdatedAt @@ -257,6 +263,8 @@ func (r *role) fillFieldMap() { r.fieldMap["can_mention_others"] = r.CanMentionOthers r.fieldMap["has_mention_count_limit"] = r.HasMentionCountLimit r.fieldMap["mention_limit"] = r.MentionLimit + r.fieldMap["max_individual_file_size"] = r.MaxIndividualFileSize + r.fieldMap["max_total_file_size"] = r.MaxTotalFileSize r.fieldMap["auto_nsfw_media"] = r.AutoNsfwMedia r.fieldMap["auto_cw_posts"] = r.AutoCwPosts r.fieldMap["auto_cw_posts_text"] = r.AutoCwPostsText diff --git a/storage-new/dbgen/server_metadata.gen.go b/storage-new/dbgen/server_metadata.gen.go index c46ba58..0eda76e 100644 --- a/storage-new/dbgen/server_metadata.gen.go +++ b/storage-new/dbgen/server_metadata.gen.go @@ -39,6 +39,7 @@ func newServerMetadata(db *gorm.DB, opts ...gen.DOOption) serverMetadata { _serverMetadata.LECSR = field.NewBytes(tableName, "lecsr") _serverMetadata.LELastUpdate = field.NewField(tableName, "le_last_update") _serverMetadata.LEUserPrivKey = field.NewBytes(tableName, "le_user_priv_key") + _serverMetadata.LastMigrationVersion = field.NewUint64(tableName, "last_migration_version") _serverMetadata.fillFieldMap() @@ -48,19 +49,20 @@ func newServerMetadata(db *gorm.DB, opts ...gen.DOOption) serverMetadata { type serverMetadata struct { serverMetadataDo - ALL field.Asterisk - Id field.Uint64 - CreatedAt field.Time - UpdatedAt field.Time - LEDomain field.String - LECertUrl field.String - LECertStableUrl field.String - LEPrivateKey field.Bytes - LECertificate field.Bytes - LEIssuerCertificate field.Bytes - LECSR field.Bytes - LELastUpdate field.Field - LEUserPrivKey field.Bytes + ALL field.Asterisk + Id field.Uint64 + CreatedAt field.Time + UpdatedAt field.Time + LEDomain field.String + LECertUrl field.String + LECertStableUrl field.String + LEPrivateKey field.Bytes + LECertificate field.Bytes + LEIssuerCertificate field.Bytes + LECSR field.Bytes + LELastUpdate field.Field + LEUserPrivKey field.Bytes + LastMigrationVersion field.Uint64 fieldMap map[string]field.Expr } @@ -89,6 +91,7 @@ func (s *serverMetadata) updateTableName(table string) *serverMetadata { s.LECSR = field.NewBytes(table, "lecsr") s.LELastUpdate = field.NewField(table, "le_last_update") s.LEUserPrivKey = field.NewBytes(table, "le_user_priv_key") + s.LastMigrationVersion = field.NewUint64(table, "last_migration_version") s.fillFieldMap() @@ -105,7 +108,7 @@ func (s *serverMetadata) GetFieldByName(fieldName string) (field.OrderExpr, bool } func (s *serverMetadata) fillFieldMap() { - s.fieldMap = make(map[string]field.Expr, 12) + s.fieldMap = make(map[string]field.Expr, 13) s.fieldMap["id"] = s.Id s.fieldMap["created_at"] = s.CreatedAt s.fieldMap["updated_at"] = s.UpdatedAt @@ -118,6 +121,7 @@ func (s *serverMetadata) fillFieldMap() { s.fieldMap["lecsr"] = s.LECSR s.fieldMap["le_last_update"] = s.LELastUpdate s.fieldMap["le_user_priv_key"] = s.LEUserPrivKey + s.fieldMap["last_migration_version"] = s.LastMigrationVersion } func (s serverMetadata) clone(db *gorm.DB) serverMetadata { diff --git a/storage-new/dbgen/user_to_user_relations.gen.go b/storage-new/dbgen/user_to_user_relations.gen.go index dd9b0ca..14dc727 100644 --- a/storage-new/dbgen/user_to_user_relations.gen.go +++ b/storage-new/dbgen/user_to_user_relations.gen.go @@ -594,6 +594,7 @@ type IUserToUserRelationDo interface { GetFollowingApLinksPagedForId(id string, page int) (result []string, err error) CountFollowersForId(id string) (result int, err error) CountFollowingForId(id string) (result int, err error) + GetLocalFollowerIdsOfId(id string) (result []string, err error) } // Get all inbox links for accounts following the user with the specified id @@ -720,6 +721,34 @@ func (u userToUserRelationDo) CountFollowingForId(id string) (result int, err er return } +// Get the ids of all local accounts following the user with the target id +// +// SELECT +// r.user_id +// FROM +// +// user_to_user_relations r +// LEFT JOIN users u ON r.user_id = u.id +// LEFT JOIN remote_servers s ON u.server_id = s.id +// +// WHERE +// +// s.is_self = true +// AND r.target_user_id = @id; +func (u userToUserRelationDo) GetLocalFollowerIdsOfId(id string) (result []string, err error) { + var params []interface{} + + var generateSQL strings.Builder + params = append(params, id) + generateSQL.WriteString("SELECT r.user_id FROM user_to_user_relations r LEFT JOIN users u ON r.user_id = u.id LEFT JOIN remote_servers s ON u.server_id = s.id WHERE s.is_self = true AND r.target_user_id = ?; ") + + var executeSQL *gorm.DB + executeSQL = u.UnderlyingDB().Raw(generateSQL.String(), params...).Find(&result) // ignore_security_alert + err = executeSQL.Error + + return +} + func (u userToUserRelationDo) Debug() IUserToUserRelationDo { return u.withDO(u.DO.Debug()) } diff --git a/storage-new/migrations.go b/storage-new/migrations.go index 1f8d089..864e7a1 100644 --- a/storage-new/migrations.go +++ b/storage-new/migrations.go @@ -1,17 +1,16 @@ package storage import ( - "fmt" - "strings" - "git.mstar.dev/mstar/goutils/other" - "git.mstar.dev/mstar/goutils/sliceutils" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "gorm.io/gorm" - "git.mstar.dev/mstar/linstrom/storage-new/models" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" ) +const CurrentMigrationVersion = 1 + // Auto-migrate all tables and types used func Migrate(db *gorm.DB) error { // Shut up gorm's queries during automigrate by setting log level to info during migration @@ -38,7 +37,10 @@ func Migrate(db *gorm.DB) error { return other.Error("storage", "Failed to create Activitystreams Activity type", err) } if err := createCollectionTarget(db); err != nil { - return other.Error("storage", "Failed t ocreate collections target type", err) + return other.Error("storage", "Failed to create collections target type", err) + } + if err := preTypeMigrations(db); err != nil { + return other.Error("storage", "Failed to run pre-type migrations", err) } if err := migrateTypes(db); err != nil { return other.Error("storage", "Failed to automigrate data structs", err) @@ -46,124 +48,17 @@ func Migrate(db *gorm.DB) error { return nil } -// Returns the raw error created by gorm, with no wrapping -func migrateTypes(db *gorm.DB) error { - if err := db.AutoMigrate(models.AllTypes...); err != nil { - return err - } - return nil -} - -// Ensure the being enum exists for the user -func createBeingType(db *gorm.DB) error { - return migrateEnum( - db, - "being_type", - sliceutils.Map(models.AllBeings, func(t models.BeingType) string { return string(t) }), - ) -} - -func createAccountRelationType(db *gorm.DB) error { - return migrateEnum( - db, - "relation_type", - sliceutils.Map( - models.AllRelations, - func(t models.RelationType) string { return string(t) }, - ), - ) -} - -func createAccountAuthMethodType(db *gorm.DB) error { - return migrateEnum( - db, - "auth_method_type", - sliceutils.Map( - models.AllAuthMethods, - func(t models.AuthenticationMethodType) string { return string(t) }, - ), - ) -} - -func createRemoteServerSoftwareType(db *gorm.DB) error { - return migrateEnum( - db, - "server_software_type", - sliceutils.Map( - models.AllServerSoftwareTypes, - func(t models.ServerSoftwareType) string { return string(t) }, - ), - ) -} - -func createActitiystreamsObjectType(db *gorm.DB) error { - return migrateEnum( - db, - "activitystreams_object_type", - sliceutils.Map( - models.AllActivitystreamsObjectTypes, - func(t models.ActivitystreamsObjectType) string { return string(t) }, - ), - ) -} - -func createActitiystreamsActivityType(db *gorm.DB) error { - return migrateEnum( - db, - "activitystreams_activity_type", - sliceutils.Map( - models.AllActivitystreamsActivityTypes, - func(t models.ActivitystreamsActivityType) string { return string(t) }, - ), - ) -} - -// func createActitiystreamsActivityTargetType(db *gorm.DB) error { -// return migrateEnum( -// db, -// "activitystreams_activity_target_type", -// sliceutils.Map( -// models.AllActivitystreamsActivityTargetTypes, -// func(t models.ActivitystreamsActivityTargetType) string { return string(t) }, -// ), -// ) -// } - -func createCollectionTarget(db *gorm.DB) error { - return migrateEnum( - db, - "collection_target_type", - sliceutils.Map( - models.AllCollectionTargetTypes, - func(t models.CollectionTargetType) string { return string(t) }, - ), - ) -} - -// Helper function for ensuring the existence of an enum with the given values -func migrateEnum(db *gorm.DB, name string, values []string) error { - if err := db.Exec("DROP TYPE IF EXISTS " + name + " CASCADE;").Error; err != nil { - return other.Error( - "storage", - fmt.Sprintf("Failed to remove old type %s (if it exists)", name), - err, - ) - } - queryBuilder := strings.Builder{} - queryBuilder.WriteString("CREATE TYPE ") - queryBuilder.WriteString(name) - queryBuilder.WriteString(" AS ENUM (") - blen := len(values) - for i, btype := range values { - queryBuilder.WriteString("'" + string(btype) + "'") - // Append comma everywhere except last entry - if i+1 < blen { - queryBuilder.WriteString(",") - } - } - queryBuilder.WriteString(");") - if err := db.Exec(queryBuilder.String()).Error; err != nil { - return err +func preTypeMigrations(db *gorm.DB) error { + genTmp := dbgen.Use(db) + meta, err := genTmp. + ServerMetadata.Select(dbgen.ServerMetadata.LastMigrationVersion). + First() + if err != nil { + return nil } + if meta.LastMigrationVersion == CurrentMigrationVersion { + return nil + } + log.Error().Msg("custom schema migrations not implemented yet") return nil } diff --git a/storage-new/models/FailedOutboundRequests.go b/storage-new/models/FailedOutboundRequests.go index 421d053..2c72ce9 100644 --- a/storage-new/models/FailedOutboundRequests.go +++ b/storage-new/models/FailedOutboundRequests.go @@ -16,3 +16,17 @@ type FailedOutboundRequest struct { TargetServer *RemoteServer // The remote server being targeted. Included to determine if a request still has a chance of success TargetServerId uint // Id of the target remote server } + +type IFailedOutboundRequest interface { + // KillServers marks all servers where a request has more than X failed attempts as dead + // + // WITH servers_to_kill AS ( + // SELECT target_server_id + // FROM failed_outbound_requests + // WHERE nr_of_attempts > @maxAttempts + // ) + // UPDATE remote_servers + // SET is_dead = true + // WHERE id IN (SELECT target_server_id FROM servers_to_kill) + KillServers(maxAttempts uint32) error +} diff --git a/storage-new/models/Feed.go b/storage-new/models/Feed.go index ccb8f36..d727183 100644 --- a/storage-new/models/Feed.go +++ b/storage-new/models/Feed.go @@ -34,3 +34,4 @@ type Feed struct { // Suffix added to feeds created as the default feed for a user const FeedDefaultSuffix = "-default" +const GlobalFeedName = "global" diff --git a/storage-new/models/MediaMetadata.go b/storage-new/models/MediaMetadata.go index 25f6082..2502060 100644 --- a/storage-new/models/MediaMetadata.go +++ b/storage-new/models/MediaMetadata.go @@ -22,7 +22,7 @@ type MediaMetadata struct { // OwnedBy User OwnedById sql.NullString // Account id this media belongs to Remote bool // whether the attachment is a remote one - // Where the media is stored. Url + // Where the media is stored. Url for remote files, the object name for local files Location string Type string // What media type this is following mime types, eg image/png // Name of the file diff --git a/storage-new/models/Role.go b/storage-new/models/Role.go index eb7dc71..44f6145 100644 --- a/storage-new/models/Role.go +++ b/storage-new/models/Role.go @@ -85,6 +85,9 @@ type Role struct { HasMentionCountLimit *bool // Local & remote MentionLimit *uint32 // Local & remote + MaxIndividualFileSize *uint64 // Local + MaxTotalFileSize *uint64 // Local + // CanViewBoosts *bool // CanViewQuotes *bool // CanViewMedia *bool diff --git a/storage-new/models/RolesDefaults.go b/storage-new/models/RolesDefaults.go index f5b0479..6d55b50 100644 --- a/storage-new/models/RolesDefaults.go +++ b/storage-new/models/RolesDefaults.go @@ -42,6 +42,9 @@ var DefaultUserRole = Role{ uint32(math.MaxUint32), ), // Set this to max, even if not used due to *HasMentionCountLimit == false + MaxIndividualFileSize: other.IntoPointer(uint64(10 << 20)), // 10 MB + MaxTotalFileSize: other.IntoPointer(uint64(500 << 20)), // 500 MB + AutoNsfwMedia: other.IntoPointer(false), AutoCwPosts: other.IntoPointer(false), AutoCwPostsText: nil, diff --git a/storage-new/models/ServerMetadata.go b/storage-new/models/ServerMetadata.go index 57982fd..2a397b5 100644 --- a/storage-new/models/ServerMetadata.go +++ b/storage-new/models/ServerMetadata.go @@ -12,6 +12,7 @@ type ServerMetadata struct { UpdatedAt time.Time // ---- Section TLS LetsEncrypt + LEDomain string LECertUrl string LECertStableUrl string @@ -21,4 +22,7 @@ type ServerMetadata struct { LECSR []byte // Encrypted LELastUpdate sql.NullTime LEUserPrivKey []byte // Encrypted + + // ---- Section Database + LastMigrationVersion uint64 } diff --git a/storage-new/models/UserToUserRelation.go b/storage-new/models/UserToUserRelation.go index b41b69e..2b683ee 100644 --- a/storage-new/models/UserToUserRelation.go +++ b/storage-new/models/UserToUserRelation.go @@ -71,4 +71,17 @@ type IUserToUserRelation interface { // r.user_id = @id AND // r.relation = 'follow' CountFollowingForId(id string) (int, error) + + // Get the ids of all local accounts following the user with the target id + // + // SELECT + // r.user_id + // FROM + // user_to_user_relations r + // LEFT JOIN users u ON r.user_id = u.id + // LEFT JOIN remote_servers s ON u.server_id = s.id + // WHERE + // s.is_self = true + // AND r.target_user_id = @id; + GetLocalFollowerIdsOfId(id string) ([]string, error) } diff --git a/storage-new/self.go b/storage-new/self.go index 6c2bdab..375779f 100644 --- a/storage-new/self.go +++ b/storage-new/self.go @@ -43,6 +43,9 @@ func InsertSelf() error { if err = attachUserToRole(user); err != nil { return other.Error("storage", "failed to save/update self user to full admin role", err) } + if err = insertGlobalFeed(user); err != nil { + return other.Error("storage", "failed to ensure that the global feed exists", err) + } return nil } @@ -215,3 +218,26 @@ func attachUserToRole(user *models.User) error { } return nil } + +func insertGlobalFeed(serverActor *models.User) error { + globalFeed, err := dbgen.Feed.Where(dbgen.Feed.Name.Eq(models.GlobalFeedName)).First() + switch err { + case nil: + return nil + case gorm.ErrRecordNotFound: + globalFeed = &models.Feed{ + Owner: *serverActor, + OwnerId: serverActor.ID, + IsDefault: true, + Name: models.GlobalFeedName, + PublicKey: sql.NullString{Valid: false}, + } + err = dbgen.Feed.Create(globalFeed) + if err != nil { + return err + } + return nil + default: + return err + } +} diff --git a/storage-new/typeMigrations.go b/storage-new/typeMigrations.go new file mode 100644 index 0000000..3a039ad --- /dev/null +++ b/storage-new/typeMigrations.go @@ -0,0 +1,123 @@ +package storage + +import ( + "fmt" + "strings" + + "git.mstar.dev/mstar/goutils/other" + "git.mstar.dev/mstar/goutils/sliceutils" + "gorm.io/gorm" + + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +// Returns the raw error created by gorm, with no wrapping +func migrateTypes(db *gorm.DB) error { + if err := db.AutoMigrate(models.AllTypes...); err != nil { + return err + } + return nil +} + +// Ensure the being enum exists for the user +func createBeingType(db *gorm.DB) error { + return migrateEnum( + db, + "being_type", + sliceutils.Map(models.AllBeings, func(t models.BeingType) string { return string(t) }), + ) +} + +func createAccountRelationType(db *gorm.DB) error { + return migrateEnum( + db, + "relation_type", + sliceutils.Map( + models.AllRelations, + func(t models.RelationType) string { return string(t) }, + ), + ) +} + +func createAccountAuthMethodType(db *gorm.DB) error { + return migrateEnum( + db, + "auth_method_type", + sliceutils.Map( + models.AllAuthMethods, + func(t models.AuthenticationMethodType) string { return string(t) }, + ), + ) +} + +func createRemoteServerSoftwareType(db *gorm.DB) error { + return migrateEnum( + db, + "server_software_type", + sliceutils.Map( + models.AllServerSoftwareTypes, + func(t models.ServerSoftwareType) string { return string(t) }, + ), + ) +} + +func createActitiystreamsObjectType(db *gorm.DB) error { + return migrateEnum( + db, + "activitystreams_object_type", + sliceutils.Map( + models.AllActivitystreamsObjectTypes, + func(t models.ActivitystreamsObjectType) string { return string(t) }, + ), + ) +} + +func createActitiystreamsActivityType(db *gorm.DB) error { + return migrateEnum( + db, + "activitystreams_activity_type", + sliceutils.Map( + models.AllActivitystreamsActivityTypes, + func(t models.ActivitystreamsActivityType) string { return string(t) }, + ), + ) +} + +func createCollectionTarget(db *gorm.DB) error { + return migrateEnum( + db, + "collection_target_type", + sliceutils.Map( + models.AllCollectionTargetTypes, + func(t models.CollectionTargetType) string { return string(t) }, + ), + ) +} + +// Helper function for ensuring the existence of an enum with the given values +func migrateEnum(db *gorm.DB, name string, values []string) error { + if err := db.Exec("DROP TYPE IF EXISTS " + name + " CASCADE;").Error; err != nil { + return other.Error( + "storage", + fmt.Sprintf("Failed to remove old type %s (if it exists)", name), + err, + ) + } + queryBuilder := strings.Builder{} + queryBuilder.WriteString("CREATE TYPE ") + queryBuilder.WriteString(name) + queryBuilder.WriteString(" AS ENUM (") + blen := len(values) + for i, btype := range values { + queryBuilder.WriteString("'" + string(btype) + "'") + // Append comma everywhere except last entry + if i+1 < blen { + queryBuilder.WriteString(",") + } + } + queryBuilder.WriteString(");") + if err := db.Exec(queryBuilder.String()).Error; err != nil { + return err + } + return nil +} diff --git a/temp.toml b/temp.toml deleted file mode 100644 index 7a0c008..0000000 --- a/temp.toml +++ /dev/null @@ -1,54 +0,0 @@ -[general] - protocol = "https" - domain = "serveo.net" - subdomain = "31b5744cc8c043386fe5c4ec2fb414f6" - private_port = 8080 - public_port = 443 - -[ssl] - handle_ssl = false - -[admin] - username = "server-admin" - first_time_setup_otp = "Example otp password" - profiling_password = "" - allow_registration = true - auth_fetch_for_get = false - auth_fetch_for_non_get = true - -[webauthn] - display_name = "Linstrom" - -[storage] - host = "localhost" - username = "linstrom" - password = "linstrom" - db_name = "linstrom" - port = 5432 - ssl_mode = "disable" - time_zone = "Europe/Berlin" - max_in_memory_cache_size = 1000000 - max_in_memory_cache_ttl = 5 - max_reconnect_attempts = 3 - -[mail] - host = "localhost" - port = 587 - username = "linstrom" - password = "linstrom" - -[self] - server_actor_display_name = "Server actor" - server_display_name = "Linstrom" - -[s3] - key_id = "GK458f9d7315fc6c9686c41045" - secret = "b6d1c8ec97052b8a40ae953de34f336170d85554fbe7875acce0ff51464724ee" - region = "garage" - endpoint = "http://localhost:3900" - use_ssl = false - bucket_name = "linstrom-bucket" - -[experimental] - use_ed25519_keys = false - auth_fetch_for_server_actor = false diff --git a/web/debug/media.go b/web/debug/media.go new file mode 100644 index 0000000..bd0219f --- /dev/null +++ b/web/debug/media.go @@ -0,0 +1,72 @@ +package webdebug + +import ( + "database/sql" + "net/http" + + webutils "git.mstar.dev/mstar/goutils/http" + "git.mstar.dev/mstar/goutils/sliceutils" + "github.com/rs/zerolog/hlog" + + "git.mstar.dev/mstar/linstrom/media" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" + "git.mstar.dev/mstar/linstrom/storage-new/models" +) + +func uploadMedia(w http.ResponseWriter, r *http.Request) { + log := hlog.FromRequest(r) + _ = r.ParseMultipartForm(10 << 20) // 10MB + + userId := r.FormValue("user-id") + blurred := r.FormValue("blurred") != "" + altText := r.FormValue("alt-text") + file, handler, err := r.FormFile("file") + if err != nil { + log.Warn().Err(err).Msg("Failed to get file from form") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusBadRequest) + return + } + defer func() { _ = file.Close() }() + _, err = media.GlobalServer.AddFile(file, handler.Filename, userId, blurred, altText) + if err != nil { + log.Error().Err(err).Msg("Failed to upload file to storage") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + } +} + +func forceMediaSync(w http.ResponseWriter, r *http.Request) { + go media.GlobalServer.ServiceEnsureFileSynchronisation() +} + +func getOwnedFiles(w http.ResponseWriter, r *http.Request) { + type File struct { + Name, Id, Mime string + } + type Outbound struct { + Files []File + } + log := hlog.FromRequest(r) + userId := r.FormValue("id") + + mm := dbgen.MediaMetadata + files, err := mm.Where(mm.OwnedById.Eq(sql.NullString{Valid: true, String: userId})). + Select(mm.Name, mm.Type, mm.ID). + Find() + if err != nil { + log.Error().Err(err).Str("user-id", userId).Msg("Failed to get files of user") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + err = webutils.SendJson(w, Outbound{ + Files: sliceutils.Map(files, func(t *models.MediaMetadata) File { + return File{ + Name: t.Name, + Id: t.ID, + Mime: t.Type, + } + }), + }) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal response to json") + } +} diff --git a/web/debug/posts.go b/web/debug/posts.go index 75f0bff..1f3419f 100644 --- a/web/debug/posts.go +++ b/web/debug/posts.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "slices" webutils "git.mstar.dev/mstar/goutils/http" "git.mstar.dev/mstar/goutils/sliceutils" @@ -23,10 +24,11 @@ import ( func postAs(w http.ResponseWriter, r *http.Request) { type Inbound struct { - Username string `json:"username"` - Content string `json:"content"` - ContentWarning *string `json:"content_warning"` - ReplyTo *string `json:"reply_to"` + Username string `json:"username"` + Content string `json:"content"` + ContentWarning *string `json:"content_warning"` + ReplyTo *string `json:"reply_to"` + Mentions []string `json:"mentions"` } log := hlog.FromRequest(r) dec := json.NewDecoder(r.Body) @@ -86,7 +88,9 @@ func postAs(w http.ResponseWriter, r *http.Request) { return } } - mentions := webshared.MentionsFromContent(data.Content) + mentions := sliceutils.RemoveDuplicate( + slices.Concat(webshared.MentionsFromContent(data.Content), data.Mentions), + ) dbPings := []*models.NoteToPing{} for _, mention := range mentions { accId, err := activitypub.ImportRemoteAccountByHandle(mention) @@ -122,6 +126,11 @@ func postAs(w http.ResponseWriter, r *http.Request) { return } err = tx.NoteToPing.Create(dbPings...) + if err != nil { + log.Error().Err(err).Msg("Failed to create note pings in db") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } activity := models.Activity{ Id: shared.NewId(), Type: string(models.ActivityCreate), diff --git a/web/debug/server.go b/web/debug/server.go index 99569ac..22c36f7 100644 --- a/web/debug/server.go +++ b/web/debug/server.go @@ -24,19 +24,29 @@ type Server struct { func New(addr string) *Server { handler := http.NewServeMux() - handler.HandleFunc("GET /non-deleted", getNonDeletedUsers) + handler.HandleFunc("POST /local-user", createLocalUser) - handler.HandleFunc("GET /delete", deleteUser) handler.HandleFunc("POST /post-as", postAs) - handler.HandleFunc("GET /notes-for", notesFrom) - handler.HandleFunc("GET /import-user", issueUserImport) handler.HandleFunc("GET /keys-for", returnKeypair) - handler.HandleFunc("GET /import-server", importServerInfo) + handler.HandleFunc("GET /import-user", issueUserImport) handler.HandleFunc("GET /request-follow", requestFollow) handler.HandleFunc("POST /send-as", proxyMessageToTarget) + handler.HandleFunc("POST /follow", requestFollow) + handler.HandleFunc("POST /update-user", updateUser) + + handler.HandleFunc("GET /non-deleted", getNonDeletedUsers) + handler.HandleFunc("GET /delete", deleteUser) + handler.HandleFunc("GET /notes-for", notesFrom) + handler.HandleFunc("GET /import-server", importServerInfo) handler.HandleFunc("GET /replies-to/{id}", inReplyTo) handler.HandleFunc("POST /fetch", requestAs) - handler.HandleFunc("POST /follow", requestFollow) + handler.HandleFunc("POST /like-note", likeNote) + handler.HandleFunc("POST /boost-note", boostNote) + + handler.HandleFunc("GET /files-owned-by", getOwnedFiles) + handler.HandleFunc("POST /upload-file", uploadMedia) + handler.HandleFunc("/force-media-sync", forceMediaSync) + web := http.Server{ Addr: addr, Handler: webutils.ChainMiddlewares( diff --git a/web/debug/users.go b/web/debug/users.go index 2fc8887..d0ed270 100644 --- a/web/debug/users.go +++ b/web/debug/users.go @@ -448,3 +448,53 @@ func requestAs(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(res.Body) _, _ = fmt.Fprint(w, string(body)) } + +func updateUser(w http.ResponseWriter, r *http.Request) { + type Inbound struct { + UserId string + Displayname *string + Description *string + RestrictedFollow *bool + } + log := hlog.FromRequest(r) + var data Inbound + err := json.NewDecoder(r.Body).Decode(&data) + if err != nil { + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusBadRequest) + return + } + queryStart := dbgen.User.Where(dbgen.User.ID.Eq(data.UserId)) + user, err := queryStart.First() + switch err { + case gorm.ErrRecordNotFound: + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusNotFound) + return + case nil: + default: + log.Error().Err(err).Msg("Db error while trying to fetch user for updating") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + updateNeeded := false + if data.Displayname != nil { + user.DisplayName = *data.Displayname + updateNeeded = true + } + if data.Description != nil { + user.Description = *data.Description + updateNeeded = true + } + if data.RestrictedFollow != nil { + user.RestrictedFollow = *data.RestrictedFollow + updateNeeded = true + } + if !updateNeeded { + return + } + err = queryStart.Save(user) + if err != nil { + log.Error().Err(err).Msg("Failed to update user with new data") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } +} diff --git a/web/public/api/activitypub/activityAnnounce.go b/web/public/api/activitypub/activityAnnounce.go index b8b64da..7c30024 100644 --- a/web/public/api/activitypub/activityAnnounce.go +++ b/web/public/api/activitypub/activityAnnounce.go @@ -1,23 +1 @@ package activitypub - -import ( - "context" - "time" -) - -// Announce is boost - -type activityAnnounceOut struct { - Context any `json:"@context,omitempty"` - Id string - Type string // Always "Announce" - Actor string // The one doing the boost - Published time.Time - To []string - CC []string - Object string // Link to object being boosted -} - -func announceFromStorage(ctx context.Context, id string) (*activityAnnounceOut, error) { - panic("not implemented") -} diff --git a/web/public/api/activitypub/activityLike.go b/web/public/api/activitypub/activityLike.go index 7c30024..a7dfd70 100644 --- a/web/public/api/activitypub/activityLike.go +++ b/web/public/api/activitypub/activityLike.go @@ -1 +1,5 @@ package activitypub + +import "net/http" + +func activityLike(w http.ResponseWriter, r *http.Request) {} diff --git a/web/public/api/activitypub/inbox.go b/web/public/api/activitypub/inbox.go index b425ffc..e05597c 100644 --- a/web/public/api/activitypub/inbox.go +++ b/web/public/api/activitypub/inbox.go @@ -8,12 +8,14 @@ import ( "io" "net/http" "regexp" + "slices" "strconv" "strings" "time" webutils "git.mstar.dev/mstar/goutils/http" "git.mstar.dev/mstar/goutils/other" + "git.mstar.dev/mstar/goutils/sliceutils" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog/hlog" "github.com/rs/zerolog/log" @@ -139,6 +141,108 @@ func userInbox(w http.ResponseWriter, r *http.Request) { } } +func globalInbox(w http.ResponseWriter, r *http.Request) { + log := hlog.FromRequest(r) + body, err := io.ReadAll(r.Body) + log.Trace(). + Err(err). + Bytes("body", body). + Any("headers", r.Header). + Msg("Global inbox message") + data := map[string]any{} + err = json.Unmarshal(body, &data) + if err != nil { + _ = webutils.ProblemDetails( + w, + http.StatusBadRequest, + "/errors/bad-request-data", + "Bad activity data", + other.IntoPointer("Body to inbox needs to be json"), + nil, + ) + return + } + if _, ok := data["@context"]; !ok { + _ = webutils.ProblemDetails( + w, + http.StatusBadRequest, + "/errors/bad-request-data", + "Bad activity data", + other.IntoPointer("Request data needs to contain context"), + nil, + ) + return + } + objectType, ok := data["type"].(string) + if !ok { + _ = webutils.ProblemDetails( + w, + http.StatusBadRequest, + "/errors/bad-request-data", + "Bad activity data", + other.IntoPointer(`Request data needs to contain a field "type" with a string value`), + nil, + ) + return + } + _, ok = data["id"].(string) + if !ok { + _ = webutils.ProblemDetails( + w, + http.StatusBadRequest, + "/errors/bad-request-data", + "Bad activity data", + other.IntoPointer(`Request data needs to contain a field "id" with a string value`), + nil, + ) + return + } + log.Debug().Str("object-type", objectType).Msg("Inbox message") + log.Trace().Bytes("body", body).Msg("Inbox raw message") + // TODO: Decide how to handle the handler failing for whatever reason + // Add object to unhandled message table and try again later? + // Discard it? And how would a handler return that it failed? + ok = true + switch objectType { + case "Like": + ok = handleLike(w, r, data) + case "Undo": + ok = handleUndo(w, r, data) + case "Accept": + ok = handleAccept(w, r, data) + case "Reject": + ok = handleReject(w, r, data) + case "Create": + ok = handleCreate(w, r, data) + default: + log.Warn(). + Str("object-type", objectType). + Msg("Unknown message type, storing for later processing") + err = dbgen.UnhandledMessage.Create(&models.UnhandledMessage{ + ForUserId: "", + GlobalInbox: true, + RawData: body, + }) + if err != nil { + log.Error().Err(err).Msg("Failed to store unhandled message for later") + } + _ = webutils.ProblemDetailsStatusOnly(w, 500) + } + if !ok { + err = dbgen.UnhandledMessage.Create(&models.UnhandledMessage{ + RawData: body, + ForUserId: "", + GlobalInbox: true, + }) + if err != nil { + log.Error(). + Err(err). + Bytes("body", body). + Msg("Failed to store failed global inbound message for later processing") + } + } +} + func handleLike(w http.ResponseWriter, r *http.Request, object map[string]any) bool { log := hlog.FromRequest(r) activityId := object["id"].(string) @@ -213,7 +317,7 @@ func handleLike(w http.ResponseWriter, r *http.Request, object map[string]any) b Reactor: *liker, ReactorId: liker.ID, Emote: nil, - EmoteId: sql.NullInt64{Valid: false}, + EmoteId: sql.NullString{Valid: false}, } tx := dbgen.Q.Begin() @@ -780,13 +884,94 @@ func handleCreate(w http.ResponseWriter, r *http.Request, object map[string]any) dbNote.RepliesTo = sql.NullString{Valid: true, String: replyUrl} } } - feed, err := dbgen.Feed.Where(dbgen.Feed.OwnerId.Eq(targetUserId), dbgen.Feed.IsDefault.Is(true)). - First() - if err != nil { - log.Error().Err(err).Msg("Failed to get feed for targeted user inbox") - _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) - return false + totalNoteTargets := slices.Concat(objectNote.To, objectNote.CC) + // TODO: Find all feeds this note needs to be added to + // Depends on note targets (cc & to tags) + // Includes specific user IDs, nothing else -> DMs, only the personal feeds of those users + // Includes https://www.w3.org/ns/activitystreams#Public (& creator follower list) -> + // Public feed (unlisted has same combo, but iirc doesn't get sent out by server) + // Includes only creator follower list -> Follower only -> only feeds that explicitly follow creator + + u2u := dbgen.UserToUserRelation + targetFeeds := []models.Feed{} + if sliceutils.Contains(totalNoteTargets, "https://www.w3.org/ns/activitystreams#Public") { + // Public post, add to global and following feeds + dbNote.AccessLevel = models.NOTE_TARGET_PUBLIC + followerIds, err := u2u.GetLocalFollowerIdsOfId(actingUser.ID) + if err != nil { + log.Error(). + Err(err). + Str("follow-target", actingUser.ID). + Msg("Failed to get ids for followers") + return false + } + userFeeds, err := dbgen.Feed.Where(dbgen.Feed.OwnerId.In(followerIds...)).Find() + if err != nil { + log.Error(). + Err(err). + Str("follow-target", actingUser.ID). + Strs("follower-ids", followerIds). + Msg("Failed to get feeds for followers") + return false + } + globalFeed, err := dbgen.Feed.Where(dbgen.Feed.Name.Eq(models.GlobalFeedName)).First() + if err != nil { + log.Error(). + Err(err). + Msg("Failed to get global feed") + return false + } + targetFeeds = slices.Concat( + targetFeeds, + sliceutils.Map(userFeeds, func(t *models.Feed) models.Feed { return *t }), + ) + targetFeeds = append(targetFeeds, *globalFeed) + } else { + if sliceutils.ContainsFunc(totalNoteTargets, func(x string) bool { + return strings.HasPrefix(x, actingUser.ID) + }) { + // Contains an url starting with the author's id, so assume that it's for followers + dbNote.AccessLevel = models.NOTE_TARGET_FOLLOWERS + followerIds, err := u2u.GetLocalFollowerIdsOfId(actingUser.ID) + if err != nil { + log.Error(). + Err(err). + Str("follow-target", actingUser.ID). + Msg("Failed to get ids for followers") + return false + } + userFeeds, err := dbgen.Feed.Where(dbgen.Feed.OwnerId.In(followerIds...)).Find() + if err != nil { + log.Error(). + Err(err). + Str("follow-target", actingUser.ID). + Strs("follower-ids", followerIds). + Msg("Failed to get feeds for followers") + return false + } + targetFeeds = sliceutils.Map(userFeeds, func(t *models.Feed) models.Feed { return *t }) + } else { + // Neither followers collection url nor public marker, private message + dbNote.AccessLevel = models.NOTE_TARGET_DM + userFeeds, err := dbgen.Feed. + LeftJoin(dbgen.User, dbgen.User.ID.EqCol(dbgen.Feed.OwnerId)). + LeftJoin(dbgen.RemoteServer, dbgen.RemoteServer.ID.EqCol(dbgen.User.ServerId)). + LeftJoin(dbgen.UserRemoteLinks, dbgen.UserRemoteLinks.ID.EqCol(dbgen.User.RemoteInfoId)). + Where(dbgen.RemoteServer.IsSelf.Is(true)).Where( + dbgen.User.ID.In(totalNoteTargets...), + ).Or(dbgen.UserRemoteLinks.ApLink.In(totalNoteTargets...)).Find() + if err != nil { + log.Error(). + Err(err). + Str("follow-target", actingUser.ID). + Strs("targeted-ids", totalNoteTargets). + Msg("Failed to get feeds for directly messaged users") + return false + } + targetFeeds = sliceutils.Map(userFeeds, func(t *models.Feed) models.Feed { return *t }) + } } + tx := dbgen.Q.Begin() err = tx.Note.Create(&dbNote) if err != nil { @@ -801,6 +986,7 @@ func handleCreate(w http.ResponseWriter, r *http.Request, object map[string]any) ObjectId: dbNote.ID, ObjectType: uint32(models.ActivitystreamsActivityTargetNote), } + err = tx.Activity.Create(&createActivity) if err != nil { _ = tx.Rollback() @@ -808,17 +994,25 @@ func handleCreate(w http.ResponseWriter, r *http.Request, object map[string]any) _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) return false } - err = tx.NoteToFeed.Create(&models.NoteToFeed{ - Reason: string(models.FeedAppearanceReasonFollowUser), - NoteId: dbNote.ID, - FeedId: uint64(feed.ID), + + feedRels := sliceutils.Map(targetFeeds, func(f models.Feed) *models.NoteToFeed { + return &models.NoteToFeed{ + Reason: string(models.FeedAppearanceReasonFollowUser), + NoteId: dbNote.ID, + FeedId: uint64(f.ID), + } }) + err = tx.NoteToFeed.Create(feedRels...) if err != nil { _ = tx.Rollback() - log.Error().Err(err).Any("note", dbNote).Msg("Failed to create note to feed relation in db") + log.Error(). + Err(err). + Any("note", dbNote). + Msg("Failed to create note to feed relations in db") _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) return false } + err = tx.Commit() if err != nil { log.Error().Err(err).Any("note", dbNote).Msg("Failed to submit note creation in db") diff --git a/web/public/api/activitypub/inboxUndo.go b/web/public/api/activitypub/inboxUndo.go index a793047..7e47943 100644 --- a/web/public/api/activitypub/inboxUndo.go +++ b/web/public/api/activitypub/inboxUndo.go @@ -113,8 +113,7 @@ func undoLike(w http.ResponseWriter, r *http.Request, object map[string]any, tar _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) return false } - reactionId := uint(other.Must(strconv.ParseUint(act.ObjectId, 10, 64))) - reaction, err := dbgen.Reaction.Where(dbgen.Reaction.ID.Eq(reactionId)).First() + reaction, err := dbgen.Reaction.Where(dbgen.Reaction.ID.Eq(act.ObjectId)).First() switch err { case gorm.ErrRecordNotFound: return true @@ -140,7 +139,7 @@ func undoLike(w http.ResponseWriter, r *http.Request, object map[string]any, tar _ = tx.Rollback() log.Error(). Err(err). - Uint("reaction-id", reaction.ID). + Str("reaction-id", reaction.ID). Msg("Failed to delete reaction on undo") _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) return false diff --git a/web/public/api/activitypub/objectEmote.go b/web/public/api/activitypub/objectEmote.go new file mode 100644 index 0000000..df2c00a --- /dev/null +++ b/web/public/api/activitypub/objectEmote.go @@ -0,0 +1,5 @@ +package activitypub + +import "net/http" + +func objectEmote(w http.ResponseWriter, r *http.Request) {} diff --git a/web/public/api/activitypub/server.go b/web/public/api/activitypub/server.go index bc9fbd4..b165399 100644 --- a/web/public/api/activitypub/server.go +++ b/web/public/api/activitypub/server.go @@ -16,7 +16,10 @@ func BuildActivitypubRouter() http.Handler { router.HandleFunc("GET /activity/reject/{id}", activityReject) router.HandleFunc("GET /activity/update/{id}", activityUpdate) router.HandleFunc("GET /activity/follow/{id}", activityFollow) + router.HandleFunc("GET /activity/like/{id}", activityLike) router.HandleFunc("GET /note/{id}", objectNote) router.HandleFunc("GET /note/{id}/reactions", noteReactions) + router.HandleFunc("GET /emote/{id}", objectEmote) + router.HandleFunc("POST /inbox", globalInbox) return router } diff --git a/web/public/media.go b/web/public/media.go new file mode 100644 index 0000000..e098855 --- /dev/null +++ b/web/public/media.go @@ -0,0 +1,44 @@ +package webpublic + +import ( + "io" + "net/http" + + webutils "git.mstar.dev/mstar/goutils/http" + "github.com/rs/zerolog/hlog" + + "git.mstar.dev/mstar/linstrom/media" + "git.mstar.dev/mstar/linstrom/storage-new/dbgen" +) + +func downloadMediaHander(w http.ResponseWriter, r *http.Request) { + mediaId := r.PathValue("id") + log := hlog.FromRequest(r) + mm := dbgen.MediaMetadata + fileReader, err := media.GlobalServer.ReadFileId(mediaId) + switch err { + case nil: + case media.ErrFileNotFound: + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusNotFound) + return + default: + log.Error().Err(err).Str("file-id", mediaId).Msg("Failed to get file") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + defer func() { _ = fileReader.Close() }() + meta, err := mm.Where(mm.ID.Eq(mediaId)).Select(mm.Type).First() + if err != nil { + log.Error(). + Err(err). + Str("file-id", mediaId). + Msg("Failed to get file metadata after already getting file") + _ = webutils.ProblemDetailsStatusOnly(w, http.StatusInternalServerError) + return + } + w.Header().Add("Content-Type", meta.Type) + _, err = io.Copy(w, fileReader) + if err != nil { + log.Error().Err(err).Msg("Failed to copy file to request") + } +} diff --git a/web/public/server.go b/web/public/server.go index 944f716..e3b2953 100644 --- a/web/public/server.go +++ b/web/public/server.go @@ -56,6 +56,7 @@ func New(addr string, duckFs fs.FS) *Server { handler.HandleFunc("GET /errors/{name}", errorTypeHandler) handler.HandleFunc("GET /default-image", buildServeDefaultImage(duckFs)) handler.HandleFunc("GET /default-image.webp", buildServeDefaultImage(duckFs)) + handler.HandleFunc("GET /media/{id}", downloadMediaHander) rootHandler := webutils.ChainMiddlewares( handler, webutils.BuildLoggingMiddleware( diff --git a/web/shared/clientRfc9421.go b/web/shared/clientRfc9421.go index c306611..7b142a8 100644 --- a/web/shared/clientRfc9421.go +++ b/web/shared/clientRfc9421.go @@ -236,7 +236,7 @@ func postPostRequest(resp *http.Response, reqErr error, dbId uint64) { // If response status is ok (< 400) delete entry in db to not process it again if resp.StatusCode < 400 { update = false - fr.Where(fr.Id.Eq(dbId)).Delete() + _, _ = fr.Where(fr.Id.Eq(dbId)).Delete() return } if resp.StatusCode == 429 {