package activitypub import ( "database/sql" "encoding/json" "errors" "fmt" "io" "strings" "time" "git.mstar.dev/mstar/goutils/other" "github.com/rs/zerolog/log" "gorm.io/gorm" "git.mstar.dev/mstar/linstrom/config" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" "git.mstar.dev/mstar/linstrom/storage-new/models" webshared "git.mstar.dev/mstar/linstrom/web/shared" ) const DefaultMaxImportRecursion = 100 var ErrMaxImportRecursionReached = errors.New("maximum import recursion reached") func ImportRemoteNote(noteId string, requester *models.User) (string, error) { return importRemoteNoteRecursive(noteId, requester, 0) } func importRemoteNoteRecursive( noteId string, requester *models.User, recursionDepth uint, ) (string, error) { type NoteTag struct { Type string `json:"type"` Href string `json:"href"` Name string `json:"name"` } type Note struct { Type string `json:"type"` Id string `json:"id"` Summary *string `json:"summary"` Content string `json:"content"` MkContent *string `json:"_misskey_content"` Published time.Time `json:"published"` To []string `json:"to"` Cc []string `json:"cc"` InReplyTo *string `json:"inReplyTo"` Sensitive bool `json:"sensitive"` AttributedTo string `json:"attributedTo"` Tags []NoteTag `json:"tag"` } // TODO: Decide whether the max recursion depth can be configured via config file if recursionDepth > DefaultMaxImportRecursion { return "", ErrMaxImportRecursionReached } // No need to import local notes. Ids of local notes either have the full public url as prefix // or no http prefix at all (internal id only) if strings.HasPrefix(noteId, config.GlobalConfig.General.GetFullPublicUrl()) || !strings.HasPrefix(noteId, "http") { switch _, err := dbgen.Note.Where(dbgen.Note.ID.Eq(noteId)).First(); err { case nil: return noteId, nil case gorm.ErrRecordNotFound: return "", other.Error("activitypub", "local note doesn't exist", err) default: return "", other.Error( "activitypub", "failed to check for existence of local note", err, ) } } res, _, err := webshared.RequestSigned("GET", noteId, nil, requester) if err != nil { return "", other.Error("activitypub", "failed to request object", err) } if res.StatusCode != 200 { return "", fmt.Errorf("activitypub: invalid status code: %v", res.StatusCode) } body, _ := io.ReadAll(res.Body) _ = res.Body.Close() data := Note{} err = json.Unmarshal(body, &data) if err != nil { return "", other.Error("activitypub", "json unmarshalling failed", err) } if data.Type != "Note" { return "", fmt.Errorf("activitypub: invalid object type: %q", data.Type) } // (Re-)Import account this note belongs to _, err = ImportRemoteAccountByAPUrl(data.AttributedTo) if err != nil { return "", other.Error("activitypub", "failed to import note author", err) } // If the note already exists in storage, update that dbNote, err := dbgen.Note.Where(dbgen.Note.ID.Eq(data.Id)).First() switch err { case nil: case gorm.ErrRecordNotFound: // Otherwise create a new one dbNote = &models.Note{ ID: data.Id, CreatorId: data.AttributedTo, } default: return "", other.Error("activitypub", "failed to check db for note", err) } dbNote.CreatedAt = data.Published if data.Summary != nil { dbNote.ContentWarning = sql.NullString{Valid: true, String: *data.Summary} } else { dbNote.ContentWarning = sql.NullString{Valid: false} } // Prefer raw misskey content if set since it *should* // have the most accurate content data if data.MkContent != nil { dbNote.RawContent = *data.MkContent } else { dbNote.RawContent = data.Content } if data.InReplyTo != nil { dbNote.RepliesTo = sql.NullString{Valid: true, String: *data.InReplyTo} defer func() { _, err := importRemoteNoteRecursive(*data.InReplyTo, requester, recursionDepth+1) switch err { case ErrMaxImportRecursionReached: log.Warn(). Str("reply-id", data.Id). Str("in-reply-to-id", *data.InReplyTo). Msg("Hit recursion limit while importing note") case nil: default: log.Error(). Err(err). Str("id", *data.InReplyTo). Msg("Failed to import note that was replied to") } // Don't return import errors of recursive imports since they don't affect this import }() } else { dbNote.RepliesTo = sql.NullString{Valid: false} } // TODO: Include more data like attachments err = dbgen.Note.Save(dbNote) if err != nil { return "", err } // Handle tags after the initial note since stored in separate tables and pings require more imports hashtags := []*models.NoteTag{} pings := []*models.NoteToPing{} for _, tag := range data.Tags { switch tag.Type { case "Mention": _, err := ImportRemoteAccountByAPUrl(tag.Href) if err != nil { return "", err } pings = append(pings, &models.NoteToPing{ NoteId: dbNote.ID, PingTargetId: tag.Href, }) case "Hashtag": hashtags = append(hashtags, &models.NoteTag{ NoteId: dbNote.ID, Tag: tag.Name, TagUrl: tag.Href, }) default: log.Warn().Str("tag-type", tag.Type).Msg("Unknown tag type") } } // FIXME: This is bad, what if it's a note update and not a new one? // For new notes this is fine, but existing ones might already have attachments. // In which case, you need to remove tags that don't exist anymore // and only create the ones not yet stored err = dbgen.NoteToPing.Save(pings...) err = dbgen.NoteTag.Save(hashtags...) return dbNote.ID, nil }