More work on note imports

activitypub/importNote.go

@@ -1,12 +1,15 @@
 package activitypub
 
 import (
+	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"time"
 
 	"git.mstar.dev/mstar/goutils/other"
+	"github.com/rs/zerolog/log"
 	"gorm.io/gorm"
 
 	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
@@ -14,19 +17,35 @@ import (
 	webshared "git.mstar.dev/mstar/linstrom/web/shared"
 )
 
+const DefaultMaxImportRecursion = 100
+
+var ErrMaxImportRecursionReached = errors.New("maximum import recursion reached")
+
 func ImportRemoteNote(noteId string, requester *models.User) (string, error) {
+	return importRemoteNoteRecursive(noteId, requester, 0)
+}
+
+func importRemoteNoteRecursive(
+	noteId string,
+	requester *models.User,
+	recursionDepth uint,
+) (string, error) {
 	type Note struct {
-		Type         string
+		Type         string    `json:"type"`
-		Id           string
+		Id           string    `json:"id"`
-		Summary      *string
+		Summary      *string   `json:"summary"`
-		Content      string
+		Content      string    `json:"content"`
-		MkContent    *string `json:"_misskey_content"`
+		MkContent    *string   `json:"_misskey_content"`
-		Published    time.Time
+		Published    time.Time `json:"published"`
-		To           []string
+		To           []string  `json:"to"`
-		Cc           []string
+		Cc           []string  `json:"cc"`
-		InReplyTo    *string
+		InReplyTo    *string   `json:"inReplyTo"`
-		Sensitive    bool
+		Sensitive    bool      `json:"sensitive"`
-		AttributedTo string
+		AttributedTo string    `json:"attributedTo"`
+	}
+	// TODO: Decide whether the max recursion depth can be configured via config file
+	if recursionDepth > DefaultMaxImportRecursion {
+		return "", ErrMaxImportRecursionReached
 	}
 	res, _, err := webshared.RequestSigned("GET", noteId, nil, requester)
 	if err != nil {
@@ -56,11 +75,50 @@ func ImportRemoteNote(noteId string, requester *models.User) (string, error) {
 	case gorm.ErrRecordNotFound:
 		dbNote = &models.Note{
 			ID:        data.Id,
-			CreatedAt: data.Published,
 			CreatorId: data.AttributedTo,
 		}
 	default:
 		return "", other.Error("activitypub", "failed to check db for note", err)
 	}
+	dbNote.CreatedAt = data.Published
+	if data.Summary != nil {
+		dbNote.ContentWarning = sql.NullString{Valid: true, String: *data.Summary}
+	} else {
+		dbNote.ContentWarning = sql.NullString{Valid: false}
+	}
+	// Prefer raw misskey content if set since it *should*
+	// have the most accurate content data
+	if data.MkContent != nil {
+		dbNote.RawContent = *data.MkContent
+	} else {
+		dbNote.RawContent = data.Content
+	}
+	if data.InReplyTo != nil {
+		dbNote.RepliesTo = sql.NullString{Valid: true, String: *data.InReplyTo}
+		defer func() {
+			_, err := importRemoteNoteRecursive(*data.InReplyTo, requester, recursionDepth+1)
+			switch err {
+			case ErrMaxImportRecursionReached:
+				log.Warn().
+					Str("reply-id", data.Id).
+					Str("in-reply-to-id", *data.InReplyTo).
+					Msg("Hit recursion limit while importing note")
+			case nil:
+			default:
+				log.Error().
+					Err(err).
+					Str("id", *data.InReplyTo).
+					Msg("Failed to import note that was replied to")
+			}
+			// Don't return import errors of recursive imports since they don't affect this import
+		}()
+	} else {
+		dbNote.RepliesTo = sql.NullString{Valid: false}
+	}
+	// TODO: Include more data like attachments
+	err = dbgen.Note.Save(dbNote)
+	if err != nil {
+		return "", err
+	}
 	return dbNote.ID, nil
 }