Add debug handler for fetching a remote actor

Will be used later to add to internal db

activitypub/import.go

@@ -0,0 +1,101 @@
+package activitypub
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"time"
+
+	"git.mstar.dev/mstar/goutils/sliceutils"
+	"github.com/rs/zerolog/log"
+
+	apshared "git.mstar.dev/mstar/linstrom/activitypub/shared"
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+	webshared "git.mstar.dev/mstar/linstrom/web/shared"
+)
+
+func ImportRemoteAccount(targetName string) (string, error) {
+	type InboundUserKey struct {
+		Id    string `json:"id"`
+		Owner string `json:"owner"`
+		Pem   string `json:"publicKeyPem"`
+	}
+	type InboundUserMedia struct {
+		Type      string `json:"type"`
+		Url       string `json:"url"`
+		MediaType string `json:"mediaType"`
+	}
+	type InboundUser struct {
+		Id                string            `json:"id"`
+		Type              string            `json:"type"`
+		PreferredUsername string            `json:"preferredUsername"`
+		Inbox             string            `json:"inbox"`
+		PublicKey         *InboundUserKey   `json:"publicKey"`
+		Published         *time.Time        `json:"published"`
+		DisplayName       *string           `json:"name"`
+		Description       *string           `json:"summary,omitempty"`
+		PublicUrl         *string           `json:"url"`
+		Icon              *InboundUserMedia `json:"icon,omitempty"`
+		Banner            *InboundUserMedia `json:"image,omitempty"`
+		Discoverable      *bool             `json:"discoverable"`
+		Location          *string           `json:"vcard:Address,omitempty"`
+		Birthday          *string           `json:"vcard:bday,omitempty"`
+		SpeakAsCat        bool              `json:"speakAsCat"`
+		IsCat             bool              `json:"isCat"`
+		RestrictedFollow  *bool             `json:"manuallyApprovesFollowers"`
+	}
+	// Get the target user's link first
+	webfinger, err := apshared.GetAccountWebfinger(targetName)
+	if err != nil {
+		return "", err
+	}
+	selfLinks := sliceutils.Filter(webfinger.Links, func(t apshared.LinkData) bool {
+		return t.Relation == "self"
+	})
+	if len(selfLinks) == 0 {
+		return "", errors.New("No self link")
+	}
+	APLink := selfLinks[0]
+	req, err := http.NewRequest("GET", *APLink.Href, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Accept", "application/activity+json")
+
+	// Server actor key for signing
+	linstromActor, err := dbgen.User.Where(dbgen.User.Username.Eq("linstrom")).First()
+	if err != nil {
+		return "", err
+	}
+	var keyBytes []byte
+	if config.GlobalConfig.Experimental.UseEd25519Keys {
+		keyBytes = linstromActor.PrivateKeyEd
+	} else {
+		keyBytes = linstromActor.PrivateKeyRsa
+	}
+
+	// Sign and send
+	err = webshared.SignRequest(req, linstromActor.ID+"#main-key", keyBytes, nil)
+	if err != nil {
+		return "", err
+	}
+	response, err := webshared.RequestClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 {
+		return "", errors.New("Bad status")
+	}
+	var data InboundUser
+	body, _ := io.ReadAll(response.Body)
+	log.Info().Bytes("body", body).Msg("Body from request")
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return "", err
+	}
+	log.Info().Any("received-data", data).Msg("Response data")
+	return "", nil
+}

activitypub/shared/types/context.go

@@ -0,0 +1,47 @@
+package types
+
+var BaseLdContext = []any{
+	"https://www.w3.org/ns/activitystreams",
+	"https://w3id.org/security/v1",
+	map[string]any{
+		"Key":                                   "sec:Key",
+		"manuallyApprovesFollowers":             "as:manuallyApprovesFollowers",
+		"sensitive":                             "as:sensitive",
+		"Hashtag":                               "as:Hashtag",
+		"quoteUrl":                              "as:quoteUrl",
+		"fedibird":                              "http://fedibird.com/ns#",
+		"quoteUri":                              "fedibird:quoteUri",
+		"toot":                                  "http://joinmastodon.org/ns#",
+		"Emoji":                                 "toot:Emoji",
+		"featured":                              "toot:featured",
+		"discoverable":                          "toot:discoverable",
+		"indexable":                             "toot:indexable",
+		"schema":                                "http://schema.org#",
+		"PropertyValue":                         "schema:PropertyValue",
+		"value":                                 "schema:value",
+		"misskey":                               "https://misskey-hub.net/ns#",
+		"_misskey_content":                      "misskey:_misskey_content",
+		"_misskey_quote":                        "misskey:_misskey_quote",
+		"_misskey_reaction":                     "misskey:_misskey_reaction",
+		"_misskey_votes":                        "misskey:_misskey_votes",
+		"_misskey_summary":                      "misskey:_misskey_summary",
+		"_misskey_followedMessage":              "misskey:_misskey_followedMessage",
+		"_misskey_requireSigninToViewContents":  "misskey:_misskey_requireSigninToViewContents",
+		"_misskey_makeNotesFollowersOnlyBefore": "misskey:_misskey_makeNotesFollowersOnlyBefore",
+		"_misskey_makeNotesHiddenBefore":        "misskey:_misskey_makeNotesHiddenBefore",
+		"_misskey_license":                      "misskey:_misskey_license",
+		"freeText": map[string]string{
+			"@id":   "misskey:freeText",
+			"@type": "schema:text",
+		},
+		"isCat":            "misskey:isCat",
+		"firefish":         "https://joinfirefish.org/ns#",
+		"speakAsCat":       "firefish:speakAsCat",
+		"sharkey":          "https://joinsharkey.org/ns#",
+		"hideOnlineStatus": "sharkey:hideOnlineStatus",
+		"backgroundUrl":    "sharkey:backgroundUrl",
+		"listenbrainz":     "sharkey:listenbrainz",
+		"enableRss":        "sharkey:enableRss",
+		"vcard":            "http://www.w3.org/2006/vcard/ns#",
+	},
+}

activitypub/shared/util.go

@@ -1,4 +1,4 @@
-package ap
+package apshared
 
 import "strings"
 
@@ -13,7 +13,7 @@ func (i InvalidFullHandleError) Error() string {
 func SplitFullHandle(full string) (string, string, error) {
 	splits := strings.Split(strings.TrimPrefix(full, "@"), "@")
 	if len(splits) != 2 {
-		return "", "", InvalidFullHandleError{}
+		return "", "", InvalidFullHandleError{full}
 	}
 	return splits[0], splits[1], nil
 }

activitypub/shared/webfinger.go

@@ -1,32 +1,37 @@
-package ap
+package apshared
 
 import (
 	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
-	"time"
+
+	webshared "git.mstar.dev/mstar/linstrom/web/shared"
 )
 
+var ErrNoApUrl = errors.New("no Activitypub url in webfinger")
+
+type LinkData struct {
+	// What type of link this is
+	// - `self` refers to the activitypub object and has Type and Href set
+	// - `http://webfinger.net/rel/profile-page` refers to the public webpage of the account and has Type and Href set
+	// - `http://ostatus.org/schema/1.0/subscribe` provides a template for subscribing/following the account. Has Template set
+	//   Template will contain a `{uri}` part with which to replace idk yet
+	Relation string `json:"rel"`
+	// The content type of the url
+	Type *string `json:"type"`
+	// The url
+	Href *string `json:"href"`
+	// Template to use for something
+	Template *string `json:"template"`
+}
+
 // Data returned from a webfinger response (and also sent when asked for an account via webfinger)
 type WebfingerData struct {
 	// What this webfinger data refers to. Accounts are usually `acct:username@host`
 	Subject string `json:"subject"`
 	// List of links related to the account
-	Links []struct {
+	Links []LinkData `json:"links"`
-		// What type of link this is
-		// - `self` refers to the activitypub object and has Type and Href set
-		// - `http://webfinger.net/rel/profile-page` refers to the public webpage of the account and has Type and Href set
-		// - `http://ostatus.org/schema/1.0/subscribe` provides a template for subscribing/following the account. Has Template set
-		//   Template will contain a `{uri}` part with which to replace idk yet
-		Relation string `json:"rel"`
-		// The content type of the url
-		Type *string `json:"type"`
-		// The url
-		Href *string `json:"href"`
-		// Template to use for something
-		Template *string `json:"template"`
-	} `json:"links"`
 }
 
 var ErrAccountNotFound = errors.New("account not found")
@@ -53,10 +58,8 @@ func GetAccountWebfinger(fullHandle string) (*WebfingerData, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Make a http client with a timeout limit of 30 seconds
-	client := http.Client{Timeout: time.Second * 30}
 	// Then send the request
-	result, err := client.Do(webfingerRequest)
+	result, err := webshared.RequestClient.Do(webfingerRequest)
 	if err != nil {
 		return nil, err
 	}

ap/getRemoteUser.go

@@ -1,85 +0,0 @@
-package ap
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"git.mstar.dev/mstar/goap"
-)
-
-var ErrNoApUrl = errors.New("no Activitypub url in webfinger")
-
-func GetRemoteUser(fullHandle string) (goap.BaseApChain, error) {
-	webfinger, err := GetAccountWebfinger(fullHandle)
-	if err != nil {
-		return nil, err
-	}
-
-	apUrl := ""
-	for _, link := range webfinger.Links {
-		if link.Relation == "self" {
-			apUrl = *link.Href
-		}
-	}
-	if apUrl == "" {
-		return nil, ErrNoApUrl
-	}
-	apRequest, err := http.NewRequest("GET", apUrl, nil)
-	if err != nil {
-		return nil, err
-	}
-	apRequest.Header.Add("Accept", "application/activity+json,application/ld+json,application/json")
-	client := http.Client{Timeout: time.Second * 30}
-	res, err := client.Do(apRequest)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("bad status code: %d", res.StatusCode)
-	}
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	apObject, _ := goap.Unmarshal(body, nil, nil)
-	// Check if Id exists
-	if _, ok := goap.FindAttribute[*goap.UDIdData](apObject); !ok {
-		return nil, fmt.Errorf("missing attribute for account: Id")
-	}
-	// Check that it has the correct object type for an account
-	if objTypePtr, ok := goap.FindAttribute[*goap.UDTypeData](apObject); !ok {
-		return nil, fmt.Errorf("missing attribute for account: Type")
-	} else if objType := *objTypePtr; objType.Type != goap.KEY_ACTIVITYSTREAMS_ACTOR {
-		return nil, fmt.Errorf("wrong ap object type: %s", objType.Type)
-	}
-	// And finally check for inbox
-	if _, ok := goap.FindAttribute[*goap.W3InboxData](apObject); !ok {
-		return nil, fmt.Errorf("missing attribute for account: Inbox")
-	}
-	return apObject, nil
-}
-
-func GetRemoteObject(target string) (goap.BaseApChain, error) {
-	apRequest, err := http.NewRequest("GET", target, nil)
-	if err != nil {
-		return nil, err
-	}
-	apRequest.Header.Add("Accept", "application/activity+json,application/ld+json,application/json")
-	client := http.Client{Timeout: time.Second * 30}
-	res, err := client.Do(apRequest)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("bad status code: %d", res.StatusCode)
-	}
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	apObject, _ := goap.Unmarshal(body, nil, nil)
-	return apObject, nil
-}

config/config.go

@@ -202,6 +202,9 @@ var defaultConfig Config = Config{
 	},
 }
 
+// Get the full domain name of the server, as specified via the config.
+//
+// Example: "git.mstar.dev" (with subdomain = "git" and domain = "mstar.dev")
 func (gc *ConfigGeneral) GetFullDomain() string {
 	if gc.Subdomain != nil {
 		return *gc.Subdomain + gc.Domain
@@ -209,6 +212,11 @@ func (gc *ConfigGeneral) GetFullDomain() string {
 	return gc.Domain
 }
 
+// Get the public root url of the server, including port (if needed, public takes precedence if set)
+// and set protocol
+//
+// Example: "http://git.mstar.dev:5722" (with Subdomain = "git", domain = "mstar.dev",
+// privatePort = 34546, publicPort = 5722 and protocol = "http")
 func (gc *ConfigGeneral) GetFullPublicUrl() string {
 	str := gc.Protocol + "://" + gc.GetFullDomain()
 	if gc.PublicPort != nil {

storage/user.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"git.mstar.dev/mstar/linstrom/ap"
+	ap "git.mstar.dev/mstar/linstrom/activitypub/shared"
 	"git.mstar.dev/mstar/linstrom/shared"
 	"github.com/go-webauthn/webauthn/webauthn"
 	"github.com/google/uuid"

web/debug/server.go

@@ -27,6 +27,7 @@ func New(addr string) *Server {
 	handler.HandleFunc("GET /delete", deleteUser)
 	handler.HandleFunc("POST /post-as", postAs)
 	handler.HandleFunc("GET /notes-for", notesFrom)
+	handler.HandleFunc("GET /import", issueUserImport)
 	web := http.Server{
 		Addr: addr,
 		Handler: webutils.ChainMiddlewares(

web/debug/users.go

@@ -13,6 +13,7 @@ import (
 	"git.mstar.dev/mstar/goutils/sliceutils"
 	"github.com/rs/zerolog/hlog"
 
+	"git.mstar.dev/mstar/linstrom/activitypub"
 	"git.mstar.dev/mstar/linstrom/shared"
 	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
 	"git.mstar.dev/mstar/linstrom/storage-new/models"
@@ -171,3 +172,19 @@ func deleteUser(w http.ResponseWriter, r *http.Request) {
 	dbgen.User.Where(dbgen.User.ID.Eq(id)).Delete()
 	w.WriteHeader(http.StatusOK)
 }
+
+func issueUserImport(w http.ResponseWriter, r *http.Request) {
+	target := r.FormValue("target")
+	_, err := activitypub.ImportRemoteAccount(target)
+	hlog.FromRequest(r).Info().Err(err).Msg("Err from import request")
+}
+
+func kickoffFollow(w http.ResponseWriter, r *http.Request) {
+	type Inbound struct {
+		Id     string
+		Target string
+	}
+	var data Inbound
+	dec := json.NewDecoder(r.Body)
+	dec.Decode(&data)
+}

web/public/api/activitypub/user.go

@@ -12,6 +12,7 @@ import (
 	"git.mstar.dev/mstar/goutils/sliceutils"
 	"github.com/rs/zerolog/hlog"
 
+	"git.mstar.dev/mstar/linstrom/activitypub/shared/types"
 	"git.mstar.dev/mstar/linstrom/config"
 	"git.mstar.dev/mstar/linstrom/storage-new"
 	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
@@ -19,52 +20,6 @@ import (
 	webshared "git.mstar.dev/mstar/linstrom/web/shared"
 )
 
-var baseLdContext = []any{
-	"https://www.w3.org/ns/activitystreams",
-	"https://w3id.org/security/v1",
-	map[string]any{
-		"Key":                                   "sec:Key",
-		"manuallyApprovesFollowers":             "as:manuallyApprovesFollowers",
-		"sensitive":                             "as:sensitive",
-		"Hashtag":                               "as:Hashtag",
-		"quoteUrl":                              "as:quoteUrl",
-		"fedibird":                              "http://fedibird.com/ns#",
-		"quoteUri":                              "fedibird:quoteUri",
-		"toot":                                  "http://joinmastodon.org/ns#",
-		"Emoji":                                 "toot:Emoji",
-		"featured":                              "toot:featured",
-		"discoverable":                          "toot:discoverable",
-		"indexable":                             "toot:indexable",
-		"schema":                                "http://schema.org#",
-		"PropertyValue":                         "schema:PropertyValue",
-		"value":                                 "schema:value",
-		"misskey":                               "https://misskey-hub.net/ns#",
-		"_misskey_content":                      "misskey:_misskey_content",
-		"_misskey_quote":                        "misskey:_misskey_quote",
-		"_misskey_reaction":                     "misskey:_misskey_reaction",
-		"_misskey_votes":                        "misskey:_misskey_votes",
-		"_misskey_summary":                      "misskey:_misskey_summary",
-		"_misskey_followedMessage":              "misskey:_misskey_followedMessage",
-		"_misskey_requireSigninToViewContents":  "misskey:_misskey_requireSigninToViewContents",
-		"_misskey_makeNotesFollowersOnlyBefore": "misskey:_misskey_makeNotesFollowersOnlyBefore",
-		"_misskey_makeNotesHiddenBefore":        "misskey:_misskey_makeNotesHiddenBefore",
-		"_misskey_license":                      "misskey:_misskey_license",
-		"freeText": map[string]string{
-			"@id":   "misskey:freeText",
-			"@type": "schema:text",
-		},
-		"isCat":            "misskey:isCat",
-		"firefish":         "https://joinfirefish.org/ns#",
-		"speakAsCat":       "firefish:speakAsCat",
-		"sharkey":          "https://joinsharkey.org/ns#",
-		"hideOnlineStatus": "sharkey:hideOnlineStatus",
-		"backgroundUrl":    "sharkey:backgroundUrl",
-		"listenbrainz":     "sharkey:listenbrainz",
-		"enableRss":        "sharkey:enableRss",
-		"vcard":            "http://www.w3.org/2006/vcard/ns#",
-	},
-}
-
 func users(w http.ResponseWriter, r *http.Request) {
 	type OutboundKey struct {
 		Id    string `json:"id"`
@@ -120,7 +75,7 @@ func users(w http.ResponseWriter, r *http.Request) {
 		keyBytes = keyBytesToPem(user.PublicKeyRsa)
 	}
 	data := Outbound{
-		Context:           baseLdContext,
+		Context:           types.BaseLdContext,
 		Id:                apUrl,
 		Type:              "Person",
 		PreferredUsername: user.Username,

web/shared/client.go

@@ -0,0 +1,79 @@
+package webshared
+
+import (
+	"crypto/sha256"
+	"net/http"
+	"time"
+
+	"git.mstar.dev/mstar/goutils/maputils"
+
+	"git.mstar.dev/mstar/linstrom/config"
+)
+
+// No init needed, zero value is good
+
+var RequestClient http.Client
+
+const xRandomHeader = "X-Auth-Random"
+
+// Sign a given outbound request for authorized fetch.
+// At the end, the Signature header will have the signature needed,
+// nothing else is modified.
+// If the request is POST, the postBody must contain the raw body of
+// the request and the Digest header will also be added
+func SignRequest(r *http.Request, keyId string, privateKeyBytes, postBody []byte) error {
+	method := r.Method
+	headers := r.Header
+	var nowString string
+	if dateString := headers.Get("Date"); dateString != "" {
+		nowString = dateString
+	} else {
+		nowString = time.Now().Format("Mon, 02 Jan 2006 15:04:05 MST")
+		headers.Set("Date", nowString)
+	}
+	var host string
+	if hostString := headers.Get("Host"); hostString != "" {
+		host = hostString
+	} else {
+		host = config.GlobalConfig.General.GetFullDomain()
+		headers.Set("Date", host)
+	}
+	applyBodyHash(headers, postBody)
+	mappedHeaders := maputils.MapSameKeys(headers, func(k string, v []string) string {
+		if len(v) > 0 {
+			return v[0]
+		} else {
+			return ""
+		}
+	})
+	var signedString string
+	if config.GlobalConfig.Experimental.UseEd25519Keys {
+		tmp, err := CreateSignatureED(method, r.URL.RawPath, mappedHeaders, privateKeyBytes)
+		if err != nil {
+			return err
+		}
+		signedString = tmp
+	} else {
+		tmp, err := CreateSignatureRSA(method, r.URL.RawPath, mappedHeaders, privateKeyBytes)
+		if err != nil {
+			return err
+		}
+		signedString = tmp
+	}
+	signature := CreateSignatureHeaderContent(
+		keyId,
+		signedString,
+		maputils.KeysFromMap(mappedHeaders)...,
+	)
+	headers.Set("Signature", signature)
+	return nil
+}
+
+func applyBodyHash(headers http.Header, body []byte) error {
+	if body == nil {
+		return nil
+	}
+	hash := sha256.Sum256(body)
+	headers.Set("Digest", string(hash[:]))
+	return nil
+}

web/shared/signing.go

@@ -1,11 +1,15 @@
 package webshared
 
 import (
+	"encoding/base64"
 	"strings"
 
 	"git.mstar.dev/mstar/linstrom/shared"
 )
 
+// Generate the signed string of the headers, method and target
+// and sign it using the given RSA key. Returns the base64 encoded
+// result
 func CreateSignatureRSA(
 	method string,
 	target string,
@@ -14,9 +18,12 @@ func CreateSignatureRSA(
 ) (string, error) {
 	message := genPreSignatureString(method, target, headers)
 	signed, err := shared.Sign(message, privateKeyBytes, true)
-	return string(signed), err
+	return base64.StdEncoding.EncodeToString(signed), err
 }
 
+// Generate the signed string of the headers, method and target
+// and sign it using the given ED25519 key. Returns the base64
+// encoded result
 func CreateSignatureED(
 	method string,
 	target string,
@@ -25,7 +32,10 @@ func CreateSignatureED(
 ) (string, error) {
 	message := genPreSignatureString(method, target, headers)
 	signed, err := shared.Sign(message, privateKeyBytes, false)
-	return string(signed), err
+	if err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(signed), nil
 }
 
 func genPreSignatureString(method, target string, headers map[string]string) string {
@@ -38,3 +48,24 @@ func genPreSignatureString(method, target string, headers map[string]string) str
 	}
 	return dataBuilder.String()
 }
+
+// Generate the content of the "Signature" header based on
+// The user who's key was used, the hashed and base64 encoded
+// signed string, as returned by CreateSignatureED/RSA
+func CreateSignatureHeaderContent(userId string, hash string, headerNames ...string) string {
+	builder := strings.Builder{}
+	builder.WriteString("keyId=\"")
+	builder.WriteString(userId)
+	builder.WriteString("\",headers=\"")
+	for i, header := range headerNames {
+		builder.WriteString(header)
+		if i+1 < len(headerNames) {
+			builder.WriteRune(' ')
+		}
+	}
+	builder.WriteString("\",signature=\"")
+	builder.WriteString(hash)
+	builder.WriteRune('"')
+
+	return builder.String()
+}