mstar/goap
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Delete old stuff, added full parser

Browse source
This commit is contained in:
Melody 2024-08-26 17:17:08 +02:00
parent 926e328986
commit b6071c50ec
12 changed files with 98 additions and 505 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
go.mod
View file

@ -2,11 +2,9 @@ module gitlab.com/mstarongitlab/goap
go 1.22.5 go 1.22.5
require github.com/piprate/json-gold v0.5.0
require ( require (
github.com/datainq/xml-date-time v0.0.0-20170820214645-2292f08baa38 // indirect github.com/datainq/xml-date-time v0.0.0-20170820214645-2292f08baa38
github.com/samber/go-type-to-string v1.7.0 // indirect github.com/piprate/json-gold v0.5.0
) )
require ( require (

2
go.sum
View file

@ -8,8 +8,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/samber/go-type-to-string v1.7.0 h1:FiSstaAikHMUSLt5bhVlsvCnD7bbQzC8L0UkkGS3Bj8=
github.com/samber/go-type-to-string v1.7.0/go.mod h1:jpU77vIDoIxkahknKDoEx9C8bQ1ADnh2sotZ8I4QqBU=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
gitlab.com/mstarongitlab/goutils v1.3.0 h1:uuxPHjIU36lyJ8/z4T2xI32zOyh53Xj0Au8K12qkaJ4= gitlab.com/mstarongitlab/goutils v1.3.0 h1:uuxPHjIU36lyJ8/z4T2xI32zOyh53Xj0Au8K12qkaJ4=

34
old/actor.go
View file

@ -1,34 +0,0 @@
package ap
type MinimalActor struct {
Id string
Username string
Inbox string
}
type MastoActor struct {
Id string
Devices MastoDevices
Discoverable MastoDiscoverable
Featured MastoFeatured
FeaturedTags MastoFeaturedTags
Indexable MastoIndexable
Memorial MastoMemorial
Inbox W3Inbox
PublicKey W3SecurityPublicKey
AlsoKnownAs ActivityStreamsAlsoKnownAs
Attachments []ActivityStreamsAttachment
Endpoints []map[string]IdType
Followers ActivityStreamsFollowers
Following ActivityStreamsFollowing
Icon ActivityStreamsIcon
Image ActivityStreamsImage
RestrictedFollow ActivityStreamsRestrictedFollow
Name ActivityStreamsName
Outbox ActivityStreamsOutbox
PrefferedUsername ActivityStreamsPrefferedUsername
Published ActivityStreamsPublished
Summary ActivityStreamsSummary
Tags []ActivityStreamsTag
Url ActivityStreamsUrl
}

24
old/checkHeader.go
View file

@ -1,24 +0,0 @@
package ap
import "strings"
// Header used for making requests for AP resources
// Not used yet
const OUTBOUND_REQUEST_HEADER = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1"
var contentHeadersToCheck = []string{
"application/json",
"application/ld+json",
}
// Check a given string if it contains any of the content types specified in
// the contentHeadersToCheck slice
// Used for differentiating requests for the ActivityPub version of some data vs frontend version
func ContainsApContentHeader(toCheck string) bool {
for _, h := range contentHeadersToCheck {
if strings.Contains(toCheck, h) {
return true
}
}
return false
}

18
old/common_types.go
View file

@ -1,18 +0,0 @@
package ap
import "net/url"
type IdType struct {
Id *url.URL
}
type ValueType[T any] struct {
Value T
}
type Media struct {
Type url.URL
MediaType *ValueType[string]
Url IdType
Sensitive *ValueType[bool]
}

81
old/constants.go
View file

@ -1,81 +0,0 @@
package ap
const (
KEY_ID = "@id" // Value of type string
KEY_TYPE = "@type" // Value of type string slice / activitystreams object url slice
KEY_VALUE = "@value" // Could be any type really
)
const (
KEY_MASTO_DEVICES = "http://joinmastodon.org/ns#devices" // No idea what this is, but Masto includes it
KEY_MASTO_DISCOVERABLE = "http://joinmastodon.org/ns#discoverable" // iirc this is whether the object can be found by crawlers
KEY_MASTO_FEATURED = "http://joinmastodon.org/ns#featured" // Users tagged? I think
KEY_MASTO_FEATURED_TAGS = "http://joinmastodon.org/ns#featuredTags" // Hashtags included, I think
KEY_MASTO_INDEXABLE = "http://joinmastodon.org/ns#indexable" // Is the object crawlable round 2
KEY_MASTO_MEMORIAL = "http://joinmastodon.org/ns#memorial" // Account dead and disabled?
)
const (
KEY_W3_INBOX = "http://www.w3.org/ns/ldp#inbox" // Where to send activities to
KEY_W3_SECURITY_PUBLICKEY = "https://w3id.org/security#publicKey" // Public key of the account
KEY_W3_SECURITY_OWNER = "https://w3id.org/security#owner" // Owner of the public key
KEY_W3_SECURITY_PUBLICKEYPEM = "https://w3id.org/security#publicKeyPem" // Pem content of the key
KEY_W3_VCARD_ADRESS = "http://www.w3.org/2006/vcard/ns#Address" // Vcard address of an account
KEY_W3_VCARD_BIRTHDAY = "http://www.w3.org/2006/vcard/ns#bday" // Vcard birthday of an account
)
const (
KEY_ACTIVITYSTREAMS_ALSOKNOWNAS = "https://www.w3.org/ns/activitystreams#alsoKnownAs" // Lists of usernames?
KEY_ACTIVITYSTREAMS_ATTACHMENTS = "https://www.w3.org/ns/activitystreams#attachment" // Attached elements like emotes and hashtags
KEY_ACTIVITYSTREAMS_NAME = "https://www.w3.org/ns/activitystreams#name" // AP name
KEY_ACTIVITYSTREAMS_ENDPOINTS = "https://www.w3.org/ns/activitystreams#endpoints" // list of assocciated endpoints
KEY_ACTIVITYSTREAMS_SHAREDINBOX = "https://www.w3.org/ns/activitystreams#sharedInbox" // Link to the shared inbox of the server
KEY_ACTIVITYSTREAMS_FOLLOWERS = "https://www.w3.org/ns/activitystreams#followers" // Who is following the account
KEY_ACTIVITYSTREAMS_FOLLOWING = "https://www.w3.org/ns/activitystreams#following" // Who is this account following
KEY_ACTIVITYSTREAMS_ICON = "https://www.w3.org/ns/activitystreams#icon" // User icon
KEY_ACTIVITYSTREAMS_MEDIATYPE = "https://www.w3.org/ns/activitystreams#mediaType" // What type of media is this? Example image/jpeg
KEY_ACTIVITYSTREAMS_URL = "https://www.w3.org/ns/activitystreams#url" // Some url
KEY_ACTIVITYSTREAMS_IMAGE = "https://www.w3.org/ns/activitystreams#image" // Account banner
KEY_ACTIVITYSTREAMS_RESTRICTED_FOLLOW = "https://www.w3.org/ns/activitystreams#manuallyApprovesFollowers" // Does the account manually approve follow requests
KEY_ACTIVITYSTREAMS_OUTBOX = "https://www.w3.org/ns/activitystreams#outbox" // Link to the account's outbox
KEY_ACTIVITYSTREAMS_PREFFEREDUSERNAME = "https://www.w3.org/ns/activitystreams#preferredUsername" // What the shown username is
KEY_ACTIVITYSTREAMS_PUBLISHED = "https://www.w3.org/ns/activitystreams#published" // When an object was created
KEY_ACTIVITYSTREAMS_SUMMARY = "https://www.w3.org/ns/activitystreams#summary" // Summary of an account
KEY_ACTIVITYSTREAMS_TAG = "https://www.w3.org/ns/activitystreams#tag" // A tag, no idea what those are
KEY_ACTIVITYSTREAMS_CC = "https://www.w3.org/ns/activitystreams#cc" // Urls also included in the object
KEY_ACTIVITYSTREAMS_TO = "https://www.w3.org/ns/activitystreams#to" // Urls to send an activity to
KEY_ACTIVITYSTREAMS_OBJECT = "https://www.w3.org/ns/activitystreams#object" // Object url and sometimes value
// Object types (I think)
// Those are values the object type can have
KEY_ACTIVITYSTREAMS_ACTOR = "https://www.w3.org/ns/activitystreams#actor"
KEY_ACTIVITYSTREAMS_FOLLOW = "https://www.w3.org/ns/activitystreams#Follow" // Object is an activity of type follow
KEY_ACTIVITYSTREAMS_PERSON = "https://www.w3.org/ns/activitystreams#Person" // Object is of type Person
KEY_ACTIVITYSTREAMS_CREATE = "https://www.w3.org/ns/activitystreams#Create" // Object is an activity of type Create
KEY_ACTIVITYSTREAMS_OAUTHAUTHORIZATION = "https://www.w3.org/ns/activitystreams#oauthAuthorizationEndpoint" // Endpoint url for oauth login?
KEY_ACTIVITYSTREAMS_OAUTHTOKEN = "https://www.w3.org/ns/activitystreams#oauthTokenEndpoint" // Endpoint url for oauth token verification?
KEY_ACTIVITYSTREAMS_UPLOADMEDIA = "https://www.w3.org/ns/activitystreams#uploadMedia" // Endpoint url to upload media to?
)
const (
KEY_SCHEMA_VALUE = "http://schema.org#value" // The value for some field
)
const (
KEY_MISSKEY_MKSUMMARY = "https://misskey-hub.net/ns#_misskey_summary" // Misskey specific formatted summary
KEY_MISSKEY_ISCAT = "https://misskey-hub.net/ns#isCat" // Does the account identify as cat?
KEY_FIREFISH_SPEAKASCAT = "https://joinfirefish.org/ns#speakAsCat" // Does the account speak like a cat?
)
const (
KEY_LITEPUB_CAPABILITIES = "http://litepub.social/ns#capabilities"
KEY_LITEPUB_OAUTHREGISTRATION = "http://litepub.social/ns#oauthRegistrationEndpoint"
)
const (
KEY_MYSTERIOUS_NOINDEX = "_:noindex"
KEY_MYSTERIOUS_BACKGROUNDURL = "_:backgroundUrl"
KEY_MYSTERIOUS_FEATURED = "_:featured"
)

49
old/context.go
View file

@ -1,49 +0,0 @@
package ap
// Just steal this one from sharkey
// Also use this one for all AP objects for now
// Can make a function to extract context from an expanded object later
// TODO: Consider if such a function is worth it or if it would hinder learning
var allContext = map[string]any{
"@context": []any{
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
map[string]any{
"fedibird": "http://fedibird.com/ns#",
"toot": "http://joinmastodon.org/ns#",
"schema": "http://schema.org#",
"misskey": "https://misskey-hub.net/ns#",
"firefish": "https://joinfirefish.org/ns#",
"sharkey": "https://joinsharkey.org/ns#",
"vcard": "http://www.w3.org/2006/vcard/ns#",
"Key": "sec:Key",
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"sensitive": "as:sensitive",
"Hashtag": "as:Hashtag",
"quoteUrl": "as:quoteUrl",
"quoteUri": "fedibird:quoteUri",
"Emoji": "toot:Emoji",
"featured": "toot:featured",
"discoverable": "toot:discoverable",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
"_misskey_content": "misskey:_misskey_content",
"_misskey_quote": "misskey:_misskey_quote",
"_misskey_reaction": "misskey:_misskey_reaction",
"_misskey_votes": "misskey:_misskey_votes",
"_misskey_summary": "misskey:_misskey_summary",
"isCat": "misskey:isCat",
"speakAsCat": "firefish:speakAsCat",
"backgroundUrl": "sharkey:backgroundUrl",
"listenbrainz": "sharkey:listenbrainz",
},
},
}

61
old/explicit_types.go
View file

@ -1,61 +0,0 @@
package ap
import (
"net/url"
"time"
)
type (
ActivityStreamsAlsoKnownAs IdType
ActivityStreamsAttachment struct {
Type *url.URL
Value ValueType[string]
Name ValueType[string]
}
ActivityStreamsSharedInbox IdType
ActivityStreamsFollowers IdType
ActivityStreamsFollowing IdType
ActivityStreamsImage Media
ActivityStreamsIcon Media
ActivityStreamsRestrictedFollow ValueType[bool]
ActivityStreamsName ValueType[string]
ActivityStreamsOutbox IdType
ActivityStreamsPrefferedUsername ValueType[string]
ActivityStreamsPublished struct {
Type string
Value time.Time
}
ActivityStreamsSummary ValueType[string]
ActivityStreamsUrl IdType
// NOTE: Technically can contain a lot more values, but tags are not consistent in what those are. Only consistent one is the type
ActivityStreamsTag struct {
Type string
}
ActivityStreamsTo IdType
ActivityStreamsCC IdType
ActivityStreamsActor IdType
// NOTE: Technically, objects can have a LOT of data. I don't care. Treat them as ID type
// Just fetch whatever the ID is later on separately and throw away anything else but the ID
ActivityStreamsObject IdType
ActivityStreamsFollow IdType
)
type (
MastoDevices IdType
MastoDiscoverable ValueType[bool]
MastoFeatured IdType
MastoFeaturedTags IdType
MastoIndexable ValueType[bool]
MastoMemorial ValueType[bool]
)
type (
W3Inbox IdType
W3SecurityPublicKey struct {
Id *url.URL
Owner *url.URL
KeyPem string
}
)

58
old/parser.go
View file

@ -1,58 +0,0 @@
package ap
import (
"encoding/json"
"fmt"
"net/url"
"github.com/piprate/json-gold/ld"
)
type ApThing map[string]any
type RemoteDocumentLoader struct{}
var ldOpts = ld.NewJsonLdOptions("")
var ldProcessor = ld.NewJsonLdProcessor()
// Try and parse a remote ActivityPub object into a more usable form
// Result is a map[string]any where the keys defined in /ap/constants.go should be usable,
// depending on the type of object
// The general approach for fetching an object will be to fetch the main object
// and only store the ID for sub-objects to fetch them later
func ParseFromUrl(u *url.URL) (ApThing, error) {
// TODO: Add custom document parser (copy default implementation) that includes verification
// Explanation:
// Expansion removes the context from a document (json-ld activitypub data)
// and turns every field into something along the lines of
// "https://example.com/ns#ObjectType": <Insert data for that object here>
// This makes it easier to deal with things as they now have a very consistent naming scheme
// See /ap/constants.go for those
parsed, err := ldProcessor.Expand(u.String(), ldOpts)
if err != nil {
return nil, fmt.Errorf("failed to process remote document: %w", err)
}
if len(parsed) == 0 {
return nil, fmt.Errorf("document has a length of 0")
}
typed, ok := parsed[0].(ApThing)
if !ok {
return nil, fmt.Errorf("couldn't cast data to ApThing")
}
return typed, nil
}
// Compact an AP object down into a compressed json-ld form
// That compacted form should be accepted by all AP servers
// It also handles context for any fields
// Content should only use keys defined in /ap/constants.go though
// Other things might get lost in translation
func Compact(content map[string]any) ([]byte, error) {
res, err := ldProcessor.Compact(content, allContext, ldOpts)
if err != nil {
return nil, fmt.Errorf("failed to compact data: %w", err)
}
return json.Marshal(res)
}

172
old/type_parsers.go
View file

@ -1,172 +0,0 @@
package ap
import (
"net/url"
"time"
)
// Try and parse a value into an IdType
// Returns nil if failed
func TryParseIdType(rawIn any) *IdType {
switch in := rawIn.(type) {
case []any:
if len(in) == 0 {
return nil
}
m, ok := in[0].(map[string]any)
if !ok {
return nil
}
return TryParseIdType(m)
case map[string]any:
vRaw, ok := in[KEY_ID]
if !ok {
return nil
}
v, ok := vRaw.(string)
if !ok {
return nil
}
u, err := url.Parse(v)
if err != nil {
return nil
}
return &IdType{
Id: u,
}
default:
return nil
}
}
func TryParseValueType[T any](rawIn any) *ValueType[T] {
switch in := rawIn.(type) {
case map[string]any:
vRaw, ok := in[KEY_ID]
if !ok {
return nil
}
v, ok := vRaw.(T)
if !ok {
return nil
}
return &ValueType[T]{
Value: v,
}
case []any:
if len(in) == 0 {
return nil
}
v, ok := in[0].(map[string]any)
if !ok {
return nil
}
return TryParseValueType[T](v)
default:
return nil
}
}
func TryParseActivityStreamsPublicKey(rawIn any) *W3SecurityPublicKey {
switch in := rawIn.(type) {
case map[string]any:
asIdType := TryParseIdType(in)
if asIdType == nil {
return nil
}
ownerType := TryParseIdType(in[KEY_W3_SECURITY_OWNER])
if ownerType == nil {
return nil
}
keyValue := TryParseValueType[string](in[KEY_W3_SECURITY_PUBLICKEYPEM])
if keyValue == nil {
return nil
}
return &W3SecurityPublicKey{
Id: asIdType.Id,
Owner: ownerType.Id,
KeyPem: keyValue.Value,
}
case []any:
if len(in) == 0 {
return nil
}
return TryParseActivityStreamsPublicKey(in[0])
default:
return nil
}
}
func TryParseActivityStreamsAttachment(rawIn any) *ActivityStreamsAttachment {
switch in := rawIn.(type) {
case []any:
if len(in) == 0 {
return nil
}
return TryParseActivityStreamsAttachment(in[0])
case map[string]any:
rawType, ok := in[KEY_TYPE]
if !ok {
return nil
}
strType, ok := rawType.(string)
if !ok {
return nil
}
urlType, err := url.Parse(strType)
if err != nil {
return nil
}
value := TryParseValueType[string](in[KEY_SCHEMA_VALUE])
if value == nil {
return nil
}
name := TryParseValueType[string](in[KEY_ACTIVITYSTREAMS_NAME])
if name == nil {
return nil
}
return &ActivityStreamsAttachment{
Type: urlType,
Name: *name,
Value: *value,
}
default:
return nil
}
}
func TryParseActivityStreamsPublished(rawIn any) *ActivityStreamsPublished {
switch in := rawIn.(type) {
case []any:
if len(in) == 0 {
return nil
}
return TryParseActivityStreamsPublished(in[0])
case map[string]any:
rawType, ok := in[KEY_TYPE]
if !ok {
return nil
}
strType, ok := rawType.(string)
if !ok {
return nil
}
value := TryParseValueType[string](in)
tv, err := time.Parse("2006-01-02T04:05:06Z", value.Value)
if err != nil {
return nil
}
return &ActivityStreamsPublished{
Type: strType,
Value: tv,
}
default:
return nil
}
}
// NOTE: Since I do not know if tags are consistent with the struct yet,
// This funtion does not do anything yet and should not be used
func TryParseActivityStreamsTag(rawIn any) *ActivityStreamsTag {
return nil
}

94
parser.go Normal file
View file

@ -0,0 +1,94 @@
package goap
import "slices"
var allInternalParsersExceptBase []UnmarshalFunc = []UnmarshalFunc{
ParseASActorData,
ParseASAlsoKnownAsData,
ParseASAttachmentsData,
ParseASAttributedToData,
ParseASCCData,
ParseASContentData,
ParseASEndpointsData,
ParseASFollowersData,
ParseASFollowingData,
ParseASHrefData,
ParseASIconData,
ParseASImageData,
ParseASMediaTypeData,
ParseASNameData,
ParseASOutboxData,
ParseASObjectData,
ParseASPreferredNameData,
ParseASPublishedData,
ParseASRestrictedData,
ParseASRepliesData,
ParseASSharedInboxData,
ParseASSummaryData,
ParseASSensitiveData,
ParseASTagData,
ParseASToData,
ParseASUrlData,
ParseASUpdatedData,
ParseOstatusAtomUriData,
ParseOstatusConversationData,
ParseW3InboxData,
ParseW3VcardAddressData,
ParseW3VcardBirthdayData,
ParseW3SecurityOwnerData,
ParseW3SecurityPublicKeyPemData,
ParseW3SecurityPublicKeyData,
ParseMastoDevicesData,
ParseMastoDiscoverableData,
ParseMastoFeaturedData,
ParseMastoDiscoverableData,
ParseMastoIndexableData,
ParseMastoMemorialData,
ParseSchemaValueData,
ParseMKIsCatData,
ParseMKSummaryData,
ParseFFSpeakAsCatData,
}
func chainParse(
raw map[string]any,
start BaseApChain,
parsers ...UnmarshalFunc,
) (BaseApChain, []error) {
var current BaseApChain = start
errors := []error{}
for _, p := range parsers {
next, err := p(raw, current)
current = next
errors = append(errors, err)
}
return current, errors
}
func ParseEmptyBase(raw map[string]any, extraParsers ...UnmarshalFunc) (BaseApChain, []error) {
base := EmptyBaseObject{}
allParsers := slices.Clone(allInternalParsersExceptBase)
allParsers = append(allParsers, extraParsers...)
return chainParse(raw, &base, allParsers...)
}
func ParseDefaultBase(raw map[string]any, extraParsers ...UnmarshalFunc) (BaseApChain, []error) {
base, err := UnmarshalBaseObject(raw, nil)
if err != nil {
return base, []error{err}
}
allParsers := slices.Concat(allInternalParsersExceptBase, extraParsers)
res, errs := chainParse(raw, base, allParsers...)
return res, slices.Insert(errs, 0, err)
}

4
tags.go
View file

@ -134,7 +134,7 @@ func parseMentionTag(raw map[string]any) (*TagMention, error) {
name := rawName.(*ASNameData) name := rawName.(*ASNameData)
return &TagMention{ return &TagMention{
TargetId: href.Id, TargetId: href.Id,
TargetName: name.Name.Value, TargetName: name.Value.Value,
}, nil }, nil
} }
@ -151,6 +151,6 @@ func parseHashtagTag(raw map[string]any) (*TagHashtag, error) {
name := rawName.(*ASNameData) name := rawName.(*ASNameData)
return &TagHashtag{ return &TagHashtag{
Url: href.Id, Url: href.Id,
Name: name.Name.Value, Name: name.Value.Value,
}, nil }, nil
} }