From 9a7e420c83be3daa7b400bb93462f32cd1d7e555 Mon Sep 17 00:00:00 2001 From: mStar Date: Sun, 15 Sep 2024 15:18:05 +0200 Subject: [PATCH] More stuff --- config/config.go | 44 +++++++++++++++++++++++++++++++++++++---- docker-compose.yml | 1 + main.go | 14 ++++--------- storage/cache/cache.go | 12 +++++++++-- storage/housekeeping.go | 5 +++++ storage/notes.go | 14 +++---------- storage/storage.go | 6 ------ 7 files changed, 63 insertions(+), 33 deletions(-) diff --git a/config/config.go b/config/config.go index 27a1d30..50b70b1 100644 --- a/config/config.go +++ b/config/config.go @@ -48,12 +48,21 @@ type ConfigAdmin struct { } type ConfigStorage struct { - // Url to the postgres database. Must contain credentials and stuff - DatabaseUrl string `toml:"database_url"` + Host string `toml:"host"` + Username string `toml:"username"` + Password string `toml:"password"` + DbName string `toml:"db_name"` + Port int `toml:"port"` + SslMode *string `toml:"ssl_mode"` + TimeZone *string `toml:"time_zone"` // Url to redis server. If empty, no redis is used RedisUrl *string `toml:"redis_url,omitempty"` // The maximum size of the in-memory cache in bytes MaxInMemoryCacheSize int64 `toml:"max_in_memory_cache_size"` + // The time to live for in app in memory cache items, in seconds + MaxInMemoryCacheTTL int + // The time to live for items in redis, in seconds + MaxRedisCacheTTL *int } type ConfigMail struct { @@ -74,7 +83,7 @@ type Config struct { Admin ConfigAdmin `toml:"admin"` Webauthn ConfigWebAuthn `toml:"webauthn"` Storage ConfigStorage `toml:"storage"` - Mail ConfigMail `toml:"mail"` + Mail ConfigMail `toml:"mail"` } var GlobalConfig Config @@ -102,9 +111,17 @@ var defaultConfig Config = Config{ HashingSecret: "some super secure secret that should never be changed or else password storage breaks", }, Storage: ConfigStorage{ - DatabaseUrl: "db.sqlite", + Host: "localhost", + Username: "linstrom", + Password: "linstrom", + DbName: "linstrom", + Port: 5432, + SslMode: other.IntoPointer("disable"), + TimeZone: other.IntoPointer("Europe/Berlin"), RedisUrl: nil, MaxInMemoryCacheSize: 1e6, // 1 Megabyte + MaxInMemoryCacheTTL: 5, + MaxRedisCacheTTL: nil, }, } @@ -125,6 +142,25 @@ func (gc *ConfigGeneral) GetFullPublicUrl() string { return str } +func (sc *ConfigStorage) BuildPostgresDSN() string { + dsn := fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s port=%d", + sc.Host, + sc.Username, + sc.Password, + sc.DbName, + sc.Port, + ) + if sc.SslMode != nil { + dsn += fmt.Sprintf(" sslmode=%s", *sc.SslMode) + } + if sc.TimeZone != nil { + dsn += fmt.Sprintf(" TimeZone=%s", *sc.TimeZone) + } + + return dsn +} + func WriteDefaultConfig(toFile string) error { log.Trace().Caller().Send() log.Info().Str("config-file", toFile).Msg("Writing default config to file") diff --git a/docker-compose.yml b/docker-compose.yml index e69de29..bb18e5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -0,0 +1 @@ +services: name diff --git a/main.go b/main.go index 0181f58..4d6de79 100644 --- a/main.go +++ b/main.go @@ -35,21 +35,15 @@ func main() { if err != nil { log.Fatal().Err(err).Msg("Failed to start cache") } - var store *storage.Storage - if config.GlobalConfig.Storage.DbIsPostgres != nil && - *config.GlobalConfig.Storage.DbIsPostgres { - store, err = storage.NewStoragePostgres( - config.GlobalConfig.Storage.DatabaseUrl, - storageCache, - ) - } else { - store, err = storage.NewStorageSqlite(config.GlobalConfig.Storage.DatabaseUrl, storageCache) - } + + store, err := storage.NewStorage(config.GlobalConfig.Storage.BuildPostgresDSN(), storageCache) + if err != nil { log.Fatal().Err(err).Msg("Failed to setup storage") } _ = store // TODO: Set up media server + // TODO: Set up queues // TODO: Set up http server // TODO: Set up plugins // TODO: Start everything diff --git a/storage/cache/cache.go b/storage/cache/cache.go index 243f5b6..b7b8f7a 100644 --- a/storage/cache/cache.go +++ b/storage/cache/cache.go @@ -12,6 +12,7 @@ import ( ristretto_store "github.com/eko/gocache/store/ristretto/v4" "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" + "gitlab.com/mstarongitlab/linstrom/config" ) type Cache struct { @@ -37,7 +38,9 @@ func NewCache(maxSize int64, redisUrl *string) (*Cache, error) { } ristrettoStore := ristretto_store.NewRistretto( ristrettoCache, - store.WithExpiration(time.Second*10), + store.WithExpiration( + time.Second*time.Duration(config.GlobalConfig.Storage.MaxInMemoryCacheSize), + ), ) var cacheManager *cache.ChainCache[[]byte] @@ -47,7 +50,12 @@ func NewCache(maxSize int64, redisUrl *string) (*Cache, error) { return nil, err } redisClient := redis.NewClient(opts) - redisStore := redis_store.NewRedis(redisClient, store.WithExpiration(time.Minute)) + redisStore := redis_store.NewRedis( + redisClient, + store.WithExpiration( + time.Second*time.Duration(*config.GlobalConfig.Storage.MaxRedisCacheTTL), + ), + ) cacheManager = cache.NewChain( cache.New[[]byte](ristrettoStore), cache.New[[]byte](redisStore), diff --git a/storage/housekeeping.go b/storage/housekeeping.go index aa7a815..a0a3d10 100644 --- a/storage/housekeeping.go +++ b/storage/housekeeping.go @@ -4,3 +4,8 @@ package storage // Things like true deletion of soft deleted data after some time // Or removing inactive access tokens // All of this will be handled by goroutines + +// TODO: Delete everything soft deleted and older than a month +// TODO: Delete old tokens not in active use anymore +// TODO: Start jobs where the last check-in was more than an hour ago +// diff --git a/storage/notes.go b/storage/notes.go index b56daa1..cc01f5a 100644 --- a/storage/notes.go +++ b/storage/notes.go @@ -107,7 +107,9 @@ func (s *Storage) UpdateNote(note *Note) error { } err = s.cache.Set(cacheNoteIdToNotePrefix+note.ID, note) if err != nil { - log.Warn().Err(err).Msg("Failed to update note into cache. Cache and db might be out of sync, a force sync is recommended") + log.Warn(). + Err(err). + Msg("Failed to update note into cache. Cache and db might be out of sync, a force sync is recommended") } return nil } @@ -121,13 +123,3 @@ func (s *Storage) DeleteNote(id string) { s.cache.Delete(cacheNoteIdToNotePrefix + id) s.db.Delete(Note{ID: id}) } - -// Try and find a note with a given ID -func (store *Storage) FindNoteById(id string) (*Note, error) { - note := Note{} - res := store.db.First(¬e, id) - if res.Error != nil { - return nil, res.Error - } - return ¬e, nil -} diff --git a/storage/storage.go b/storage/storage.go index 913926d..b4529c9 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -7,7 +7,6 @@ package storage import ( "errors" - "fmt" "github.com/rs/zerolog/log" "gitlab.com/mstarongitlab/linstrom/storage/cache" @@ -47,8 +46,3 @@ func NewStorage(dbUrl string, cache *cache.Cache) (*Storage, error) { return &Storage{db, cache}, nil } - -// TODO: Placeholder. Update to proper implementation later. Including signature -func (s *Storage) FindLocalAccount(handle string) (string, error) { - return handle, nil -}