package storage import ( "time" "github.com/jackc/pgx/v5/pgconn" "github.com/rs/zerolog/log" "gorm.io/driver/postgres" "gorm.io/gorm" "git.mstar.dev/mstar/linstrom/config" "git.mstar.dev/mstar/linstrom/shared" "git.mstar.dev/mstar/linstrom/storage-new/dbgen" ) //go:generate go run ../cmd/NewRoleHelperGenerator/main.go -input ./models/Role.go -output role_generated.go -mod storage // "Lock" to only have one active reconnection attempt going at any time var activeReconnectionAttempt = false // Attempt to connect to the configured DB. // If successful, also configures dbgen to use that connection func AttemptReconnect() { // Ensure that only one attempt is going at the same time // since a connection failure could easily cause multiple calls if activeReconnectionAttempt { log.Info().Msg("Already attempting to reconnect") return } activeReconnectionAttempt = true defer func() { activeReconnectionAttempt = false }() log.Warn().Msg("DB connection failure! Attempting to reconnect") maxAttempts := config.GlobalConfig.Storage.MaxReconnectAttempts for i := range maxAttempts { // If not the first attempt, sleep for 5 seconds and hope that this helps if i > 0 { time.Sleep(time.Second * 5) } log.Warn().Msg("Attempting to reconnect to db") db, err := gorm.Open( postgres.Open(config.GlobalConfig.Storage.BuildPostgresDSN()), &gorm.Config{ Logger: shared.NewGormLogger(log.Logger), }, ) // If reconnect failed, log, then enter next loop iteration if err != nil { log.Error(). Err(err). Int("remaining-attempts", maxAttempts-(i+1)). Msg("Reconnect attempt failed") continue } // No errors, reconnect successful. Give dbgen the new connection and return dbgen.SetDefault(db) return } // All attempts to reconnect have failed. // The situation is not recoverable. // Log it and exit // This is not a panic reason as it is expected that a db connection // could always drop due to outside influence log.Fatal().Msg("Failed to reconnect to the database! Exiting") } // HandleReconnectError checks if the given error requires // a reconnect attempt. If it does, it also starts // a reconnection attempt. func HandleReconnectError(errToCheck error) bool { if _, ok := errToCheck.(*pgconn.ConnectError); ok { go AttemptReconnect() return true } else { return false } }