78 lines
2.3 KiB
Go
78 lines
2.3 KiB
Go
package storage
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
"github.com/rs/zerolog/log"
|
|
"gorm.io/driver/postgres"
|
|
"gorm.io/gorm"
|
|
|
|
"git.mstar.dev/mstar/linstrom/config"
|
|
"git.mstar.dev/mstar/linstrom/shared"
|
|
"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
|
|
)
|
|
|
|
//go:generate go run ../cmd/NewRoleHelperGenerator/main.go -input ./models/Role.go -output role_generated.go -mod storage
|
|
|
|
// "Lock" to only have one active reconnection attempt going at any time
|
|
var activeReconnectionAttempt = false
|
|
|
|
// Attempt to connect to the configured DB.
|
|
// If successful, also configures dbgen to use that connection
|
|
func AttemptReconnect() {
|
|
// Ensure that only one attempt is going at the same time
|
|
// since a connection failure could easily cause multiple calls
|
|
if activeReconnectionAttempt {
|
|
log.Info().Msg("Already attempting to reconnect")
|
|
return
|
|
}
|
|
activeReconnectionAttempt = true
|
|
defer func() {
|
|
activeReconnectionAttempt = false
|
|
}()
|
|
log.Warn().Msg("DB connection failure! Attempting to reconnect")
|
|
maxAttempts := config.GlobalConfig.Storage.MaxReconnectAttempts
|
|
for i := range maxAttempts {
|
|
// If not the first attempt, sleep for 5 seconds and hope that this helps
|
|
if i > 0 {
|
|
time.Sleep(time.Second * 5)
|
|
}
|
|
log.Warn().Msg("Attempting to reconnect to db")
|
|
db, err := gorm.Open(
|
|
postgres.Open(config.GlobalConfig.Storage.BuildPostgresDSN()),
|
|
&gorm.Config{
|
|
Logger: shared.NewGormLogger(log.Logger),
|
|
},
|
|
)
|
|
// If reconnect failed, log, then enter next loop iteration
|
|
if err != nil {
|
|
log.Error().
|
|
Err(err).
|
|
Int("remaining-attempts", maxAttempts-(i+1)).
|
|
Msg("Reconnect attempt failed")
|
|
continue
|
|
}
|
|
// No errors, reconnect successful. Give dbgen the new connection and return
|
|
dbgen.SetDefault(db)
|
|
return
|
|
}
|
|
// All attempts to reconnect have failed.
|
|
// The situation is not recoverable.
|
|
// Log it and exit
|
|
// This is not a panic reason as it is expected that a db connection
|
|
// could always drop due to outside influence
|
|
log.Fatal().Msg("Failed to reconnect to the database! Exiting")
|
|
}
|
|
|
|
// HandleReconnectError checks if the given error requires
|
|
// a reconnect attempt. If it does, it also starts
|
|
// a reconnection attempt.
|
|
func HandleReconnectError(errToCheck error) bool {
|
|
if _, ok := errToCheck.(*pgconn.ConnectError); ok {
|
|
go AttemptReconnect()
|
|
return true
|
|
} else {
|
|
return false
|
|
}
|
|
}
|