AP stuff almost works

storage-new/storage.go

@@ -1,3 +1,78 @@
 package storage
 
+import (
+	"time"
+
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/rs/zerolog/log"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+
+	"git.mstar.dev/mstar/linstrom/config"
+	"git.mstar.dev/mstar/linstrom/shared"
+	"git.mstar.dev/mstar/linstrom/storage-new/dbgen"
+)
+
 //go:generate go run ../cmd/NewRoleHelperGenerator/main.go -input ./models/Role.go -output role_generated.go -mod storage
+
+// "Lock" to only have one active reconnection attempt going at any time
+var activeReconnectionAttempt = false
+
+// Attempt to connect to the configured DB.
+// If successful, also configures dbgen to use that connection
+func AttemptReconnect() {
+	// Ensure that only one attempt is going at the same time
+	// since a connection failure could easily cause multiple calls
+	if activeReconnectionAttempt {
+		log.Info().Msg("Already attempting to reconnect")
+		return
+	}
+	activeReconnectionAttempt = true
+	defer func() {
+		activeReconnectionAttempt = false
+	}()
+	log.Warn().Msg("DB connection failure! Attempting to reconnect")
+	maxAttempts := config.GlobalConfig.Storage.MaxReconnectAttempts
+	for i := range maxAttempts {
+		// If not the first attempt, sleep for 5 seconds and hope that this helps
+		if i > 0 {
+			time.Sleep(time.Second * 5)
+		}
+		log.Warn().Msg("Attempting to reconnect to db")
+		db, err := gorm.Open(
+			postgres.Open(config.GlobalConfig.Storage.BuildPostgresDSN()),
+			&gorm.Config{
+				Logger: shared.NewGormLogger(log.Logger),
+			},
+		)
+		// If reconnect failed, log, then enter next loop iteration
+		if err != nil {
+			log.Error().
+				Err(err).
+				Int("remaining-attempts", maxAttempts-(i+1)).
+				Msg("Reconnect attempt failed")
+			continue
+		}
+		// No errors, reconnect successful. Give dbgen the new connection and return
+		dbgen.SetDefault(db)
+		return
+	}
+	// All attempts to reconnect have failed.
+	// The situation is not recoverable.
+	// Log it and exit
+	// This is not a panic reason as it is expected that a db connection
+	// could always drop due to outside influence
+	log.Fatal().Msg("Failed to reconnect to the database! Exiting")
+}
+
+// HandleReconnectError checks if the given error requires
+// a reconnect attempt. If it does, it also starts
+// a reconnection attempt.
+func HandleReconnectError(errToCheck error) bool {
+	if _, ok := errToCheck.(*pgconn.ConnectError); ok {
+		go AttemptReconnect()
+		return true
+	} else {
+		return false
+	}
+}