Skip to content

Commit

Permalink
Move EvaluateClusterState into zombie.go (#195)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ederene20 authored May 1, 2023
1 parent 16283e4 commit 8d3e650
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 30 deletions.
31 changes: 1 addition & 30 deletions internal/api/handle_event.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"net/http"

"github.com/fly-apps/postgres-flex/internal/flypg"
"github.com/jackc/pgx/v5"
)

type EventRequest struct {
Expand Down Expand Up @@ -72,38 +71,10 @@ func processEvent(ctx context.Context, event EventRequest) error {
return nil
}

if err := evaluateClusterState(ctx, conn, node); err != nil {
if err := flypg.EvaluateClusterState(ctx, conn, node); err != nil {
return fmt.Errorf("failed to evaluate cluster state: %s", err)
}
}

return nil
}

// TODO Move this into zombie.go
func evaluateClusterState(ctx context.Context, conn *pgx.Conn, node *flypg.Node) error {
primary, err := flypg.PerformScreening(ctx, conn, node)
if errors.Is(err, flypg.ErrZombieDiagnosisUndecided) || errors.Is(err, flypg.ErrZombieDiscovered) {
if err := flypg.Quarantine(ctx, node, primary); err != nil {
return fmt.Errorf("failed to quarantine failed primary: %s", err)
}
log.Println("[WARN] Primary is going read-only to protect against potential split-brain")
return nil
} else if err != nil {
return fmt.Errorf("failed to run zombie diagnosis: %s", err)
}

// Clear zombie lock if it exists
if flypg.ZombieLockExists() {
log.Println("Quorom has been reached. Disabling read-only mode.")
if err := flypg.RemoveZombieLock(); err != nil {
return fmt.Errorf("failed to remove zombie lock file: %s", err)
}

if err := flypg.BroadcastReadonlyChange(ctx, node, false); err != nil {
log.Printf("failed to disable readonly: %s", err)
}
}

return nil
}
27 changes: 27 additions & 0 deletions internal/flypg/zombie.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,3 +254,30 @@ func handleZombieLock(ctx context.Context, n *Node) error {

return nil
}

func EvaluateClusterState(ctx context.Context, conn *pgx.Conn, node *Node) error {
primary, err := PerformScreening(ctx, conn, node)
if errors.Is(err, ErrZombieDiagnosisUndecided) || errors.Is(err, ErrZombieDiscovered) {
if err := Quarantine(ctx, node, primary); err != nil {
return fmt.Errorf("failed to quarantine failed primary: %s", err)
}
log.Println("[WARN] Primary is going read-only to protect against potential split-brain")
return nil
} else if err != nil {
return fmt.Errorf("failed to run zombie diagnosis: %s", err)
}

// Clear zombie lock if it exists
if ZombieLockExists() {
log.Println("Quorom has been reached. Disabling read-only mode.")
if err := RemoveZombieLock(); err != nil {
return fmt.Errorf("failed to remove zombie lock file: %s", err)
}

if err := BroadcastReadonlyChange(ctx, node, false); err != nil {
log.Printf("failed to disable readonly: %s", err)
}
}

return nil
}

0 comments on commit 8d3e650

Please sign in to comment.