From 10b5a28862d7291787e589ac2801c053eeb08a5d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sat, 18 Oct 2014 15:14:45 +0200 Subject: [PATCH 1/2] Combine sequential update events We move the haproxy update and reload logic into a go routine which listens for update requests on updateChan. If an update is already queued, the channel is drained and another update request is sent. This way there can only be one pending update queued. At the same time, no update can be missed. By moving the real haproxy reload work into a go routine, the actual HTTP handler remains fast and slim. Before this patch in real world environements it took over 2 seconds to return. Marathon sometimes sends a lot of events sequentially, leading to HTTP POST timeouts on Marathon's side if the HTTP handler takes so long. This patch will fix this and at the same time reduce the number of reloads to a minimum. --- services/event_bus/event_handler.go | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/services/event_bus/event_handler.go b/services/event_bus/event_handler.go index 628265f..d057e18 100644 --- a/services/event_bus/event_handler.go +++ b/services/event_bus/event_handler.go @@ -33,17 +33,43 @@ type Handlers struct { func (h *Handlers) MarathonEventHandler(event MarathonEvent) { log.Printf("%s => %s\n", event.EventType, event.Timestamp) - handleHAPUpdate(h.Conf, h.Zookeeper) + queueUpdate(h) h.Conf.StatsD.Increment(1.0, "reload.marathon", 1) } func (h *Handlers) ServiceEventHandler(event ServiceEvent) { log.Println("Domain mapping: Stated changed") - handleHAPUpdate(h.Conf, h.Zookeeper) + queueUpdate(h) h.Conf.StatsD.Increment(1.0, "reload.domain", 1) } -var execSem = make(chan int, 1) +var updateChan = make(chan *Handlers, 1) + +func init() { + go func () { + log.Println("Starting update loop") + for { + h := <-updateChan + handleHAPUpdate(h.Conf, h.Zookeeper) + } + } () +} + +var queueUpdateSem = make(chan int, 1) + +func queueUpdate(h *Handlers) { + queueUpdateSem <- 1 + + select { + case _ = <-updateChan: + log.Println("Found pending update request. Don't start another one.") + default: + log.Println("Queuing an haproxy update.") + } + updateChan <- h + + <-queueUpdateSem +} func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { currentContent, _ := ioutil.ReadFile(conf.HAProxy.OutputPath) @@ -61,9 +87,7 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { err := ioutil.WriteFile(conf.HAProxy.OutputPath, []byte(newContent), 0666) if err != nil { log.Fatalf("Failed to write template on path: %s", err) } - execSem <- 1 execCommand(conf.HAProxy.ReloadCommand) - <-execSem log.Println("HAProxy: Configuration updated") return true From 6c0e875ef25fed68f1dcb506ea780b40ffd251e4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 20 Oct 2014 14:56:22 +0200 Subject: [PATCH 2/2] Make bamboo verbose on haproxy reload errors --- services/event_bus/event_handler.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/services/event_bus/event_handler.go b/services/event_bus/event_handler.go index d057e18..d7d0e90 100644 --- a/services/event_bus/event_handler.go +++ b/services/event_bus/event_handler.go @@ -87,9 +87,12 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { err := ioutil.WriteFile(conf.HAProxy.OutputPath, []byte(newContent), 0666) if err != nil { log.Fatalf("Failed to write template on path: %s", err) } - execCommand(conf.HAProxy.ReloadCommand) - - log.Println("HAProxy: Configuration updated") + err = execCommand(conf.HAProxy.ReloadCommand) + if err != nil { + log.Fatalf("HAProxy: update failed\n") + } else { + log.Println("HAProxy: Configuration updated") + } return true } else { log.Println("HAProxy: Same content, no need to reload") @@ -97,10 +100,12 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { } } -func execCommand(cmd string) { - _, err := exec.Command("sh", "-c", cmd).Output() +func execCommand(cmd string) error { + log.Printf("Exec cmd: %s \n", cmd) + output, err := exec.Command("sh", "-c", cmd).CombinedOutput() if err != nil { log.Println(err.Error()) + log.Println("Output:\n" + string(output[:])) } - log.Printf("Exec cmd: %s \n", cmd) + return err }