From c741ade0062a29747420584ade4ccec4c2cf212d Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Sun, 6 Sep 2015 00:18:06 -0400
Subject: [PATCH 01/12] Initial attempt at SRV based cluster discovery

---
 command/agent/command.go | 17 ++++++++
 command/agent/config.go  |  8 ++++
 command/agent/srv.go     | 86 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+)
 create mode 100644 command/agent/srv.go

diff --git a/command/agent/command.go b/command/agent/command.go
index e89e54490..d8380a5b9 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -76,6 +76,7 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.Var((*AppendSliceValue)(&tags), "tag",
 		"tag pair, specified as key=value")
 	cmdFlags.StringVar(&cmdConfig.Discover, "discover", "", "mDNS discovery name")
+	cmdFlags.StringVar(&cmdConfig.SRVName, "srvname", "", "SRV record to lookup")
 	cmdFlags.StringVar(&cmdConfig.Interface, "iface", "", "interface to bind to")
 	cmdFlags.StringVar(&cmdConfig.TagsFile, "tags-file", "", "tag persistence file")
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -381,6 +382,16 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	bindIP, bindPort, err := config.AddrParts(config.BindAddr)
 	bindAddr := &net.TCPAddr{IP: net.ParseIP(bindIP), Port: bindPort}
 
+	// Start the SRV lookup layer
+	if config.SRVName != "" {
+
+		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVName)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error starting SRV resolver: %s", err))
+			return nil
+		}
+	}
+
 	// Start the discovery layer
 	if config.Discover != "" {
 		// Use the advertise addr and port
@@ -427,6 +438,11 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	if config.Discover != "" {
 		c.Ui.Info(fmt.Sprintf("  mDNS cluster: %s", config.Discover))
 	}
+
+	if config.SRVName != "" {
+		c.Ui.Info(fmt.Sprintf("  SRV record: %s", config.SRVName))
+	}
+
 	return ipc
 }
 
@@ -737,6 +753,7 @@ Options:
                            can be reloaded during later agent starts. This option
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
+  -srvname                 SRV record to discover peers
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/config.go b/command/agent/config.go
index 09903131d..b7c60e64d 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -132,6 +132,11 @@ type Config struct {
 	// allows Serf agents to join each other with zero configuration.
 	Discover string `mapstructure:"discover"`
 
+	// SRVName is used look for other agents using DNS SRV records.
+	// When this is set, the agent will periodically look up the SRV record
+	// and attempt to add any hosts it finds
+	SRVName string `mapstructure:"srvname"`
+
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the
 	// address of the provided interface. It is also used to set the multicast
@@ -380,6 +385,9 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Discover != "" {
 		result.Discover = b.Discover
 	}
+	if b.SRVName != "" {
+		result.SRVName = b.SRVName
+	}
 	if b.Interface != "" {
 		result.Interface = b.Interface
 	}
diff --git a/command/agent/srv.go b/command/agent/srv.go
new file mode 100644
index 000000000..10e85458a
--- /dev/null
+++ b/command/agent/srv.go
@@ -0,0 +1,86 @@
+package agent
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"time"
+)
+
+const (
+	srvPollInterval  = 60 * time.Second
+	srvQuietInterval = 100 * time.Millisecond
+)
+
+// AgentSRV periodically polls an SRV record
+// And attempts to join the hosts supplied
+type AgentSRV struct {
+	agent   *Agent
+	srvname string
+	logger  *log.Logger
+	replay  bool
+}
+
+// NewAgentSRV is used to create a new AgentSRV
+func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvname string) (*AgentSRV, error) {
+
+	// Initialize the AgentSRV
+	m := &AgentSRV{
+		agent:   agent,
+		srvname: srvname,
+		logger:  log.New(logOutput, "", log.LstdFlags),
+		replay:  replay,
+	}
+
+	// Start the background workers
+	go m.run()
+	return m, nil
+}
+
+// run is a long running goroutine that scans for new hosts periodically
+func (m *AgentSRV) run() {
+	hosts := make(chan *net.SRV)
+	poll := time.After(0)
+	var quiet <-chan time.Time
+	var join []string
+
+	for {
+		select {
+		case h := <-hosts:
+			// Format the host address
+			addr := fmt.Sprintf("%s:%d", h.Target, h.Port)
+
+			// Queue for handling
+			join = append(join, addr)
+			quiet = time.After(srvQuietInterval)
+
+		case <-quiet:
+			// Attempt the join
+			n, err := m.agent.Join(join, m.replay)
+			if err != nil {
+				m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
+			}
+			if n > 0 {
+				m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+			}
+
+			join = nil
+
+		case <-poll:
+			poll = time.After(srvPollInterval)
+			go m.poll(hosts)
+		}
+	}
+}
+
+// poll is invoked periodically to check for new hosts
+func (m *AgentSRV) poll(hosts chan *net.SRV) {
+	_, results, err := net.LookupSRV("", "", m.srvname)
+	if err != nil {
+		m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
+	}
+	for _, host := range results {
+		hosts <- host
+	}
+}

From 8fce3a11fe96754abe6d3fd5c530be77021eac77 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Sun, 6 Sep 2015 09:47:26 -0400
Subject: [PATCH 02/12] Support multiple SRV records

---
 command/agent/command.go | 12 ++++++------
 command/agent/config.go  | 11 ++++++-----
 command/agent/srv.go     | 23 ++++++++++++++---------
 3 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index d8380a5b9..5dc4c1915 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -76,7 +76,7 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.Var((*AppendSliceValue)(&tags), "tag",
 		"tag pair, specified as key=value")
 	cmdFlags.StringVar(&cmdConfig.Discover, "discover", "", "mDNS discovery name")
-	cmdFlags.StringVar(&cmdConfig.SRVName, "srvname", "", "SRV record to lookup")
+	cmdFlags.StringVar(&cmdConfig.SRVRecords, "srvrecords", "", "SRV record to lookup")
 	cmdFlags.StringVar(&cmdConfig.Interface, "iface", "", "interface to bind to")
 	cmdFlags.StringVar(&cmdConfig.TagsFile, "tags-file", "", "tag persistence file")
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -383,9 +383,9 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	bindAddr := &net.TCPAddr{IP: net.ParseIP(bindIP), Port: bindPort}
 
 	// Start the SRV lookup layer
-	if config.SRVName != "" {
+	if config.SRVRecords != "" {
 
-		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVName)
+		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVRecords)
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("Error starting SRV resolver: %s", err))
 			return nil
@@ -439,8 +439,8 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 		c.Ui.Info(fmt.Sprintf("  mDNS cluster: %s", config.Discover))
 	}
 
-	if config.SRVName != "" {
-		c.Ui.Info(fmt.Sprintf("  SRV record: %s", config.SRVName))
+	if config.SRVRecords != "" {
+		c.Ui.Info(fmt.Sprintf("  SRV record: %s", config.SRVRecords))
 	}
 
 	return ipc
@@ -753,7 +753,7 @@ Options:
                            can be reloaded during later agent starts. This option
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
-  -srvname                 SRV record to discover peers
+  -srvrecords              SRV record(s) to discover peers. Accepts comma separated list.
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/config.go b/command/agent/config.go
index b7c60e64d..9b51348fd 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -132,10 +132,11 @@ type Config struct {
 	// allows Serf agents to join each other with zero configuration.
 	Discover string `mapstructure:"discover"`
 
-	// SRVName is used look for other agents using DNS SRV records.
+	// SRVRecords is used look for other agents using DNS SRV records.
 	// When this is set, the agent will periodically look up the SRV record
-	// and attempt to add any hosts it finds
-	SRVName string `mapstructure:"srvname"`
+	// and attempt to add any hosts it finds. You may pass a comma separated
+	// list if you wish to check multiple records
+	SRVRecords string `mapstructure:"srvrecords"`
 
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the
@@ -385,8 +386,8 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Discover != "" {
 		result.Discover = b.Discover
 	}
-	if b.SRVName != "" {
-		result.SRVName = b.SRVName
+	if b.SRVRecords != "" {
+		result.SRVRecords = b.SRVRecords
 	}
 	if b.Interface != "" {
 		result.Interface = b.Interface
diff --git a/command/agent/srv.go b/command/agent/srv.go
index 10e85458a..28f9d91ba 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"strings"
 	"time"
 )
 
@@ -17,18 +18,18 @@ const (
 // And attempts to join the hosts supplied
 type AgentSRV struct {
 	agent   *Agent
-	srvname string
+	srvrecords string
 	logger  *log.Logger
 	replay  bool
 }
 
 // NewAgentSRV is used to create a new AgentSRV
-func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvname string) (*AgentSRV, error) {
+func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords string) (*AgentSRV, error) {
 
 	// Initialize the AgentSRV
 	m := &AgentSRV{
 		agent:   agent,
-		srvname: srvname,
+		srvrecords: srvrecords,
 		logger:  log.New(logOutput, "", log.LstdFlags),
 		replay:  replay,
 	}
@@ -76,11 +77,15 @@ func (m *AgentSRV) run() {
 
 // poll is invoked periodically to check for new hosts
 func (m *AgentSRV) poll(hosts chan *net.SRV) {
-	_, results, err := net.LookupSRV("", "", m.srvname)
-	if err != nil {
-		m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
-	}
-	for _, host := range results {
-		hosts <- host
+	for _, record := range strings.Split(m.srvrecords, ",") {
+		_, results, err := net.LookupSRV("", "", record)
+
+		if err != nil {
+			m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
+		}
+
+		for _, host := range results {
+			hosts <- host
+		}
 	}
 }

From 84efa2ae5e8998130e00cb002c91e9b244b61176 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Sun, 6 Sep 2015 14:11:13 -0400
Subject: [PATCH 03/12] Specify SRV records individually

---
 command/agent/command.go | 11 ++++++-----
 command/agent/config.go  |  8 ++++----
 command/agent/srv.go     | 19 +++++++++----------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index 5dc4c1915..6461f0363 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -76,7 +76,8 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.Var((*AppendSliceValue)(&tags), "tag",
 		"tag pair, specified as key=value")
 	cmdFlags.StringVar(&cmdConfig.Discover, "discover", "", "mDNS discovery name")
-	cmdFlags.StringVar(&cmdConfig.SRVRecords, "srvrecords", "", "SRV record to lookup")
+	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.SRVRecords), "srvrecord",
+		"SRV record to lookup")
 	cmdFlags.StringVar(&cmdConfig.Interface, "iface", "", "interface to bind to")
 	cmdFlags.StringVar(&cmdConfig.TagsFile, "tags-file", "", "tag persistence file")
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -383,7 +384,7 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	bindAddr := &net.TCPAddr{IP: net.ParseIP(bindIP), Port: bindPort}
 
 	// Start the SRV lookup layer
-	if config.SRVRecords != "" {
+	if len(config.SRVRecords) > 0 {
 
 		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVRecords)
 		if err != nil {
@@ -439,8 +440,8 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 		c.Ui.Info(fmt.Sprintf("  mDNS cluster: %s", config.Discover))
 	}
 
-	if config.SRVRecords != "" {
-		c.Ui.Info(fmt.Sprintf("  SRV record: %s", config.SRVRecords))
+	if len(config.SRVRecords) > 0 {
+		c.Ui.Info(fmt.Sprintf("  SRV records: %s", strings.Join(config.SRVRecords, ", ")))
 	}
 
 	return ipc
@@ -753,7 +754,7 @@ Options:
                            can be reloaded during later agent starts. This option
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
-  -srvrecords              SRV record(s) to discover peers. Accepts comma separated list.
+  -srvrecord               SRV record to discover peers. Can be specified multiple times
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/config.go b/command/agent/config.go
index 9b51348fd..8b77233dd 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -134,9 +134,9 @@ type Config struct {
 
 	// SRVRecords is used look for other agents using DNS SRV records.
 	// When this is set, the agent will periodically look up the SRV record
-	// and attempt to add any hosts it finds. You may pass a comma separated
-	// list if you wish to check multiple records
-	SRVRecords string `mapstructure:"srvrecords"`
+	// and attempt to add any hosts it finds. You can specify multiple times
+	// to look up multiple SRV records.
+	SRVRecords []string `mapstructure:"srvrecord"`
 
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the
@@ -386,7 +386,7 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Discover != "" {
 		result.Discover = b.Discover
 	}
-	if b.SRVRecords != "" {
+	if len(b.SRVRecords) > 0 {
 		result.SRVRecords = b.SRVRecords
 	}
 	if b.Interface != "" {
diff --git a/command/agent/srv.go b/command/agent/srv.go
index 28f9d91ba..7ebc7decb 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -5,7 +5,6 @@ import (
 	"io"
 	"log"
 	"net"
-	"strings"
 	"time"
 )
 
@@ -17,21 +16,21 @@ const (
 // AgentSRV periodically polls an SRV record
 // And attempts to join the hosts supplied
 type AgentSRV struct {
-	agent   *Agent
-	srvrecords string
-	logger  *log.Logger
-	replay  bool
+	agent      *Agent
+	srvrecords []string
+	logger     *log.Logger
+	replay     bool
 }
 
 // NewAgentSRV is used to create a new AgentSRV
-func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords string) (*AgentSRV, error) {
+func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []string) (*AgentSRV, error) {
 
 	// Initialize the AgentSRV
 	m := &AgentSRV{
-		agent:   agent,
+		agent:      agent,
 		srvrecords: srvrecords,
-		logger:  log.New(logOutput, "", log.LstdFlags),
-		replay:  replay,
+		logger:     log.New(logOutput, "", log.LstdFlags),
+		replay:     replay,
 	}
 
 	// Start the background workers
@@ -77,7 +76,7 @@ func (m *AgentSRV) run() {
 
 // poll is invoked periodically to check for new hosts
 func (m *AgentSRV) poll(hosts chan *net.SRV) {
-	for _, record := range strings.Split(m.srvrecords, ",") {
+	for _, record := range m.srvrecords {
 		_, results, err := net.LookupSRV("", "", record)
 
 		if err != nil {

From d8dfc9c2b4309651bf6880496aa189326bbc630f Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Wed, 9 Sep 2015 17:56:15 -0400
Subject: [PATCH 04/12] Simplify to use a single loop with sleep

---
 command/agent/command.go |  2 +-
 command/agent/srv.go     | 55 +++++++++++++++-------------------------
 2 files changed, 22 insertions(+), 35 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index 6461f0363..d6e41f9d0 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -754,7 +754,7 @@ Options:
                            can be reloaded during later agent starts. This option
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
-  -srvrecord               SRV record to discover peers. Can be specified multiple times
+  -srvrecord               SRV record to discover peers. Can be specified multiple times.
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/srv.go b/command/agent/srv.go
index 7ebc7decb..c10771124 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -9,8 +9,7 @@ import (
 )
 
 const (
-	srvPollInterval  = 60 * time.Second
-	srvQuietInterval = 100 * time.Millisecond
+	srvPollInterval = 60 * time.Second
 )
 
 // AgentSRV periodically polls an SRV record
@@ -33,58 +32,46 @@ func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []st
 		replay:     replay,
 	}
 
-	// Start the background workers
+	// Start the poller in the background
 	go m.run()
 	return m, nil
 }
 
 // run is a long running goroutine that scans for new hosts periodically
 func (m *AgentSRV) run() {
-	hosts := make(chan *net.SRV)
-	poll := time.After(0)
-	var quiet <-chan time.Time
-	var join []string
 
 	for {
-		select {
-		case h := <-hosts:
-			// Format the host address
-			addr := fmt.Sprintf("%s:%d", h.Target, h.Port)
+		// Format the host address
+		records := m.querySRV()
 
-			// Queue for handling
-			join = append(join, addr)
-			quiet = time.After(srvQuietInterval)
+		n, err := m.agent.Join(records, m.replay)
 
-		case <-quiet:
-			// Attempt the join
-			n, err := m.agent.Join(join, m.replay)
-			if err != nil {
-				m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
-			}
-			if n > 0 {
-				m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
-			}
-
-			join = nil
-
-		case <-poll:
-			poll = time.After(srvPollInterval)
-			go m.poll(hosts)
+		// Attempt the join
+		if err != nil {
+			m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
 		}
+		if n > 0 {
+			m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+		}
+
+		time.Sleep(srvPollInterval)
 	}
 }
 
-// poll is invoked periodically to check for new hosts
-func (m *AgentSRV) poll(hosts chan *net.SRV) {
+// querySRV looks up the SRV records and returns a slice of all SRV records
+func (m *AgentSRV) querySRV() []string {
+	var hosts []string
 	for _, record := range m.srvrecords {
-		_, results, err := net.LookupSRV("", "", record)
+		_, srvhosts, err := net.LookupSRV("", "", record)
 
 		if err != nil {
 			m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
 		}
 
-		for _, host := range results {
-			hosts <- host
+		for _, host := range srvhosts {
+			addr := fmt.Sprintf("%s:%d", host.Target, host.Port)
+			hosts = append(hosts, addr)
 		}
 	}
+	return hosts
 }

From 9aacf784b1ae893394da7fe4ee7251c76acef9f4 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Wed, 9 Sep 2015 19:39:40 -0400
Subject: [PATCH 05/12] Don't attempt to rejoin hosts

---
 command/agent/srv.go | 52 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 12 deletions(-)

diff --git a/command/agent/srv.go b/command/agent/srv.go
index c10771124..164d26720 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -42,25 +42,38 @@ func (m *AgentSRV) run() {
 
 	for {
 		// Format the host address
-		records := m.querySRV()
+		records := m.findSRV()
 
-		n, err := m.agent.Join(records, m.replay)
+		// Attempt the join only if there are new records
+		if len(records) > 0 {
+			n, err := m.agent.Join(records, m.replay)
 
-		// Attempt the join
-		if err != nil {
-			m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
-		}
-		if n > 0 {
-			m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+			if err != nil {
+				m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
+			}
+			if n > 0 {
+				m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+			}
 		}
 
+		// Sleep until it's time to poll again
 		time.Sleep(srvPollInterval)
 	}
 }
 
-// querySRV looks up the SRV records and returns a slice of all SRV records
-func (m *AgentSRV) querySRV() []string {
+// findSRV looks up the SRV records and returns a slice of all SRV records
+// that are not currently cluster members
+func (m *AgentSRV) findSRV() []string {
 	var hosts []string
+
+	// map the members so that we only do a single O(n) search through members
+	known_members := make(map[string]bool)
+	for _, v := range m.agent.Serf().Members() {
+		member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
+		known_members[member] = true
+	}
+
+	// Look up each SRV record and check if it's already in the cluster
 	for _, record := range m.srvrecords {
 		_, srvhosts, err := net.LookupSRV("", "", record)
 
@@ -68,9 +81,24 @@ func (m *AgentSRV) querySRV() []string {
 			m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
 		}
 
+		// Filter each hosts in the SRV record
 		for _, host := range srvhosts {
-			addr := fmt.Sprintf("%s:%d", host.Target, host.Port)
-			hosts = append(hosts, addr)
+			// Find its addresses, as it's the only unique ID we can rely on
+			ipaddrs, err := net.LookupIP(host.Target)
+
+			if err != nil {
+				m.logger.Printf("[ERR] agent.srv: resolve SRV record %s to IP %v", host.Target, err)
+			}
+
+			// For each address the host has, check if it's already in the cluster
+			for _, ipaddr := range ipaddrs {
+				addr := fmt.Sprintf("%s:%d", ipaddr.String(), host.Port)
+				if _, known := known_members[addr]; !known {
+					// If the host is not already in the cluster,
+					// Add it to the list of hosts to try to join
+					hosts = append(hosts, addr)
+				}
+			}
 		}
 	}
 	return hosts

From ec1c534a37891414f901f930396c43a26be8919a Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Wed, 9 Sep 2015 20:16:03 -0400
Subject: [PATCH 06/12] Add a timeout to findSRV

---
 command/agent/srv.go | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/command/agent/srv.go b/command/agent/srv.go
index 164d26720..a5c833c8a 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -10,6 +10,7 @@ import (
 
 const (
 	srvPollInterval = 60 * time.Second
+	srvFindTimeout  = srvPollInterval / 2
 )
 
 // AgentSRV periodically polls an SRV record
@@ -41,21 +42,27 @@ func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []st
 func (m *AgentSRV) run() {
 
 	for {
-		// Format the host address
-		records := m.findSRV()
-
-		// Attempt the join only if there are new records
-		if len(records) > 0 {
-			n, err := m.agent.Join(records, m.replay)
-
-			if err != nil {
-				m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
-			}
-			if n > 0 {
-				m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+		// Set up a channel and select so we can timeout if findSRV takes too long
+		// set channel to 1 so that goroutines can't pile-up
+		c := make(chan []string, 1)
+		go func() { c <- m.findSRV() }()
+		select {
+		case records := <-c:
+			// Attempt the join only if there are new records
+			if len(records) > 0 {
+				n, err := m.agent.Join(records, m.replay)
+
+				if err != nil {
+					m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
+				}
+				if n > 0 {
+					m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+				}
 			}
+		// Report the timeout
+		case <-time.After(srvFindTimeout):
+			m.logger.Printf("[ERR] agent.srv: findSRV timed out")
 		}
-
 		// Sleep until it's time to poll again
 		time.Sleep(srvPollInterval)
 	}

From 485105008856fe1be6bacc826a655d4e3e773a88 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Wed, 9 Sep 2015 20:30:45 -0400
Subject: [PATCH 07/12] Only check against members that are alive

---
 command/agent/srv.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/command/agent/srv.go b/command/agent/srv.go
index a5c833c8a..4eb4637e8 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -76,8 +76,10 @@ func (m *AgentSRV) findSRV() []string {
 	// map the members so that we only do a single O(n) search through members
 	known_members := make(map[string]bool)
 	for _, v := range m.agent.Serf().Members() {
-		member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
-		known_members[member] = true
+		if v.Status.String() == "alive" {
+			member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
+			known_members[member] = true
+		}
 	}
 
 	// Look up each SRV record and check if it's already in the cluster

From e1c8fefdd0a97f95151b0d70498630aef1fb2196 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Thu, 10 Sep 2015 20:11:54 -0400
Subject: [PATCH 08/12] Remove useless retries for SRV, make rejoin optional

---
 command/agent/command.go |  6 +++-
 command/agent/config.go  |  7 +++++
 command/agent/srv.go     | 60 ++++++++++++++++++++++------------------
 3 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index d6e41f9d0..fa0651428 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -78,6 +78,8 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.StringVar(&cmdConfig.Discover, "discover", "", "mDNS discovery name")
 	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.SRVRecords), "srvrecord",
 		"SRV record to lookup")
+	cmdFlags.BoolVar(&cmdConfig.RetrySRV, "retry-srv", false,
+		"poll the SRV record for changes and keep trying to join")
 	cmdFlags.StringVar(&cmdConfig.Interface, "iface", "", "interface to bind to")
 	cmdFlags.StringVar(&cmdConfig.TagsFile, "tags-file", "", "tag persistence file")
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -386,7 +388,7 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	// Start the SRV lookup layer
 	if len(config.SRVRecords) > 0 {
 
-		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVRecords)
+		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVRecords, config.RetrySRV)
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("Error starting SRV resolver: %s", err))
 			return nil
@@ -755,6 +757,8 @@ Options:
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
   -srvrecord               SRV record to discover peers. Can be specified multiple times.
+  -retry-srv               When provided, continuously try to join SRV hosts
+                           that are not already members of the cluster.
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/config.go b/command/agent/config.go
index 8b77233dd..d511415a3 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -138,6 +138,10 @@ type Config struct {
 	// to look up multiple SRV records.
 	SRVRecords []string `mapstructure:"srvrecord"`
 
+	// If set, continuously poll the SRV records supplied to try and join
+	// all hosts that are not already members of the cluster.
+	RetrySRV bool `mapstructure:"retry-srv"`
+
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the
 	// address of the provided interface. It is also used to set the multicast
@@ -389,6 +393,9 @@ func MergeConfig(a, b *Config) *Config {
 	if len(b.SRVRecords) > 0 {
 		result.SRVRecords = b.SRVRecords
 	}
+	if b.RetrySRV {
+		result.RetrySRV = true
+	}
 	if b.Interface != "" {
 		result.Interface = b.Interface
 	}
diff --git a/command/agent/srv.go b/command/agent/srv.go
index 4eb4637e8..5f79b758e 100644
--- a/command/agent/srv.go
+++ b/command/agent/srv.go
@@ -6,11 +6,12 @@ import (
 	"log"
 	"net"
 	"time"
+
+	"github.com/hashicorp/serf/serf"
 )
 
 const (
 	srvPollInterval = 60 * time.Second
-	srvFindTimeout  = srvPollInterval / 2
 )
 
 // AgentSRV periodically polls an SRV record
@@ -20,10 +21,11 @@ type AgentSRV struct {
 	srvrecords []string
 	logger     *log.Logger
 	replay     bool
+	retry      bool
 }
 
 // NewAgentSRV is used to create a new AgentSRV
-func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []string) (*AgentSRV, error) {
+func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []string, retry bool) (*AgentSRV, error) {
 
 	// Initialize the AgentSRV
 	m := &AgentSRV{
@@ -31,43 +33,47 @@ func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []st
 		srvrecords: srvrecords,
 		logger:     log.New(logOutput, "", log.LstdFlags),
 		replay:     replay,
+		retry:      retry,
 	}
 
-	// Start the poller in the background
-	go m.run()
+	if m.retry {
+		// Start the poller in the background
+		m.logger.Printf("[INFO] Starting SRV background poller.")
+		go m.poll()
+	} else {
+		m.logger.Printf("[INFO] Starting SRV for one-shot join.")
+		// A one-shot attempt to try to join the cluster via SRV
+		go m.joinSRV()
+	}
 	return m, nil
 }
 
 // run is a long running goroutine that scans for new hosts periodically
-func (m *AgentSRV) run() {
+func (m *AgentSRV) poll() {
 
 	for {
-		// Set up a channel and select so we can timeout if findSRV takes too long
-		// set channel to 1 so that goroutines can't pile-up
-		c := make(chan []string, 1)
-		go func() { c <- m.findSRV() }()
-		select {
-		case records := <-c:
-			// Attempt the join only if there are new records
-			if len(records) > 0 {
-				n, err := m.agent.Join(records, m.replay)
-
-				if err != nil {
-					m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
-				}
-				if n > 0 {
-					m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
-				}
-			}
-		// Report the timeout
-		case <-time.After(srvFindTimeout):
-			m.logger.Printf("[ERR] agent.srv: findSRV timed out")
-		}
+		m.joinSRV()
 		// Sleep until it's time to poll again
 		time.Sleep(srvPollInterval)
 	}
 }
 
+// Attempt to any hosts in the SRV record not already in the cluster
+func (m *AgentSRV) joinSRV() {
+	records := m.findSRV()
+	// Attempt the join only if there are new records
+	if len(records) > 0 {
+		n, err := m.agent.Join(records, m.replay)
+
+		if err != nil {
+			m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
+		}
+		if n > 0 {
+			m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
+		}
+	}
+}
+
 // findSRV looks up the SRV records and returns a slice of all SRV records
 // that are not currently cluster members
 func (m *AgentSRV) findSRV() []string {
@@ -76,7 +82,7 @@ func (m *AgentSRV) findSRV() []string {
 	// map the members so that we only do a single O(n) search through members
 	known_members := make(map[string]bool)
 	for _, v := range m.agent.Serf().Members() {
-		if v.Status.String() == "alive" {
+		if v.Status.String() == serf.SerfAlive.String() {
 			member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
 			known_members[member] = true
 		}

From 4faefaf8bdc788812de431a116124d20341fb23d Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Sat, 12 Sep 2015 17:40:37 -0400
Subject: [PATCH 09/12] Mirror behavior of join and retry-join more closely

---
 command/agent/command.go | 153 +++++++++++++++++++++++++++++++++------
 command/agent/config.go  |  17 ++---
 command/agent/srv.go     | 120 ------------------------------
 3 files changed, 140 insertions(+), 150 deletions(-)
 delete mode 100644 command/agent/srv.go

diff --git a/command/agent/command.go b/command/agent/command.go
index fa0651428..08fc8fef8 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -76,10 +77,10 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.Var((*AppendSliceValue)(&tags), "tag",
 		"tag pair, specified as key=value")
 	cmdFlags.StringVar(&cmdConfig.Discover, "discover", "", "mDNS discovery name")
-	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.SRVRecords), "srvrecord",
-		"SRV record to lookup")
-	cmdFlags.BoolVar(&cmdConfig.RetrySRV, "retry-srv", false,
-		"poll the SRV record for changes and keep trying to join")
+	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.JoinSRV), "join-srv",
+		"SRV record to join on startup")
+	cmdFlags.Var((*AppendSliceValue)(&cmdConfig.RetryJoinSRV), "retry-join-srv",
+		"SRV record to join on startup with retry")
 	cmdFlags.StringVar(&cmdConfig.Interface, "iface", "", "interface to bind to")
 	cmdFlags.StringVar(&cmdConfig.TagsFile, "tags-file", "", "tag persistence file")
 	cmdFlags.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
@@ -385,16 +386,6 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	bindIP, bindPort, err := config.AddrParts(config.BindAddr)
 	bindAddr := &net.TCPAddr{IP: net.ParseIP(bindIP), Port: bindPort}
 
-	// Start the SRV lookup layer
-	if len(config.SRVRecords) > 0 {
-
-		_, err := NewAgentSRV(agent, logOutput, config.ReplayOnJoin, config.SRVRecords, config.RetrySRV)
-		if err != nil {
-			c.Ui.Error(fmt.Sprintf("Error starting SRV resolver: %s", err))
-			return nil
-		}
-	}
-
 	// Start the discovery layer
 	if config.Discover != "" {
 		// Use the advertise addr and port
@@ -442,11 +433,126 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 		c.Ui.Info(fmt.Sprintf("  mDNS cluster: %s", config.Discover))
 	}
 
-	if len(config.SRVRecords) > 0 {
-		c.Ui.Info(fmt.Sprintf("  SRV records: %s", strings.Join(config.SRVRecords, ", ")))
+	return ipc
+}
+
+// startupJoin is invoked to handle any joins specified to take place at start time
+func (c *Command) startupJoinSRV(config *Config, agent *Agent) error {
+	if len(config.JoinSRV) == 0 {
+		return nil
+	}
+
+	c.Ui.Output(fmt.Sprintf("Joining cluster via SRV...(replay: %v)", config.ReplayOnJoin))
+	n, err := c.joinSRV(agent, config.ReplayOnJoin, config.JoinSRV)
+	if err != nil {
+		return err
+	}
+
+	if n == 0 {
+		return errors.New("Failed to join any hosts via SRV")
 	}
 
-	return ipc
+	c.Ui.Info(fmt.Sprintf("Join completed. Synced with %d initial agents", n))
+	return nil
+
+}
+
+// retryJoinSRV is invoked to handle joins with retries. This runs until at least a
+// single successful join or RetryMaxAttempts is reached
+func (c *Command) retryJoinSRV(config *Config, agent *Agent, errCh chan struct{}) {
+	// Quit fast if there is no nodes to join
+	if len(config.RetryJoinSRV) == 0 {
+		return
+	}
+
+	// Track the number of join attempts
+	attempt := 0
+	for {
+		// Try to perform the join
+		n, err := c.joinSRV(agent, config.ReplayOnJoin, config.RetryJoinSRV)
+
+		if err != nil {
+			c.logger.Printf("[ERR] agent: Failed to join via SRV: %v", err)
+		}
+
+		if err == nil && n > 0 {
+			c.logger.Printf("[INFO] agent: Join completed. Synced with %d initial agents", n)
+		}
+
+		// Check if the maximum attempts has been exceeded
+		attempt++
+		if config.RetryMaxAttempts > 0 && attempt > config.RetryMaxAttempts {
+			c.logger.Printf("[ERR] agent: maximum retry SRV join attempts made, exiting")
+			close(errCh)
+			return
+		}
+
+		c.logger.Printf("[INFO] agent: Will check SRV again in %v", config.RetryInterval)
+		time.Sleep(config.RetryInterval)
+	}
+}
+
+func (c *Command) joinSRV(agent *Agent, replay bool, srvrecords []string) (int, error) {
+	records := c.findSRV(agent, srvrecords)
+	// Attempt the join only if there are new records
+	if len(records) > 0 {
+		n, err := agent.Join(records, replay)
+
+		if err != nil {
+			return n, err
+		}
+
+		if n > 0 {
+			c.logger.Printf("[INFO] agent: Joined %d hosts via SRV", n)
+		}
+
+	}
+	return 0, nil
+}
+
+// findSRV looks up the SRV records and returns a slice of all SRV records
+// that are not currently cluster members
+func (c *Command) findSRV(agent *Agent, srvrecords []string) []string {
+	var hosts []string
+
+	// map the members so that we only do a single O(n) search through members
+	known_members := make(map[string]bool)
+	for _, v := range agent.Serf().Members() {
+		if v.Status.String() == serf.SerfAlive.String() {
+			member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
+			known_members[member] = true
+		}
+	}
+
+	// Look up each SRV record and check if it's already in the cluster
+	for _, record := range srvrecords {
+		_, srvhosts, err := net.LookupSRV("", "", record)
+
+		if err != nil {
+			c.logger.Printf("[ERR] agent: Failed to poll for new SRV hosts: %v", err)
+		}
+
+		// Filter each hosts in the SRV record
+		for _, host := range srvhosts {
+			// Find its addresses, as it's the only unique ID we can rely on
+			ipaddrs, err := net.LookupIP(host.Target)
+
+			if err != nil {
+				c.logger.Printf("[ERR] agent: resolve SRV record %s to IP %v", host.Target, err)
+			}
+
+			// For each address the host has, check if it's already in the cluster
+			for _, ipaddr := range ipaddrs {
+				addr := fmt.Sprintf("%s:%d", ipaddr.String(), host.Port)
+				if _, known := known_members[addr]; !known {
+					// If the host is not already in the cluster,
+					// Add it to the list of hosts to try to join
+					hosts = append(hosts, addr)
+				}
+			}
+		}
+	}
+	return hosts
 }
 
 // startupJoin is invoked to handle any joins specified to take place at start time
@@ -578,6 +684,11 @@ func (c *Command) Run(args []string) int {
 		return 1
 	}
 
+	if err := c.startupJoinSRV(config, agent); err != nil {
+		c.Ui.Error(err.Error())
+		return 1
+	}
+
 	// Enable log streaming
 	c.Ui.Info("")
 	c.Ui.Output("Log data will now stream in as it occurs:\n")
@@ -586,6 +697,7 @@ func (c *Command) Run(args []string) int {
 	// Start the retry joins
 	retryJoinCh := make(chan struct{})
 	go c.retryJoin(config, agent, retryJoinCh)
+	go c.retryJoinSRV(config, agent, retryJoinCh)
 
 	// Wait for exit
 	return c.handleSignals(config, agent, retryJoinCh)
@@ -739,7 +851,9 @@ Options:
   -retry-join=addr         An agent to join with. This flag be specified multiple times.
                            Does not exit on failure like -join, used to retry until success.
   -retry-interval=30s      Sets the interval on which a node will attempt to retry joining
-                           nodes provided by -retry-join. Defaults to 30s.
+                           nodes provided by -retry-join or -retry-join-srv. Defaults to 30s.
+  -join-srv=record         SRV record to discover peers. Can be specified multiple times.
+  -retry-join-srv=record   Like join-srv, but retry on failure.
   -retry-max=0             Limits the number of retry events. Defaults to 0 for unlimited.
   -role=foo                The role of this node, if any. This can be used
                            by event scripts to differentiate different types
@@ -756,9 +870,6 @@ Options:
                            can be reloaded during later agent starts. This option
                            is incompatible with the '-tag' option and requires there
                            be no tags in the agent configuration file, if given.
-  -srvrecord               SRV record to discover peers. Can be specified multiple times.
-  -retry-srv               When provided, continuously try to join SRV hosts
-                           that are not already members of the cluster.
   -syslog                  When provided, logs will also be sent to syslog.
 
 Event handlers:
diff --git a/command/agent/config.go b/command/agent/config.go
index d511415a3..b13bb088f 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -133,14 +133,13 @@ type Config struct {
 	Discover string `mapstructure:"discover"`
 
 	// SRVRecords is used look for other agents using DNS SRV records.
-	// When this is set, the agent will periodically look up the SRV record
+	// When this is set, the agent will look up the SRV record
 	// and attempt to add any hosts it finds. You can specify multiple times
 	// to look up multiple SRV records.
-	SRVRecords []string `mapstructure:"srvrecord"`
+	JoinSRV []string `mapstructure:"join-srv"`
 
-	// If set, continuously poll the SRV records supplied to try and join
-	// all hosts that are not already members of the cluster.
-	RetrySRV bool `mapstructure:"retry-srv"`
+	// Exactly like join-srv, only keep trying until a successful join happens.
+	RetryJoinSRV []string `mapstructure:"retry-join-srv"`
 
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the
@@ -390,11 +389,11 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Discover != "" {
 		result.Discover = b.Discover
 	}
-	if len(b.SRVRecords) > 0 {
-		result.SRVRecords = b.SRVRecords
+	if len(b.JoinSRV) > 0 {
+		result.JoinSRV = b.JoinSRV
 	}
-	if b.RetrySRV {
-		result.RetrySRV = true
+	if len(b.RetryJoinSRV) > 0 {
+		result.RetryJoinSRV = b.RetryJoinSRV
 	}
 	if b.Interface != "" {
 		result.Interface = b.Interface
diff --git a/command/agent/srv.go b/command/agent/srv.go
deleted file mode 100644
index 5f79b758e..000000000
--- a/command/agent/srv.go
+++ /dev/null
@@ -1,120 +0,0 @@
-package agent
-
-import (
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"time"
-
-	"github.com/hashicorp/serf/serf"
-)
-
-const (
-	srvPollInterval = 60 * time.Second
-)
-
-// AgentSRV periodically polls an SRV record
-// And attempts to join the hosts supplied
-type AgentSRV struct {
-	agent      *Agent
-	srvrecords []string
-	logger     *log.Logger
-	replay     bool
-	retry      bool
-}
-
-// NewAgentSRV is used to create a new AgentSRV
-func NewAgentSRV(agent *Agent, logOutput io.Writer, replay bool, srvrecords []string, retry bool) (*AgentSRV, error) {
-
-	// Initialize the AgentSRV
-	m := &AgentSRV{
-		agent:      agent,
-		srvrecords: srvrecords,
-		logger:     log.New(logOutput, "", log.LstdFlags),
-		replay:     replay,
-		retry:      retry,
-	}
-
-	if m.retry {
-		// Start the poller in the background
-		m.logger.Printf("[INFO] Starting SRV background poller.")
-		go m.poll()
-	} else {
-		m.logger.Printf("[INFO] Starting SRV for one-shot join.")
-		// A one-shot attempt to try to join the cluster via SRV
-		go m.joinSRV()
-	}
-	return m, nil
-}
-
-// run is a long running goroutine that scans for new hosts periodically
-func (m *AgentSRV) poll() {
-
-	for {
-		m.joinSRV()
-		// Sleep until it's time to poll again
-		time.Sleep(srvPollInterval)
-	}
-}
-
-// Attempt to any hosts in the SRV record not already in the cluster
-func (m *AgentSRV) joinSRV() {
-	records := m.findSRV()
-	// Attempt the join only if there are new records
-	if len(records) > 0 {
-		n, err := m.agent.Join(records, m.replay)
-
-		if err != nil {
-			m.logger.Printf("[ERR] agent.srv: Failed to join: %v", err)
-		}
-		if n > 0 {
-			m.logger.Printf("[INFO] agent.srv: Joined %d hosts", n)
-		}
-	}
-}
-
-// findSRV looks up the SRV records and returns a slice of all SRV records
-// that are not currently cluster members
-func (m *AgentSRV) findSRV() []string {
-	var hosts []string
-
-	// map the members so that we only do a single O(n) search through members
-	known_members := make(map[string]bool)
-	for _, v := range m.agent.Serf().Members() {
-		if v.Status.String() == serf.SerfAlive.String() {
-			member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
-			known_members[member] = true
-		}
-	}
-
-	// Look up each SRV record and check if it's already in the cluster
-	for _, record := range m.srvrecords {
-		_, srvhosts, err := net.LookupSRV("", "", record)
-
-		if err != nil {
-			m.logger.Printf("[ERR] agent.srv: Failed to poll for new hosts: %v", err)
-		}
-
-		// Filter each hosts in the SRV record
-		for _, host := range srvhosts {
-			// Find its addresses, as it's the only unique ID we can rely on
-			ipaddrs, err := net.LookupIP(host.Target)
-
-			if err != nil {
-				m.logger.Printf("[ERR] agent.srv: resolve SRV record %s to IP %v", host.Target, err)
-			}
-
-			// For each address the host has, check if it's already in the cluster
-			for _, ipaddr := range ipaddrs {
-				addr := fmt.Sprintf("%s:%d", ipaddr.String(), host.Port)
-				if _, known := known_members[addr]; !known {
-					// If the host is not already in the cluster,
-					// Add it to the list of hosts to try to join
-					hosts = append(hosts, addr)
-				}
-			}
-		}
-	}
-	return hosts
-}

From 49d0e690389410884b13ef22edfab747217b7eff Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Tue, 15 Sep 2015 22:11:19 -0400
Subject: [PATCH 10/12] Address most comments

---
 command/agent/command.go | 20 ++++++++------------
 command/agent/config.go  |  4 ++--
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index 08fc8fef8..031e52021 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -436,7 +436,7 @@ func (c *Command) startAgent(config *Config, agent *Agent,
 	return ipc
 }
 
-// startupJoin is invoked to handle any joins specified to take place at start time
+// startupJoinSRV attempts to join a cluster provided by SRV records
 func (c *Command) startupJoinSRV(config *Config, agent *Agent) error {
 	if len(config.JoinSRV) == 0 {
 		return nil
@@ -495,19 +495,14 @@ func (c *Command) retryJoinSRV(config *Config, agent *Agent, errCh chan struct{}
 func (c *Command) joinSRV(agent *Agent, replay bool, srvrecords []string) (int, error) {
 	records := c.findSRV(agent, srvrecords)
 	// Attempt the join only if there are new records
-	if len(records) > 0 {
-		n, err := agent.Join(records, replay)
+	if len(records) == 0 {
+		return 0, nil
+	}
 
-		if err != nil {
-			return n, err
-		}
+	n, err := agent.Join(records, replay)
 
-		if n > 0 {
-			c.logger.Printf("[INFO] agent: Joined %d hosts via SRV", n)
-		}
+	return n, err
 
-	}
-	return 0, nil
 }
 
 // findSRV looks up the SRV records and returns a slice of all SRV records
@@ -529,7 +524,8 @@ func (c *Command) findSRV(agent *Agent, srvrecords []string) []string {
 		_, srvhosts, err := net.LookupSRV("", "", record)
 
 		if err != nil {
-			c.logger.Printf("[ERR] agent: Failed to poll for new SRV hosts: %v", err)
+			c.logger.Printf("[ERR] agent: Failed to poll %s for new SRV hosts: %v", record, err)
+			continue
 		}
 
 		// Filter each hosts in the SRV record
diff --git a/command/agent/config.go b/command/agent/config.go
index b13bb088f..4fdede32e 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -136,10 +136,10 @@ type Config struct {
 	// When this is set, the agent will look up the SRV record
 	// and attempt to add any hosts it finds. You can specify multiple times
 	// to look up multiple SRV records.
-	JoinSRV []string `mapstructure:"join-srv"`
+	JoinSRV []string `mapstructure:"join_srv"`
 
 	// Exactly like join-srv, only keep trying until a successful join happens.
-	RetryJoinSRV []string `mapstructure:"retry-join-srv"`
+	RetryJoinSRV []string `mapstructure:"retry_join_srv"`
 
 	// Interface is used to provide a binding interface to use. It can be
 	// used instead of providing a bind address, as Serf will discover the

From 9872524ad27af600c266463e9d2054bb36b15a6f Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Tue, 15 Sep 2015 22:21:08 -0400
Subject: [PATCH 11/12] Simplify as requested

---
 command/agent/command.go | 33 +++++----------------------------
 1 file changed, 5 insertions(+), 28 deletions(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index 031e52021..8b30bb995 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -477,6 +477,7 @@ func (c *Command) retryJoinSRV(config *Config, agent *Agent, errCh chan struct{}
 
 		if err == nil && n > 0 {
 			c.logger.Printf("[INFO] agent: Join completed. Synced with %d initial agents", n)
+			return
 		}
 
 		// Check if the maximum attempts has been exceeded
@@ -505,20 +506,10 @@ func (c *Command) joinSRV(agent *Agent, replay bool, srvrecords []string) (int,
 
 }
 
-// findSRV looks up the SRV records and returns a slice of all SRV records
-// that are not currently cluster members
+// findSRV looks up the SRV records and returns a slice of all hosts in the records
 func (c *Command) findSRV(agent *Agent, srvrecords []string) []string {
 	var hosts []string
 
-	// map the members so that we only do a single O(n) search through members
-	known_members := make(map[string]bool)
-	for _, v := range agent.Serf().Members() {
-		if v.Status.String() == serf.SerfAlive.String() {
-			member := fmt.Sprintf("%s:%d", v.Addr.String(), v.Port)
-			known_members[member] = true
-		}
-	}
-
 	// Look up each SRV record and check if it's already in the cluster
 	for _, record := range srvrecords {
 		_, srvhosts, err := net.LookupSRV("", "", record)
@@ -528,24 +519,10 @@ func (c *Command) findSRV(agent *Agent, srvrecords []string) []string {
 			continue
 		}
 
-		// Filter each hosts in the SRV record
+		// Add the hosts from the SRV record
 		for _, host := range srvhosts {
-			// Find its addresses, as it's the only unique ID we can rely on
-			ipaddrs, err := net.LookupIP(host.Target)
-
-			if err != nil {
-				c.logger.Printf("[ERR] agent: resolve SRV record %s to IP %v", host.Target, err)
-			}
-
-			// For each address the host has, check if it's already in the cluster
-			for _, ipaddr := range ipaddrs {
-				addr := fmt.Sprintf("%s:%d", ipaddr.String(), host.Port)
-				if _, known := known_members[addr]; !known {
-					// If the host is not already in the cluster,
-					// Add it to the list of hosts to try to join
-					hosts = append(hosts, addr)
-				}
-			}
+			addr := fmt.Sprintf("%s:%d", host.Target, host.Port)
+			hosts = append(hosts, addr)
 		}
 	}
 	return hosts

From dfaaa7b096a6329d2098b0019243d35219d7d3f5 Mon Sep 17 00:00:00 2001
From: Dale Hamel <dale.hamel@shopify.com>
Date: Tue, 15 Sep 2015 22:30:13 -0400
Subject: [PATCH 12/12] Return a reasonable error message

---
 command/agent/command.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/command/agent/command.go b/command/agent/command.go
index 8b30bb995..3bc4f9c8f 100644
--- a/command/agent/command.go
+++ b/command/agent/command.go
@@ -497,7 +497,7 @@ func (c *Command) joinSRV(agent *Agent, replay bool, srvrecords []string) (int,
 	records := c.findSRV(agent, srvrecords)
 	// Attempt the join only if there are new records
 	if len(records) == 0 {
-		return 0, nil
+		return 0, errors.New("No hosts found in SRV record")
 	}
 
 	n, err := agent.Join(records, replay)