Secure cluster traffic via mutual TLS (#2237)

* Add TLS option to gossip cluster

Co-authored-by: Sharad Gaur <sharadgaur@gmail.com>
Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* generate new certs that expire in 100 years

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Fix tls_connection attributes

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Improve error message

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Fix tls client config docs

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Add capacity arg to message buffer

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* fix formatting

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Update version; add version validation

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* use lru cache for connection pool

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* lock reading from the connection

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* when extracting net.Conn from tlsConn, lock and throw away wrapper

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* Add mutex to connection pool to protect cache

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

* fix linting

Signed-off-by: Dustin Hooten <dustinhooten@gmail.com>

Co-authored-by: Sharad Gaur <sharadgaur@gmail.com>

.gitignore

@@ -11,7 +11,9 @@
 !.golangci.yml
 !/cli/testdata/*.yml
 !/cli/config/testdata/*.yml
+!/cluster/testdata/*.yml
 !/config/testdata/*.yml
+!/examples/ha/tls/*.yml
 !/notify/email/testdata/*.yml
 !/doc/examples/simple.yml
 !/circle.yml

cluster/cluster.go

@@ -29,7 +29,6 @@ import (
 	"github.com/hashicorp/memberlist"
 	"github.com/oklog/ulid"
 	"github.com/pkg/errors"
-
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -141,6 +140,7 @@ func Create(
 	tcpTimeout time.Duration,
 	probeTimeout time.Duration,
 	probeInterval time.Duration,
+	tlsTransportConfig *TLSTransportConfig,
 ) (*Peer, error) {
 	bindHost, bindPortStr, err := net.SplitHostPort(bindAddr)
 	if err != nil {
@@ -235,6 +235,14 @@ func Create(
 		p.setInitialFailed(resolvedPeers, bindAddr)
 	}
 
+	if tlsTransportConfig != nil {
+		level.Info(l).Log("msg", "using TLS for gossip")
+		cfg.Transport, err = NewTLSTransport(context.Background(), l, reg, cfg.BindAddr, cfg.BindPort, tlsTransportConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "tls transport")
+		}
+	}
+
 	ml, err := memberlist.Create(cfg)
 	if err != nil {
 		return nil, errors.Wrap(err, "create memberlist")

cluster/cluster_test.go

@@ -35,6 +35,7 @@ func TestClusterJoinAndReconnect(t *testing.T) {
 	t.Run("TestReconnect", testReconnect)
 	t.Run("TestRemoveFailedPeers", testRemoveFailedPeers)
 	t.Run("TestInitiallyFailingPeers", testInitiallyFailingPeers)
+	t.Run("TestTLSConnection", testTLSConnection)
 }
 
 func testJoinLeave(t *testing.T) {
@@ -51,6 +52,7 @@ func testJoinLeave(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p)
@@ -83,6 +85,7 @@ func testJoinLeave(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p2)
@@ -116,6 +119,7 @@ func testReconnect(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p)
@@ -139,6 +143,7 @@ func testReconnect(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p2)
@@ -177,6 +182,7 @@ func testRemoveFailedPeers(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p)
@@ -226,6 +232,7 @@ func testInitiallyFailingPeers(t *testing.T) {
 		DefaultTcpTimeout,
 		DefaultProbeTimeout,
 		DefaultProbeInterval,
+		nil,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, p)
@@ -254,3 +261,68 @@ func testInitiallyFailingPeers(t *testing.T) {
 		require.Equal(t, expectedLen, len(p.failedPeers))
 	}
 }
+
+func testTLSConnection(t *testing.T) {
+	logger := log.NewNopLogger()
+	tlsTransportConfig1, err := GetTLSTransportConfig("./testdata/tls_config_node1.yml")
+	require.NoError(t, err)
+	p1, err := Create(
+		logger,
+		prometheus.NewRegistry(),
+		"127.0.0.1:0",
+		"",
+		[]string{},
+		true,
+		DefaultPushPullInterval,
+		DefaultGossipInterval,
+		DefaultTcpTimeout,
+		DefaultProbeTimeout,
+		DefaultProbeInterval,
+		tlsTransportConfig1,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, p1)
+	err = p1.Join(
+		DefaultReconnectInterval,
+		DefaultReconnectTimeout,
+	)
+	require.NoError(t, err)
+	require.False(t, p1.Ready())
+	require.Equal(t, p1.Status(), "settling")
+	go p1.Settle(context.Background(), 0*time.Second)
+	p1.WaitReady(context.Background())
+	require.Equal(t, p1.Status(), "ready")
+
+	// Create the peer who joins the first.
+	tlsTransportConfig2, err := GetTLSTransportConfig("./testdata/tls_config_node2.yml")
+	require.NoError(t, err)
+	p2, err := Create(
+		logger,
+		prometheus.NewRegistry(),
+		"127.0.0.1:0",
+		"",
+		[]string{p1.Self().Address()},
+		true,
+		DefaultPushPullInterval,
+		DefaultGossipInterval,
+		DefaultTcpTimeout,
+		DefaultProbeTimeout,
+		DefaultProbeInterval,
+		tlsTransportConfig2,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, p2)
+	err = p2.Join(
+		DefaultReconnectInterval,
+		DefaultReconnectTimeout,
+	)
+	require.NoError(t, err)
+	go p2.Settle(context.Background(), 0*time.Second)
+
+	require.Equal(t, 2, p1.ClusterSize())
+	p2.Leave(0 * time.Second)
+	require.Equal(t, 1, p1.ClusterSize())
+	require.Equal(t, 1, len(p1.failedPeers))
+	require.Equal(t, p2.Self().Address(), p1.peers[p2.Self().Address()].Node.Address())
+	require.Equal(t, p2.Name(), p1.failedPeers[0].Name)
+}

cluster/clusterpb/cluster.pb.go

@@ -24,6 +24,31 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
+type MemberlistMessage_Kind int32
+
+const (
+	MemberlistMessage_STREAM MemberlistMessage_Kind = 0
+	MemberlistMessage_PACKET MemberlistMessage_Kind = 1
+)
+
+var MemberlistMessage_Kind_name = map[int32]string{
+	0: "STREAM",
+	1: "PACKET",
+}
+
+var MemberlistMessage_Kind_value = map[string]int32{
+	"STREAM": 0,
+	"PACKET": 1,
+}
+
+func (x MemberlistMessage_Kind) String() string {
+	return proto.EnumName(MemberlistMessage_Kind_name, int32(x))
+}
+
+func (MemberlistMessage_Kind) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_3cfb3b8ec240c376, []int{2, 0}
+}
+
 type Part struct {
 	Key                  string   `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
 	Data                 []byte   `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
@@ -105,26 +130,79 @@ func (m *FullState) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_FullState proto.InternalMessageInfo
 
+type MemberlistMessage struct {
+	Version              string                 `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	Kind                 MemberlistMessage_Kind `protobuf:"varint,2,opt,name=kind,proto3,enum=clusterpb.MemberlistMessage_Kind" json:"kind,omitempty"`
+	FromAddr             string                 `protobuf:"bytes,3,opt,name=from_addr,json=fromAddr,proto3" json:"from_addr,omitempty"`
+	Msg                  []byte                 `protobuf:"bytes,4,opt,name=msg,proto3" json:"msg,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}               `json:"-"`
+	XXX_unrecognized     []byte                 `json:"-"`
+	XXX_sizecache        int32                  `json:"-"`
+}
+
+func (m *MemberlistMessage) Reset()         { *m = MemberlistMessage{} }
+func (m *MemberlistMessage) String() string { return proto.CompactTextString(m) }
+func (*MemberlistMessage) ProtoMessage()    {}
+func (*MemberlistMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor_3cfb3b8ec240c376, []int{2}
+}
+func (m *MemberlistMessage) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *MemberlistMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_MemberlistMessage.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *MemberlistMessage) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MemberlistMessage.Merge(m, src)
+}
+func (m *MemberlistMessage) XXX_Size() int {
+	return m.Size()
+}
+func (m *MemberlistMessage) XXX_DiscardUnknown() {
+	xxx_messageInfo_MemberlistMessage.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MemberlistMessage proto.InternalMessageInfo
+
 func init() {
+	proto.RegisterEnum("clusterpb.MemberlistMessage_Kind", MemberlistMessage_Kind_name, MemberlistMessage_Kind_value)
 	proto.RegisterType((*Part)(nil), "clusterpb.Part")
 	proto.RegisterType((*FullState)(nil), "clusterpb.FullState")
+	proto.RegisterType((*MemberlistMessage)(nil), "clusterpb.MemberlistMessage")
 }
 
 func init() { proto.RegisterFile("cluster.proto", fileDescriptor_3cfb3b8ec240c376) }
 
 var fileDescriptor_3cfb3b8ec240c376 = []byte{
-	// 168 bytes of a gzipped FileDescriptorProto
+	// 289 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4d, 0xce, 0x29, 0x2d,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xdf, 0x4a, 0xc3, 0x30,
-	0x2e, 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x84, 0x72, 0x0b, 0x92, 0xa4,
+	0x14, 0xc6, 0x17, 0x17, 0xa7, 0x3d, 0xfe, 0xab, 0xc1, 0x8b, 0xa0, 0x50, 0x67, 0xaf, 0x06, 0x4a,
-	0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0xa2, 0xfa, 0x20, 0x16, 0x44, 0x81, 0x92, 0x0e, 0x17, 0x4b,
+	0x85, 0x89, 0xe0, 0xed, 0x26, 0xf3, 0x66, 0x14, 0x46, 0xb7, 0x7b, 0x49, 0x4d, 0x2c, 0x65, 0x6d,
-	0x40, 0x62, 0x51, 0x89, 0x90, 0x00, 0x17, 0x73, 0x76, 0x6a, 0xa5, 0x04, 0xa3, 0x02, 0xa3, 0x06,
+	0x53, 0x92, 0x4c, 0xf0, 0xb9, 0x7c, 0x89, 0x5e, 0xfa, 0x04, 0xa2, 0x7d, 0x12, 0x49, 0x36, 0x45,
-	0x67, 0x10, 0x88, 0x29, 0x24, 0xc4, 0xc5, 0x92, 0x92, 0x58, 0x92, 0x28, 0xc1, 0xa4, 0xc0, 0xa8,
+	0xf0, 0xee, 0x77, 0x0e, 0x5f, 0x7e, 0x1f, 0x27, 0x70, 0xf0, 0x54, 0xac, 0xb4, 0x11, 0x2a, 0xaa,
-	0xc1, 0x13, 0x04, 0x66, 0x2b, 0x59, 0x70, 0x71, 0xba, 0x95, 0xe6, 0xe4, 0x04, 0x97, 0x24, 0x96,
+	0x95, 0x34, 0x92, 0x78, 0x9b, 0xb1, 0x4e, 0x4f, 0x4f, 0x32, 0x99, 0x49, 0xb7, 0xbd, 0xb6, 0xb4,
-	0xa4, 0x0a, 0x69, 0x73, 0xb1, 0x16, 0x24, 0x16, 0x95, 0x14, 0x4b, 0x30, 0x2a, 0x30, 0x6b, 0x70,
+	0x0e, 0x84, 0x57, 0x80, 0x67, 0x4c, 0x19, 0xe2, 0x43, 0x77, 0x29, 0x5e, 0x29, 0xea, 0xa3, 0x81,
-	0x1b, 0xf1, 0xeb, 0xc1, 0xed, 0xd2, 0x03, 0x19, 0xe9, 0xc4, 0x72, 0xe2, 0x9e, 0x3c, 0x43, 0x10,
+	0x97, 0x58, 0x24, 0x04, 0x30, 0x67, 0x86, 0xd1, 0xad, 0x3e, 0x1a, 0xec, 0x27, 0x8e, 0xc3, 0x3b,
-	0x44, 0x8d, 0x93, 0xc0, 0x89, 0x87, 0x72, 0x0c, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7,
+	0xf0, 0x1e, 0x56, 0x45, 0x31, 0x37, 0xcc, 0x08, 0x72, 0x09, 0xdb, 0x35, 0x53, 0x46, 0x53, 0xd4,
-	0xf8, 0xe0, 0x91, 0x1c, 0x63, 0x12, 0x1b, 0xd8, 0x01, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff,
+	0xef, 0x0e, 0xf6, 0x86, 0x47, 0xd1, 0x6f, 0x57, 0x64, 0x95, 0x63, 0xdc, 0x7c, 0x9c, 0x77, 0x92,
-	0xfd, 0x3c, 0xdb, 0xe7, 0xb2, 0x00, 0x00, 0x00,
+	0x75, 0x26, 0x7c, 0x43, 0x70, 0x1c, 0x8b, 0x32, 0x15, 0xaa, 0xc8, 0xb5, 0x89, 0x85, 0xd6, 0x2c,
+	0x13, 0x84, 0xc2, 0xce, 0x8b, 0x50, 0x3a, 0x97, 0xd5, 0xa6, 0xf9, 0x67, 0x24, 0xb7, 0x80, 0x97,
+	0x79, 0xc5, 0x5d, 0xfb, 0xe1, 0xf0, 0xe2, 0x8f, 0xfb, 0x9f, 0x25, 0x9a, 0xe6, 0x15, 0x4f, 0x5c,
+	0x9c, 0x9c, 0x81, 0xf7, 0xac, 0x64, 0xf9, 0xc8, 0x38, 0x57, 0xb4, 0xeb, 0x94, 0xbb, 0x76, 0x31,
+	0xe2, 0x5c, 0xd9, 0x1b, 0x4b, 0x9d, 0x51, 0xec, 0x0e, 0xb2, 0x18, 0x06, 0x80, 0xed, 0x63, 0x02,
+	0xd0, 0x9b, 0x2f, 0x92, 0xc9, 0x28, 0xf6, 0x3b, 0x96, 0x67, 0xa3, 0xfb, 0xe9, 0x64, 0xe1, 0xa3,
+	0xb1, 0xdf, 0x7c, 0x05, 0x9d, 0xa6, 0x0d, 0xd0, 0x7b, 0x1b, 0xa0, 0xcf, 0x36, 0x40, 0x69, 0xcf,
+	0x7d, 0xdb, 0xcd, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaa, 0x3f, 0xca, 0x45, 0x68, 0x01, 0x00,
+	0x00,
 }
 
 func (m *Part) Marshal() (dAtA []byte, err error) {
@@ -209,6 +287,59 @@ func (m *FullState) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *MemberlistMessage) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *MemberlistMessage) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *MemberlistMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.XXX_unrecognized != nil {
+		i -= len(m.XXX_unrecognized)
+		copy(dAtA[i:], m.XXX_unrecognized)
+	}
+	if len(m.Msg) > 0 {
+		i -= len(m.Msg)
+		copy(dAtA[i:], m.Msg)
+		i = encodeVarintCluster(dAtA, i, uint64(len(m.Msg)))
+		i--
+		dAtA[i] = 0x22
+	}
+	if len(m.FromAddr) > 0 {
+		i -= len(m.FromAddr)
+		copy(dAtA[i:], m.FromAddr)
+		i = encodeVarintCluster(dAtA, i, uint64(len(m.FromAddr)))
+		i--
+		dAtA[i] = 0x1a
+	}
+	if m.Kind != 0 {
+		i = encodeVarintCluster(dAtA, i, uint64(m.Kind))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Version) > 0 {
+		i -= len(m.Version)
+		copy(dAtA[i:], m.Version)
+		i = encodeVarintCluster(dAtA, i, uint64(len(m.Version)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintCluster(dAtA []byte, offset int, v uint64) int {
 	offset -= sovCluster(v)
 	base := offset
@@ -258,6 +389,33 @@ func (m *FullState) Size() (n int) {
 	return n
 }
 
+func (m *MemberlistMessage) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Version)
+	if l > 0 {
+		n += 1 + l + sovCluster(uint64(l))
+	}
+	if m.Kind != 0 {
+		n += 1 + sovCluster(uint64(m.Kind))
+	}
+	l = len(m.FromAddr)
+	if l > 0 {
+		n += 1 + l + sovCluster(uint64(l))
+	}
+	l = len(m.Msg)
+	if l > 0 {
+		n += 1 + l + sovCluster(uint64(l))
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
 func sovCluster(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -466,6 +624,174 @@ func (m *FullState) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *MemberlistMessage) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowCluster
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: MemberlistMessage: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: MemberlistMessage: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCluster
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthCluster
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthCluster
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Version = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType)
+			}
+			m.Kind = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCluster
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Kind |= MemberlistMessage_Kind(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FromAddr", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCluster
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthCluster
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthCluster
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.FromAddr = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Msg", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowCluster
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthCluster
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthCluster
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Msg = append(m.Msg[:0], dAtA[iNdEx:postIndex]...)
+			if m.Msg == nil {
+				m.Msg = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipCluster(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthCluster
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipCluster(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

cluster/clusterpb/cluster.proto

@@ -16,3 +16,14 @@ message Part {
 message FullState {
   repeated Part parts = 1 [(gogoproto.nullable) = false];
 }
+
+message MemberlistMessage {
+  string version = 1;
+  enum Kind {
+    STREAM = 0;
+    PACKET = 1;
+  }
+  Kind kind = 2;
+  string from_addr = 3;
+  bytes msg = 4;
+}

cluster/connection_pool.go

@@ -0,0 +1,84 @@
+// Copyright 2020 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"crypto/tls"
+	"fmt"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/pkg/errors"
+)
+
+const capacity = 1024
+
+type connectionPool struct {
+	mtx       sync.Mutex
+	cache     *lru.Cache
+	tlsConfig *tls.Config
+}
+
+func newConnectionPool(tlsClientCfg *tls.Config) (*connectionPool, error) {
+	cache, err := lru.NewWithEvict(
+		capacity, func(_ interface{}, value interface{}) {
+			conn, ok := value.(*tlsConn)
+			if ok {
+				_ = conn.Close()
+			}
+		},
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create new LRU")
+	}
+	return &connectionPool{
+		cache:     cache,
+		tlsConfig: tlsClientCfg,
+	}, nil
+}
+
+// borrowConnection returns a *tlsConn from the pool. The connection does not
+// need to be returned to the pool because each connection has its own locking.
+func (pool *connectionPool) borrowConnection(addr string, timeout time.Duration) (*tlsConn, error) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+	if pool.cache == nil {
+		return nil, errors.New("connection pool closed")
+	}
+	key := fmt.Sprintf("%s/%d", addr, int64(timeout))
+	value, exists := pool.cache.Get(key)
+	if exists {
+		conn, ok := value.(*tlsConn)
+		if ok && conn.alive() {
+			return conn, nil
+		}
+	}
+	conn, err := dialTLSConn(addr, timeout, pool.tlsConfig)
+	if err != nil {
+		return nil, err
+	}
+	pool.cache.Add(key, conn)
+	return conn, nil
+}
+
+func (pool *connectionPool) shutdown() {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+	if pool.cache == nil {
+		return
+	}
+	pool.cache.Purge()
+	pool.cache = nil
+}

cluster/testdata/certs/ca-config.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "876000h"
+    },
+    "profiles": {
+      "massl": {
+        "usages": ["signing", "key encipherment", "server auth", "client auth"],
+        "expiry": "876000h"
+      }
+    }
+  }
+}

cluster/testdata/certs/ca-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "massl",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "massl",
+      "OU": "VIC",
+      "ST": "Victoria"
+    }
+  ]
+}

cluster/testdata/certs/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAuljDjKVGwlyiuKTSHc1QpoZPX9dbgwU/9113ctI8U/ZwMWLp
+nZ4f/zVpf4LW5foM9zSEUGPiyJe/NaTZUOXkRBSIQ13QroK4OJ1XGacQKpTxewCb
+ChESZEfKWEhnP/Y7BYc4z1Li6Dkxh4TIElHwOVe62jbhNnzYlr4evmSuiuItAc8u
+hEYxncThPzmHEWPXKw8CFNhxCSYsjbb72UAIht0knMHQ7VXBX1VuuL0rolJBiToC
+va+I6CjG0c6qfi9/BcPsuW6cNjmQnwTg6SaSoGO/5zgbxBgy9MZQEot88d1T2XH6
+rBANYsfojvyCXuytWnj04mvdAWwmFh0hhq+nxQIDAQABAoIBAQCwcL1vXUq7W4UD
+OaRtbWrQ0dk0ETBnxT/E0y33fRJ8GZovWM2EXSVEuukSP+uEQ5elNYeWqo0fi3cT
+ruvJSnMw9xPyXVDq+4C8slW3R1TqTK683VzvUizM4KC5qIyCpn1KBbgHrh6E7Sp1
+e4cIuaawVN3qIg5qThmx2YA4nBIcEt68q9cpy3NgEe+EQf44zM/St+y8kSkDUOVw
+fNKX0WfZ/hPL1TAYpWiIgSf+m/V3d/1l/scvMYONcuSjXSORCyoeAWYtOQgf78wW
+9j3kiBTaqDYCUZFnY/ltlZrm8ltAaKVJ0MmPKjVh8GJBXZp9fSVU8Y4ZIZRSeuBA
+OoStHGAdAoGBAMluMIE33hGny2V0dNzW23D84eXQK38AsdP632jQeuzxBknItg45
+qAfzh8F8W10DQnSv5tj0bmOHfo0mG09bu9eo5nLLINOE7Ju/7ly/76RNJNJ4ADjx
+JKZi/PpvfP+s/fzel0X3OPgA+CJKzUHuqlU4V9BLc7focZAYtaM2w7rHAoGBAOzU
+eXpapkqYhbYRcsrVV57nZV0rLzsLVJBpJg2zC8un95ALrr0rlZfuPJfOCY/uuS1w
+f8ixRz2MkRWGreLHy35NB4GV0sF9VPn1jMp9SuBNvO0JRUMWuDAdVe8SCjXadrOh
++m3yKJSkFKDchglUYnZKV1skgA/b9jjjnu2fvd0TAoGAVUTnFZxvzmuIp78fxWjS
+5ka23hE8iHvjy4e00WsHzovNjKiBoQ35Orx16ItbJcm+dSUNhSQcItf104yhHPwJ
+Tab7PvcMQ15OxzP9lJfPu27Iuqv/9Bro1+Kpkt5lPNqffk9AHGcmX54RbHrb3yBI
+TOEYE14Nc3nbsRM0uQ3y13sCgYB5Om4QZpSWvKo9P4M+NqTKb3JglblwhOU9osVa
+39ra3dkIgCJrLQM/KTEVF9+nMLDThLG0fqKT6/9cQHuECXet6Co+d/3RE6HK7Zmr
+ESWh2ckqoMM2i0uvPWT+ooJdfL2kR/bUDtAc/jyc9yUZY3ufR4Cd4/o1pAfOqR1y
+T4G1xwKBgQChE4VWawCVg2qanRjvZcdNk0zpZx4dxqqKYq/VHuSfjNLQixIZsgXT
+xx9BHuORn6c/nurqEStLwN3BzbpPU/j6YjMUmTslSH2sKhHwWNYGBZC52aJiOOda
+Bz6nAkihG0n2PjYt2T84w6FWHgLJuSsmiEVJcb+AOdyKh1MlzJiwMQ==
+-----END RSA PRIVATE KEY-----

cluster/testdata/certs/ca.csr

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICpzCCAY8CAQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMx
+DjAMBgNVBAMTBW1hc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+uljDjKVGwlyiuKTSHc1QpoZPX9dbgwU/9113ctI8U/ZwMWLpnZ4f/zVpf4LW5foM
+9zSEUGPiyJe/NaTZUOXkRBSIQ13QroK4OJ1XGacQKpTxewCbChESZEfKWEhnP/Y7
+BYc4z1Li6Dkxh4TIElHwOVe62jbhNnzYlr4evmSuiuItAc8uhEYxncThPzmHEWPX
+Kw8CFNhxCSYsjbb72UAIht0knMHQ7VXBX1VuuL0rolJBiToCva+I6CjG0c6qfi9/
+BcPsuW6cNjmQnwTg6SaSoGO/5zgbxBgy9MZQEot88d1T2XH6rBANYsfojvyCXuyt
+Wnj04mvdAWwmFh0hhq+nxQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAJFmooMt
+TocElxCb3DGJTRUXxr4DqcATASIX35a2wV3MmPqUHHXr6BQkO/FRho66EsZf3DE/
+mumou01K+KByxgsmw04CACjSeZ2t/g6pAsDCKrx/BwL3tAo09lG2Y2Ah0BND2Cta
+EZpTliU2MimZlk7UZb8VIXh2Tx56fZRoHLzO4U4+FY8ZR+tspxPRM7hLg/aUqA5D
+zGj6kByX8aYjxsmQokP4rx/w2mz6vwt4cZ1pXwr0RderkMIh9Har/0k9X1WIAP61
+PNQx74qnaq+icjtN2+8gvJE/CJL/wfcwW6kQwEtX1xsTpnzyFaRoYpSPQrvkCtiW
++WzgnOh7RvKyAYI=
+-----END CERTIFICATE REQUEST-----

cluster/testdata/certs/ca.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDlDCCAnygAwIBAgIUetcc8yLasSAxvyUWtgTVKR6jfBIwDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMB4XDTIxMDUwNTE2MTYwMFoXDTI2MDUwNDE2MTYwMFowYjELMAkGA1UE
+BhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlNZWxib3VybmUxDjAM
+BgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMTBW1hc3NsMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuljDjKVGwlyiuKTSHc1QpoZPX9db
+gwU/9113ctI8U/ZwMWLpnZ4f/zVpf4LW5foM9zSEUGPiyJe/NaTZUOXkRBSIQ13Q
+roK4OJ1XGacQKpTxewCbChESZEfKWEhnP/Y7BYc4z1Li6Dkxh4TIElHwOVe62jbh
+NnzYlr4evmSuiuItAc8uhEYxncThPzmHEWPXKw8CFNhxCSYsjbb72UAIht0knMHQ
+7VXBX1VuuL0rolJBiToCva+I6CjG0c6qfi9/BcPsuW6cNjmQnwTg6SaSoGO/5zgb
+xBgy9MZQEot88d1T2XH6rBANYsfojvyCXuytWnj04mvdAWwmFh0hhq+nxQIDAQAB
+o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
+d4DTElKq6gnGYDJZgJvC+4flrZAwDQYJKoZIhvcNAQELBQADggEBAHYaMxcyVdFP
+QZJHDP6x+A1hHeZy/kaL2VPzUIOgt3ONEvjhsJblFjLaTWwXFJwxVHXRsQjqrURj
+dD2VIkMbCJM4WN8odi6PdHscL0DwHKzwZDapQxsOUYcPBbR2EsBPRyWSoJmBnEyZ
+0TANtyUcZNWc7/C4GG66SWypu1YRdtcaDiEOVkdetGWuo5bw1J7B7isQ4J4a3+Qn
+iTKId/7wWBY95JS2xJuOeEjk/ty74ruSapk2Aip1GRSRXMDtD0XeO5dpxc6eIU3C
+jbRpUGqCdSAfzxQTlxQAMEKdgRlt7lzMFX/PAXagDOMC9D/6APn5fquEV2d9wMWs
+0N9UCwKftAU=
+-----END CERTIFICATE-----

cluster/testdata/certs/node1-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:server",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "system:node1",
+      "OU": "massl",
+      "ST": "Victoria"
+    }
+  ]
+}

cluster/testdata/certs/node1-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA1b9bm4rvDtpYsqgtCC52+L535d4/Q2O10fWD2i2CfRXXfYJQ
+5cr4AV2iqScFsJSs7KwyQde/c4VWj/vEA2/SJHZFBlknKdCcrgHVebrvnzm6Guze
+ICutZSKocFXy9Kw+YmWuA64nHVfmSCKG07GhXhEsLsSCn4PTDYOiGAUm1GdSDxUp
+8yUXec13Eb20mld0xE9kQnCnEWRnxMXtQJXoz9lpLc7DgXtN6nCXSG/CqdDPOduU
+nmseaxyAGpAFnUmxcqUuYAJUQ1hUOJhk0RVSsLTmu+FGdOxk79AxmmKQ2z9l/GuA
+VikVJGTxY4jRPezxHQ3bdqzzCIdJxTxLinftZQIDAQABAoIBADpxQtvphemau8vF
+feKRycfDVEcOmF+VoL4SkgWSke4fjbbsbbAW6e59qp7zY3PfgtSHVIp6Mgek+oEN
+xo9mAKAlkkPlFncxadWN/M921FPF1ePMxgMnzhYr/sAQUAikG76NrKGm+VzljrpE
+bnbtR4DP0zPKWSjCQ2+bgTNuHSrPwUtEngVT6ugjfWU1RitlvjTsZ9hSuOSBlS7P
+rjbQGaEh53PraDut8PIlF4wIF+nLeERFP/a6DC8Btpbv9P50YRosag6yU/G+OYX9
+spvBPvRJGrubslKnNRz9AcjbVd3QhL+Tm7mV7iakK918jLWb95Ro4WW+9lT6IAi6
+xRSOr9UCgYEA5wI3JhKkYa4PST7ALqmJSDkPH8+tctiEx+ovmnqBufFuLWFoO/rc
+EOYslnaZM3UVCnhrFv7+LxezSI5DyQu8dBEzf0RMICvXUNBkGC7ZJQL428fjXPhX
+8mZIoJ0ol4hbamr8yTYlK0vGTwqN1bDj71w6NszuN4ecN1cKNWsMbnMCgYEA7N8Y
+MzHWNijMr7xZ1lXl4Ye9+kp3aTUjUYBBaxFr4bQ8y0fH48lzq3qOso5wgvp0DKYo
+uemD5QKbo81LKoTRLa+nxPq0UqKm9FiSWmnrcxMuph989oZ1ZFHA2B+nvbuMTF8J
+8sESclTSbgkG87DpycJOUwG3XAcXM+80pXuzJscCgYB+Dzxu/09KqoRW8PJIxGVQ
+zypMrrS07iiPO2FcyCtQf8oi43vQ91Ttt91vAisZ5HNl8k5mDyJAKovANToSVOAy
+6kwSz/9GswXdaMqmU7JVOyj4Lj0JN9AuS9ioJPrIrjVMfjORzYU8+i2uZlD94niP
+3uE5lF0OWmdJ36qHefIftwKBgQDcPQZcO19H1iGS2FbTYeSvEK5ENM7YRG8FTXIF
+4hnjrtjDzYb+tYVWEErznFrifYo/ZJMDYSqgWQ9reusDqqBvkR41mUDmgJMpJ91U
+MZ2YzmIWVbqz4QrvbtAWY0Bsuh/VtpwiWQAUy+coJj6PgJOvY3m91h+tcm5RfHz/
+zIcjawKBgA6kDcOLOnWcvhP3XwtW5dcWlNuBBNEKna+kIT/5Vlrh91y2w7F54DNK
+i0w5CZCpbTugJmZ67XLHnfongC7e2vAQ3atoT96RU4mf9614qs9LMtGAbnuCLB8+
+sT2rnaZKtzr83ensbYkbBxP/zmPBfFQ9FKcIYIA7En8zAIr2T3vJ
+-----END RSA PRIVATE KEY-----

cluster/testdata/certs/node1.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC5TCCAc0CAQAwczELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxFTATBgNVBAoTDHN5c3RlbTpub2RlMTEOMAwGA1UE
+CxMFbWFzc2wxFjAUBgNVBAMTDXN5c3RlbTpzZXJ2ZXIwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDVv1ubiu8O2liyqC0ILnb4vnfl3j9DY7XR9YPaLYJ9
+Fdd9glDlyvgBXaKpJwWwlKzsrDJB179zhVaP+8QDb9IkdkUGWScp0JyuAdV5uu+f
+Oboa7N4gK61lIqhwVfL0rD5iZa4DricdV+ZIIobTsaFeESwuxIKfg9MNg6IYBSbU
+Z1IPFSnzJRd5zXcRvbSaV3TET2RCcKcRZGfExe1AlejP2WktzsOBe03qcJdIb8Kp
+0M8525Seax5rHIAakAWdSbFypS5gAlRDWFQ4mGTRFVKwtOa74UZ07GTv0DGaYpDb
+P2X8a4BWKRUkZPFjiNE97PEdDdt2rPMIh0nFPEuKd+1lAgMBAAGgLTArBgkqhkiG
+9w0BCQ4xHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0B
+AQsFAAOCAQEAW/tTyJaBfWtbC9hYUmhh8lxUztv2+WT4xaR/jdQ46sk/87vKuwI6
+4AkkGfiPLLqgW3xbQOwk5/ynRabttbsgTUHt744RtRFLzfcQKEBZoNPvrfHvmDil
+YqHIOx2SJ5hzIBwVlVSBn50hdSSED1Ip22DaU8GukzuacB8+2rhg3MOWJbKVt5aR
+03H4XkAynLS1FHNOraDIv1eT58D3l4hanrNOZIa0xAuChd25qLO/JHvU/3wccGUA
+KNg3vGOy2Q8qVBrTFLn+yQHuOr/wSupXESO1jiI/h+txsBQnZ6oYfZnVJ+7o3Oln
+3Hguw77aYeTAeZQPPbmJbDLegLG0ZC6RmA==
+-----END CERTIFICATE REQUEST-----

cluster/testdata/certs/node1.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEAjCCAuqgAwIBAgIUbYMGwSgQF8iRZ5xmhflInj8VZ0owDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMCAXDTIxMDUwNTE2MTYwMFoYDzIxMjEwNDExMTYxNjAwWjBzMQswCQYD
+VQQGEwJBVTERMA8GA1UECBMIVmljdG9yaWExEjAQBgNVBAcTCU1lbGJvdXJuZTEV
+MBMGA1UEChMMc3lzdGVtOm5vZGUxMQ4wDAYDVQQLEwVtYXNzbDEWMBQGA1UEAxMN
+c3lzdGVtOnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANW/
+W5uK7w7aWLKoLQgudvi+d+XeP0NjtdH1g9otgn0V132CUOXK+AFdoqknBbCUrOys
+MkHXv3OFVo/7xANv0iR2RQZZJynQnK4B1Xm67585uhrs3iArrWUiqHBV8vSsPmJl
+rgOuJx1X5kgihtOxoV4RLC7Egp+D0w2DohgFJtRnUg8VKfMlF3nNdxG9tJpXdMRP
+ZEJwpxFkZ8TF7UCV6M/ZaS3Ow4F7Tepwl0hvwqnQzznblJ5rHmscgBqQBZ1JsXKl
+LmACVENYVDiYZNEVUrC05rvhRnTsZO/QMZpikNs/ZfxrgFYpFSRk8WOI0T3s8R0N
+23as8wiHScU8S4p37WUCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l
+BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE
+FGprx5v+KrO4DeOtA6kps4BL/zKyMB8GA1UdIwQYMBaAFHeA0xJSquoJxmAyWYCb
+wvuH5a2QMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF
+AAOCAQEAmWTdMLyWOrNAS0uY+u3FUV3Hm50xF1PfxbT6wK1hu6vH6B63E0o9K2/1
+U25Ie8Y2IzFocKMvbqC+mrY56G0bWoUlMONhthYqm8uTKtjlFO33A9I7WIT9Tw+B
+nnwZZO7+Ljkd30qSzBinCjrIEx31Vq2pr54ungd8+wK8nfz/zdZnJcqxcN9zvCXB
+GTE8yCuqGWKk/oDuIzVjr73U0QaWi+vThqJtBjhOIWQHHVJwbIyhuYzUaivgZPYB
+8eKXWk4JH3eAcq5z5koNGyCcZd/k4WnvxZYxNBAkoQ6AWVfEMGOCaRjD1FTnMbpG
+BW79ndJqLmn8OH+DeCnSWhTWxAgg+Q==
+-----END CERTIFICATE-----

cluster/testdata/certs/node2-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:server",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "system:node2",
+      "OU": "massl",
+      "ST": "Victoria"
+    }
+  ]
+}

cluster/testdata/certs/node2-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtCtzT9vhRMTbhAg/pm8eBn+4IvVQeVqnHoEon9IKIx5fyvqS
+Q6Ui3xSik9kJq5FSAa1mScajJwfB1o6ycaSP6n+Q88Py4v7q65n0stCHoJCH0uPw
+MQyEhwX7nNilV9C4UZTyZ2StDdAjmMBHiN81EJAqH2d4Xtgrd/IIWhljSXm+aPbu
+QjSz8BtR/7+MswrCdlJ8y6gWi020kt6GSHjmaxI1jStGvBxxksK86v3J97wfNwWY
+7GJi70uBrvO0pk5bYckDzUTKeN1QGvBnZ8uDXs7pPysvftJr85GzX0iE9YLMDxO3
+qc/PlwCdxM8H6gHTTkLPizGZtpMF9Z497pW9YQIDAQABAoIBAFfQwdCPxHmnVbNB
+7fwqNsFGKTLozMOJeuE0ZN+ZGZXKbTha70WHTLrcrO1RIRR9rTHiGXQmHEmez0zL
+mpAnfHn4mWcm/9DCHTCehpVNbH3HVFxm+yB9EG9bbCsjsVtfASfKaGgauvp7k44V
+UgiVeqDLE6zg2tunk3BQCOAZdbpOiXrdvoZiGx2Q4SMLPfzmfIyH4BUT836pLTmp
+o6/yNiFqQWfCgjeEAOQor4TcdzYIT+3wP51HfAjhZKMIvmjwL16ov1/QpmWRD4ni
+4svzYpeMYpl5OrZkKeDS4ZIQBGjxk+fzPmfFUbfVRSI2gDORsah8HoRVI4LnwKWn
+7kQDv0ECgYEA6V+KVb8bPzCZNbroEZFdug6YtT4yv5Mj3/kpMTIvA3vtu02v8e7F
+O56yT43QfUZA0Ar37O0HQ6mbpPsRE5RSr70i40RR+slMZVHX/AQViG7oQJGBijPt
+1tFdLnb+1wSON3jYt2975Kw2IfgOXprWtEmL5zGuplEUjx9Lbdf1HjkCgYEAxaNe
+XgXdAiWFoY4Qq6xBRO/WNZCdn3Ysqx6snCtDRilxeNyDoE/6x2Ma9/NRBtIiulAb
+s09vDRfJKLbzocUhIn8BQ+GkbAS/A6+x2vcuGhK3F84xqZdbrCqvqdJS8K824jug
+vUCfCBJlyNRDz8kEsN5odLM1xkij93Jv23HvGGkCgYEAptcz6ctfalSPI9eEs5KO
+REbNK73UwBssaaISreYnsED4G5EVuUuvW8k/xxomtHj2OwWsa4ilSd1GtbL8aVf/
+qT35ZCrixP0GjeTuGXC+CDTp+8dKqggoAAzbpi1SUVwjZEsT/EhKdZgcdzqE42Ol
+HWz7BQUCzEpo/U0tOtFKnxkCgYEAi05Vy8wyNbsg7/jlAzyNXPv4bxUaJTX00kDy
+xbkw2BmKI/i6xprZVwUiEzdsG3SuicjBXahVzFLBtXMPUy1R57DBwYkgjgriYMTM
+hlzIIBSk/aCXHMTVFwuXegoH8CJwexIwgHU2I0hkeiQ0EBfOuKRr2CYhdzvoZxhA
+g9tQ/lECgYAjPYoXfNI3rHCWUmaD5eDJZpE0xuJeiiy5auojykdAc7vVapNaIyMK
+G3EaU44RtXcSwH19TlH9UCm3MH1QiIwaBOzGcKj3Ut6ZyFKuWDUk4yqvps3uZU/h
+h16Tp49Ja7/4LY1uuEngg1KMEiWgk5jiU7G0H9zrtEiTj9c3FDKDvg==
+-----END RSA PRIVATE KEY-----

cluster/testdata/certs/node2.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC5TCCAc0CAQAwczELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxFTATBgNVBAoTDHN5c3RlbTpub2RlMjEOMAwGA1UE
+CxMFbWFzc2wxFjAUBgNVBAMTDXN5c3RlbTpzZXJ2ZXIwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0K3NP2+FExNuECD+mbx4Gf7gi9VB5WqcegSif0goj
+Hl/K+pJDpSLfFKKT2QmrkVIBrWZJxqMnB8HWjrJxpI/qf5Dzw/Li/urrmfSy0Ieg
+kIfS4/AxDISHBfuc2KVX0LhRlPJnZK0N0COYwEeI3zUQkCofZ3he2Ct38ghaGWNJ
+eb5o9u5CNLPwG1H/v4yzCsJ2UnzLqBaLTbSS3oZIeOZrEjWNK0a8HHGSwrzq/cn3
+vB83BZjsYmLvS4Gu87SmTlthyQPNRMp43VAa8Gdny4Nezuk/Ky9+0mvzkbNfSIT1
+gswPE7epz8+XAJ3EzwfqAdNOQs+LMZm2kwX1nj3ulb1hAgMBAAGgLTArBgkqhkiG
+9w0BCQ4xHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0B
+AQsFAAOCAQEARh0Pi36mNmyprU4j25GWNqQYCJ6cBGnaPeiwr8/F3rsGsF4LTQdP
+xW2oBrEWyYRidNCkSMrPkcSiXu1Loy9APwSAXgJZWMYy0Ccdbd3P7dtGNOZkKaLA
+QKntGA5E1YAbzNhlt7NviGpqZ49K2aOgcGBTnDZ7xDzmg4uo3tcHgzOCwarYZT8l
+qVpc3jAyxRBOrxVKPZNFb4hAFvUm8k6/Etn5n4otN0JT3KGewbfQY50CxW5ShK52
+QCs2PmFMYHHmG11FD3W755MxzhL6UmMy20GUgWWthGmR1LugcBgDtWO/7bqqC9tT
+XYDTDJ1j0g3Y0cvy2+kltrams4lGE3xs6g==
+-----END CERTIFICATE REQUEST-----

cluster/testdata/certs/node2.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEAjCCAuqgAwIBAgIUex5xEYsDJPUg8idU0Sql2ixGdTwwDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMCAXDTIxMDUwNTE2MTYwMFoYDzIxMjEwNDExMTYxNjAwWjBzMQswCQYD
+VQQGEwJBVTERMA8GA1UECBMIVmljdG9yaWExEjAQBgNVBAcTCU1lbGJvdXJuZTEV
+MBMGA1UEChMMc3lzdGVtOm5vZGUyMQ4wDAYDVQQLEwVtYXNzbDEWMBQGA1UEAxMN
+c3lzdGVtOnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALQr
+c0/b4UTE24QIP6ZvHgZ/uCL1UHlapx6BKJ/SCiMeX8r6kkOlIt8UopPZCauRUgGt
+ZknGoycHwdaOsnGkj+p/kPPD8uL+6uuZ9LLQh6CQh9Lj8DEMhIcF+5zYpVfQuFGU
+8mdkrQ3QI5jAR4jfNRCQKh9neF7YK3fyCFoZY0l5vmj27kI0s/AbUf+/jLMKwnZS
+fMuoFotNtJLehkh45msSNY0rRrwccZLCvOr9yfe8HzcFmOxiYu9Lga7ztKZOW2HJ
+A81EynjdUBrwZ2fLg17O6T8rL37Sa/ORs19IhPWCzA8Tt6nPz5cAncTPB+oB005C
+z4sxmbaTBfWePe6VvWECAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l
+BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE
+FDNgivphLRqKzV8n29GJq6S2I+CQMB8GA1UdIwQYMBaAFHeA0xJSquoJxmAyWYCb
+wvuH5a2QMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF
+AAOCAQEAnNG3nzycALGf+N8PuG4sUIkD+SYA1nOEgfD2KiGNyuTYHhGgFXTw8KzB
+olH05VidldBvC0+pl5EqZAp9qdzpw6Z5Mb0gdoZY6TeKDUo022G3BHLMUGLp8y+i
+KE6+awwgdJZ6vPbdnWAh7VM/HCUrGIIPmLFan13j/2RiMfaDxdMAowPmbVc8MLgA
+JHI6pPo8D1DacEvMM09qGtwQEUoREOWJ/SzTWl1nc/IAS1yOL1LCyKLcoj/HWqjG
+3LXficQ7rf+Cpn1GnrKwMziT0OLDLxOs/+5d3nFSLxqF1lpykhPPkmHOHnuY8sMX
+Qdndn9QILdp5GNvqiVNQYcQa/gOb6g==
+-----END CERTIFICATE-----

cluster/testdata/tls_config_node1.yml

@@ -0,0 +1,9 @@
+tls_server_config:
+  cert_file: "certs/node1.pem"
+  key_file: "certs/node1-key.pem"
+  client_ca_file: "certs/ca.pem"
+  client_auth_type: "VerifyClientCertIfGiven"
+tls_client_config:
+  cert_file: "certs/node1.pem"
+  key_file: "certs/node1-key.pem"
+  ca_file: "certs/ca.pem"

cluster/testdata/tls_config_node2.yml

@@ -0,0 +1,9 @@
+tls_server_config:
+  cert_file: "certs/node2.pem"
+  key_file: "certs/node2-key.pem"
+  client_ca_file: "certs/ca.pem"
+  client_auth_type: "VerifyClientCertIfGiven"
+tls_client_config:
+  cert_file: "certs/node2.pem"
+  key_file: "certs/node2-key.pem"
+  ca_file: "certs/ca.pem"

cluster/tls_config.go

@@ -0,0 +1,45 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/exporter-toolkit/web"
+	"gopkg.in/yaml.v2"
+)
+
+type TLSTransportConfig struct {
+	TLSServerConfig *web.TLSStruct    `yaml:"tls_server_config"`
+	TLSClientConfig *config.TLSConfig `yaml:"tls_client_config"`
+}
+
+func GetTLSTransportConfig(configPath string) (*TLSTransportConfig, error) {
+	if configPath == "" {
+		return nil, nil
+	}
+	bytes, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, err
+	}
+	cfg := &TLSTransportConfig{}
+	if err := yaml.UnmarshalStrict(bytes, cfg); err != nil {
+		return nil, err
+	}
+	cfg.TLSServerConfig.SetDirectory(filepath.Dir(configPath))
+	cfg.TLSClientConfig.SetDirectory(filepath.Dir(configPath))
+	return cfg, nil
+}

cluster/tls_connection.go

@@ -0,0 +1,188 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/hashicorp/memberlist"
+	"github.com/pkg/errors"
+
+	"github.com/prometheus/alertmanager/cluster/clusterpb"
+)
+
+const (
+	version      = "v0.1.0"
+	uint32length = 4
+)
+
+// tlsConn wraps net.Conn with connection pooling data.
+type tlsConn struct {
+	mtx        sync.Mutex
+	connection net.Conn
+	live       bool
+}
+
+func dialTLSConn(addr string, timeout time.Duration, tlsConfig *tls.Config) (*tlsConn, error) {
+	dialer := &net.Dialer{Timeout: timeout}
+	conn, err := tls.DialWithDialer(dialer, network, addr, tlsConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tlsConn{
+		connection: conn,
+		live:       true,
+	}, nil
+}
+
+func rcvTLSConn(conn net.Conn) *tlsConn {
+	return &tlsConn{
+		connection: conn,
+		live:       true,
+	}
+}
+
+// Write writes a byte array into the connection. It returns the number of bytes written and an error.
+func (conn *tlsConn) Write(b []byte) (int, error) {
+	conn.mtx.Lock()
+	defer conn.mtx.Unlock()
+	n, err := conn.connection.Write(b)
+
+	if err != nil {
+		conn.live = false
+	}
+	return n, err
+}
+
+func (conn *tlsConn) alive() bool {
+	conn.mtx.Lock()
+	defer conn.mtx.Unlock()
+	return conn.live
+}
+
+func (conn *tlsConn) getRawConn() net.Conn {
+	conn.mtx.Lock()
+	defer conn.mtx.Unlock()
+	raw := conn.connection
+	conn.live = false
+	conn.connection = nil
+	return raw
+}
+
+// writePacket writes all the bytes in one operation so no concurrent write happens in between.
+// It prefixes the message length.
+func (conn *tlsConn) writePacket(fromAddr string, b []byte) error {
+	msg, err := proto.Marshal(
+		&clusterpb.MemberlistMessage{
+			Version:  version,
+			Kind:     clusterpb.MemberlistMessage_PACKET,
+			FromAddr: fromAddr,
+			Msg:      b,
+		},
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to marshal memeberlist packet message")
+	}
+	buf := make([]byte, uint32length, uint32length+len(msg))
+	binary.LittleEndian.PutUint32(buf, uint32(len(msg)))
+	_, err = conn.Write(append(buf, msg...))
+	return err
+}
+
+// writeStream simply signals that this is a stream connection by sending the connection type.
+func (conn *tlsConn) writeStream() error {
+	msg, err := proto.Marshal(
+		&clusterpb.MemberlistMessage{
+			Version: version,
+			Kind:    clusterpb.MemberlistMessage_STREAM,
+		},
+	)
+	if err != nil {
+		return errors.Wrap(err, "unable to marshal memeberlist stream message")
+	}
+	buf := make([]byte, uint32length, uint32length+len(msg))
+	binary.LittleEndian.PutUint32(buf, uint32(len(msg)))
+	_, err = conn.Write(append(buf, msg...))
+	return err
+}
+
+// read returns a packet for packet connections or an error if there is one.
+// It returns nothing if the connection is meant to be streamed.
+func (conn *tlsConn) read() (*memberlist.Packet, error) {
+	if conn.connection == nil {
+		return nil, errors.New("nil connection")
+	}
+
+	conn.mtx.Lock()
+	reader := bufio.NewReader(conn.connection)
+	lenBuf := make([]byte, uint32length)
+	_, err := io.ReadFull(reader, lenBuf)
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading message length")
+	}
+	msgLen := binary.LittleEndian.Uint32(lenBuf)
+	msgBuf := make([]byte, msgLen)
+	_, err = io.ReadFull(reader, msgBuf)
+	conn.mtx.Unlock()
+
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading message")
+	}
+	pb := clusterpb.MemberlistMessage{}
+	err = proto.Unmarshal(msgBuf, &pb)
+	if err != nil {
+		return nil, errors.Wrap(err, "error parsing message")
+	}
+	if pb.Version != version {
+		return nil, errors.New("tls memberlist message version incompatible")
+	}
+	switch pb.Kind {
+	case clusterpb.MemberlistMessage_STREAM:
+		return nil, nil
+	case clusterpb.MemberlistMessage_PACKET:
+		return toPacket(pb)
+	default:
+		return nil, errors.New("could not read from either stream or packet channel")
+	}
+}
+
+func toPacket(pb clusterpb.MemberlistMessage) (*memberlist.Packet, error) {
+	addr, err := net.ResolveTCPAddr(network, pb.FromAddr)
+	if err != nil {
+		return nil, errors.Wrap(err, "error parsing packet sender address")
+	}
+	return &memberlist.Packet{
+		Buf:       pb.Msg,
+		From:      addr,
+		Timestamp: time.Now(),
+	}, nil
+}
+
+func (conn *tlsConn) Close() error {
+	conn.mtx.Lock()
+	defer conn.mtx.Unlock()
+	conn.live = false
+	if conn.connection == nil {
+		return nil
+	}
+	return conn.connection.Close()
+}

cluster/tls_connection_test.go

@@ -0,0 +1,126 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"errors"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteStream(t *testing.T) {
+	w, r := net.Pipe()
+	conn := &tlsConn{
+		connection: w,
+	}
+	defer r.Close()
+	go func() {
+		conn.writeStream()
+		w.Close()
+	}()
+	packet, err := rcvTLSConn(r).read()
+	require.Nil(t, err)
+	require.Nil(t, packet)
+}
+
+func TestWritePacket(t *testing.T) {
+	testCases := []struct {
+		fromAddr string
+		msg      string
+	}{
+		{fromAddr: "127.0.0.1:8001", msg: ""},
+		{fromAddr: "10.0.0.4:9094", msg: "hello"},
+		{fromAddr: "127.0.0.1:8001", msg: "0"},
+	}
+	for _, tc := range testCases {
+		w, r := net.Pipe()
+		defer r.Close()
+		go func() {
+			conn := &tlsConn{connection: w}
+			conn.writePacket(tc.fromAddr, []byte(tc.msg))
+			w.Close()
+		}()
+		packet, err := rcvTLSConn(r).read()
+		require.Nil(t, err)
+		require.Equal(t, tc.msg, string(packet.Buf))
+		require.Equal(t, tc.fromAddr, packet.From.String())
+
+	}
+}
+
+func TestRead_Nil(t *testing.T) {
+	packet, err := (&tlsConn{}).read()
+	require.Nil(t, packet)
+	require.NotNil(t, err)
+}
+
+func TestTLSConn_Close(t *testing.T) {
+	testCases := []string{
+		"foo",
+		"bar",
+	}
+	for _, tc := range testCases {
+		c := &tlsConn{
+			connection: &mockConn{
+				errMsg: tc,
+			},
+			live: true,
+		}
+		err := c.Close()
+		require.Equal(t, errors.New(tc), err, tc)
+		require.False(t, c.alive())
+		require.True(t, c.connection.(*mockConn).closed)
+	}
+}
+
+type mockConn struct {
+	closed bool
+	errMsg string
+}
+
+func (m *mockConn) Read(b []byte) (n int, err error) {
+	panic("implement me")
+}
+
+func (m *mockConn) Write(b []byte) (n int, err error) {
+	panic("implement me")
+}
+
+func (m *mockConn) Close() error {
+	m.closed = true
+	return errors.New(m.errMsg)
+}
+
+func (m *mockConn) LocalAddr() net.Addr {
+	panic("implement me")
+}
+
+func (m *mockConn) RemoteAddr() net.Addr {
+	panic("implement me")
+}
+
+func (m *mockConn) SetDeadline(t time.Time) error {
+	panic("implement me")
+}
+
+func (m *mockConn) SetReadDeadline(t time.Time) error {
+	panic("implement me")
+}
+
+func (m *mockConn) SetWriteDeadline(t time.Time) error {
+	panic("implement me")
+}

cluster/tls_transport.go

@@ -0,0 +1,346 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Forked from https://github.com/mxinden/memberlist-tls-transport.
+
+// Implements Transport interface so that all gossip communications occur via TLS over TCP.
+
+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/hashicorp/go-sockaddr"
+	"github.com/hashicorp/memberlist"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	common "github.com/prometheus/common/config"
+	"github.com/prometheus/exporter-toolkit/web"
+)
+
+const (
+	metricNamespace = "alertmanager"
+	metricSubsystem = "tls_transport"
+	network         = "tcp"
+)
+
+// TLSTransport is a Transport implementation that uses TLS over TCP for both
+// packet and stream operations.
+type TLSTransport struct {
+	ctx          context.Context
+	cancel       context.CancelFunc
+	logger       log.Logger
+	bindAddr     string
+	bindPort     int
+	done         chan struct{}
+	listener     net.Listener
+	packetCh     chan *memberlist.Packet
+	streamCh     chan net.Conn
+	connPool     *connectionPool
+	tlsServerCfg *tls.Config
+	tlsClientCfg *tls.Config
+
+	packetsSent prometheus.Counter
+	packetsRcvd prometheus.Counter
+	streamsSent prometheus.Counter
+	streamsRcvd prometheus.Counter
+	readErrs    prometheus.Counter
+	writeErrs   *prometheus.CounterVec
+}
+
+// NewTLSTransport returns a TLS transport with the given configuration.
+// On successful initialization, a tls listener will be created and listening.
+// A valid bindAddr is required. If bindPort == 0, the system will assign
+// a free port automatically.
+func NewTLSTransport(
+	ctx context.Context,
+	logger log.Logger,
+	reg prometheus.Registerer,
+	bindAddr string,
+	bindPort int,
+	cfg *TLSTransportConfig,
+) (*TLSTransport, error) {
+	if cfg == nil {
+		return nil, errors.New("must specify TLSTransportConfig")
+	}
+	tlsServerCfg, err := web.ConfigToTLSConfig(cfg.TLSServerConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid TLS server config")
+	}
+	tlsClientCfg, err := common.NewTLSConfig(cfg.TLSClientConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid TLS client config")
+	}
+	ip := net.ParseIP(bindAddr)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid bind address \"%s\"", bindAddr)
+	}
+	addr := &net.TCPAddr{IP: ip, Port: bindPort}
+	listener, err := tls.Listen(network, addr.String(), tlsServerCfg)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("failed to start TLS listener on %q port %d", bindAddr, bindPort))
+	}
+	connPool, err := newConnectionPool(tlsClientCfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize tls transport connection pool")
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	t := &TLSTransport{
+		ctx:          ctx,
+		cancel:       cancel,
+		logger:       logger,
+		bindAddr:     bindAddr,
+		bindPort:     bindPort,
+		done:         make(chan struct{}),
+		listener:     listener,
+		packetCh:     make(chan *memberlist.Packet),
+		streamCh:     make(chan net.Conn),
+		connPool:     connPool,
+		tlsServerCfg: tlsServerCfg,
+		tlsClientCfg: tlsClientCfg,
+	}
+
+	t.registerMetrics(reg)
+
+	go func() {
+		t.listen()
+		close(t.done)
+	}()
+	return t, nil
+}
+
+// FinalAdvertiseAddr is given the user's configured values (which
+// might be empty) and returns the desired IP and port to advertise to
+// the rest of the cluster.
+func (t *TLSTransport) FinalAdvertiseAddr(ip string, port int) (net.IP, int, error) {
+	var advertiseAddr net.IP
+	var advertisePort int
+	if ip != "" {
+		advertiseAddr = net.ParseIP(ip)
+		if advertiseAddr == nil {
+			return nil, 0, fmt.Errorf("failed to parse advertise address %q", ip)
+		}
+
+		if ip4 := advertiseAddr.To4(); ip4 != nil {
+			advertiseAddr = ip4
+		}
+		advertisePort = port
+	} else {
+		if t.bindAddr == "0.0.0.0" {
+			// Otherwise, if we're not bound to a specific IP, let's
+			// use a suitable private IP address.
+			var err error
+			ip, err = sockaddr.GetPrivateIP()
+			if err != nil {
+				return nil, 0, fmt.Errorf("failed to get interface addresses: %v", err)
+			}
+			if ip == "" {
+				return nil, 0, fmt.Errorf("no private IP address found, and explicit IP not provided")
+			}
+
+			advertiseAddr = net.ParseIP(ip)
+			if advertiseAddr == nil {
+				return nil, 0, fmt.Errorf("failed to parse advertise address: %q", ip)
+			}
+		} else {
+			advertiseAddr = t.listener.Addr().(*net.TCPAddr).IP
+		}
+		advertisePort = t.GetAutoBindPort()
+	}
+	return advertiseAddr, advertisePort, nil
+}
+
+// PacketCh returns a channel that can be read to receive incoming
+// packets from other peers.
+func (t *TLSTransport) PacketCh() <-chan *memberlist.Packet {
+	return t.packetCh
+}
+
+// StreamCh returns a channel that can be read to handle incoming stream
+// connections from other peers.
+func (t *TLSTransport) StreamCh() <-chan net.Conn {
+	return t.streamCh
+}
+
+// Shutdown is called when memberlist is shutting down; this gives the
+// TLS Transport a chance to clean up the listener and other goroutines.
+func (t *TLSTransport) Shutdown() error {
+	level.Debug(t.logger).Log("msg", "shutting down tls transport")
+	t.cancel()
+	err := t.listener.Close()
+	t.connPool.shutdown()
+	<-t.done
+	return err
+}
+
+// WriteTo is a packet-oriented interface that borrows a connection
+// from the pool, and writes to it. It also returns a timestamp of when
+// the packet was written.
+func (t *TLSTransport) WriteTo(b []byte, addr string) (time.Time, error) {
+	conn, err := t.connPool.borrowConnection(addr, DefaultTcpTimeout)
+	if err != nil {
+		t.writeErrs.WithLabelValues("packet").Inc()
+		return time.Now(), errors.Wrap(err, "failed to dial")
+	}
+	fromAddr := t.listener.Addr().String()
+	err = conn.writePacket(fromAddr, b)
+	if err != nil {
+		t.writeErrs.WithLabelValues("packet").Inc()
+		return time.Now(), errors.Wrap(err, "failed to write packet")
+	}
+	t.packetsSent.Add(float64(len(b)))
+	return time.Now(), nil
+}
+
+// DialTimeout is used to create a connection that allows memberlist
+// to perform two-way communications with a peer.
+func (t *TLSTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) {
+	conn, err := dialTLSConn(addr, timeout, t.tlsClientCfg)
+	if err != nil {
+		t.writeErrs.WithLabelValues("stream").Inc()
+		return nil, errors.Wrap(err, "failed to dial")
+	}
+	err = conn.writeStream()
+	netConn := conn.getRawConn()
+	if err != nil {
+		t.writeErrs.WithLabelValues("stream").Inc()
+		return netConn, errors.Wrap(err, "failed to create stream connection")
+	}
+	t.streamsSent.Inc()
+	return netConn, nil
+}
+
+// GetAutoBindPort returns the bind port that was automatically given by the system
+// if a bindPort of 0 was specified during instantiation.
+func (t *TLSTransport) GetAutoBindPort() int {
+	return t.listener.Addr().(*net.TCPAddr).Port
+}
+
+// listen starts up multiple handlers accepting concurrent connections.
+func (t *TLSTransport) listen() {
+	for {
+		select {
+		case <-t.ctx.Done():
+
+			return
+		default:
+			conn, err := t.listener.Accept()
+			if err != nil {
+				// The error "use of closed network connection" is returned when the listener is closed.
+				// It is not exported in a more reasonable way. See https://github.com/golang/go/issues/4373.
+				if strings.Contains(err.Error(), "use of closed network connection") {
+					return
+				}
+				t.readErrs.Inc()
+				level.Debug(t.logger).Log("msg", "error accepting connection", "err", err)
+
+			} else {
+				go t.handle(conn)
+			}
+		}
+	}
+}
+
+func (t *TLSTransport) handle(conn net.Conn) {
+	for {
+		packet, err := rcvTLSConn(conn).read()
+		if err != nil {
+			level.Debug(t.logger).Log("msg", "error reading from connection", "err", err)
+			t.readErrs.Inc()
+			return
+		}
+		select {
+		case <-t.ctx.Done():
+			return
+		default:
+			if packet != nil {
+				n := len(packet.Buf)
+				t.packetCh <- packet
+				t.packetsRcvd.Add(float64(n))
+			} else {
+				t.streamCh <- conn
+				t.streamsRcvd.Inc()
+				return
+			}
+		}
+	}
+}
+
+func (t *TLSTransport) registerMetrics(reg prometheus.Registerer) {
+	t.packetsSent = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "packet_bytes_sent_total",
+			Help:      "The number of packet bytes sent to outgoing connections (excluding internal metadata).",
+		},
+	)
+	t.packetsRcvd = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "packet_bytes_received_total",
+			Help:      "The number of packet bytes received from incoming connections (excluding internal metadata).",
+		},
+	)
+	t.streamsSent = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "stream_connections_sent_total",
+			Help:      "The number of stream connections sent.",
+		},
+	)
+
+	t.streamsRcvd = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "stream_connections_received_total",
+			Help:      "The number of stream connections received.",
+		},
+	)
+	t.readErrs = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "read_errors_total",
+			Help:      "The number of errors encountered while reading from incoming connections.",
+		},
+	)
+	t.writeErrs = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "write_errors_total",
+			Help:      "The number of errors encountered while writing to outgoing connections.",
+		},
+		[]string{"connection_type"},
+	)
+
+	if reg != nil {
+		reg.MustRegister(t.packetsSent)
+		reg.MustRegister(t.packetsRcvd)
+		reg.MustRegister(t.streamsSent)
+		reg.MustRegister(t.streamsRcvd)
+		reg.MustRegister(t.readErrs)
+		reg.MustRegister(t.writeErrs)
+	}
+}

cluster/tls_transport_test.go

@@ -0,0 +1,213 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"bufio"
+	context2 "context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/stretchr/testify/require"
+)
+
+var logger = log.NewNopLogger()
+
+func TestNewTLSTransport(t *testing.T) {
+	testCases := []struct {
+		bindAddr    string
+		bindPort    int
+		tlsConfFile string
+		err         string
+	}{
+		{err: "must specify TLSTransportConfig"},
+		{err: "invalid bind address \"\"", tlsConfFile: "testdata/tls_config_node1.yml"},
+		{bindAddr: "abc123", err: "invalid bind address \"abc123\"", tlsConfFile: "testdata/tls_config_node1.yml"},
+		{bindAddr: localhost, bindPort: 0, tlsConfFile: "testdata/tls_config_node1.yml"},
+		{bindAddr: localhost, bindPort: 9094, tlsConfFile: "testdata/tls_config_node2.yml"},
+	}
+	l := log.NewNopLogger()
+	for _, tc := range testCases {
+		cfg := mustTLSTransportConfig(tc.tlsConfFile)
+		transport, err := NewTLSTransport(context2.Background(), l, nil, tc.bindAddr, tc.bindPort, cfg)
+		if len(tc.err) > 0 {
+			require.Equal(t, tc.err, err.Error())
+			require.Nil(t, transport)
+		} else {
+			require.Nil(t, err)
+			require.Equal(t, tc.bindAddr, transport.bindAddr)
+			require.Equal(t, tc.bindPort, transport.bindPort)
+			require.Equal(t, l, transport.logger)
+			require.NotNil(t, transport.listener)
+			transport.Shutdown()
+		}
+	}
+}
+
+const localhost = "127.0.0.1"
+
+func TestFinalAdvertiseAddr(t *testing.T) {
+	testCases := []struct {
+		bindAddr      string
+		bindPort      int
+		inputIp       string
+		inputPort     int
+		expectedIp    string
+		expectedPort  int
+		expectedError string
+	}{
+		{bindAddr: localhost, bindPort: 9094, inputIp: "10.0.0.5", inputPort: 54231, expectedIp: "10.0.0.5", expectedPort: 54231},
+		{bindAddr: localhost, bindPort: 9093, inputIp: "invalid", inputPort: 54231, expectedError: "failed to parse advertise address \"invalid\""},
+		{bindAddr: "0.0.0.0", bindPort: 0, inputIp: "", inputPort: 0, expectedIp: "random"},
+		{bindAddr: localhost, bindPort: 0, inputIp: "", inputPort: 0, expectedIp: localhost},
+		{bindAddr: localhost, bindPort: 9095, inputIp: "", inputPort: 0, expectedIp: localhost, expectedPort: 9095},
+	}
+	for _, tc := range testCases {
+		tlsConf := mustTLSTransportConfig("testdata/tls_config_node1.yml")
+		transport, err := NewTLSTransport(context2.Background(), logger, nil, tc.bindAddr, tc.bindPort, tlsConf)
+		require.Nil(t, err)
+		ip, port, err := transport.FinalAdvertiseAddr(tc.inputIp, tc.inputPort)
+		if len(tc.expectedError) > 0 {
+			require.Equal(t, tc.expectedError, err.Error())
+		} else {
+			require.Nil(t, err)
+			if tc.expectedPort == 0 {
+				require.True(t, tc.expectedPort < port)
+				require.True(t, uint32(port) <= uint32(1<<32-1))
+			} else {
+				require.Equal(t, tc.expectedPort, port)
+			}
+			if tc.expectedIp == "random" {
+				require.NotNil(t, ip)
+			} else {
+				require.Equal(t, tc.expectedIp, ip.String())
+			}
+		}
+		transport.Shutdown()
+	}
+}
+
+func TestWriteTo(t *testing.T) {
+	tlsConf1 := mustTLSTransportConfig("testdata/tls_config_node1.yml")
+	t1, _ := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf1)
+	defer t1.Shutdown()
+
+	tlsConf2 := mustTLSTransportConfig("testdata/tls_config_node2.yml")
+	t2, _ := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf2)
+	defer t2.Shutdown()
+
+	from := fmt.Sprintf("%s:%d", t1.bindAddr, t1.GetAutoBindPort())
+	to := fmt.Sprintf("%s:%d", t2.bindAddr, t2.GetAutoBindPort())
+	sent := []byte(("test packet"))
+	_, err := t1.WriteTo(sent, to)
+	require.Nil(t, err)
+	packet := <-t2.PacketCh()
+	require.Equal(t, sent, packet.Buf)
+	require.Equal(t, from, packet.From.String())
+}
+
+func BenchmarkWriteTo(b *testing.B) {
+	tlsConf1 := mustTLSTransportConfig("testdata/tls_config_node1.yml")
+	t1, _ := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf1)
+	defer t1.Shutdown()
+
+	tlsConf2 := mustTLSTransportConfig("testdata/tls_config_node2.yml")
+	t2, _ := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf2)
+	defer t2.Shutdown()
+
+	b.ResetTimer()
+	from := fmt.Sprintf("%s:%d", t1.bindAddr, t1.GetAutoBindPort())
+	to := fmt.Sprintf("%s:%d", t2.bindAddr, t2.GetAutoBindPort())
+	sent := []byte(("test packet"))
+
+	_, err := t1.WriteTo(sent, to)
+	require.Nil(b, err)
+	packet := <-t2.PacketCh()
+
+	require.Equal(b, sent, packet.Buf)
+	require.Equal(b, from, packet.From.String())
+}
+
+func TestDialTimout(t *testing.T) {
+	tlsConf1 := mustTLSTransportConfig("testdata/tls_config_node1.yml")
+	t1, err := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf1)
+	require.Nil(t, err)
+	defer t1.Shutdown()
+
+	tlsConf2 := mustTLSTransportConfig("testdata/tls_config_node2.yml")
+	t2, err := NewTLSTransport(context2.Background(), logger, nil, "127.0.0.1", 0, tlsConf2)
+	require.Nil(t, err)
+	defer t2.Shutdown()
+
+	addr := fmt.Sprintf("%s:%d", t2.bindAddr, t2.GetAutoBindPort())
+	from, err := t1.DialTimeout(addr, 5*time.Second)
+	require.Nil(t, err)
+	defer from.Close()
+
+	var to net.Conn
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		to = <-t2.StreamCh()
+		wg.Done()
+	}()
+
+	sent := []byte(("test stream"))
+	m, err := from.Write(sent)
+	require.Nil(t, err)
+	require.Greater(t, m, 0)
+
+	wg.Wait()
+
+	reader := bufio.NewReader(to)
+	buf := make([]byte, len(sent))
+	n, err := io.ReadFull(reader, buf)
+	require.Nil(t, err)
+	require.Equal(t, len(sent), n)
+	require.Equal(t, sent, buf)
+}
+
+type logWr struct {
+	bytes []byte
+}
+
+func (l *logWr) Write(p []byte) (n int, err error) {
+	l.bytes = append(l.bytes, p...)
+	return len(p), nil
+}
+
+func TestShutdown(t *testing.T) {
+	tlsConf1 := mustTLSTransportConfig("testdata/tls_config_node1.yml")
+	l := &logWr{}
+	t1, _ := NewTLSTransport(context2.Background(), log.NewLogfmtLogger(l), nil, "127.0.0.1", 0, tlsConf1)
+	// Sleeping to make sure listeners have started and can subsequently be shut down gracefully.
+	time.Sleep(500 * time.Millisecond)
+	err := t1.Shutdown()
+	require.Nil(t, err)
+	require.NotContains(t, string(l.bytes), "use of closed network connection")
+	require.Contains(t, string(l.bytes), "shutting down tls transport")
+}
+
+func mustTLSTransportConfig(filename string) *TLSTransportConfig {
+	config, err := GetTLSTransportConfig(filename)
+	if err != nil {
+		panic(err)
+	}
+	return config
+}

cmd/alertmanager/main.go

@@ -211,6 +211,7 @@ func run() int {
 		settleTimeout        = kingpin.Flag("cluster.settle-timeout", "Maximum time to wait for cluster connections to settle before evaluating notifications.").Default(cluster.DefaultPushPullInterval.String()).Duration()
 		reconnectInterval    = kingpin.Flag("cluster.reconnect-interval", "Interval between attempting to reconnect to lost peers.").Default(cluster.DefaultReconnectInterval.String()).Duration()
 		peerReconnectTimeout = kingpin.Flag("cluster.reconnect-timeout", "Length of time to attempt to reconnect to a lost peer.").Default(cluster.DefaultReconnectTimeout.String()).Duration()
+		tlsConfigFile        = kingpin.Flag("cluster.tls-config", "[EXPERIMENTAL] Path to config yaml file that can enable mutual TLS within the gossip protocol.").Default("").String()
 	)
 
 	promlogflag.AddFlags(kingpin.CommandLine, &promlogConfig)
@@ -231,6 +232,11 @@ func run() int {
 		return 1
 	}
 
+	tlsTransportConfig, err := cluster.GetTLSTransportConfig(*tlsConfigFile)
+	if err != nil {
+		level.Error(logger).Log("msg", "unable to initialize TLS transport configuration for gossip mesh", "err", err)
+		return 1
+	}
 	var peer *cluster.Peer
 	if *clusterBindAddr != "" {
 		peer, err = cluster.Create(
@@ -245,6 +251,7 @@ func run() int {
 			*tcpTimeout,
 			*probeTimeout,
 			*probeInterval,
+			tlsTransportConfig,
 		)
 		if err != nil {
 			level.Error(logger).Log("msg", "unable to initialize gossip mesh", "err", err)

docs/https.md

@@ -8,8 +8,9 @@ sort_rank: 11
 Alertmanager supports basic authentication and TLS.
 This is **experimental** and might change in the future.
 
-Currently TLS is only supported for the HTTP traffic. Gossip traffic does not
+Currently TLS is supported for the HTTP traffic and gossip traffic.
-support encryption yet.
+
+## HTTP Traffic
 
 To specify which web configuration file to load, use the `--web.config.file` flag.
 
@@ -82,3 +83,63 @@ basic_auth_users:
   [ <string>: <secret> ... ]
 ```
 
+## Gossip Traffic
+
+To specify whether to use mutual TLS for gossip, use the `--cluster.tls-config` flag.
+
+The server and client sides of the gossip are configurable.
+
+```
+tls_server_config:
+  # Certificate and key files for server to use to authenticate to client.
+  cert_file: <filename>
+  key_file: <filename>
+
+  # Server policy for client authentication. Maps to ClientAuth Policies.
+  # For more detail on clientAuth options:
+  # https://golang.org/pkg/crypto/tls/#ClientAuthType
+  [ client_auth_type: <string> | default = "NoClientCert" ]
+
+  # CA certificate for client certificate authentication to the server.
+  [ client_ca_file: <filename> ]
+
+  # Minimum TLS version that is acceptable.
+  [ min_version: <string> | default = "TLS12" ]
+
+  # Maximum TLS version that is acceptable.
+  [ max_version: <string> | default = "TLS13" ]
+
+  # List of supported cipher suites for TLS versions up to TLS 1.2. If empty,
+  # Go default cipher suites are used. Available cipher suites are documented
+  # in the go documentation:
+  # https://golang.org/pkg/crypto/tls/#pkg-constants
+  [ cipher_suites:
+    [ - <string> ] ]
+
+  # prefer_server_cipher_suites controls whether the server selects the
+  # client's most preferred ciphersuite, or the server's most preferred
+  # ciphersuite. If true then the server's preference, as expressed in
+  # the order of elements in cipher_suites, is used.
+  [ prefer_server_cipher_suites: <bool> | default = true ]
+
+  # Elliptic curves that will be used in an ECDHE handshake, in preference
+  # order. Available curves are documented in the go documentation:
+  # https://golang.org/pkg/crypto/tls/#CurveID
+  [ curve_preferences:
+    [ - <string> ] ]
+
+tls_client_config:
+  # Path to the CA certificate with which to validate the server certificate.
+  [ ca_file: <filepath> ]
+    
+  # Certificate and key files for client cert authentication to the server.
+  [ cert_file: <filepath> ]
+  [ key_file: <filepath> ]
+  
+  # Server name extension to indicate the name of the server.
+  # http://tools.ietf.org/html/rfc4366#section-3.1
+  [ server_name: <string> ]
+  
+  # Disable validation of the server certificate.
+  [ insecure_skip_verify: <boolean> | default = false]
+```

examples/ha/tls/Makefile

@@ -0,0 +1,27 @@
+# Based on https://github.com/wolfeidau/golang-massl/
+
+.PHONY: start
+start:
+	goreman start
+
+.PHONY: gen-certs
+gen-certs: certs/ca.pem certs/node1.pem certs/node1-key.pem certs/node2.pem certs/node2-key.pem
+
+certs/ca.pem certs/ca-key.pem: certs/ca-csr.json
+	cd certs; cfssl gencert -initca ca-csr.json | cfssljson -bare ca
+
+certs/node1.pem certs/node1-key.pem: certs/ca-config.json certs/ca.pem certs/ca-key.pem certs/node1-csr.json
+	cd certs; cfssl gencert  \
+    -ca=ca.pem \
+    -ca-key=ca-key.pem \
+    -config=ca-config.json \
+    -hostname=localhost,127.0.0.1 \
+    -profile=massl node1-csr.json | cfssljson -bare node1
+
+certs/node2.pem certs/node2-key.pem: certs/ca-config.json certs/ca.pem certs/ca-key.pem certs/node2-csr.json
+	cd certs; cfssl gencert  \
+    -ca=ca.pem \
+    -ca-key=ca-key.pem \
+    -config=ca-config.json \
+    -hostname=localhost,127.0.0.1 \
+    -profile=massl node2-csr.json | cfssljson -bare node2

examples/ha/tls/Procfile

@@ -0,0 +1,7 @@
+a1: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a1 --web.listen-address=:9093  --cluster.listen-address=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node1.yml
+a2: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a2 --web.listen-address=:9094  --cluster.listen-address=127.0.0.1:8002 --cluster.peer=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node2.yml
+a3: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a3 --web.listen-address=:9095  --cluster.listen-address=127.0.0.1:8003 --cluster.peer=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node1.yml
+a4: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a4 --web.listen-address=:9096  --cluster.listen-address=127.0.0.1:8004 --cluster.peer=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node2.yml
+a5: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a5 --web.listen-address=:9097  --cluster.listen-address=127.0.0.1:8005 --cluster.peer=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node1.yml
+a6: ./../../../alertmanager --log.level=debug --storage.path=$TMPDIR/a6 --web.listen-address=:9098  --cluster.listen-address=127.0.0.1:8006 --cluster.peer=127.0.0.1:8001 --config.file=../alertmanager.yml --cluster.tls-config=./tls_config_node2.yml
+wh: go run ../../webhook/echo.go

examples/ha/tls/README.md

@@ -0,0 +1,18 @@
+# TLS Transport Config Example
+
+## Usage
+1. Install dependencies:
+   1. `go install github.com/cloudflare/cfssl/cmd/cfssl`
+   2. `go install github.com/mattn/goreman`
+2. Build Alertmanager (root of repository):
+   1. `go mod download`
+   1. `make build`.
+2. `make start` (inside this directory).
+
+## Testing
+1. Start the cluster (as explained above)
+2. Navigate to one of the Alertmanager instances at `localhost:9093`.
+3. Create a silence.
+4. Navigate to the other Alertmanager instance at `localhost:9094`.
+5. Observe that the silence created in the other Alertmanager instance has been synchronized over to this instance.
+6. Repeat.

examples/ha/tls/certs/ca-config.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+      "expiry": "876000h"
+    },
+    "profiles": {
+      "massl": {
+        "usages": ["signing", "key encipherment", "server auth", "client auth"],
+        "expiry": "876000h"
+      }
+    }
+  }
+}

examples/ha/tls/certs/ca-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "massl",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "massl",
+      "OU": "VIC",
+      "ST": "Victoria"
+    }
+  ]
+}

examples/ha/tls/certs/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAuljDjKVGwlyiuKTSHc1QpoZPX9dbgwU/9113ctI8U/ZwMWLp
+nZ4f/zVpf4LW5foM9zSEUGPiyJe/NaTZUOXkRBSIQ13QroK4OJ1XGacQKpTxewCb
+ChESZEfKWEhnP/Y7BYc4z1Li6Dkxh4TIElHwOVe62jbhNnzYlr4evmSuiuItAc8u
+hEYxncThPzmHEWPXKw8CFNhxCSYsjbb72UAIht0knMHQ7VXBX1VuuL0rolJBiToC
+va+I6CjG0c6qfi9/BcPsuW6cNjmQnwTg6SaSoGO/5zgbxBgy9MZQEot88d1T2XH6
+rBANYsfojvyCXuytWnj04mvdAWwmFh0hhq+nxQIDAQABAoIBAQCwcL1vXUq7W4UD
+OaRtbWrQ0dk0ETBnxT/E0y33fRJ8GZovWM2EXSVEuukSP+uEQ5elNYeWqo0fi3cT
+ruvJSnMw9xPyXVDq+4C8slW3R1TqTK683VzvUizM4KC5qIyCpn1KBbgHrh6E7Sp1
+e4cIuaawVN3qIg5qThmx2YA4nBIcEt68q9cpy3NgEe+EQf44zM/St+y8kSkDUOVw
+fNKX0WfZ/hPL1TAYpWiIgSf+m/V3d/1l/scvMYONcuSjXSORCyoeAWYtOQgf78wW
+9j3kiBTaqDYCUZFnY/ltlZrm8ltAaKVJ0MmPKjVh8GJBXZp9fSVU8Y4ZIZRSeuBA
+OoStHGAdAoGBAMluMIE33hGny2V0dNzW23D84eXQK38AsdP632jQeuzxBknItg45
+qAfzh8F8W10DQnSv5tj0bmOHfo0mG09bu9eo5nLLINOE7Ju/7ly/76RNJNJ4ADjx
+JKZi/PpvfP+s/fzel0X3OPgA+CJKzUHuqlU4V9BLc7focZAYtaM2w7rHAoGBAOzU
+eXpapkqYhbYRcsrVV57nZV0rLzsLVJBpJg2zC8un95ALrr0rlZfuPJfOCY/uuS1w
+f8ixRz2MkRWGreLHy35NB4GV0sF9VPn1jMp9SuBNvO0JRUMWuDAdVe8SCjXadrOh
++m3yKJSkFKDchglUYnZKV1skgA/b9jjjnu2fvd0TAoGAVUTnFZxvzmuIp78fxWjS
+5ka23hE8iHvjy4e00WsHzovNjKiBoQ35Orx16ItbJcm+dSUNhSQcItf104yhHPwJ
+Tab7PvcMQ15OxzP9lJfPu27Iuqv/9Bro1+Kpkt5lPNqffk9AHGcmX54RbHrb3yBI
+TOEYE14Nc3nbsRM0uQ3y13sCgYB5Om4QZpSWvKo9P4M+NqTKb3JglblwhOU9osVa
+39ra3dkIgCJrLQM/KTEVF9+nMLDThLG0fqKT6/9cQHuECXet6Co+d/3RE6HK7Zmr
+ESWh2ckqoMM2i0uvPWT+ooJdfL2kR/bUDtAc/jyc9yUZY3ufR4Cd4/o1pAfOqR1y
+T4G1xwKBgQChE4VWawCVg2qanRjvZcdNk0zpZx4dxqqKYq/VHuSfjNLQixIZsgXT
+xx9BHuORn6c/nurqEStLwN3BzbpPU/j6YjMUmTslSH2sKhHwWNYGBZC52aJiOOda
+Bz6nAkihG0n2PjYt2T84w6FWHgLJuSsmiEVJcb+AOdyKh1MlzJiwMQ==
+-----END RSA PRIVATE KEY-----

examples/ha/tls/certs/ca.csr

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICpzCCAY8CAQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMx
+DjAMBgNVBAMTBW1hc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+uljDjKVGwlyiuKTSHc1QpoZPX9dbgwU/9113ctI8U/ZwMWLpnZ4f/zVpf4LW5foM
+9zSEUGPiyJe/NaTZUOXkRBSIQ13QroK4OJ1XGacQKpTxewCbChESZEfKWEhnP/Y7
+BYc4z1Li6Dkxh4TIElHwOVe62jbhNnzYlr4evmSuiuItAc8uhEYxncThPzmHEWPX
+Kw8CFNhxCSYsjbb72UAIht0knMHQ7VXBX1VuuL0rolJBiToCva+I6CjG0c6qfi9/
+BcPsuW6cNjmQnwTg6SaSoGO/5zgbxBgy9MZQEot88d1T2XH6rBANYsfojvyCXuyt
+Wnj04mvdAWwmFh0hhq+nxQIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAJFmooMt
+TocElxCb3DGJTRUXxr4DqcATASIX35a2wV3MmPqUHHXr6BQkO/FRho66EsZf3DE/
+mumou01K+KByxgsmw04CACjSeZ2t/g6pAsDCKrx/BwL3tAo09lG2Y2Ah0BND2Cta
+EZpTliU2MimZlk7UZb8VIXh2Tx56fZRoHLzO4U4+FY8ZR+tspxPRM7hLg/aUqA5D
+zGj6kByX8aYjxsmQokP4rx/w2mz6vwt4cZ1pXwr0RderkMIh9Har/0k9X1WIAP61
+PNQx74qnaq+icjtN2+8gvJE/CJL/wfcwW6kQwEtX1xsTpnzyFaRoYpSPQrvkCtiW
++WzgnOh7RvKyAYI=
+-----END CERTIFICATE REQUEST-----

examples/ha/tls/certs/ca.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDlDCCAnygAwIBAgIUetcc8yLasSAxvyUWtgTVKR6jfBIwDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMB4XDTIxMDUwNTE2MTYwMFoXDTI2MDUwNDE2MTYwMFowYjELMAkGA1UE
+BhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlNZWxib3VybmUxDjAM
+BgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMTBW1hc3NsMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuljDjKVGwlyiuKTSHc1QpoZPX9db
+gwU/9113ctI8U/ZwMWLpnZ4f/zVpf4LW5foM9zSEUGPiyJe/NaTZUOXkRBSIQ13Q
+roK4OJ1XGacQKpTxewCbChESZEfKWEhnP/Y7BYc4z1Li6Dkxh4TIElHwOVe62jbh
+NnzYlr4evmSuiuItAc8uhEYxncThPzmHEWPXKw8CFNhxCSYsjbb72UAIht0knMHQ
+7VXBX1VuuL0rolJBiToCva+I6CjG0c6qfi9/BcPsuW6cNjmQnwTg6SaSoGO/5zgb
+xBgy9MZQEot88d1T2XH6rBANYsfojvyCXuytWnj04mvdAWwmFh0hhq+nxQIDAQAB
+o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
+d4DTElKq6gnGYDJZgJvC+4flrZAwDQYJKoZIhvcNAQELBQADggEBAHYaMxcyVdFP
+QZJHDP6x+A1hHeZy/kaL2VPzUIOgt3ONEvjhsJblFjLaTWwXFJwxVHXRsQjqrURj
+dD2VIkMbCJM4WN8odi6PdHscL0DwHKzwZDapQxsOUYcPBbR2EsBPRyWSoJmBnEyZ
+0TANtyUcZNWc7/C4GG66SWypu1YRdtcaDiEOVkdetGWuo5bw1J7B7isQ4J4a3+Qn
+iTKId/7wWBY95JS2xJuOeEjk/ty74ruSapk2Aip1GRSRXMDtD0XeO5dpxc6eIU3C
+jbRpUGqCdSAfzxQTlxQAMEKdgRlt7lzMFX/PAXagDOMC9D/6APn5fquEV2d9wMWs
+0N9UCwKftAU=
+-----END CERTIFICATE-----

examples/ha/tls/certs/node1-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:server",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "system:node1",
+      "OU": "massl",
+      "ST": "Victoria"
+    }
+  ]
+}

examples/ha/tls/certs/node1-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA1b9bm4rvDtpYsqgtCC52+L535d4/Q2O10fWD2i2CfRXXfYJQ
+5cr4AV2iqScFsJSs7KwyQde/c4VWj/vEA2/SJHZFBlknKdCcrgHVebrvnzm6Guze
+ICutZSKocFXy9Kw+YmWuA64nHVfmSCKG07GhXhEsLsSCn4PTDYOiGAUm1GdSDxUp
+8yUXec13Eb20mld0xE9kQnCnEWRnxMXtQJXoz9lpLc7DgXtN6nCXSG/CqdDPOduU
+nmseaxyAGpAFnUmxcqUuYAJUQ1hUOJhk0RVSsLTmu+FGdOxk79AxmmKQ2z9l/GuA
+VikVJGTxY4jRPezxHQ3bdqzzCIdJxTxLinftZQIDAQABAoIBADpxQtvphemau8vF
+feKRycfDVEcOmF+VoL4SkgWSke4fjbbsbbAW6e59qp7zY3PfgtSHVIp6Mgek+oEN
+xo9mAKAlkkPlFncxadWN/M921FPF1ePMxgMnzhYr/sAQUAikG76NrKGm+VzljrpE
+bnbtR4DP0zPKWSjCQ2+bgTNuHSrPwUtEngVT6ugjfWU1RitlvjTsZ9hSuOSBlS7P
+rjbQGaEh53PraDut8PIlF4wIF+nLeERFP/a6DC8Btpbv9P50YRosag6yU/G+OYX9
+spvBPvRJGrubslKnNRz9AcjbVd3QhL+Tm7mV7iakK918jLWb95Ro4WW+9lT6IAi6
+xRSOr9UCgYEA5wI3JhKkYa4PST7ALqmJSDkPH8+tctiEx+ovmnqBufFuLWFoO/rc
+EOYslnaZM3UVCnhrFv7+LxezSI5DyQu8dBEzf0RMICvXUNBkGC7ZJQL428fjXPhX
+8mZIoJ0ol4hbamr8yTYlK0vGTwqN1bDj71w6NszuN4ecN1cKNWsMbnMCgYEA7N8Y
+MzHWNijMr7xZ1lXl4Ye9+kp3aTUjUYBBaxFr4bQ8y0fH48lzq3qOso5wgvp0DKYo
+uemD5QKbo81LKoTRLa+nxPq0UqKm9FiSWmnrcxMuph989oZ1ZFHA2B+nvbuMTF8J
+8sESclTSbgkG87DpycJOUwG3XAcXM+80pXuzJscCgYB+Dzxu/09KqoRW8PJIxGVQ
+zypMrrS07iiPO2FcyCtQf8oi43vQ91Ttt91vAisZ5HNl8k5mDyJAKovANToSVOAy
+6kwSz/9GswXdaMqmU7JVOyj4Lj0JN9AuS9ioJPrIrjVMfjORzYU8+i2uZlD94niP
+3uE5lF0OWmdJ36qHefIftwKBgQDcPQZcO19H1iGS2FbTYeSvEK5ENM7YRG8FTXIF
+4hnjrtjDzYb+tYVWEErznFrifYo/ZJMDYSqgWQ9reusDqqBvkR41mUDmgJMpJ91U
+MZ2YzmIWVbqz4QrvbtAWY0Bsuh/VtpwiWQAUy+coJj6PgJOvY3m91h+tcm5RfHz/
+zIcjawKBgA6kDcOLOnWcvhP3XwtW5dcWlNuBBNEKna+kIT/5Vlrh91y2w7F54DNK
+i0w5CZCpbTugJmZ67XLHnfongC7e2vAQ3atoT96RU4mf9614qs9LMtGAbnuCLB8+
+sT2rnaZKtzr83ensbYkbBxP/zmPBfFQ9FKcIYIA7En8zAIr2T3vJ
+-----END RSA PRIVATE KEY-----

examples/ha/tls/certs/node1.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC5TCCAc0CAQAwczELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxFTATBgNVBAoTDHN5c3RlbTpub2RlMTEOMAwGA1UE
+CxMFbWFzc2wxFjAUBgNVBAMTDXN5c3RlbTpzZXJ2ZXIwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQDVv1ubiu8O2liyqC0ILnb4vnfl3j9DY7XR9YPaLYJ9
+Fdd9glDlyvgBXaKpJwWwlKzsrDJB179zhVaP+8QDb9IkdkUGWScp0JyuAdV5uu+f
+Oboa7N4gK61lIqhwVfL0rD5iZa4DricdV+ZIIobTsaFeESwuxIKfg9MNg6IYBSbU
+Z1IPFSnzJRd5zXcRvbSaV3TET2RCcKcRZGfExe1AlejP2WktzsOBe03qcJdIb8Kp
+0M8525Seax5rHIAakAWdSbFypS5gAlRDWFQ4mGTRFVKwtOa74UZ07GTv0DGaYpDb
+P2X8a4BWKRUkZPFjiNE97PEdDdt2rPMIh0nFPEuKd+1lAgMBAAGgLTArBgkqhkiG
+9w0BCQ4xHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0B
+AQsFAAOCAQEAW/tTyJaBfWtbC9hYUmhh8lxUztv2+WT4xaR/jdQ46sk/87vKuwI6
+4AkkGfiPLLqgW3xbQOwk5/ynRabttbsgTUHt744RtRFLzfcQKEBZoNPvrfHvmDil
+YqHIOx2SJ5hzIBwVlVSBn50hdSSED1Ip22DaU8GukzuacB8+2rhg3MOWJbKVt5aR
+03H4XkAynLS1FHNOraDIv1eT58D3l4hanrNOZIa0xAuChd25qLO/JHvU/3wccGUA
+KNg3vGOy2Q8qVBrTFLn+yQHuOr/wSupXESO1jiI/h+txsBQnZ6oYfZnVJ+7o3Oln
+3Hguw77aYeTAeZQPPbmJbDLegLG0ZC6RmA==
+-----END CERTIFICATE REQUEST-----

examples/ha/tls/certs/node1.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEAjCCAuqgAwIBAgIUbYMGwSgQF8iRZ5xmhflInj8VZ0owDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMCAXDTIxMDUwNTE2MTYwMFoYDzIxMjEwNDExMTYxNjAwWjBzMQswCQYD
+VQQGEwJBVTERMA8GA1UECBMIVmljdG9yaWExEjAQBgNVBAcTCU1lbGJvdXJuZTEV
+MBMGA1UEChMMc3lzdGVtOm5vZGUxMQ4wDAYDVQQLEwVtYXNzbDEWMBQGA1UEAxMN
+c3lzdGVtOnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANW/
+W5uK7w7aWLKoLQgudvi+d+XeP0NjtdH1g9otgn0V132CUOXK+AFdoqknBbCUrOys
+MkHXv3OFVo/7xANv0iR2RQZZJynQnK4B1Xm67585uhrs3iArrWUiqHBV8vSsPmJl
+rgOuJx1X5kgihtOxoV4RLC7Egp+D0w2DohgFJtRnUg8VKfMlF3nNdxG9tJpXdMRP
+ZEJwpxFkZ8TF7UCV6M/ZaS3Ow4F7Tepwl0hvwqnQzznblJ5rHmscgBqQBZ1JsXKl
+LmACVENYVDiYZNEVUrC05rvhRnTsZO/QMZpikNs/ZfxrgFYpFSRk8WOI0T3s8R0N
+23as8wiHScU8S4p37WUCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l
+BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE
+FGprx5v+KrO4DeOtA6kps4BL/zKyMB8GA1UdIwQYMBaAFHeA0xJSquoJxmAyWYCb
+wvuH5a2QMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF
+AAOCAQEAmWTdMLyWOrNAS0uY+u3FUV3Hm50xF1PfxbT6wK1hu6vH6B63E0o9K2/1
+U25Ie8Y2IzFocKMvbqC+mrY56G0bWoUlMONhthYqm8uTKtjlFO33A9I7WIT9Tw+B
+nnwZZO7+Ljkd30qSzBinCjrIEx31Vq2pr54ungd8+wK8nfz/zdZnJcqxcN9zvCXB
+GTE8yCuqGWKk/oDuIzVjr73U0QaWi+vThqJtBjhOIWQHHVJwbIyhuYzUaivgZPYB
+8eKXWk4JH3eAcq5z5koNGyCcZd/k4WnvxZYxNBAkoQ6AWVfEMGOCaRjD1FTnMbpG
+BW79ndJqLmn8OH+DeCnSWhTWxAgg+Q==
+-----END CERTIFICATE-----

examples/ha/tls/certs/node2-csr.json

@@ -0,0 +1,16 @@
+{
+  "CN": "system:server",
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "AU",
+      "L": "Melbourne",
+      "O": "system:node2",
+      "OU": "massl",
+      "ST": "Victoria"
+    }
+  ]
+}

examples/ha/tls/certs/node2-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtCtzT9vhRMTbhAg/pm8eBn+4IvVQeVqnHoEon9IKIx5fyvqS
+Q6Ui3xSik9kJq5FSAa1mScajJwfB1o6ycaSP6n+Q88Py4v7q65n0stCHoJCH0uPw
+MQyEhwX7nNilV9C4UZTyZ2StDdAjmMBHiN81EJAqH2d4Xtgrd/IIWhljSXm+aPbu
+QjSz8BtR/7+MswrCdlJ8y6gWi020kt6GSHjmaxI1jStGvBxxksK86v3J97wfNwWY
+7GJi70uBrvO0pk5bYckDzUTKeN1QGvBnZ8uDXs7pPysvftJr85GzX0iE9YLMDxO3
+qc/PlwCdxM8H6gHTTkLPizGZtpMF9Z497pW9YQIDAQABAoIBAFfQwdCPxHmnVbNB
+7fwqNsFGKTLozMOJeuE0ZN+ZGZXKbTha70WHTLrcrO1RIRR9rTHiGXQmHEmez0zL
+mpAnfHn4mWcm/9DCHTCehpVNbH3HVFxm+yB9EG9bbCsjsVtfASfKaGgauvp7k44V
+UgiVeqDLE6zg2tunk3BQCOAZdbpOiXrdvoZiGx2Q4SMLPfzmfIyH4BUT836pLTmp
+o6/yNiFqQWfCgjeEAOQor4TcdzYIT+3wP51HfAjhZKMIvmjwL16ov1/QpmWRD4ni
+4svzYpeMYpl5OrZkKeDS4ZIQBGjxk+fzPmfFUbfVRSI2gDORsah8HoRVI4LnwKWn
+7kQDv0ECgYEA6V+KVb8bPzCZNbroEZFdug6YtT4yv5Mj3/kpMTIvA3vtu02v8e7F
+O56yT43QfUZA0Ar37O0HQ6mbpPsRE5RSr70i40RR+slMZVHX/AQViG7oQJGBijPt
+1tFdLnb+1wSON3jYt2975Kw2IfgOXprWtEmL5zGuplEUjx9Lbdf1HjkCgYEAxaNe
+XgXdAiWFoY4Qq6xBRO/WNZCdn3Ysqx6snCtDRilxeNyDoE/6x2Ma9/NRBtIiulAb
+s09vDRfJKLbzocUhIn8BQ+GkbAS/A6+x2vcuGhK3F84xqZdbrCqvqdJS8K824jug
+vUCfCBJlyNRDz8kEsN5odLM1xkij93Jv23HvGGkCgYEAptcz6ctfalSPI9eEs5KO
+REbNK73UwBssaaISreYnsED4G5EVuUuvW8k/xxomtHj2OwWsa4ilSd1GtbL8aVf/
+qT35ZCrixP0GjeTuGXC+CDTp+8dKqggoAAzbpi1SUVwjZEsT/EhKdZgcdzqE42Ol
+HWz7BQUCzEpo/U0tOtFKnxkCgYEAi05Vy8wyNbsg7/jlAzyNXPv4bxUaJTX00kDy
+xbkw2BmKI/i6xprZVwUiEzdsG3SuicjBXahVzFLBtXMPUy1R57DBwYkgjgriYMTM
+hlzIIBSk/aCXHMTVFwuXegoH8CJwexIwgHU2I0hkeiQ0EBfOuKRr2CYhdzvoZxhA
+g9tQ/lECgYAjPYoXfNI3rHCWUmaD5eDJZpE0xuJeiiy5auojykdAc7vVapNaIyMK
+G3EaU44RtXcSwH19TlH9UCm3MH1QiIwaBOzGcKj3Ut6ZyFKuWDUk4yqvps3uZU/h
+h16Tp49Ja7/4LY1uuEngg1KMEiWgk5jiU7G0H9zrtEiTj9c3FDKDvg==
+-----END RSA PRIVATE KEY-----

examples/ha/tls/certs/node2.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC5TCCAc0CAQAwczELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIw
+EAYDVQQHEwlNZWxib3VybmUxFTATBgNVBAoTDHN5c3RlbTpub2RlMjEOMAwGA1UE
+CxMFbWFzc2wxFjAUBgNVBAMTDXN5c3RlbTpzZXJ2ZXIwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0K3NP2+FExNuECD+mbx4Gf7gi9VB5WqcegSif0goj
+Hl/K+pJDpSLfFKKT2QmrkVIBrWZJxqMnB8HWjrJxpI/qf5Dzw/Li/urrmfSy0Ieg
+kIfS4/AxDISHBfuc2KVX0LhRlPJnZK0N0COYwEeI3zUQkCofZ3he2Ct38ghaGWNJ
+eb5o9u5CNLPwG1H/v4yzCsJ2UnzLqBaLTbSS3oZIeOZrEjWNK0a8HHGSwrzq/cn3
+vB83BZjsYmLvS4Gu87SmTlthyQPNRMp43VAa8Gdny4Nezuk/Ky9+0mvzkbNfSIT1
+gswPE7epz8+XAJ3EzwfqAdNOQs+LMZm2kwX1nj3ulb1hAgMBAAGgLTArBgkqhkiG
+9w0BCQ4xHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0B
+AQsFAAOCAQEARh0Pi36mNmyprU4j25GWNqQYCJ6cBGnaPeiwr8/F3rsGsF4LTQdP
+xW2oBrEWyYRidNCkSMrPkcSiXu1Loy9APwSAXgJZWMYy0Ccdbd3P7dtGNOZkKaLA
+QKntGA5E1YAbzNhlt7NviGpqZ49K2aOgcGBTnDZ7xDzmg4uo3tcHgzOCwarYZT8l
+qVpc3jAyxRBOrxVKPZNFb4hAFvUm8k6/Etn5n4otN0JT3KGewbfQY50CxW5ShK52
+QCs2PmFMYHHmG11FD3W755MxzhL6UmMy20GUgWWthGmR1LugcBgDtWO/7bqqC9tT
+XYDTDJ1j0g3Y0cvy2+kltrams4lGE3xs6g==
+-----END CERTIFICATE REQUEST-----

examples/ha/tls/certs/node2.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEAjCCAuqgAwIBAgIUex5xEYsDJPUg8idU0Sql2ixGdTwwDQYJKoZIhvcNAQEL
+BQAwYjELMAkGA1UEBhMCQVUxETAPBgNVBAgTCFZpY3RvcmlhMRIwEAYDVQQHEwlN
+ZWxib3VybmUxDjAMBgNVBAoTBW1hc3NsMQwwCgYDVQQLEwNWSUMxDjAMBgNVBAMT
+BW1hc3NsMCAXDTIxMDUwNTE2MTYwMFoYDzIxMjEwNDExMTYxNjAwWjBzMQswCQYD
+VQQGEwJBVTERMA8GA1UECBMIVmljdG9yaWExEjAQBgNVBAcTCU1lbGJvdXJuZTEV
+MBMGA1UEChMMc3lzdGVtOm5vZGUyMQ4wDAYDVQQLEwVtYXNzbDEWMBQGA1UEAxMN
+c3lzdGVtOnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALQr
+c0/b4UTE24QIP6ZvHgZ/uCL1UHlapx6BKJ/SCiMeX8r6kkOlIt8UopPZCauRUgGt
+ZknGoycHwdaOsnGkj+p/kPPD8uL+6uuZ9LLQh6CQh9Lj8DEMhIcF+5zYpVfQuFGU
+8mdkrQ3QI5jAR4jfNRCQKh9neF7YK3fyCFoZY0l5vmj27kI0s/AbUf+/jLMKwnZS
+fMuoFotNtJLehkh45msSNY0rRrwccZLCvOr9yfe8HzcFmOxiYu9Lga7ztKZOW2HJ
+A81EynjdUBrwZ2fLg17O6T8rL37Sa/ORs19IhPWCzA8Tt6nPz5cAncTPB+oB005C
+z4sxmbaTBfWePe6VvWECAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l
+BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE
+FDNgivphLRqKzV8n29GJq6S2I+CQMB8GA1UdIwQYMBaAFHeA0xJSquoJxmAyWYCb
+wvuH5a2QMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF
+AAOCAQEAnNG3nzycALGf+N8PuG4sUIkD+SYA1nOEgfD2KiGNyuTYHhGgFXTw8KzB
+olH05VidldBvC0+pl5EqZAp9qdzpw6Z5Mb0gdoZY6TeKDUo022G3BHLMUGLp8y+i
+KE6+awwgdJZ6vPbdnWAh7VM/HCUrGIIPmLFan13j/2RiMfaDxdMAowPmbVc8MLgA
+JHI6pPo8D1DacEvMM09qGtwQEUoREOWJ/SzTWl1nc/IAS1yOL1LCyKLcoj/HWqjG
+3LXficQ7rf+Cpn1GnrKwMziT0OLDLxOs/+5d3nFSLxqF1lpykhPPkmHOHnuY8sMX
+Qdndn9QILdp5GNvqiVNQYcQa/gOb6g==
+-----END CERTIFICATE-----

examples/ha/tls/tls_config_node1.yml

@@ -0,0 +1,9 @@
+tls_server_config:
+  cert_file: "certs/node1.pem"
+  key_file: "certs/node1-key.pem"
+  client_ca_file: "certs/ca.pem"
+  client_auth_type: "VerifyClientCertIfGiven"
+tls_client_config:
+  cert_file: "certs/node1.pem"
+  key_file: "certs/node1-key.pem"
+  ca_file: "certs/ca.pem"

examples/ha/tls/tls_config_node2.yml

@@ -0,0 +1,9 @@
+tls_server_config:
+  cert_file: "certs/node2.pem"
+  key_file: "certs/node2-key.pem"
+  client_ca_file: "certs/ca.pem"
+  client_auth_type: "VerifyClientCertIfGiven"
+tls_client_config:
+  cert_file: "certs/node2.pem"
+  key_file: "certs/node2-key.pem"
+  ca_file: "certs/ca.pem"

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/hashicorp/go-sockaddr v1.0.2
 	github.com/hashicorp/go-uuid v1.0.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4
 	github.com/hashicorp/memberlist v0.2.4
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/kylelemons/godebug v1.1.0

go.sum

@@ -306,8 +306,9 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/memberlist v0.2.4 h1:OOhYzSvFnkFQXm1ysE8RjXTHsqSRDyP4emusC9K7DYg=
 github.com/hashicorp/memberlist v0.2.4/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=