Merge pull request #80 from tserong/wip-EMFILE

Terminate exporter process if maximum open files exceeded

exporter.go

@@ -18,8 +18,12 @@ package main
 import (
 	"flag"
 	"log"
+	"net"
 	"net/http"
+	"os"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/ceph/go-ceph/rados"
 	"github.com/digitalocean/ceph_exporter/collectors"
@@ -27,6 +31,31 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// This horrible thing is a copy of tcpKeepAliveListener, tweaked to
+// specifically check if it hits EMFILE when doing an accept, and if so,
+// terminate the process.
+
+type emfileAwareTcpListener struct {
+	*net.TCPListener
+}
+
+func (ln emfileAwareTcpListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		if oerr, ok := err.(*net.OpError); ok {
+			if serr, ok := oerr.Err.(*os.SyscallError); ok && serr.Err == syscall.EMFILE {
+				// This calls os.Exit(1) and terminates the process
+				log.Fatalf("%v", err)
+			}
+		}
+		// Default return
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(3 * time.Minute)
+	return tc, nil
+}
+
 // CephExporter wraps all the ceph collectors and provides a single global
 // exporter to extracts metrics out of. It also ensures that the collection
 // is done in a thread-safe manner, the necessary requirement stated by
@@ -152,7 +181,16 @@ func main() {
 	})
 
 	log.Printf("Starting ceph exporter on %q", *addr)
-	if err := http.ListenAndServe(*addr, nil); err != nil {
-		log.Fatalf("cannot start ceph exporter: %s", err)
+	// Below is essentially http.ListenAndServe(), but using our custom
+	// emfileAwareTcpListener that will die if we run out of file descriptors
+	ln, err := net.Listen("tcp", *addr)
+	if err == nil {
+		err := http.Serve(emfileAwareTcpListener{ln.(*net.TCPListener)}, nil)
+		if err != nil {
+			log.Fatalf("unable to serve requests: %s", err)
+		}
+	}
+	if err != nil {
+		log.Fatalf("unable to create listener: %s", err)
 	}
 }