Add discovery endpoint for Redis cluster (#989)

* Add discovery endpoint for Redis cluster


---------

Co-authored-by: Oliver <[email protected]>

Author: steveteuber

README.md

@@ -135,6 +135,33 @@ The `targets-redis-instances.json` should look something like this:
 
 Prometheus uses file watches and all changes to the json file are applied immediately.
 
+### Prometheus Configuration to Scrape All Nodes in a Redis Cluster
+
+When using a Redis Cluster, the exporter provides a discovery endpoint that can be used to discover all nodes in the cluster.
+To use this feature, the exporter must be started with the `--is-cluster` flag.\
+The discovery endpoint is available at `/discover-cluster-nodes` and can be used in the Prometheus configuration like this:
+
+```yaml
+scrape_configs:
+  - job_name: 'redis_exporter_cluster_nodes'
+    http_sd_configs:
+      - url: http://<<REDIS-EXPORTER-HOSTNAME>>:9121/discover-cluster-nodes
+        refresh_interval: 10m
+    metrics_path: /scrape
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: <<REDIS-EXPORTER-HOSTNAME>>:9121
+
+  ## config for scraping the exporter itself
+  - job_name: 'redis_exporter'
+    static_configs:
+      - targets:
+        - <<REDIS-EXPORTER-HOSTNAME>>:9121
+```
 
 ### Command line flags
 
@@ -207,7 +234,7 @@ Alternatively, you can provide the username and/or password using the `--redis.u
 If you want to use a dedicated Redis user for the redis_exporter (instead of the default user) then you need enable a list of commands for that user.
 You can use the following Redis command to set up the user, just replace `<<<USERNAME>>>` and `<<<PASSWORD>>>` with your desired values.
 ```
-ACL SETUSER <<<USERNAME>>> -@all +@connection +memory -readonly +strlen +config|get +xinfo +pfcount -quit +zcard +type +xlen -readwrite -command +client -wait +scard +llen +hlen +get +eval +slowlog +cluster|info -hello -echo +info +latency +scan -reset -auth -asking ><<<PASSWORD>>>
+ACL SETUSER <<<USERNAME>>> -@all +@connection +memory -readonly +strlen +config|get +xinfo +pfcount -quit +zcard +type +xlen -readwrite -command +client -wait +scard +llen +hlen +get +eval +slowlog +cluster|info +cluster|slots +cluster|nodes -hello -echo +info +latency +scan -reset -auth -asking ><<<PASSWORD>>>
 ```
 
 For monitoring a Sentinel-node you may use the following command with the right ACL:

exporter/exporter.go

@@ -509,6 +509,7 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) {
 
 	e.mux.HandleFunc("/", e.indexHandler)
 	e.mux.HandleFunc("/scrape", e.scrapeHandler)
+	e.mux.HandleFunc("/discover-cluster-nodes", e.discoverClusterNodesHandler)
 	e.mux.HandleFunc("/health", e.healthHandler)
 	e.mux.HandleFunc("/-/reload", e.reloadPwdFile)
 

exporter/http.go

@@ -2,6 +2,7 @@ package exporter
 
 import (
 	"crypto/subtle"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -103,6 +104,54 @@ func (e *Exporter) scrapeHandler(w http.ResponseWriter, r *http.Request) {
 	).ServeHTTP(w, r)
 }
 
+func (e *Exporter) discoverClusterNodesHandler(w http.ResponseWriter, r *http.Request) {
+	if !e.options.IsCluster {
+		http.Error(w, "The discovery endpoint is only available on a redis cluster", http.StatusBadRequest)
+		return
+	}
+
+	c, err := e.connectToRedisCluster()
+	if err != nil {
+		http.Error(w, "Couldn't connect to redis cluster", http.StatusInternalServerError)
+		return
+	}
+	defer c.Close()
+
+	nodes, err := e.getClusterNodes(c)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Failed to fetch cluster nodes: %s", err), http.StatusInternalServerError)
+		return
+	}
+
+	discovery := []struct {
+		Targets []string          `json:"targets"`
+		Labels  map[string]string `json:"labels"`
+	}{
+		{
+			Targets: make([]string, len(nodes)),
+			Labels:  make(map[string]string, 0),
+		},
+	}
+
+	isTls := strings.HasPrefix(e.redisAddr, "rediss://")
+	for i, node := range nodes {
+		if isTls {
+			discovery[0].Targets[i] = "rediss://" + node
+		} else {
+			discovery[0].Targets[i] = "redis://" + node
+		}
+	}
+
+	data, err := json.MarshalIndent(discovery, "", "  ")
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Failed to marshal discovery data: %s", err), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write(data)
+}
+
 func (e *Exporter) reloadPwdFile(w http.ResponseWriter, r *http.Request) {
 	if e.options.RedisPwdFile == "" {
 		http.Error(w, "There is no pwd file specified", http.StatusBadRequest)

exporter/http_test.go

@@ -321,6 +321,56 @@ func TestHttpHandlers(t *testing.T) {
 	}
 }
 
+func TestHttpDiscoverClusterNodesHandlers(t *testing.T) {
+	if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" {
+		t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
+	}
+
+	tests := []struct {
+		path      string
+		want      string
+		isCluster bool
+	}{
+		{
+			path:      "/discover-cluster-nodes",
+			want:      "redis://127.0.0.1:7000",
+			isCluster: true,
+		},
+		{
+			path:      "/discover-cluster-nodes",
+			want:      "redis://127.0.0.1:7001",
+			isCluster: true,
+		},
+		{
+			path:      "/discover-cluster-nodes",
+			want:      "redis://127.0.0.1:7002",
+			isCluster: true,
+		},
+		{
+			path:      "/discover-cluster-nodes",
+			want:      "The discovery endpoint is only available on a redis cluster",
+			isCluster: false,
+		},
+	}
+
+	for _, tst := range tests {
+		t.Run(fmt.Sprintf("path: %s, isCluster: %v", tst.path, tst.isCluster), func(t *testing.T) {
+			e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"), Options{
+				Namespace: "test",
+				Registry:  prometheus.NewRegistry(),
+				IsCluster: tst.isCluster,
+			})
+			ts := httptest.NewServer(e)
+			defer ts.Close()
+
+			body := downloadURL(t, ts.URL+tst.path)
+			if !strings.Contains(body, tst.want) {
+				t.Fatalf(`error, expected string "%s" in body, got body: \n\n%s`, tst.want, body)
+			}
+		})
+	}
+}
+
 func TestReloadHandlers(t *testing.T) {
 	if os.Getenv("TEST_PWD_REDIS_URI") == "" {
 		t.Skipf("TEST_PWD_REDIS_URI not set - skipping")

exporter/nodes.go

@@ -0,0 +1,52 @@
+package exporter
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/gomodule/redigo/redis"
+	log "github.com/sirupsen/logrus"
+)
+
+var reNodeAddress = regexp.MustCompile(`^(?P<ip>.+):(?P<port>\d+)@(?P<cport>\d+)(?:,(?P<hostname>.+))?`)
+
+func (e *Exporter) getClusterNodes(c redis.Conn) ([]string, error) {
+	output, err := redis.String(doRedisCmd(c, "CLUSTER", "NODES"))
+	if err != nil {
+		log.Errorf("Error getting cluster nodes: %s", err)
+		return nil, err
+	}
+
+	lines := strings.Split(output, "\n")
+	nodes := []string{}
+
+	for _, line := range lines {
+		if node, ok := parseClusterNodeString(line); ok {
+			nodes = append(nodes, node)
+		}
+	}
+
+	return nodes, nil
+}
+
+/*
+<id> <ip:port@cport[,hostname]> <flags> <master> <ping-sent> <pong-recv> <config-epoch> <link-state> <slot> <slot> ... <slot>
+eaf69c70d876558a948ba62af0884a37d42c9627 127.0.0.1:7002@17002 master - 0 1742836359057 3 connected 10923-16383
+*/
+func parseClusterNodeString(node string) (string, bool) {
+	log.Debugf("parseClusterNodeString node: [%s]", node)
+
+	fields := strings.Fields(node)
+	if len(fields) < 2 {
+		log.Debugf("Invalid field count for node: %s", node)
+		return "", false
+	}
+
+	address := reNodeAddress.FindStringSubmatch(fields[1])
+	if len(address) < 3 {
+		log.Debugf("Invalid format for node address, got: %s", fields[1])
+		return "", false
+	}
+
+	return address[1] + ":" + address[2], true
+}

exporter/nodes_test.go

@@ -0,0 +1,89 @@
+package exporter
+
+import (
+	"os"
+	"slices"
+	"testing"
+)
+
+func TestNodesGetClusterNodes(t *testing.T) {
+	if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" {
+		t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
+	}
+
+	host := os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI")
+	e, _ := NewRedisExporter(host, Options{})
+	c, err := e.connectToRedisCluster()
+	if err != nil {
+		t.Fatalf("connectToRedisCluster() err: %s", err)
+	}
+	defer c.Close()
+
+	nodes, err := e.getClusterNodes(c)
+	if err != nil {
+		t.Fatalf("getClusterNodes() err: %s", err)
+	}
+
+	tsts := []struct {
+		node string
+		ok   bool
+	}{
+		{node: "127.0.0.1:7003", ok: true},
+		{node: "127.0.0.1:7002", ok: true},
+		{node: "127.0.0.1:7005", ok: true},
+		{node: "127.0.0.1:7001", ok: true},
+		{node: "127.0.0.1:7004", ok: true},
+		{node: "127.0.0.1:7000", ok: true},
+
+		{node: "", ok: false},
+		{node: " ", ok: false},
+		{node: "127.0.0.1", ok: false},
+		{node: "127.0.0.1:8000", ok: false},
+	}
+
+	for _, tst := range tsts {
+		t.Run(tst.node, func(t *testing.T) {
+			found := slices.Contains(nodes, tst.node)
+			if found != tst.ok {
+				t.Errorf("Test failed for node: %s expected: %t, got: %t", tst.node, tst.ok, found)
+			}
+		})
+	}
+}
+
+func TestParseClusterNodeString(t *testing.T) {
+	tsts := []struct {
+		line string
+		node string
+		ok   bool
+	}{
+		// The following are examples of the output of the CLUSTER NODES command.
+		// https://redis.io/docs/latest/commands/cluster-nodes/
+		{line: "07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004,hostname4 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected", node: "127.0.0.1:30004", ok: true},
+		{line: "67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002,hostname2 master - 0 1426238316232 2 connected 5461-10922", node: "127.0.0.1:30002", ok: true},
+		{line: "292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003,hostname3 master - 0 1426238318243 3 connected 10923-16383", node: "127.0.0.1:30003", ok: true},
+		{line: "6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005,hostname5 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected", node: "127.0.0.1:30005", ok: true},
+		{line: "824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006,hostname6 slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected", node: "127.0.0.1:30006", ok: true},
+		{line: "e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001,hostname1 myself,master - 0 0 1 connected 0-5460", node: "127.0.0.1:30001", ok: true},
+		{line: "e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-5460", node: "127.0.0.1:30001", ok: true},
+
+		{line: "07c37dfeb235213a872192d90877d0cd55635b91", ok: false},
+		{line: "07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004,hostname4 slave", ok: false},
+		{line: "127.0.0.1:30005,hostname5", ok: false},
+	}
+
+	for _, tst := range tsts {
+		t.Run(tst.line, func(t *testing.T) {
+			node, ok := parseClusterNodeString(tst.line)
+
+			if ok != tst.ok {
+				t.Errorf("Test failed for line: %s", tst.line)
+				return
+			}
+			if node != tst.node {
+				t.Errorf("Node not matching, expected: %s, got: %s", tst.node, node)
+				return
+			}
+		})
+	}
+}