From e92865193cc44bc64cdd5e54cd11ec8eddb6dd03 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 29 Apr 2019 12:19:40 -0400 Subject: [PATCH] wip --- .drone.yml | 9 +- exporter/discovery.go | 75 +++--- exporter/discovery_test.go | 107 +++++--- exporter/redis.go | 529 ++++++++++++++++--------------------- exporter/redis_test.go | 404 +++++++++++----------------- main.go | 58 ++-- 6 files changed, 527 insertions(+), 655 deletions(-) diff --git a/.drone.yml b/.drone.yml index 823ff420..cf3e30c9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -38,16 +38,17 @@ steps: environment: GO111MODULE: on LOG_LEVEL: "info" - TEST_TILE38_URI: "tile38:9851" + TEST_TILE38_URI: "redis://tile38:9851" + TEST_REDIS_URI: "redis://redis:6379" TEST_SECOND_REDIS_URI: "redis://moar-redis:6380" - TEST_REDIS_CLUSTER_MASTER_URI: "redis-cluster:7000" - TEST_REDIS_CLUSTER_SLAVE_URI: "redis-cluster:7005" + TEST_REDIS_CLUSTER_MASTER_URI: "redis://redis-cluster:7000" + TEST_REDIS_CLUSTER_SLAVE_URI: "redis://redis-cluster:7005" COVERALLS_TOKEN: from_secret: coveralls-token commands: - 'go build -mod=vendor' - "sleep 10" # let the redis test instances all come up first - - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/... --redis.addr=redis' + - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/...' - 'echo "checking gofmt"' - 'echo " ! gofmt -d main.go 2>&1 | read " | bash' - 'echo " ! gofmt -d exporter/*.go 2>&1 | read " | bash' diff --git a/exporter/discovery.go b/exporter/discovery.go index 962776c6..66c614d3 100644 --- a/exporter/discovery.go +++ b/exporter/discovery.go @@ -12,93 +12,94 @@ import ( // loadRedisArgs loads the configuration for which redis hosts to monitor from either // the environment or as passed from program arguments. Returns the list of host addrs, // passwords, and their aliases. -func LoadRedisArgs(addr, password, alias, separator string) ([]string, []string, []string) { +func LoadRedisArgs(addr, password, alias, separator string) []RedisHost { if addr == "" { addr = "redis://localhost:6379" } + var res []RedisHost addrs := strings.Split(addr, separator) passwords := strings.Split(password, separator) - for len(passwords) < len(addrs) { - passwords = append(passwords, passwords[0]) - } aliases := strings.Split(alias, separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) + for idx, addr := range addrs { + var pwd, alias string + if idx < len(passwords) { + pwd = passwords[idx] + } + if idx < len(aliases) { + alias = aliases[idx] + } + res = append(res, RedisHost{Addr: addr, Password: pwd, Alias: alias}) } - return addrs, passwords, aliases + return res } // loadRedisFile opens the specified file and loads the configuration for which redis // hosts to monitor. Returns the list of hosts addrs, passwords, and their aliases. -func LoadRedisFile(fileName string) ([]string, []string, []string, error) { - var addrs []string - var passwords []string - var aliases []string +func LoadRedisFile(fileName string) ([]RedisHost, error) { file, err := os.Open(fileName) if err != nil { - return nil, nil, nil, err + return nil, err } r := csv.NewReader(file) r.FieldsPerRecord = -1 records, err := r.ReadAll() if err != nil { - return nil, nil, nil, err + return nil, err } file.Close() // For each line, test if it contains an optional password and alias and provide them, // else give them empty strings + + var res []RedisHost for _, record := range records { - length := len(record) - switch length { + var addr, pwd, alias string + switch len(record) { case 3: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, record[2]) + addr = record[0] + pwd = record[1] + alias = record[2] case 2: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, "") + addr = record[0] + pwd = record[1] case 1: - addrs = append(addrs, record[0]) - passwords = append(passwords, "") - aliases = append(aliases, "") + addr = record[0] + + default: + continue } + res = append(res, RedisHost{Addr: addr, Password: pwd, Alias: alias}) } - return addrs, passwords, aliases, nil + return res, nil } -func GetCloudFoundryRedisBindings() (addrs, passwords, aliases []string) { +func GetCloudFoundryRedisBindings() []RedisHost { + var res []RedisHost if !cfenv.IsRunningOnCF() { - return + return res } appEnv, err := cfenv.Current() if err != nil { log.Warnln("Unable to get current CF environment", err) - return + return res } redisServices, err := appEnv.Services.WithTag("redis") if err != nil { log.Warnln("Error while getting redis services", err) - return + return res } for _, redisService := range redisServices { credentials := redisService.Credentials host := getAlternative(credentials, "host", "hostname") port := getAlternative(credentials, "port") - password := getAlternative(credentials, "password") - addr := host + ":" + port - alias := redisService.Name - - addrs = append(addrs, addr) - passwords = append(passwords, password) - aliases = append(aliases, alias) + pwd := getAlternative(credentials, "pwd") + res = append(res, RedisHost{Addr: addr, Password: pwd, Alias: redisService.Name}) } - return + return res } func getAlternative(credentials map[string]interface{}, alternatives ...string) string { diff --git a/exporter/discovery_test.go b/exporter/discovery_test.go index 4a09f2e5..3883bc9e 100644 --- a/exporter/discovery_test.go +++ b/exporter/discovery_test.go @@ -1,60 +1,48 @@ package exporter import ( - "log" "testing" -) -func cmpStringArrays(a1, a2 []string) bool { - if len(a1) != len(a2) { - return false - } - for n := range a1 { - if a1[n] != a2[n] { - return false - } - } - return true -} + log "github.com/sirupsen/logrus" +) func TestLoadRedisArgs(t *testing.T) { - log.Println("TestLoadRedisArgs()") tests := []struct { - addr, pwd, alias, sep string - wantAddr, wantPwds, wantAliases []string + addr, pwd, alias, sep string + wantAddrs, wantPwds, wantAliases []string }{ { addr: "", sep: ",", - wantAddr: []string{"redis://localhost:6379"}, + wantAddrs: []string{"redis://localhost:6379"}, wantPwds: []string{""}, wantAliases: []string{""}, }, { addr: "redis://localhost:6379", sep: ",", - wantAddr: []string{"redis://localhost:6379"}, + wantAddrs: []string{"redis://localhost:6379"}, wantPwds: []string{""}, wantAliases: []string{""}, }, { addr: "redis://localhost:6379,redis://localhost:7000", sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000"}, + wantAddrs: []string{"redis://localhost:6379", "redis://localhost:7000"}, wantPwds: []string{"", ""}, wantAliases: []string{"", ""}, }, { addr: "redis://localhost:6379,redis://localhost:7000,redis://localhost:7001", sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7001"}, + wantAddrs: []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7001"}, wantPwds: []string{"", "", ""}, wantAliases: []string{"", "", ""}, }, { alias: "host-1", sep: ",", - wantAddr: []string{"redis://localhost:6379"}, + wantAddrs: []string{"redis://localhost:6379"}, wantPwds: []string{""}, wantAliases: []string{"host-1"}, }, @@ -62,39 +50,78 @@ func TestLoadRedisArgs(t *testing.T) { for _, test := range tests { sep := test.sep - addrs, pwds, aliases := LoadRedisArgs(test.addr, test.pwd, test.alias, sep) - if !cmpStringArrays(addrs, test.wantAddr) { - t.Errorf("addrs not matching wantAliases, got: %v want: %v", addrs, test.wantAddr) - } - if !cmpStringArrays(pwds, test.wantPwds) { - t.Errorf("pwds not matching wantAliases, got: %v want: %v", pwds, test.wantPwds) - } - if !cmpStringArrays(aliases, test.wantAliases) { - t.Errorf("aliases not matching wantAliases, got: %v want: %v", aliases, test.wantAliases) - } + hosts := LoadRedisArgs(test.addr, test.pwd, test.alias, sep) + checkHosts( + t, hosts, + test.wantAddrs, + test.wantPwds, + test.wantAliases) } } func TestLoadRedisFile(t *testing.T) { - if _, _, _, err := LoadRedisFile("doesnt-exist.txt"); err == nil { + if _, err := LoadRedisFile("doesnt-exist.txt"); err == nil { t.Errorf("should have failed opening non existing file") return } - addrs, pwds, aliases, err := LoadRedisFile("../contrib/sample_redis_hosts_file.txt") + hosts, err := LoadRedisFile("../contrib/sample_redis_hosts_file.txt") if err != nil { t.Errorf("LoadRedisFile() failed, err: %s", err) return } - log.Printf("aliases: %v \n", aliases) - if !cmpStringArrays(addrs, []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7000"}) { - t.Errorf("addrs not matching want") + + log.Debugf("hosts: %v \n", hosts) + + checkHosts( + t, hosts, + []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7000"}, + []string{"", "password", "second-pwd"}, + []string{"", "alias", ""}, + ) +} + +func checkHosts(t *testing.T, hosts []RedisHost, addrs, pwds, aliases []string) { + for _, addr := range addrs { + found := false + for _, host := range hosts { + if host.Addr == addr { + found = true + break + } + } + if !found { + t.Errorf("Didn't find addr: %s, got hosts: %#v", addr, hosts) + return + } } - if !cmpStringArrays(pwds, []string{"", "password", "second-pwd"}) { - t.Errorf("pwds not matching want") + + for _, pwd := range pwds { + found := false + for _, host := range hosts { + if host.Password == pwd { + found = true + break + } + } + if !found { + t.Errorf("Didn't find pwd: %s, got hosts: %#v", pwd, hosts) + return + } } - if !cmpStringArrays(aliases, []string{"", "alias", ""}) { - t.Errorf("aliases not matching want") + + for _, alias := range aliases { + found := false + for _, host := range hosts { + if host.Alias == alias { + found = true + break + } + } + if !found { + t.Errorf("Didn't find alias: %s, got hosts: %#v", alias, hosts) + return + } } } diff --git a/exporter/redis.go b/exporter/redis.go index 133c43c8..04fa9964 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -1,6 +1,7 @@ package exporter import ( + "crypto/tls" "errors" "fmt" "net/url" @@ -16,11 +17,14 @@ import ( log "github.com/sirupsen/logrus" ) -// RedisHost represents a set of Redis Hosts to health check. type RedisHost struct { - Addrs []string - Passwords []string - Aliases []string + Addr string + Password string + Alias string +} + +func (r *RedisHost) Label() []string { + return []string{r.Addr, r.Alias} } type dbKeyPair struct { @@ -34,24 +38,23 @@ type keyInfo struct { // Exporter implements the prometheus.Exporter interface, and exports Redis metrics. type Exporter struct { - redis RedisHost - namespace string - keys []dbKeyPair - singleKeys []dbKeyPair - keyValues *prometheus.GaugeVec - keySizes *prometheus.GaugeVec - scriptValues *prometheus.GaugeVec - duration prometheus.Gauge - scrapeErrors prometheus.Gauge + hosts []RedisHost + namespace string + keys []dbKeyPair + singleKeys []dbKeyPair + totalScrapes prometheus.Counter - metrics map[string]*prometheus.GaugeVec + + metricDescriptions map[string]*prometheus.Desc + + options Options LuaScript []byte - metricsMtx sync.RWMutex sync.RWMutex } +/* type scrapeResult struct { Name string Value float64 @@ -59,9 +62,10 @@ type scrapeResult struct { Alias string DB string } +*/ var ( - metricMap = map[string]string{ + metricMapGauges = map[string]string{ // # Server "uptime_in_seconds": "uptime_in_seconds", "process_id": "process_id", @@ -80,7 +84,6 @@ var ( "used_memory_rss": "memory_used_rss_bytes", "used_memory_peak": "memory_used_peak_bytes", "used_memory_lua": "memory_used_lua_bytes", - "total_system_memory": "total_system_memory_bytes", "maxmemory": "memory_max_bytes", // # Persistence @@ -108,21 +111,19 @@ var ( "aof_last_write_status": "aof_last_write_status", // # Stats - "total_connections_received": "connections_received_total", - "total_commands_processed": "commands_processed_total", - "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", - "total_net_input_bytes": "net_input_bytes_total", - "total_net_output_bytes": "net_output_bytes_total", - "instantaneous_input_kbps": "instantaneous_input_kbps", - "instantaneous_output_kbps": "instantaneous_output_kbps", - "rejected_connections": "rejected_connections_total", - "expired_keys": "expired_keys_total", - "evicted_keys": "evicted_keys_total", - "keyspace_hits": "keyspace_hits_total", - "keyspace_misses": "keyspace_misses_total", - "pubsub_channels": "pubsub_channels", - "pubsub_patterns": "pubsub_patterns", - "latest_fork_usec": "latest_fork_usec", + "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", + "total_net_input_bytes": "net_input_bytes_total", + "total_net_output_bytes": "net_output_bytes_total", + "instantaneous_input_kbps": "instantaneous_input_kbps", + "instantaneous_output_kbps": "instantaneous_output_kbps", + "rejected_connections": "rejected_connections_total", + "expired_keys": "expired_keys_total", + "evicted_keys": "evicted_keys_total", + "keyspace_hits": "keyspace_hits_total", + "keyspace_misses": "keyspace_misses_total", + "pubsub_channels": "pubsub_channels", + "pubsub_patterns": "pubsub_patterns", + "latest_fork_usec": "latest_fork_usec", // # Replication "loading": "loading_dump_file", @@ -131,12 +132,6 @@ var ( "master_last_io_seconds_ago": "master_last_io_seconds", "master_repl_offset": "master_repl_offset", - // # CPU - "used_cpu_sys": "used_cpu_sys", - "used_cpu_user": "used_cpu_user", - "used_cpu_sys_children": "used_cpu_sys_children", - "used_cpu_user_children": "used_cpu_user_children", - // # Cluster "cluster_stats_messages_sent": "cluster_messages_sent_total", "cluster_stats_messages_received": "cluster_messages_received_total", @@ -162,98 +157,22 @@ var ( "version": "version", // since tile38 version 1.14.1 } + metricMapCounters = map[string]string{ + // # Stats + "total_connections_received": "connections_received_total", + "total_commands_processed": "commands_processed_total", + + // # CPU + "used_cpu_sys": "used_cpu_sys", + "used_cpu_user": "used_cpu_user", + "used_cpu_sys_children": "used_cpu_sys_children", + "used_cpu_user_children": "used_cpu_user_children", + } + instanceInfoFields = map[string]bool{"role": true, "redis_version": true, "redis_build_id": true, "redis_mode": true, "os": true} slaveInfoFields = map[string]bool{"master_host": true, "master_port": true, "slave_read_only": true} ) -func (e *Exporter) initGauges() { - - e.metrics = map[string]*prometheus.GaugeVec{} - e.metrics["instance_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "instance_info", - Help: "Information about the Redis instance", - }, []string{"addr", "alias", "role", "redis_version", "redis_build_id", "redis_mode", "os"}) - e.metrics["slave_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slave_info", - Help: "Information about the Redis slave", - }, []string{"addr", "alias", "master_host", "master_port", "read_only"}) - e.metrics["start_time_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "start_time_seconds", - Help: "Start time of the Redis instance since unix epoch in seconds.", - }, []string{"addr", "alias"}) - e.metrics["master_link_up"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "master_link_up", - Help: "Master link status on Redis slave", - }, []string{"addr", "alias"}) - e.metrics["connected_slave_offset"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_offset", - Help: "Offset of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["connected_slave_lag_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_lag_seconds", - Help: "Lag of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["db_keys"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys", - Help: "Total number of keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_keys_expiring"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys_expiring", - Help: "Total number of expiring keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_avg_ttl_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_avg_ttl_seconds", - Help: "Avg TTL in seconds", - }, []string{"addr", "alias", "db"}) - - // Latency info - e.metrics["latency_spike_last"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_last", - Help: "When the latency spike last occurred", - }, []string{"addr", "alias", "event_name"}) - e.metrics["latency_spike_milliseconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_milliseconds", - Help: "Length of the last latency spike in milliseconds", - }, []string{"addr", "alias", "event_name"}) - - e.metrics["commands_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_total", - Help: "Total number of calls per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["commands_duration_seconds_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_duration_seconds_total", - Help: "Total amount of time in seconds spent per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["slowlog_length"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_length", - Help: "Total slowlog", - }, []string{"addr", "alias"}) - e.metrics["slowlog_last_id"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_last_id", - Help: "Last id of slowlog", - }, []string{"addr", "alias"}) - e.metrics["last_slow_execution_duration_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "last_slow_execution_duration_seconds", - Help: "The amount of time needed for last slow execution, in seconds", - }, []string{"addr", "alias"}) -} - // splitKeyArgs splits a command-line supplied argument into a slice of dbKeyPairs. func parseKeyArg(keysArgString string) (keys []dbKeyPair, err error) { if keysArgString == "" { @@ -282,104 +201,120 @@ func parseKeyArg(keysArgString string) (keys []dbKeyPair, err error) { return keys, err } +type Options struct { + Namespace string + CheckSingleKeys string + CheckKeys string + IncludeVerbotenMetrics bool + SkipTLSVerification bool +} + +func newMetricDescr(namespace string, metricName string, docString string, labels []string) *prometheus.Desc { + return prometheus.NewDesc(prometheus.BuildFQName(namespace, "", metricName), docString, labels, nil) +} + // NewRedisExporter returns a new exporter of Redis metrics. // note to self: next time we add an argument, instead add a RedisExporter struct -func NewRedisExporter(host RedisHost, namespace, checkSingleKeys, checkKeys string) (*Exporter, error) { +func NewRedisExporter(hosts []RedisHost, opts Options) (*Exporter, error) { e := Exporter{ - redis: host, - namespace: namespace, - keyValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_value", - Help: "The value of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - keySizes: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_size", - Help: "The length or size of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - scriptValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "script_value", - Help: "Values returned by the collect script", - }, []string{"addr", "alias", "key"}), - duration: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_duration_seconds", - Help: "The last scrape duration.", - }), + hosts: hosts, + options: opts, + namespace: opts.Namespace, + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, + Namespace: opts.Namespace, Name: "exporter_scrapes_total", Help: "Current total redis scrapes.", }), - scrapeErrors: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_error", - Help: "The last scrape error status.", - }), } var err error - if e.keys, err = parseKeyArg(checkKeys); err != nil { + if e.keys, err = parseKeyArg(opts.CheckKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-keys: %#v", err) } log.Debugf("keys: %#v", e.keys) - if e.singleKeys, err = parseKeyArg(checkSingleKeys); err != nil { + if e.singleKeys, err = parseKeyArg(opts.CheckSingleKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-single-keys: %#v", err) } log.Debugf("singleKeys: %#v", e.singleKeys) - e.initGauges() + if opts.IncludeVerbotenMetrics { + metricMapGauges["total_system_memory"] = "total_system_memory_bytes" + } + + l := []string{"addr", "alias"} + e.metricDescriptions = map[string]*prometheus.Desc{} + e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", l) + e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"addr", "alias", "role", "redis_version", "redis_build_id", "redis_mode", "os"}) + e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", l) + e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"addr", "alias", "err"}) + + e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"addr", "alias", "key"}) + e.metricDescriptions["key_values"] = newMetricDescr(opts.Namespace, "key_value", `The value of "key"`, []string{"addr", "alias", "db", "key"}) + e.metricDescriptions["key_sizes"] = newMetricDescr(opts.Namespace, "key_size", `The length or size of "key"`, []string{"addr", "alias", "db", "key"}) + + e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"addr", "alias", "cmd"}) + e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"addr", "alias", "cmd"}) + e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, l) + e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, l) + e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, l) + + e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"addr", "alias", "event_name"}) + e.metricDescriptions["latency_spike_milliseconds"] = newMetricDescr(opts.Namespace, "latency_spike_milliseconds", `Length of the last latency spike in milliseconds`, []string{"addr", "alias", "event_name"}) + + e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"addr", "alias", "master_host", "master_port", "read_only"}) + + e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", l) + e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", l) + e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) + e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) + + e.metricDescriptions["db_keys"] = newMetricDescr(opts.Namespace, "db_keys", "Total number of keys by DB", []string{"addr", "alias", "db"}) + e.metricDescriptions["db_keys_expiring"] = newMetricDescr(opts.Namespace, "db_keys_expiring", "Total number of expiring keys by DB", []string{"addr", "alias", "db"}) + e.metricDescriptions["db_avg_ttl_seconds"] = newMetricDescr(opts.Namespace, "db_avg_ttl_seconds", "Avg TTL in seconds", []string{"addr", "alias", "db"}) + return &e, nil } // Describe outputs Redis metric descriptions. func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { + for _, desc := range e.metricDescriptions { + ch <- desc + } - for _, m := range e.metrics { - m.Describe(ch) + for _, v := range metricMapGauges { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{"addr", "alias"}) + } + + for _, v := range metricMapCounters { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{"addr", "alias"}) } - e.keySizes.Describe(ch) - e.keyValues.Describe(ch) - ch <- e.duration.Desc() ch <- e.totalScrapes.Desc() - ch <- e.scrapeErrors.Desc() } // Collect fetches new metrics from the RedisHost and updates the appropriate metrics. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - scrapes := make(chan scrapeResult) - e.Lock() defer e.Unlock() - e.keySizes.Reset() - e.keyValues.Reset() + e.scrapeAllHosts(ch) - e.initGauges() - go e.scrape(scrapes) - e.setMetrics(scrapes) - - e.keySizes.Collect(ch) - e.keyValues.Collect(ch) - e.scriptValues.Collect(ch) - - ch <- e.duration ch <- e.totalScrapes - ch <- e.scrapeErrors - e.collectMetrics(ch) } func includeMetric(s string) bool { if strings.HasPrefix(s, "db") || strings.HasPrefix(s, "cmdstat_") || strings.HasPrefix(s, "cluster_") { return true } - _, ok := metricMap[s] + if _, ok := metricMapGauges[s]; ok { + return true + } + + _, ok := metricMapCounters[s] return ok } @@ -478,7 +413,7 @@ func parseConnectedSlaveString(slaveName string, slaveInfo string) (offset float return } -func extractConfigMetrics(config []string, addr string, alias string, scrapes chan<- scrapeResult) (dbCount int, err error) { +func (e *Exporter) extractConfigMetrics(ch chan<- prometheus.Metric, config []string, host RedisHost) (dbCount int, err error) { if len(config)%2 != 0 { return 0, fmt.Errorf("invalid config: %#v", config) } @@ -502,13 +437,27 @@ func extractConfigMetrics(config []string, addr string, alias string, scrapes ch } if val, err := strconv.ParseFloat(strVal, 64); err == nil { - scrapes <- scrapeResult{Name: fmt.Sprintf("config_%s", config[pos*2]), Addr: addr, Alias: alias, Value: val} + // todo: this needs a test + e.registerGaugeValue(ch, fmt.Sprintf("config_%s", config[pos*2]), val, host.Label()) } } return } -func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string, val float64, labels []string) { + e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels) +} + +func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labels []string) { + descr := e.metricDescriptions[metric] + if descr == nil { + descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", []string{"addr", "alias"}) + } + + ch <- prometheus.MustNewConstMetric(descr, valType, val, labels...) +} + +func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string, host RedisHost) error { for i := 0; i < len(info); i += 2 { log.Debugf("tile38: %s:%s", info[i], info[i+1]) @@ -519,13 +468,13 @@ func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, host, fieldKey, fieldValue) } return nil } -func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, host RedisHost, fieldKey string, fieldValue string) { /* Format: cmdstat_get:calls=21,usec=175,usec_per_call=8.33 @@ -552,46 +501,36 @@ func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey return } - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - cmd := splitKey[1] - e.metrics["commands_total"].WithLabelValues(addr, alias, cmd).Set(calls) - e.metrics["commands_duration_seconds_total"].WithLabelValues(addr, alias, cmd).Set(usecTotal / 1e6) + e.registerMetricValue(ch, "commands_total", calls, prometheus.CounterValue, []string{host.Addr, host.Alias, cmd}) + e.registerMetricValue(ch, "commands_duration_seconds_total", usecTotal/1e6, prometheus.CounterValue, []string{host.Addr, host.Alias, cmd}) } -func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey string, fieldValue string) bool { - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - +func (e *Exporter) handleMetricsReplication(ch chan<- prometheus.Metric, host RedisHost, fieldKey string, fieldValue string) bool { // only slaves have this field if fieldKey == "master_link_status" { if fieldValue == "up" { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(1) + e.registerGaugeValue(ch, "master_link_up", 1, host.Label()) } else { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(0) + e.registerGaugeValue(ch, "master_link_up", 0, host.Label()) } return true } // not a slave, try extracting master metrics - if slaveOffset, slaveIp, slavePort, slaveState, lag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { - e.metrics["connected_slave_offset"].WithLabelValues( - addr, - alias, + if slaveOffset, slaveIp, slavePort, slaveState, slaveLag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { + e.registerGaugeValue(ch, "connected_slave_offset", slaveOffset, []string{host.Addr, host.Alias, slaveIp, slavePort, slaveState, - ).Set(slaveOffset) + }) - if lag > -1 { - e.metrics["connected_slave_lag_seconds"].WithLabelValues( - addr, - alias, + if slaveLag > -1 { + e.registerGaugeValue(ch, "connected_slave_lag_seconds", slaveLag, []string{host.Addr, host.Alias, slaveIp, slavePort, slaveState, - ).Set(lag) + }) } return true } @@ -599,17 +538,15 @@ func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey return false } -func (e *Exporter) handleMetricsServer(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsServer(ch chan<- prometheus.Metric, host RedisHost, fieldKey string, fieldValue string) { if fieldKey == "uptime_in_seconds" { if uptime, err := strconv.ParseFloat(fieldValue, 64); err == nil { - e.metricsMtx.RLock() - e.metrics["start_time_seconds"].WithLabelValues(addr, alias).Set(float64(time.Now().Unix()) - uptime) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "start_time_seconds", float64(time.Now().Unix())-uptime, host.Label()) } } } -func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes chan<- scrapeResult, dbCount int) error { +func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, host RedisHost, info string, dbCount int) error { instanceInfo := map[string]string{} slaveInfo := map[string]string{} handledDBs := map[string]bool{} @@ -644,24 +581,26 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c switch fieldClass { case "Replication": - if ok := e.handleMetricsReplication(addr, alias, fieldKey, fieldValue); ok { + if ok := e.handleMetricsReplication(ch, host, fieldKey, fieldValue); ok { continue } case "Server": - e.handleMetricsServer(addr, alias, fieldKey, fieldValue) + e.handleMetricsServer(ch, host, fieldKey, fieldValue) case "Commandstats": - e.handleMetricsCommandStats(addr, alias, fieldKey, fieldValue) + e.handleMetricsCommandStats(ch, host, fieldKey, fieldValue) continue case "Keyspace": if keysTotal, keysEx, avgTTL, ok := parseDBKeyspaceString(fieldKey, fieldValue); ok { dbName := fieldKey - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: keysTotal} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: keysEx} + + e.registerGaugeValue(ch, "db_keys", keysTotal, []string{host.Addr, host.Alias, dbName}) + e.registerGaugeValue(ch, "db_keys_expiring", keysEx, []string{host.Addr, host.Alias, dbName}) + if avgTTL > -1 { - scrapes <- scrapeResult{Name: "db_avg_ttl_seconds", Addr: addr, Alias: alias, DB: dbName, Value: avgTTL} + e.registerGaugeValue(ch, "db_avg_ttl_seconds", avgTTL, []string{host.Addr, host.Alias, dbName}) } handledDBs[dbName] = true continue @@ -672,40 +611,35 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, host, fieldKey, fieldValue) } for dbIndex := 0; dbIndex < dbCount; dbIndex++ { dbName := "db" + strconv.Itoa(dbIndex) if _, exists := handledDBs[dbName]; !exists { - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: 0} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: 0} + e.registerGaugeValue(ch, "db_keys", 0, []string{host.Addr, host.Alias, dbName}) + e.registerGaugeValue(ch, "db_keys_expiring", 0, []string{host.Addr, host.Alias, dbName}) } } - e.metricsMtx.RLock() - e.metrics["instance_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "instance_info", 1, []string{host.Addr, host.Alias, instanceInfo["role"], instanceInfo["redis_version"], instanceInfo["redis_build_id"], instanceInfo["redis_mode"], - instanceInfo["os"], - ).Set(1) + instanceInfo["os"]}) + if instanceInfo["role"] == "slave" { - e.metrics["slave_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "slave_info", 1, []string{host.Addr, host.Alias, slaveInfo["master_host"], slaveInfo["master_port"], - slaveInfo["slave_read_only"], - ).Set(1) + slaveInfo["slave_read_only"]}) } - e.metricsMtx.RUnlock() return nil } -func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, host RedisHost, info string) error { lines := strings.Split(info, "\r\n") for _, line := range lines { @@ -722,16 +656,20 @@ func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, host, fieldKey, fieldValue) } return nil } -func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scrapeResult) error { +func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, host RedisHost, fieldKey, fieldValue string) error { metricName := sanitizeMetricName(fieldKey) - if newName, ok := metricMap[metricName]; ok { + if newName, ok := metricMapGauges[metricName]; ok { metricName = newName + } else { + if newName, ok := metricMapCounters[metricName]; ok { + metricName = newName + } } var err error @@ -753,7 +691,12 @@ func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scr log.Debugf("couldn't parse %s, err: %s", fieldValue, err) } - scrapes <- scrapeResult{Name: metricName, Addr: addr, Alias: alias, Value: val} + t := prometheus.GaugeValue + if metricMapCounters[metricName] != "" { + t = prometheus.CounterValue + } + + e.registerMetricValue(ch, metricName, val, t, host.Label()) return nil } @@ -864,43 +807,47 @@ func getKeysFromPatterns(c redis.Conn, keys []dbKeyPair) (expandedKeys []dbKeyPa return expandedKeys, err } -func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error { +func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric, host RedisHost) error { options := []redis.DialOption{ redis.DialConnectTimeout(5 * time.Second), redis.DialReadTimeout(5 * time.Second), redis.DialWriteTimeout(5 * time.Second), + + redis.DialTLSConfig(&tls.Config{ + InsecureSkipVerify: e.options.SkipTLSVerification, + }), } - if len(e.redis.Passwords) > idx && e.redis.Passwords[idx] != "" { - options = append(options, redis.DialPassword(e.redis.Passwords[idx])) + if host.Password != "" { + options = append(options, redis.DialPassword(host.Password)) } - log.Debugf("Trying DialURL(): %s", addr) - c, err := redis.DialURL(addr, options...) + log.Debugf("Trying DialURL(): %s", host.Addr) + c, err := redis.DialURL(host.Addr, options...) if err != nil { log.Debugf("DialURL() failed, err: %s", err) - if frags := strings.Split(addr, "://"); len(frags) == 2 { + if frags := strings.Split(host.Addr, "://"); len(frags) == 2 { log.Debugf("Trying: Dial(): %s %s", frags[0], frags[1]) c, err = redis.Dial(frags[0], frags[1], options...) } else { - log.Debugf("Trying: Dial(): tcp %s", addr) - c, err = redis.Dial("tcp", addr, options...) + log.Debugf("Trying: Dial(): tcp %s", host.Addr) + c, err = redis.Dial("tcp", host.Addr, options...) } } if err != nil { - log.Debugf("aborting for addr: %s - redis err: %s", addr, err) + log.Debugf("aborting for addr: %s - redis err: %s", host.Addr, err) return err } defer c.Close() - log.Debugf("connected to: %s", addr) + log.Debugf("connected to: %s", host.Addr) dbCount := 0 if config, err := redis.Strings(c.Do("CONFIG", "GET", "*")); err == nil { - dbCount, err = extractConfigMetrics(config, addr, e.redis.Aliases[idx], scrapes) + dbCount, err = e.extractConfigMetrics(ch, config, host) if err != nil { log.Errorf("Redis CONFIG err: %s", err) return err @@ -921,7 +868,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx if isClusterEnabled { if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil { - e.extractClusterInfoMetrics(clusterInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractClusterInfoMetrics(ch, host, clusterInfo) // in cluster mode Redis only supports one database so no extra padding beyond that needed dbCount = 1 @@ -936,12 +883,12 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx } } - e.extractInfoMetrics(infoAll, addr, e.redis.Aliases[idx], scrapes, dbCount) + e.extractInfoMetrics(ch, host, infoAll, dbCount) // SERVER command only works on tile38 database. check the following link to // find out more: https://tile38.com/ if serverInfo, err := redis.Strings(doRedisCmd(c, "SERVER")); err == nil { - e.extractTile38Metrics(serverInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractTile38Metrics(ch, serverInfo, host) } else { log.Debugf("Tile38 SERVER err: %s", err) } @@ -952,10 +899,8 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 { latencyResult := tempVal[0].([]interface{}) if _, err := redis.Scan(latencyResult, &eventName, &spikeLast, &milliseconds, &max); err == nil { - e.metricsMtx.RLock() - e.metrics["latency_spike_last"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(spikeLast)) - e.metrics["latency_spike_milliseconds"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(milliseconds)) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "latency_spike_last", float64(spikeLast), []string{host.Addr, host.Alias, eventName}) + e.registerGaugeValue(ch, "latency_spike_milliseconds", float64(milliseconds), []string{host.Addr, host.Alias, eventName}) } } } @@ -989,11 +934,12 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx continue } dbLabel := "db" + k.db - e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(info.size) + + e.registerGaugeValue(ch, "key_sizes", info.size, []string{host.Addr, host.Alias, dbLabel, k.key}) // Only record value metric if value is float-y - if value, err := redis.Float64(c.Do("GET", k.key)); err == nil { - e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(value) + if val, err := redis.Float64(c.Do("GET", k.key)); err == nil { + e.registerGaugeValue(ch, "key_values", val, []string{host.Addr, host.Alias, dbLabel, k.key}) } } @@ -1005,16 +951,14 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx } else if kv != nil { for key, stringVal := range kv { if val, err := strconv.ParseFloat(stringVal, 64); err == nil { - e.scriptValues.WithLabelValues(addr, e.redis.Aliases[idx], key).Set(val) + e.registerGaugeValue(ch, "script_values", val, []string{host.Addr, host.Alias, key}) } } } } if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { - e.metricsMtx.RLock() - e.metrics["slowlog_length"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(reply.(int64))) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64)), host.Label()) } if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { @@ -1030,58 +974,37 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx } } - e.metricsMtx.RLock() - e.metrics["slowlog_last_id"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(slowlogLastId)) - e.metrics["last_slow_execution_duration_seconds"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(lastSlowExecutionDurationSeconds) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastId), host.Label()) + e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds, host.Label()) } log.Debugf("scrapeRedisHost() done") return nil } -func (e *Exporter) scrape(scrapes chan<- scrapeResult) { - defer close(scrapes) - +func (e *Exporter) scrapeHost(ch chan<- prometheus.Metric, host RedisHost) { now := time.Now().UnixNano() - e.totalScrapes.Inc() - - errorCount := 0 - for idx, addr := range e.redis.Addrs { - var up float64 = 1 - if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil { - errorCount++ - up = 0 - } - scrapes <- scrapeResult{Name: "up", Addr: addr, Alias: e.redis.Aliases[idx], Value: up} + var up float64 = 1 + if err := e.scrapeRedisHost(ch, host); err != nil { + up = 0 + e.registerGaugeValue(ch, "scrape_error", 1.0, []string{host.Addr, host.Alias, fmt.Sprintf("%s", err)}) + } else { + e.registerGaugeValue(ch, "scrape_error", 0, []string{host.Addr, host.Alias, ""}) } - e.scrapeErrors.Set(float64(errorCount)) - e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000) -} - -func (e *Exporter) setMetrics(scrapes <-chan scrapeResult) { - for scr := range scrapes { - name := scr.Name - if _, ok := e.metrics[name]; !ok { - e.metricsMtx.Lock() - e.metrics[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: name, - Help: name + "metric", // needs to be set for prometheus >= 2.3.1 - }, []string{"addr", "alias"}) - e.metricsMtx.Unlock() - } - var labels prometheus.Labels = map[string]string{"addr": scr.Addr, "alias": scr.Alias} - if len(scr.DB) > 0 { - labels["db"] = scr.DB - } - e.metrics[name].With(labels).Set(scr.Value) - } + e.registerGaugeValue(ch, "up", up, host.Label()) + e.registerGaugeValue(ch, "last_scrape_duration", float64(time.Now().UnixNano()-now)/1000000000, host.Label()) } -func (e *Exporter) collectMetrics(metrics chan<- prometheus.Metric) { - for _, m := range e.metrics { - m.Collect(metrics) - } +func (e *Exporter) scrapeAllHosts(ch chan<- prometheus.Metric) { + e.totalScrapes.Inc() + wg := sync.WaitGroup{} + wg.Add(len(e.hosts)) + for _, host := range e.hosts { + go func(h RedisHost) { + e.scrapeHost(ch, h) + wg.Done() + }(host) + } + wg.Wait() } diff --git a/exporter/redis_test.go b/exporter/redis_test.go index dcd8bdd3..4f222f8c 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -9,7 +9,6 @@ package exporter */ import ( - "flag" "fmt" "io/ioutil" "net/http" @@ -35,27 +34,25 @@ const ( ) var ( - redisAddr = flag.String("redis.addr", ":6379", "Address of the test instance, without `redis://`") - redisAlias = flag.String("redis.alias", "foo", "Alias of the test instance") - separator = flag.String("separator", ",", "separator used to split redis.addr, redis.password and redis.alias into several elements.") - - keys = []string{} - keysExpiring = []string{} - listKeys = []string{} - ts = int32(time.Now().Unix()) - defaultRedisHost = RedisHost{} + keys = []string{} + keysExpiring = []string{} + listKeys = []string{} + ts = int32(time.Now().Unix()) dbNumStr = "11" altDBNumStr = "12" dbNumStrFull = fmt.Sprintf("db%s", dbNumStr) - - TestServerURL = "" ) const ( TestSetName = "test-set" ) +func getTestExporter() *Exporter { + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test"}) + return e +} + func setupLatency(t *testing.T, addr string) error { c, err := redis.DialURL(addr) @@ -166,10 +163,10 @@ func downloadUrl(t *testing.T, url string) string { } func TestLatencySpike(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() - setupLatency(t, defaultRedisHost.Addrs[0]) - defer resetLatency(t, defaultRedisHost.Addrs[0]) + setupLatency(t, os.Getenv("TEST_REDIS_URI")) + defer resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -178,24 +175,21 @@ func TestLatencySpike(t *testing.T) { }() for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { - got := &dto.Metric{} - m.Write(got) - - val := got.GetGauge().GetValue() - // Because we're dealing with latency, there might be a slight delay - // even after sleeping for a specific amount of time so checking - // to see if we're between +-5 of our expected value - if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { - t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) - } + if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { + got := &dto.Metric{} + m.Write(got) + + val := got.GetGauge().GetValue() + // Because we're dealing with latency, there might be a slight delay + // even after sleeping for a specific amount of time so checking + // to see if we're between +-5 of our expected value + if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { + t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) } } } - resetLatency(t, defaultRedisHost.Addrs[0]) + resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -218,10 +212,7 @@ func TestTile38(t *testing.T) { t.SkipNow() } - e, _ := NewRedisExporter( - RedisHost{Addrs: []string{os.Getenv("TEST_TILE38_URI")}, Aliases: []string{"tile"}}, - "test", "", "", - ) + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_TILE38_URI"), Alias: "tile"}}, Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -229,22 +220,21 @@ func TestTile38(t *testing.T) { close(chM) }() - find := false + found := false for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "cpus_total") { - find = true - } + if strings.Contains(m.Desc().String(), "cpus_total") { + found = true + log.Debugf("type: %T", m) } + } - if !find { + if !found { t.Errorf("cpus_total was not found in tile38 metrics") } } func TestSlowLog(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() chM := make(chan prometheus.Metric) go func() { @@ -266,8 +256,8 @@ func TestSlowLog(t *testing.T) { } } - setupSlowLog(t, defaultRedisHost.Addrs[0]) - defer resetSlowLog(t, defaultRedisHost.Addrs[0]) + setupSlowLog(t, os.Getenv("TEST_REDIS_URI")) + defer resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -300,7 +290,7 @@ func TestSlowLog(t *testing.T) { } } - resetSlowLog(t, defaultRedisHost.Addrs[0]) + resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -403,14 +393,14 @@ func deleteKeysFromDB(t *testing.T, addr string) error { return nil } +/* func TestHostVariations(t *testing.T) { for _, prefix := range []string{"", "redis://", "tcp://"} { - addr := prefix + *redisAddr - host := RedisHost{Addrs: []string{addr}, Aliases: []string{""}} - e, _ := NewRedisExporter(host, "test", "", "") + addr := prefix + os.Getenv("TEST_REDIS_URI") + e, _ := NewRedisExporter([]RedisHost{{Addr: addr}}, Options{Namespace: "test"}) - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) + scrapes := make(chan prometheus.Metric, 10000) + e.scrapeAllHosts(scrapes) found := 0 for range scrapes { found++ @@ -423,10 +413,10 @@ func TestHostVariations(t *testing.T) { } func TestCountingKeys(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) + e.scrapeAllHosts(scrapes) var keysTestDB float64 for s := range scrapes { @@ -436,11 +426,11 @@ func TestCountingKeys(t *testing.T) { } } - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) scrapes = make(chan scrapeResult, 1000) - e.scrape(scrapes) + e.scrapeAllHosts(scrapes) // +1 for the one SET key want := keysTestDB + float64(len(keys)) + float64(len(keysExpiring)) + 1 + float64(len(listKeys)) @@ -454,9 +444,9 @@ func TestCountingKeys(t *testing.T) { } } - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) scrapes = make(chan scrapeResult, 10000) - e.scrape(scrapes) + e.scrapeAllHosts(scrapes) for s := range scrapes { if s.Name == "db_keys" && s.DB == dbNumStrFull { @@ -474,51 +464,15 @@ func TestCountingKeys(t *testing.T) { } } -func TestExporterMetrics(t *testing.T) { - - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - - e.setMetrics(scrapes) - - want := 25 - if len(e.metrics) < want { - t.Errorf("need moar metrics, found: %d, want: %d", len(e.metrics), want) - } - - wantKeys := []string{ - "db_keys", - "db_avg_ttl_seconds", - "used_cpu_sys", - "loading_dump_file", // testing renames - "config_maxmemory", // testing config extraction - "config_maxclients", // testing config extraction - "slowlog_length", - "slowlog_last_id", - "start_time_seconds", - "uptime_in_seconds", - } - - for _, k := range wantKeys { - if _, ok := e.metrics[k]; !ok { - t.Errorf("missing metrics key: %s", k) - } - } -} func TestExporterValues(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) + e.scrapeAllHosts(scrapes) wantValues := map[string]float64{ "db_keys_total": float64(len(keys)+len(keysExpiring)) + 1, // + 1 for the SET key @@ -533,6 +487,7 @@ func TestExporterValues(t *testing.T) { } } } +*/ type tstData struct { db string @@ -609,10 +564,13 @@ func TestParseConnectedSlaveString(t *testing.T) { } func TestKeyValuesAndSizes(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter( + []RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, + Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}, + ) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -623,15 +581,10 @@ func TestKeyValuesAndSizes(t *testing.T) { want := map[string]bool{"test_key_size": false, "test_key_value": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -702,7 +655,7 @@ func TestScanForKeys(t *testing.T) { fixtures = append(fixtures, newKeyFixture("SET", key, "Rats!")) } - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -739,7 +692,7 @@ func TestScanForKeys(t *testing.T) { } func TestGetKeysFromPatterns(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") dbMain := dbNumStr dbAlt := altDBNumStr @@ -823,7 +776,7 @@ func TestGetKeysFromPatterns(t *testing.T) { } func TestGetKeyInfo(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -891,10 +844,10 @@ func TestGetKeyInfo(t *testing.T) { func TestKeySizeList(t *testing.T) { s := dbNumStrFull + "=" + listKeys[0] - e, _ := NewRedisExporter(defaultRedisHost, "test", s, "") + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test", CheckSingleKeys: s}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -905,14 +858,9 @@ func TestKeySizeList(t *testing.T) { found := false for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_key_size") { - found = true - break - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_key_size") { + found = true + break } } @@ -922,12 +870,12 @@ func TestKeySizeList(t *testing.T) { } func TestScript(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() e.LuaScript = []byte(`return {"a", "11", "b", "12", "c", "13"}`) nKeys := 3 - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -936,13 +884,8 @@ func TestScript(t *testing.T) { }() for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_script_value") { - nKeys-- - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_script_value") { + nKeys-- } } if nKeys != 0 { @@ -951,7 +894,7 @@ func TestScript(t *testing.T) { } func TestKeyValueInvalidDB(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "999="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test", CheckSingleKeys: "999=" + url.QueryEscape(keys[0])}) chM := make(chan prometheus.Metric) go func() { @@ -970,7 +913,7 @@ func TestKeyValueInvalidDB(t *testing.T) { } } default: - log.Printf("default: m: %#v", m) + log.Debugf("default: m: %#v", m) } } for k, found := range dontWant { @@ -982,10 +925,10 @@ func TestKeyValueInvalidDB(t *testing.T) { } func TestCommandStats(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e := getTestExporter() - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -996,15 +939,10 @@ func TestCommandStats(t *testing.T) { want := map[string]bool{"test_commands_duration_seconds_total": false, "test_commands_total": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -1023,10 +961,10 @@ func TestHTTPEndpoint(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) body := downloadUrl(t, ts.URL+"/metrics") @@ -1038,11 +976,25 @@ func TestHTTPEndpoint(t *testing.T) { `test_key_size`, `test_instance_info`, + "db_keys", + "db_avg_ttl_seconds", + "used_cpu_sys", + "loading_dump_file", // testing renames + "config_maxmemory", // testing config extraction + "config_maxclients", // testing config extraction + "slowlog_length", + "slowlog_last_id", + "start_time_seconds", + "uptime_in_seconds", + // labels and label values - `addr="redis://` + *redisAddr, + `addr="` + os.Getenv("TEST_REDIS_URI"), `redis_mode`, `standalone`, `cmd="get"`, + + `test_db_keys{addr="` + os.Getenv("TEST_REDIS_URI") + `",alias="",db="db11"} 11`, + `test_db_keys_expiring{addr="` + os.Getenv("TEST_REDIS_URI") + `",alias="",db="db11"} `, } for _, test := range tests { if !strings.Contains(body, test) { @@ -1052,8 +1004,7 @@ func TestHTTPEndpoint(t *testing.T) { } func TestNonExistingHost(t *testing.T) { - rr := RedisHost{Addrs: []string{"unix:///tmp/doesnt.exist"}, Aliases: []string{""}} - e, _ := NewRedisExporter(rr, "test", "", "") + e, _ := NewRedisExporter([]RedisHost{{Addr: "unix:///tmp/doesnt.exist"}}, Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -1066,35 +1017,26 @@ func TestNonExistingHost(t *testing.T) { for m := range chM { descString := m.Desc().String() + for k := range want { + if strings.Contains(descString, k) { - switch m.(type) { - case prometheus.Counter: - - for k := range want { - if strings.Contains(descString, k) { - - g := &dto.Metric{} - m.Write(g) - - val := 0.0 - - if g.GetGauge() != nil { - val = *g.GetGauge().Value - } else if g.GetCounter() != nil { - val = *g.GetCounter().Value - } else { - continue - } - if val == want[k] { - want[k] = -1.0 - } + g := &dto.Metric{} + m.Write(g) + + val := 0.0 + + if g.GetGauge() != nil { + val = *g.GetGauge().Value + } else if g.GetCounter() != nil { + val = *g.GetCounter().Value + } else { + continue + } + if val == want[k] { + want[k] = -1.0 } } - - default: - log.Printf("default: m: %#v", m) } - } for k, v := range want { if v > 0 { @@ -1104,7 +1046,7 @@ func TestNonExistingHost(t *testing.T) { } func TestMoreThanOneHost(t *testing.T) { - firstHost := defaultRedisHost.Addrs[0] + firstHostURI := os.Getenv("TEST_REDIS_URI") secondHostURI := os.Getenv("TEST_SECOND_REDIS_URI") if secondHostURI == "" { log.Printf("TEST_SECOND_REDIS_URI not set - skipping test") @@ -1126,8 +1068,8 @@ func TestMoreThanOneHost(t *testing.T) { } defer c.Close() - setupDBKeys(t, firstHost) - defer deleteKeysFromDB(t, firstHost) + setupDBKeys(t, firstHostURI) + defer deleteKeysFromDB(t, firstHostURI) setupDBKeys(t, secondHostURI) defer deleteKeysFromDB(t, secondHostURI) @@ -1145,9 +1087,13 @@ func TestMoreThanOneHost(t *testing.T) { return } - twoHostCfg := RedisHost{Addrs: []string{firstHost, secondHostURI}, Aliases: []string{"", ""}} - checkKey := dbNumStrFull + "=" + url.QueryEscape(keys[0]) - e, _ := NewRedisExporter(twoHostCfg, "test", checkKey, "") + e, _ := NewRedisExporter( + []RedisHost{ + {Addr: firstHostURI}, + {Addr: secondHostURI}, + }, + Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}, + ) chM := make(chan prometheus.Metric) go func() { @@ -1156,36 +1102,30 @@ func TestMoreThanOneHost(t *testing.T) { }() want := map[string]float64{ - firstHost: TestValue, + firstHostURI: TestValue, secondHostURI: secondHostValue, } for m := range chM { + pb := &dto.Metric{} + m.Write(pb) - switch m.(type) { - case prometheus.Gauge: - pb := &dto.Metric{} - m.Write(pb) - - if !strings.Contains(m.Desc().String(), "test_key_value") { - continue - } + if !strings.Contains(m.Desc().String(), "test_key_value") { + continue + } - for _, l := range pb.GetLabel() { - for lbl, val := range want { - if l.GetName() == "addr" && l.GetValue() == lbl && pb.GetGauge().GetValue() == val { - want[lbl] = -1 - } + for _, l := range pb.GetLabel() { + for lbl, val := range want { + if l.GetName() == "addr" && l.GetValue() == lbl && pb.GetGauge().GetValue() == val { + want[lbl] = -1 } } - default: - log.Printf("default: m: %#v", m) } } for lbl, val := range want { if val > 0 { - t.Errorf("Never found value for: %s", lbl) + t.Errorf("Never found value [%f] for label [%s]", val, lbl) } } } @@ -1211,10 +1151,10 @@ func TestKeysReset(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+keys[0], "") + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + keys[0]}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) @@ -1229,7 +1169,7 @@ func TestKeysReset(t *testing.T) { t.Errorf("Did not found key %q\n%s", keys[0], body) } - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) body = downloadUrl(t, ts.URL+"/metrics") if strings.Contains(body, keys[0]) { @@ -1251,10 +1191,8 @@ func TestClusterMaster(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"master"}} - log.Printf("master - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") + e, _ := NewRedisExporter([]RedisHost{{Addr: addr, Alias: "master"}}, Options{Namespace: "test"}) prometheus.Register(e) @@ -1265,7 +1203,7 @@ func TestClusterMaster(t *testing.T) { }() body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("master - body: %s", body) + log.Debugf("master - body: %s", body) for _, want := range []string{ "test_instance_info{addr=\"redis://redis-cluster:7000\",alias=\"master\"", "test_master_repl_offset", @@ -1281,13 +1219,11 @@ func TestPasswordProtectedInstance(t *testing.T) { defer ts.Close() testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{testPwd} - setupDBKeys(t, host.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) + c, err := redis.DialURL(os.Getenv("TEST_REDIS_URI")) if err != nil { t.Errorf("couldn't setup redis, err: %s ", err) return @@ -1306,10 +1242,10 @@ func TestPasswordProtectedInstance(t *testing.T) { if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { t.Fatalf("error unsetting password, err: %s", err) } - deleteKeysFromDB(t, host.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) }() - e, _ := NewRedisExporter(host, "test", "", "") + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI"), Password: testPwd}}, Options{Namespace: "test"}) prometheus.Register(e) chM := make(chan prometheus.Metric, 10000) @@ -1334,13 +1270,11 @@ func TestPasswordInvalid(t *testing.T) { defer ts.Close() testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{"wrong_password"} - setupDBKeys(t, host.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) + c, err := redis.DialURL(os.Getenv("TEST_REDIS_URI")) if err != nil { t.Errorf("couldn't setup redis, err: %s ", err) return @@ -1359,9 +1293,10 @@ func TestPasswordInvalid(t *testing.T) { if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { t.Fatalf("error unsetting password, err: %s", err) } - deleteKeysFromDB(t, host.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) }() - e, _ := NewRedisExporter(host, "test", "", "") + + e, _ := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI"), Password: "wrong_password"}}, Options{Namespace: "test"}) prometheus.Register(e) @@ -1371,9 +1306,10 @@ func TestPasswordInvalid(t *testing.T) { close(chM) }() + want := `test_exporter_last_scrape_error{addr="` + os.Getenv("TEST_REDIS_URI") + `",alias="",err="dial redis: unknown network redis"} 1` body := downloadUrl(t, ts.URL+"/metrics") - if !strings.Contains(body, "test_exporter_last_scrape_error 1") { - t.Errorf(`error, expected string "test_exporter_last_scrape_error 1" in body`) + if !strings.Contains(body, want) { + t.Errorf(`error, expected string "%s" in body, got body: \n\n%s`, want, body) } } @@ -1391,10 +1327,8 @@ func TestClusterSlave(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"slave"}} - log.Printf("slave - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") + e, _ := NewRedisExporter([]RedisHost{{Addr: addr, Alias: "slave"}}, Options{Namespace: "test"}) prometheus.Register(e) @@ -1405,7 +1339,7 @@ func TestClusterSlave(t *testing.T) { }() body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("slave - body: %s", body) + log.Debugf("slave - body: %s", body) for _, want := range []string{ "test_instance_info", "test_master_last_io_seconds", @@ -1430,8 +1364,7 @@ func TestCheckKeys(t *testing.T) { {"wrong=wrong=1", "", false}, {"", "wrong=wrong=2", false}, } { - - _, err := NewRedisExporter(defaultRedisHost, "test", tst.SingleCheckKey, tst.CheckKeys) + _, err := NewRedisExporter([]RedisHost{{Addr: os.Getenv("TEST_REDIS_URI")}}, Options{Namespace: "test", CheckSingleKeys: tst.SingleCheckKey, CheckKeys: tst.CheckKeys}) if tst.ExpectSuccess && err != nil { t.Errorf("Expected success for test: %#v, got err: %s", tst, err) return @@ -1445,8 +1378,6 @@ func TestCheckKeys(t *testing.T) { } func init() { - flag.Parse() - ll := strings.ToLower(os.Getenv("LOG_LEVEL")) if pl, err := log.ParseLevel(ll); err == nil { log.Printf("Setting log level to: %s", ll) @@ -1466,17 +1397,4 @@ func init() { key := fmt.Sprintf("key_exp_%s_%d", n, ts) keysExpiring = append(keysExpiring, key) } - - addrs := strings.Split(*redisAddr, *separator) - if len(addrs) == 0 || len(addrs[0]) == 0 { - log.Fatal("Invalid parameter --redis.addr") - } - - aliases := strings.Split(*redisAlias, *separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) - } - - log.Printf("Using redis addrs: %#v", addrs) - defaultRedisHost = RedisHost{Addrs: []string{"redis://" + *redisAddr}, Aliases: aliases} } diff --git a/main.go b/main.go index 86a83343..7540f7b2 100644 --- a/main.go +++ b/main.go @@ -37,23 +37,24 @@ func getEnvBool(key string) (envValBool bool) { func main() { var ( - redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of one or more redis nodes, separated by separator") - redisFile = flag.String("redis.file", getEnv("REDIS_FILE", ""), "Path to file containing one or more redis nodes, separated by newline. NOTE: mutually exclusive with redis.addr") - redisPassword = flag.String("redis.password", getEnv("REDIS_PASSWORD", ""), "Password for one or more redis nodes, separated by separator") - redisPasswordFile = flag.String("redis.password-file", getEnv("REDIS_PASSWORD_FILE", ""), "File containing the password for one or more redis nodes, separated by separator. NOTE: mutually exclusive with redis.password") - redisAlias = flag.String("redis.alias", getEnv("REDIS_ALIAS", ""), "Redis instance alias for one or more redis nodes, separated by separator") - namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") - checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") - checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") - scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") - separator = flag.String("separator", getEnv("REDIS_EXPORTER_SEPARATOR", ","), "separator used to split redis.addr, redis.password and redis.alias into several elements.") - listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") - metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") - logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") - isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") - showVersion = flag.Bool("version", false, "Show version information and exit") - useCfBindings = flag.Bool("use-cf-bindings", getEnvBool("REDIS_EXPORTER_USE-CF-BINDINGS"), "Use Cloud Foundry service bindings") - redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to export go runtime metrics also") + redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of one or more redis nodes, separated by separator") + redisFile = flag.String("redis.file", getEnv("REDIS_FILE", ""), "Path to file containing one or more redis nodes, separated by newline. NOTE: mutually exclusive with redis.addr") + redisPassword = flag.String("redis.password", getEnv("REDIS_PASSWORD", ""), "Password for one or more redis nodes, separated by separator") + redisPasswordFile = flag.String("redis.password-file", getEnv("REDIS_PASSWORD_FILE", ""), "File containing the password for one or more redis nodes, separated by separator. NOTE: mutually exclusive with redis.password") + redisAlias = flag.String("redis.alias", getEnv("REDIS_ALIAS", ""), "Redis instance alias for one or more redis nodes, separated by separator") + namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") + checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") + checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") + scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") + separator = flag.String("separator", getEnv("REDIS_EXPORTER_SEPARATOR", ","), "separator used to split redis.addr, redis.password and redis.alias into several elements.") + listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") + metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") + logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") + isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") + showVersion = flag.Bool("version", false, "Show version information and exit") + useCfBindings = flag.Bool("use-cf-bindings", getEnvBool("REDIS_EXPORTER_USE-CF-BINDINGS"), "Use Cloud Foundry service bindings") + redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to export go runtime metrics also") + inclVerbotenMetrics = flag.Bool("incl-verboten-metrics", getEnvBool("REDIS_EXPORTER_INCL_VERBOTEN_METRICS"), "Whether to include metrics that are not considered kosher by Prometheus standards") ) flag.Parse() @@ -97,25 +98,27 @@ func main() { parsedRedisPassword = *redisPassword } - var addrs, passwords, aliases []string - + var hosts []exporter.RedisHost switch { case *redisFile != "": var err error - if addrs, passwords, aliases, err = exporter.LoadRedisFile(*redisFile); err != nil { + if hosts, err = exporter.LoadRedisFile(*redisFile); err != nil { log.Fatal(err) } case *useCfBindings: - addrs, passwords, aliases = exporter.GetCloudFoundryRedisBindings() + hosts = exporter.GetCloudFoundryRedisBindings() default: - addrs, passwords, aliases = exporter.LoadRedisArgs(*redisAddr, parsedRedisPassword, *redisAlias, *separator) + hosts = exporter.LoadRedisArgs(*redisAddr, parsedRedisPassword, *redisAlias, *separator) } exp, err := exporter.NewRedisExporter( - exporter.RedisHost{Addrs: addrs, Passwords: passwords, Aliases: aliases}, - *namespace, - *checkSingleKeys, - *checkKeys, + hosts, + exporter.Options{ + Namespace: *namespace, + CheckKeys: *checkKeys, + CheckSingleKeys: *checkSingleKeys, + IncludeVerbotenMetrics: *inclVerbotenMetrics, + }, ) if err != nil { log.Fatal(err) @@ -157,7 +160,6 @@ func main() { }) log.Printf("Providing metrics at %s%s", *listenAddress, *metricPath) - log.Printf("Connecting to redis hosts: %#v", addrs) - log.Printf("Using alias: %#v", aliases) + log.Printf("Connecting to redis hosts: %#v", hosts) log.Fatal(http.ListenAndServe(*listenAddress, nil)) }