diff --git a/.drone.yml b/.drone.yml index 823ff420..5db822a2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,10 +11,10 @@ services: image: redis:5 ports: - 6379 - - name: moar-redis + - name: pwd-redis image: redis:5 - commands: - - /usr/local/bin/redis-server --port 6380 + commands: + - "/usr/local/bin/redis-server --port 6380 --requirepass redis-password" ports: - 6380 - name: redis-cluster @@ -38,16 +38,17 @@ steps: environment: GO111MODULE: on LOG_LEVEL: "info" - TEST_TILE38_URI: "tile38:9851" - TEST_SECOND_REDIS_URI: "redis://moar-redis:6380" - TEST_REDIS_CLUSTER_MASTER_URI: "redis-cluster:7000" - TEST_REDIS_CLUSTER_SLAVE_URI: "redis-cluster:7005" + TEST_TILE38_URI: "redis://tile38:9851" + TEST_REDIS_URI: "redis://redis:6379" + TEST_PWD_REDIS_URI: "redis://h:redis-password@pwd-redis:6380" + TEST_REDIS_CLUSTER_MASTER_URI: "redis://redis-cluster:7000" + TEST_REDIS_CLUSTER_SLAVE_URI: "redis://redis-cluster:7005" COVERALLS_TOKEN: from_secret: coveralls-token commands: - 'go build -mod=vendor' - "sleep 10" # let the redis test instances all come up first - - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/... --redis.addr=redis' + - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/...' - 'echo "checking gofmt"' - 'echo " ! gofmt -d main.go 2>&1 | read " | bash' - 'echo " ! gofmt -d exporter/*.go 2>&1 | read " | bash' diff --git a/exporter/discovery.go b/exporter/discovery.go deleted file mode 100644 index 962776c6..00000000 --- a/exporter/discovery.go +++ /dev/null @@ -1,111 +0,0 @@ -package exporter - -import ( - "encoding/csv" - "os" - "strings" - - "github.com/cloudfoundry-community/go-cfenv" - log "github.com/sirupsen/logrus" -) - -// loadRedisArgs loads the configuration for which redis hosts to monitor from either -// the environment or as passed from program arguments. Returns the list of host addrs, -// passwords, and their aliases. -func LoadRedisArgs(addr, password, alias, separator string) ([]string, []string, []string) { - if addr == "" { - addr = "redis://localhost:6379" - } - addrs := strings.Split(addr, separator) - passwords := strings.Split(password, separator) - for len(passwords) < len(addrs) { - passwords = append(passwords, passwords[0]) - } - aliases := strings.Split(alias, separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) - } - return addrs, passwords, aliases -} - -// loadRedisFile opens the specified file and loads the configuration for which redis -// hosts to monitor. Returns the list of hosts addrs, passwords, and their aliases. -func LoadRedisFile(fileName string) ([]string, []string, []string, error) { - var addrs []string - var passwords []string - var aliases []string - file, err := os.Open(fileName) - if err != nil { - return nil, nil, nil, err - } - r := csv.NewReader(file) - r.FieldsPerRecord = -1 - records, err := r.ReadAll() - if err != nil { - return nil, nil, nil, err - } - file.Close() - // For each line, test if it contains an optional password and alias and provide them, - // else give them empty strings - for _, record := range records { - length := len(record) - switch length { - case 3: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, record[2]) - case 2: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, "") - case 1: - addrs = append(addrs, record[0]) - passwords = append(passwords, "") - aliases = append(aliases, "") - } - } - return addrs, passwords, aliases, nil -} - -func GetCloudFoundryRedisBindings() (addrs, passwords, aliases []string) { - if !cfenv.IsRunningOnCF() { - return - } - - appEnv, err := cfenv.Current() - if err != nil { - log.Warnln("Unable to get current CF environment", err) - return - } - - redisServices, err := appEnv.Services.WithTag("redis") - if err != nil { - log.Warnln("Error while getting redis services", err) - return - } - - for _, redisService := range redisServices { - credentials := redisService.Credentials - host := getAlternative(credentials, "host", "hostname") - port := getAlternative(credentials, "port") - password := getAlternative(credentials, "password") - - addr := host + ":" + port - alias := redisService.Name - - addrs = append(addrs, addr) - passwords = append(passwords, password) - aliases = append(aliases, alias) - } - - return -} - -func getAlternative(credentials map[string]interface{}, alternatives ...string) string { - for _, key := range alternatives { - if value, ok := credentials[key]; ok { - return value.(string) - } - } - return "" -} diff --git a/exporter/discovery_test.go b/exporter/discovery_test.go deleted file mode 100644 index 4a09f2e5..00000000 --- a/exporter/discovery_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package exporter - -import ( - "log" - "testing" -) - -func cmpStringArrays(a1, a2 []string) bool { - if len(a1) != len(a2) { - return false - } - for n := range a1 { - if a1[n] != a2[n] { - return false - } - } - return true -} - -func TestLoadRedisArgs(t *testing.T) { - log.Println("TestLoadRedisArgs()") - tests := []struct { - addr, pwd, alias, sep string - wantAddr, wantPwds, wantAliases []string - }{ - { - addr: "", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{""}, - }, - { - addr: "redis://localhost:6379", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{""}, - }, - { - addr: "redis://localhost:6379,redis://localhost:7000", - sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000"}, - wantPwds: []string{"", ""}, - wantAliases: []string{"", ""}, - }, - { - addr: "redis://localhost:6379,redis://localhost:7000,redis://localhost:7001", - sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7001"}, - wantPwds: []string{"", "", ""}, - wantAliases: []string{"", "", ""}, - }, - { - alias: "host-1", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{"host-1"}, - }, - } - - for _, test := range tests { - sep := test.sep - addrs, pwds, aliases := LoadRedisArgs(test.addr, test.pwd, test.alias, sep) - if !cmpStringArrays(addrs, test.wantAddr) { - t.Errorf("addrs not matching wantAliases, got: %v want: %v", addrs, test.wantAddr) - } - if !cmpStringArrays(pwds, test.wantPwds) { - t.Errorf("pwds not matching wantAliases, got: %v want: %v", pwds, test.wantPwds) - } - if !cmpStringArrays(aliases, test.wantAliases) { - t.Errorf("aliases not matching wantAliases, got: %v want: %v", aliases, test.wantAliases) - } - } -} - -func TestLoadRedisFile(t *testing.T) { - if _, _, _, err := LoadRedisFile("doesnt-exist.txt"); err == nil { - t.Errorf("should have failed opening non existing file") - return - } - - addrs, pwds, aliases, err := LoadRedisFile("../contrib/sample_redis_hosts_file.txt") - if err != nil { - t.Errorf("LoadRedisFile() failed, err: %s", err) - return - } - log.Printf("aliases: %v \n", aliases) - if !cmpStringArrays(addrs, []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7000"}) { - t.Errorf("addrs not matching want") - } - if !cmpStringArrays(pwds, []string{"", "password", "second-pwd"}) { - t.Errorf("pwds not matching want") - } - if !cmpStringArrays(aliases, []string{"", "alias", ""}) { - t.Errorf("aliases not matching want") - } -} - -func TestGetCloudFoundryRedisBindings(t *testing.T) { - GetCloudFoundryRedisBindings() -} diff --git a/exporter/redis.go b/exporter/redis.go index 133c43c8..9e078ef9 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -1,8 +1,10 @@ package exporter import ( + "crypto/tls" "errors" "fmt" + "net/http" "net/url" "regexp" "strconv" @@ -12,17 +14,11 @@ import ( "github.com/gomodule/redigo/redis" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" prom_strutil "github.com/prometheus/prometheus/util/strutil" log "github.com/sirupsen/logrus" ) -// RedisHost represents a set of Redis Hosts to health check. -type RedisHost struct { - Addrs []string - Passwords []string - Aliases []string -} - type dbKeyPair struct { db, key string } @@ -34,34 +30,33 @@ type keyInfo struct { // Exporter implements the prometheus.Exporter interface, and exports Redis metrics. type Exporter struct { - redis RedisHost - namespace string - keys []dbKeyPair - singleKeys []dbKeyPair - keyValues *prometheus.GaugeVec - keySizes *prometheus.GaugeVec - scriptValues *prometheus.GaugeVec - duration prometheus.Gauge - scrapeErrors prometheus.Gauge - totalScrapes prometheus.Counter - metrics map[string]*prometheus.GaugeVec + sync.Mutex + redisAddr string + namespace string + keys []dbKeyPair + singleKeys []dbKeyPair - LuaScript []byte + totalScrapes prometheus.Counter + targetScrapeDuration prometheus.Summary + targetScrapeRequestErrors prometheus.Counter + + metricDescriptions map[string]*prometheus.Desc - metricsMtx sync.RWMutex - sync.RWMutex + options Options + LuaScript []byte } -type scrapeResult struct { - Name string - Value float64 - Addr string - Alias string - DB string +type Options struct { + Namespace string + ConfigCommandName string + CheckSingleKeys string + CheckKeys string + IncludeVerbotenMetrics bool + SkipTLSVerification bool } var ( - metricMap = map[string]string{ + metricMapGauges = map[string]string{ // # Server "uptime_in_seconds": "uptime_in_seconds", "process_id": "process_id", @@ -80,7 +75,6 @@ var ( "used_memory_rss": "memory_used_rss_bytes", "used_memory_peak": "memory_used_peak_bytes", "used_memory_lua": "memory_used_lua_bytes", - "total_system_memory": "total_system_memory_bytes", "maxmemory": "memory_max_bytes", // # Persistence @@ -108,21 +102,12 @@ var ( "aof_last_write_status": "aof_last_write_status", // # Stats - "total_connections_received": "connections_received_total", - "total_commands_processed": "commands_processed_total", - "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", - "total_net_input_bytes": "net_input_bytes_total", - "total_net_output_bytes": "net_output_bytes_total", - "instantaneous_input_kbps": "instantaneous_input_kbps", - "instantaneous_output_kbps": "instantaneous_output_kbps", - "rejected_connections": "rejected_connections_total", - "expired_keys": "expired_keys_total", - "evicted_keys": "evicted_keys_total", - "keyspace_hits": "keyspace_hits_total", - "keyspace_misses": "keyspace_misses_total", - "pubsub_channels": "pubsub_channels", - "pubsub_patterns": "pubsub_patterns", - "latest_fork_usec": "latest_fork_usec", + "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", + "instantaneous_input_kbps": "instantaneous_input_kbps", + "instantaneous_output_kbps": "instantaneous_output_kbps", + "pubsub_channels": "pubsub_channels", + "pubsub_patterns": "pubsub_patterns", + "latest_fork_usec": "latest_fork_usec", // # Replication "loading": "loading_dump_file", @@ -131,12 +116,6 @@ var ( "master_last_io_seconds_ago": "master_last_io_seconds", "master_repl_offset": "master_repl_offset", - // # CPU - "used_cpu_sys": "used_cpu_sys", - "used_cpu_user": "used_cpu_user", - "used_cpu_sys_children": "used_cpu_sys_children", - "used_cpu_user_children": "used_cpu_user_children", - // # Cluster "cluster_stats_messages_sent": "cluster_messages_sent_total", "cluster_stats_messages_received": "cluster_messages_received_total", @@ -162,96 +141,62 @@ var ( "version": "version", // since tile38 version 1.14.1 } + metricMapCounters = map[string]string{ + "total_connections_received": "connections_received_total", + "total_commands_processed": "commands_processed_total", + + "rejected_connections": "rejected_connections_total", + "total_net_input_bytes": "net_input_bytes_total", + "total_net_output_bytes": "net_output_bytes_total", + + "expired_keys": "expired_keys_total", + "evicted_keys": "evicted_keys_total", + "keyspace_hits": "keyspace_hits_total", + "keyspace_misses": "keyspace_misses_total", + + "used_cpu_sys": "used_cpu_sys_seconds_total", + "used_cpu_user": "used_cpu_user_seconds_total", + "used_cpu_sys_children": "used_cpu_sys_children_seconds_total", + "used_cpu_user_children": "used_cpu_user_children_seconds_total", + } + instanceInfoFields = map[string]bool{"role": true, "redis_version": true, "redis_build_id": true, "redis_mode": true, "os": true} slaveInfoFields = map[string]bool{"master_host": true, "master_port": true, "slave_read_only": true} ) -func (e *Exporter) initGauges() { - - e.metrics = map[string]*prometheus.GaugeVec{} - e.metrics["instance_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "instance_info", - Help: "Information about the Redis instance", - }, []string{"addr", "alias", "role", "redis_version", "redis_build_id", "redis_mode", "os"}) - e.metrics["slave_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slave_info", - Help: "Information about the Redis slave", - }, []string{"addr", "alias", "master_host", "master_port", "read_only"}) - e.metrics["start_time_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "start_time_seconds", - Help: "Start time of the Redis instance since unix epoch in seconds.", - }, []string{"addr", "alias"}) - e.metrics["master_link_up"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "master_link_up", - Help: "Master link status on Redis slave", - }, []string{"addr", "alias"}) - e.metrics["connected_slave_offset"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_offset", - Help: "Offset of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["connected_slave_lag_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_lag_seconds", - Help: "Lag of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["db_keys"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys", - Help: "Total number of keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_keys_expiring"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys_expiring", - Help: "Total number of expiring keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_avg_ttl_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_avg_ttl_seconds", - Help: "Avg TTL in seconds", - }, []string{"addr", "alias", "db"}) - - // Latency info - e.metrics["latency_spike_last"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_last", - Help: "When the latency spike last occurred", - }, []string{"addr", "alias", "event_name"}) - e.metrics["latency_spike_milliseconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_milliseconds", - Help: "Length of the last latency spike in milliseconds", - }, []string{"addr", "alias", "event_name"}) - - e.metrics["commands_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_total", - Help: "Total number of calls per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["commands_duration_seconds_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_duration_seconds_total", - Help: "Total amount of time in seconds spent per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["slowlog_length"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_length", - Help: "Total slowlog", - }, []string{"addr", "alias"}) - e.metrics["slowlog_last_id"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_last_id", - Help: "Last id of slowlog", - }, []string{"addr", "alias"}) - e.metrics["last_slow_execution_duration_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "last_slow_execution_duration_seconds", - Help: "The amount of time needed for last slow execution, in seconds", - }, []string{"addr", "alias"}) +func (e *Exporter) ScrapeHandler(w http.ResponseWriter, r *http.Request) { + target := r.URL.Query().Get("target") + if target == "" { + http.Error(w, "'target' parameter must be specified", 400) + e.targetScrapeRequestErrors.Inc() + return + } + + start := time.Now() + + // todo: this needs a test + checkKeys := r.URL.Query().Get("check-keys") + checkSingleKey := r.URL.Query().Get("check-single-keys") + + opts := e.options + opts.CheckKeys = checkKeys + opts.CheckSingleKeys = checkSingleKey + + exp, err := NewRedisExporter(target, opts) + if err != nil { + http.Error(w, "NewRedisExporter() err: err", 400) + e.targetScrapeRequestErrors.Inc() + return + } + registry := prometheus.NewRegistry() + registry.MustRegister(exp) + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + + h.ServeHTTP(w, r) + + duration := time.Since(start).Seconds() + e.targetScrapeDuration.Observe(duration) + log.Debugf("Scrape of target '%s' took %f seconds", target, duration) } // splitKeyArgs splits a command-line supplied argument into a slice of dbKeyPairs. @@ -282,104 +227,143 @@ func parseKeyArg(keysArgString string) (keys []dbKeyPair, err error) { return keys, err } -// NewRedisExporter returns a new exporter of Redis metrics. -// note to self: next time we add an argument, instead add a RedisExporter struct -func NewRedisExporter(host RedisHost, namespace, checkSingleKeys, checkKeys string) (*Exporter, error) { +func newMetricDescr(namespace string, metricName string, docString string, labels []string) *prometheus.Desc { + return prometheus.NewDesc(prometheus.BuildFQName(namespace, "", metricName), docString, labels, nil) +} +// NewRedisExporter returns a new exporter of Redis metrics. +func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { e := Exporter{ - redis: host, - namespace: namespace, - keyValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_value", - Help: "The value of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - keySizes: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_size", - Help: "The length or size of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - scriptValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "script_value", - Help: "Values returned by the collect script", - }, []string{"addr", "alias", "key"}), - duration: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_duration_seconds", - Help: "The last scrape duration.", - }), + redisAddr: redisURI, + options: opts, + namespace: opts.Namespace, + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, + Namespace: opts.Namespace, Name: "exporter_scrapes_total", Help: "Current total redis scrapes.", }), - scrapeErrors: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_error", - Help: "The last scrape error status.", - }), + + targetScrapeDuration: prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Name: "target_scrape_collection_duration_seconds", + Help: "Duration of collections by the SNMP exporter", + }, + ), + targetScrapeRequestErrors: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: opts.Namespace, + Name: "target_scrape_request_errors_total", + Help: "Errors in requests to the SNMP exporter", + }, + ), + } + + if e.options.ConfigCommandName == "" { + e.options.ConfigCommandName = "CONFIG" } var err error - if e.keys, err = parseKeyArg(checkKeys); err != nil { + if e.keys, err = parseKeyArg(opts.CheckKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-keys: %#v", err) } log.Debugf("keys: %#v", e.keys) - if e.singleKeys, err = parseKeyArg(checkSingleKeys); err != nil { + if e.singleKeys, err = parseKeyArg(opts.CheckSingleKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-single-keys: %#v", err) } log.Debugf("singleKeys: %#v", e.singleKeys) - e.initGauges() + if opts.IncludeVerbotenMetrics { + metricMapGauges["total_system_memory"] = "total_system_memory_bytes" + } + + e.metricDescriptions = map[string]*prometheus.Desc{} + e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", nil) + e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"role", "redis_version", "redis_build_id", "redis_mode", "os"}) + e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", nil) + e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"err"}) + + e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"key"}) + e.metricDescriptions["key_values"] = newMetricDescr(opts.Namespace, "key_value", `The value of "key"`, []string{"db", "key"}) + e.metricDescriptions["key_sizes"] = newMetricDescr(opts.Namespace, "key_size", `The length or size of "key"`, []string{"db", "key"}) + + e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"cmd"}) + e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"cmd"}) + e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, nil) + e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, nil) + e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, nil) + + e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"event_name"}) + e.metricDescriptions["latency_spike_seconds"] = newMetricDescr(opts.Namespace, "latency_spike_seconds", `Length of the last latency spike in seconds`, []string{"event_name"}) + + e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"master_host", "master_port", "read_only"}) + + e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", nil) + e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", nil) + e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) + e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) + + e.metricDescriptions["db_keys"] = newMetricDescr(opts.Namespace, "db_keys", "Total number of keys by DB", []string{"db"}) + e.metricDescriptions["db_keys_expiring"] = newMetricDescr(opts.Namespace, "db_keys_expiring", "Total number of expiring keys by DB", []string{"db"}) + e.metricDescriptions["db_avg_ttl_seconds"] = newMetricDescr(opts.Namespace, "db_avg_ttl_seconds", "Avg TTL in seconds", []string{"db"}) + return &e, nil } // Describe outputs Redis metric descriptions. func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { + for _, desc := range e.metricDescriptions { + ch <- desc + } + + for _, v := range metricMapGauges { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) + } - for _, m := range e.metrics { - m.Describe(ch) + for _, v := range metricMapCounters { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) } - e.keySizes.Describe(ch) - e.keyValues.Describe(ch) - ch <- e.duration.Desc() ch <- e.totalScrapes.Desc() - ch <- e.scrapeErrors.Desc() + ch <- e.targetScrapeDuration.Desc() + ch <- e.targetScrapeRequestErrors.Desc() } // Collect fetches new metrics from the RedisHost and updates the appropriate metrics. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - scrapes := make(chan scrapeResult) - e.Lock() defer e.Unlock() + e.totalScrapes.Inc() - e.keySizes.Reset() - e.keyValues.Reset() - - e.initGauges() - go e.scrape(scrapes) - e.setMetrics(scrapes) + now := time.Now().UnixNano() + var up float64 = 1 + if err := e.scrapeRedisHost(ch); err != nil { + up = 0 + e.registerGaugeValue(ch, "scrape_error", 1.0, fmt.Sprintf("%s", err)) + } else { + e.registerGaugeValue(ch, "scrape_error", 0, "") + } - e.keySizes.Collect(ch) - e.keyValues.Collect(ch) - e.scriptValues.Collect(ch) + e.registerGaugeValue(ch, "up", up) + e.registerGaugeValue(ch, "last_scrape_duration", float64(time.Now().UnixNano()-now)/1000000000) - ch <- e.duration ch <- e.totalScrapes - ch <- e.scrapeErrors - e.collectMetrics(ch) + ch <- e.targetScrapeDuration + ch <- e.targetScrapeRequestErrors } func includeMetric(s string) bool { if strings.HasPrefix(s, "db") || strings.HasPrefix(s, "cmdstat_") || strings.HasPrefix(s, "cluster_") { return true } - _, ok := metricMap[s] + if _, ok := metricMapGauges[s]; ok { + return true + } + + _, ok := metricMapCounters[s] return ok } @@ -478,7 +462,7 @@ func parseConnectedSlaveString(slaveName string, slaveInfo string) (offset float return } -func extractConfigMetrics(config []string, addr string, alias string, scrapes chan<- scrapeResult) (dbCount int, err error) { +func (e *Exporter) extractConfigMetrics(ch chan<- prometheus.Metric, config []string) (dbCount int, err error) { if len(config)%2 != 0 { return 0, fmt.Errorf("invalid config: %#v", config) } @@ -502,13 +486,26 @@ func extractConfigMetrics(config []string, addr string, alias string, scrapes ch } if val, err := strconv.ParseFloat(strVal, 64); err == nil { - scrapes <- scrapeResult{Name: fmt.Sprintf("config_%s", config[pos*2]), Addr: addr, Alias: alias, Value: val} + e.registerGaugeValue(ch, fmt.Sprintf("config_%s", config[pos*2]), val) } } return } -func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string, val float64, labels ...string) { + e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels...) +} + +func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labelValues ...string) { + descr := e.metricDescriptions[metric] + if descr == nil { + descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", nil) + } + + ch <- prometheus.MustNewConstMetric(descr, valType, val, labelValues...) +} + +func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) { for i := 0; i < len(info); i += 2 { log.Debugf("tile38: %s:%s", info[i], info[i+1]) @@ -519,13 +516,11 @@ func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } - - return nil } -func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { /* Format: cmdstat_get:calls=21,usec=175,usec_per_call=8.33 @@ -552,46 +547,36 @@ func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey return } - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - cmd := splitKey[1] - e.metrics["commands_total"].WithLabelValues(addr, alias, cmd).Set(calls) - e.metrics["commands_duration_seconds_total"].WithLabelValues(addr, alias, cmd).Set(usecTotal / 1e6) + e.registerMetricValue(ch, "commands_total", calls, prometheus.CounterValue, cmd) + e.registerMetricValue(ch, "commands_duration_seconds_total", usecTotal/1e6, prometheus.CounterValue, cmd) } -func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey string, fieldValue string) bool { - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - +func (e *Exporter) handleMetricsReplication(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) bool { // only slaves have this field if fieldKey == "master_link_status" { if fieldValue == "up" { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(1) + e.registerGaugeValue(ch, "master_link_up", 1) } else { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(0) + e.registerGaugeValue(ch, "master_link_up", 0) } return true } // not a slave, try extracting master metrics - if slaveOffset, slaveIp, slavePort, slaveState, lag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { - e.metrics["connected_slave_offset"].WithLabelValues( - addr, - alias, - slaveIp, - slavePort, - slaveState, - ).Set(slaveOffset) - - if lag > -1 { - e.metrics["connected_slave_lag_seconds"].WithLabelValues( - addr, - alias, - slaveIp, - slavePort, - slaveState, - ).Set(lag) + if slaveOffset, slaveIP, slavePort, slaveState, slaveLag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { + e.registerGaugeValue(ch, + "connected_slave_offset", + slaveOffset, + slaveIP, slavePort, slaveState, + ) + + if slaveLag > -1 { + e.registerGaugeValue(ch, + "connected_slave_lag_seconds", + slaveLag, + slaveIP, slavePort, slaveState, + ) } return true } @@ -599,17 +584,15 @@ func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey return false } -func (e *Exporter) handleMetricsServer(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsServer(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { if fieldKey == "uptime_in_seconds" { if uptime, err := strconv.ParseFloat(fieldValue, 64); err == nil { - e.metricsMtx.RLock() - e.metrics["start_time_seconds"].WithLabelValues(addr, alias).Set(float64(time.Now().Unix()) - uptime) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "start_time_seconds", float64(time.Now().Unix())-uptime) } } } -func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes chan<- scrapeResult, dbCount int) error { +func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string, dbCount int) error { instanceInfo := map[string]string{} slaveInfo := map[string]string{} handledDBs := map[string]bool{} @@ -644,24 +627,26 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c switch fieldClass { case "Replication": - if ok := e.handleMetricsReplication(addr, alias, fieldKey, fieldValue); ok { + if ok := e.handleMetricsReplication(ch, fieldKey, fieldValue); ok { continue } case "Server": - e.handleMetricsServer(addr, alias, fieldKey, fieldValue) + e.handleMetricsServer(ch, fieldKey, fieldValue) case "Commandstats": - e.handleMetricsCommandStats(addr, alias, fieldKey, fieldValue) + e.handleMetricsCommandStats(ch, fieldKey, fieldValue) continue case "Keyspace": if keysTotal, keysEx, avgTTL, ok := parseDBKeyspaceString(fieldKey, fieldValue); ok { dbName := fieldKey - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: keysTotal} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: keysEx} + + e.registerGaugeValue(ch, "db_keys", keysTotal, dbName) + e.registerGaugeValue(ch, "db_keys_expiring", keysEx, dbName) + if avgTTL > -1 { - scrapes <- scrapeResult{Name: "db_avg_ttl_seconds", Addr: addr, Alias: alias, DB: dbName, Value: avgTTL} + e.registerGaugeValue(ch, "db_avg_ttl_seconds", avgTTL, dbName) } handledDBs[dbName] = true continue @@ -672,40 +657,35 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } for dbIndex := 0; dbIndex < dbCount; dbIndex++ { dbName := "db" + strconv.Itoa(dbIndex) if _, exists := handledDBs[dbName]; !exists { - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: 0} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: 0} + e.registerGaugeValue(ch, "db_keys", 0, dbName) + e.registerGaugeValue(ch, "db_keys_expiring", 0, dbName) } } - e.metricsMtx.RLock() - e.metrics["instance_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "instance_info", 1, instanceInfo["role"], instanceInfo["redis_version"], instanceInfo["redis_build_id"], instanceInfo["redis_mode"], - instanceInfo["os"], - ).Set(1) + instanceInfo["os"]) + if instanceInfo["role"] == "slave" { - e.metrics["slave_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "slave_info", 1, slaveInfo["master_host"], slaveInfo["master_port"], - slaveInfo["slave_read_only"], - ).Set(1) + slaveInfo["slave_read_only"]) } - e.metricsMtx.RUnlock() return nil } -func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, info string) error { lines := strings.Split(info, "\r\n") for _, line := range lines { @@ -722,16 +702,104 @@ func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } return nil } -func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scrapeResult) error { - metricName := sanitizeMetricName(fieldKey) - if newName, ok := metricMap[metricName]; ok { +func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + log.Debugf("e.singleKeys: %#v", e.singleKeys) + allKeys := append([]dbKeyPair{}, e.singleKeys...) + + log.Debugf("e.keys: %#v", e.keys) + scannedKeys, err := getKeysFromPatterns(c, e.keys) + if err != nil { + log.Errorf("Error expanding key patterns: %#v", err) + } else { + allKeys = append(allKeys, scannedKeys...) + } + + log.Debugf("allKeys: %#v", allKeys) + for _, k := range allKeys { + if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { + log.Debugf("Couldn't select database %#v when getting key info.", k.db) + continue + } + + info, err := getKeyInfo(c, k.key) + if err != nil { + switch err { + case errNotFound: + log.Debugf("Key '%s' not found when trying to get type and size.", k.key) + default: + log.Error(err) + } + continue + } + dbLabel := "db" + k.db + e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key) + + // Only record value metric if value is float-y + if val, err := redis.Float64(c.Do("GET", k.key)); err == nil { + e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key) + } + } +} + +func (e *Exporter) extractLuaScriptMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if e.LuaScript == nil || len(e.LuaScript) == 0 { + return + } + + log.Debug("Evaluating e.LuaScript") + kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) + if err != nil { + log.Errorf("LuaScript error: %v", err) + return + } + + if kv != nil { + for key, stringVal := range kv { + if val, err := strconv.ParseFloat(stringVal, 64); err == nil { + e.registerGaugeValue(ch, "script_values", val, key) + } + } + } +} + +func (e *Exporter) extractSlowLogMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { + e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64))) + } + + if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { + var slowlogLastID int64 + var lastSlowExecutionDurationSeconds float64 + + if len(values) > 0 { + if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { + slowlogLastID = values[0].(int64) + if len(values) > 2 { + lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 + } + } + } + + e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID)) + e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds) + } +} + +func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, fieldKey, fieldValue string) error { + orgMetricName := sanitizeMetricName(fieldKey) + metricName := orgMetricName + if newName, ok := metricMapGauges[metricName]; ok { metricName = newName + } else { + if newName, ok := metricMapCounters[metricName]; ok { + metricName = newName + } } var err error @@ -753,7 +821,11 @@ func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scr log.Debugf("couldn't parse %s, err: %s", fieldValue, err) } - scrapes <- scrapeResult{Name: metricName, Addr: addr, Alias: alias, Value: val} + t := prometheus.GaugeValue + if metricMapCounters[orgMetricName] != "" { + t = prometheus.CounterValue + } + e.registerMetricValue(ch, metricName, val, t) return nil } @@ -782,7 +854,7 @@ func getKeyInfo(c redis.Conn, key string) (info keyInfo, err error) { return info, errNotFound case "string": if size, err := redis.Int64(c.Do("PFCOUNT", key)); err == nil { - info.keyType = "hyperloglog" + // hyperloglog info.size = float64(size) } else if size, err := redis.Int64(c.Do("STRLEN", key)); err == nil { info.size = float64(size) @@ -864,20 +936,23 @@ func getKeysFromPatterns(c redis.Conn, keys []dbKeyPair) (expandedKeys []dbKeyPa return expandedKeys, err } -func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error { +func connectToRedis(addr string, skipTLSVerification bool) (redis.Conn, error) { options := []redis.DialOption{ redis.DialConnectTimeout(5 * time.Second), redis.DialReadTimeout(5 * time.Second), redis.DialWriteTimeout(5 * time.Second), - } - if len(e.redis.Passwords) > idx && e.redis.Passwords[idx] != "" { - options = append(options, redis.DialPassword(e.redis.Passwords[idx])) + redis.DialTLSConfig(&tls.Config{ + InsecureSkipVerify: skipTLSVerification, + }), } - log.Debugf("Trying DialURL(): %s", addr) - c, err := redis.DialURL(addr, options...) - + uri := addr + if !strings.Contains(uri, "://") { + uri = "redis://" + uri + } + log.Debugf("Trying DialURL(): %s", uri) + c, err := redis.DialURL(uri, options...) if err != nil { log.Debugf("DialURL() failed, err: %s", err) if frags := strings.Split(addr, "://"); len(frags) == 2 { @@ -888,19 +963,22 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx c, err = redis.Dial("tcp", addr, options...) } } + return c, err +} +func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { + c, err := connectToRedis(e.redisAddr, e.options.SkipTLSVerification) if err != nil { - log.Debugf("aborting for addr: %s - redis err: %s", addr, err) + log.Debugf("aborting for addr: %s - redis err: %s", e.redisAddr, err) return err } - defer c.Close() - log.Debugf("connected to: %s", addr) - dbCount := 0 + log.Debugf("connected to: %s", e.redisAddr) - if config, err := redis.Strings(c.Do("CONFIG", "GET", "*")); err == nil { - dbCount, err = extractConfigMetrics(config, addr, e.redis.Aliases[idx], scrapes) + dbCount := 0 + if config, err := redis.Strings(c.Do(e.options.ConfigCommandName, "GET", "*")); err == nil { + dbCount, err = e.extractConfigMetrics(ch, config) if err != nil { log.Errorf("Redis CONFIG err: %s", err) return err @@ -917,11 +995,10 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx return err } } - isClusterEnabled := strings.Contains(infoAll, "cluster_enabled:1") - if isClusterEnabled { + if strings.Contains(infoAll, "cluster_enabled:1") { if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil { - e.extractClusterInfoMetrics(clusterInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractClusterInfoMetrics(ch, clusterInfo) // in cluster mode Redis only supports one database so no extra padding beyond that needed dbCount = 1 @@ -936,152 +1013,35 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx } } - e.extractInfoMetrics(infoAll, addr, e.redis.Aliases[idx], scrapes, dbCount) + e.extractInfoMetrics(ch, infoAll, dbCount) // SERVER command only works on tile38 database. check the following link to // find out more: https://tile38.com/ if serverInfo, err := redis.Strings(doRedisCmd(c, "SERVER")); err == nil { - e.extractTile38Metrics(serverInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractTile38Metrics(ch, serverInfo) } else { log.Debugf("Tile38 SERVER err: %s", err) } if reply, err := doRedisCmd(c, "LATENCY", "LATEST"); err == nil { var eventName string - var spikeLast, milliseconds, max int64 if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 { latencyResult := tempVal[0].([]interface{}) - if _, err := redis.Scan(latencyResult, &eventName, &spikeLast, &milliseconds, &max); err == nil { - e.metricsMtx.RLock() - e.metrics["latency_spike_last"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(spikeLast)) - e.metrics["latency_spike_milliseconds"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(milliseconds)) - e.metricsMtx.RUnlock() - } - } - } - - log.Debugf("e.singleKeys: %#v", e.singleKeys) - allKeys := append([]dbKeyPair{}, e.singleKeys...) - - log.Debugf("e.keys: %#v", e.keys) - scannedKeys, err := getKeysFromPatterns(c, e.keys) - if err != nil { - log.Errorf("Error expanding key patterns: %#v", err) - } else { - allKeys = append(allKeys, scannedKeys...) - } - - log.Debugf("allKeys: %#v", allKeys) - for _, k := range allKeys { - if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { - log.Debugf("Couldn't select database %#v when getting key info.", k.db) - continue - } - - info, err := getKeyInfo(c, k.key) - if err != nil { - switch err { - case errNotFound: - log.Debugf("Key '%s' not found when trying to get type and size.", k.key) - default: - log.Error(err) + var spikeLast, spikeDuration, max int64 + if _, err := redis.Scan(latencyResult, &eventName, &spikeLast, &spikeDuration, &max); err == nil { + spikeDuration = spikeDuration / 1e6 + e.registerGaugeValue(ch, "latency_spike_last", float64(spikeLast), eventName) + e.registerGaugeValue(ch, "latency_spike_seconds", float64(spikeDuration), eventName) } - continue - } - dbLabel := "db" + k.db - e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(info.size) - - // Only record value metric if value is float-y - if value, err := redis.Float64(c.Do("GET", k.key)); err == nil { - e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(value) } } - if e.LuaScript != nil && len(e.LuaScript) > 0 { - log.Debug("e.script") - kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) - if err != nil { - log.Errorf("Collect script error: %v", err) - } else if kv != nil { - for key, stringVal := range kv { - if val, err := strconv.ParseFloat(stringVal, 64); err == nil { - e.scriptValues.WithLabelValues(addr, e.redis.Aliases[idx], key).Set(val) - } - } - } - } + e.extractCheckKeyMetrics(ch, c) - if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { - e.metricsMtx.RLock() - e.metrics["slowlog_length"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(reply.(int64))) - e.metricsMtx.RUnlock() - } + e.extractLuaScriptMetrics(ch, c) - if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { - var slowlogLastId int64 = 0 - var lastSlowExecutionDurationSeconds float64 = 0 - - if len(values) > 0 { - if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { - slowlogLastId = values[0].(int64) - if len(values) > 2 { - lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 - } - } - } - - e.metricsMtx.RLock() - e.metrics["slowlog_last_id"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(slowlogLastId)) - e.metrics["last_slow_execution_duration_seconds"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(lastSlowExecutionDurationSeconds) - e.metricsMtx.RUnlock() - } + e.extractSlowLogMetrics(ch, c) log.Debugf("scrapeRedisHost() done") return nil } - -func (e *Exporter) scrape(scrapes chan<- scrapeResult) { - defer close(scrapes) - - now := time.Now().UnixNano() - e.totalScrapes.Inc() - - errorCount := 0 - for idx, addr := range e.redis.Addrs { - var up float64 = 1 - if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil { - errorCount++ - up = 0 - } - scrapes <- scrapeResult{Name: "up", Addr: addr, Alias: e.redis.Aliases[idx], Value: up} - } - - e.scrapeErrors.Set(float64(errorCount)) - e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000) -} - -func (e *Exporter) setMetrics(scrapes <-chan scrapeResult) { - for scr := range scrapes { - name := scr.Name - if _, ok := e.metrics[name]; !ok { - e.metricsMtx.Lock() - e.metrics[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: name, - Help: name + "metric", // needs to be set for prometheus >= 2.3.1 - }, []string{"addr", "alias"}) - e.metricsMtx.Unlock() - } - var labels prometheus.Labels = map[string]string{"addr": scr.Addr, "alias": scr.Alias} - if len(scr.DB) > 0 { - labels["db"] = scr.DB - } - e.metrics[name].With(labels).Set(scr.Value) - } -} - -func (e *Exporter) collectMetrics(metrics chan<- prometheus.Metric) { - for _, m := range e.metrics { - m.Collect(metrics) - } -} diff --git a/exporter/redis_test.go b/exporter/redis_test.go index dcd8bdd3..cd3e3b65 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -9,7 +9,6 @@ package exporter */ import ( - "flag" "fmt" "io/ioutil" "net/http" @@ -35,27 +34,25 @@ const ( ) var ( - redisAddr = flag.String("redis.addr", ":6379", "Address of the test instance, without `redis://`") - redisAlias = flag.String("redis.alias", "foo", "Alias of the test instance") - separator = flag.String("separator", ",", "separator used to split redis.addr, redis.password and redis.alias into several elements.") - - keys = []string{} - keysExpiring = []string{} - listKeys = []string{} - ts = int32(time.Now().Unix()) - defaultRedisHost = RedisHost{} + keys = []string{} + keysExpiring = []string{} + listKeys = []string{} + ts = int32(time.Now().Unix()) dbNumStr = "11" altDBNumStr = "12" dbNumStrFull = fmt.Sprintf("db%s", dbNumStr) - - TestServerURL = "" ) const ( TestSetName = "test-set" ) +func getTestExporter() *Exporter { + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test"}) + return e +} + func setupLatency(t *testing.T, addr string) error { c, err := redis.DialURL(addr) @@ -151,7 +148,7 @@ func resetSlowLog(t *testing.T, addr string) error { return nil } -func downloadUrl(t *testing.T, url string) string { +func downloadURL(t *testing.T, url string) string { log.Debugf("downloadURL() %s", url) resp, err := http.Get(url) if err != nil { @@ -166,10 +163,10 @@ func downloadUrl(t *testing.T, url string) string { } func TestLatencySpike(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() - setupLatency(t, defaultRedisHost.Addrs[0]) - defer resetLatency(t, defaultRedisHost.Addrs[0]) + setupLatency(t, os.Getenv("TEST_REDIS_URI")) + defer resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -178,24 +175,21 @@ func TestLatencySpike(t *testing.T) { }() for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { - got := &dto.Metric{} - m.Write(got) - - val := got.GetGauge().GetValue() - // Because we're dealing with latency, there might be a slight delay - // even after sleeping for a specific amount of time so checking - // to see if we're between +-5 of our expected value - if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { - t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) - } + if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { + got := &dto.Metric{} + m.Write(got) + + val := got.GetGauge().GetValue() + // Because we're dealing with latency, there might be a slight delay + // even after sleeping for a specific amount of time so checking + // to see if we're between +-5 of our expected value + if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { + t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) } } } - resetLatency(t, defaultRedisHost.Addrs[0]) + resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -215,13 +209,10 @@ func TestLatencySpike(t *testing.T) { func TestTile38(t *testing.T) { if os.Getenv("TEST_TILE38_URI") == "" { - t.SkipNow() + t.Skipf("TEST_TILE38_URI not set - skipping") } - e, _ := NewRedisExporter( - RedisHost{Addrs: []string{os.Getenv("TEST_TILE38_URI")}, Aliases: []string{"tile"}}, - "test", "", "", - ) + e, _ := NewRedisExporter(os.Getenv("TEST_TILE38_URI"), Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -229,22 +220,21 @@ func TestTile38(t *testing.T) { close(chM) }() - find := false + found := false for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "cpus_total") { - find = true - } + if strings.Contains(m.Desc().String(), "cpus_total") { + found = true + log.Debugf("type: %T", m) } + } - if !find { + if !found { t.Errorf("cpus_total was not found in tile38 metrics") } } func TestSlowLog(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() chM := make(chan prometheus.Metric) go func() { @@ -252,7 +242,7 @@ func TestSlowLog(t *testing.T) { close(chM) }() - oldSlowLogId := float64(0) + oldSlowLogID := float64(0) for m := range chM { switch m := m.(type) { @@ -261,13 +251,13 @@ func TestSlowLog(t *testing.T) { got := &dto.Metric{} m.Write(got) - oldSlowLogId = got.GetGauge().GetValue() + oldSlowLogID = got.GetGauge().GetValue() } } } - setupSlowLog(t, defaultRedisHost.Addrs[0]) - defer resetSlowLog(t, defaultRedisHost.Addrs[0]) + setupSlowLog(t, os.Getenv("TEST_REDIS_URI")) + defer resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -284,7 +274,7 @@ func TestSlowLog(t *testing.T) { val := got.GetGauge().GetValue() - if oldSlowLogId > val { + if oldSlowLogID > val { t.Errorf("no new slowlogs found") } } @@ -300,7 +290,7 @@ func TestSlowLog(t *testing.T) { } } - resetSlowLog(t, defaultRedisHost.Addrs[0]) + resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -324,11 +314,10 @@ func TestSlowLog(t *testing.T) { } } -func setupDBKeys(t *testing.T, addr string) error { - - c, err := redis.DialURL(addr) +func setupDBKeys(t *testing.T, uri string) error { + c, err := redis.DialURL(uri) if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) + t.Errorf("couldn't setup redis for uri %s, err: %s ", uri, err) return err } defer c.Close() @@ -404,145 +393,31 @@ func deleteKeysFromDB(t *testing.T, addr string) error { } func TestHostVariations(t *testing.T) { - for _, prefix := range []string{"", "redis://", "tcp://"} { - addr := prefix + *redisAddr - host := RedisHost{Addrs: []string{addr}, Aliases: []string{""}} - e, _ := NewRedisExporter(host, "test", "", "") - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - found := 0 - for range scrapes { - found++ - } - - if found == 0 { - t.Errorf("didn't find any scrapes for host: %s", addr) - } - } -} - -func TestCountingKeys(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) + host := strings.ReplaceAll(os.Getenv("TEST_REDIS_URI"), "redis://", "") - var keysTestDB float64 - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - keysTestDB = s.Value - break - } - } - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - - scrapes = make(chan scrapeResult, 1000) - e.scrape(scrapes) - - // +1 for the one SET key - want := keysTestDB + float64(len(keys)) + float64(len(keysExpiring)) + 1 + float64(len(listKeys)) - - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - if want != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break - } - } - - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - scrapes = make(chan scrapeResult, 10000) - e.scrape(scrapes) - - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - if keysTestDB != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break - } - if s.Name == "db_avg_ttl_seconds" && s.DB == dbNumStrFull { - if keysTestDB != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break + for _, prefix := range []string{"", "redis://", "tcp://", ""} { + addr := prefix + host + c, err := connectToRedis(addr, true) + if err != nil { + t.Errorf("connectToRedis() err: %s", err) + continue } - } -} - -func TestExporterMetrics(t *testing.T) { - - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - - e.setMetrics(scrapes) - - want := 25 - if len(e.metrics) < want { - t.Errorf("need moar metrics, found: %d, want: %d", len(e.metrics), want) - } - - wantKeys := []string{ - "db_keys", - "db_avg_ttl_seconds", - "used_cpu_sys", - "loading_dump_file", // testing renames - "config_maxmemory", // testing config extraction - "config_maxclients", // testing config extraction - "slowlog_length", - "slowlog_last_id", - "start_time_seconds", - "uptime_in_seconds", - } - for _, k := range wantKeys { - if _, ok := e.metrics[k]; !ok { - t.Errorf("missing metrics key: %s", k) + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) } - } -} - -func TestExporterValues(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - - wantValues := map[string]float64{ - "db_keys_total": float64(len(keys)+len(keysExpiring)) + 1, // + 1 for the SET key - "db_expiring_keys_total": float64(len(keysExpiring)), - } - - for s := range scrapes { - if wantVal, ok := wantValues[s.Name]; ok { - if dbNumStrFull == s.DB && wantVal != s.Value { - t.Errorf("values not matching, %f != %f", wantVal, s.Value) - } - } + c.Close() } } -type tstData struct { - db string - stats string - keysTotal, keysEx, avgTTL float64 - ok bool -} - func TestKeyspaceStringParser(t *testing.T) { - tsts := []tstData{ + tsts := []struct { + db string + stats string + keysTotal, keysEx, avgTTL float64 + ok bool + }{ {db: "xxx", stats: "", ok: false}, {db: "xxx", stats: "keys=1,expires=0,avg_ttl=0", ok: false}, {db: "db0", stats: "xxx", ok: false}, @@ -609,10 +484,13 @@ func TestParseConnectedSlaveString(t *testing.T) { } func TestKeyValuesAndSizes(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter( + os.Getenv("TEST_REDIS_URI"), + Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}, + ) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -623,15 +501,10 @@ func TestKeyValuesAndSizes(t *testing.T) { want := map[string]bool{"test_key_size": false, "test_key_value": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -702,7 +575,7 @@ func TestScanForKeys(t *testing.T) { fixtures = append(fixtures, newKeyFixture("SET", key, "Rats!")) } - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -739,7 +612,7 @@ func TestScanForKeys(t *testing.T) { } func TestGetKeysFromPatterns(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") dbMain := dbNumStr dbAlt := altDBNumStr @@ -823,7 +696,7 @@ func TestGetKeysFromPatterns(t *testing.T) { } func TestGetKeyInfo(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -891,10 +764,10 @@ func TestGetKeyInfo(t *testing.T) { func TestKeySizeList(t *testing.T) { s := dbNumStrFull + "=" + listKeys[0] - e, _ := NewRedisExporter(defaultRedisHost, "test", s, "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: s}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -903,16 +776,10 @@ func TestKeySizeList(t *testing.T) { }() found := false - for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_key_size") { - found = true - break - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_key_size") { + found = true + break } } @@ -922,12 +789,12 @@ func TestKeySizeList(t *testing.T) { } func TestScript(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() e.LuaScript = []byte(`return {"a", "11", "b", "12", "c", "13"}`) nKeys := 3 - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -936,13 +803,8 @@ func TestScript(t *testing.T) { }() for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_script_value") { - nKeys-- - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_script_value") { + nKeys-- } } if nKeys != 0 { @@ -951,7 +813,7 @@ func TestScript(t *testing.T) { } func TestKeyValueInvalidDB(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "999="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: "999=" + url.QueryEscape(keys[0])}) chM := make(chan prometheus.Metric) go func() { @@ -970,7 +832,7 @@ func TestKeyValueInvalidDB(t *testing.T) { } } default: - log.Printf("default: m: %#v", m) + log.Debugf("default: m: %#v", m) } } for k, found := range dontWant { @@ -982,10 +844,10 @@ func TestKeyValueInvalidDB(t *testing.T) { } func TestCommandStats(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e := getTestExporter() - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -996,15 +858,10 @@ func TestCommandStats(t *testing.T) { want := map[string]bool{"test_commands_duration_seconds_total": false, "test_commands_total": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -1023,13 +880,13 @@ func TestHTTPEndpoint(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") tests := []string{ // metrics @@ -1038,11 +895,24 @@ func TestHTTPEndpoint(t *testing.T) { `test_key_size`, `test_instance_info`, + "db_keys", + "db_avg_ttl_seconds", + "used_cpu_sys", + "loading_dump_file", // testing renames + "config_maxmemory", // testing config extraction + "config_maxclients", // testing config extraction + "slowlog_length", + "slowlog_last_id", + "start_time_seconds", + "uptime_in_seconds", + // labels and label values - `addr="redis://` + *redisAddr, `redis_mode`, `standalone`, `cmd="get"`, + + `test_db_keys{db="db11"} 11`, + `test_db_keys_expiring{db="db11"} `, } for _, test := range tests { if !strings.Contains(body, test) { @@ -1051,103 +921,57 @@ func TestHTTPEndpoint(t *testing.T) { } } -func TestNonExistingHost(t *testing.T) { - rr := RedisHost{Addrs: []string{"unix:///tmp/doesnt.exist"}, Aliases: []string{""}} - e, _ := NewRedisExporter(rr, "test", "", "") +func TestHTTPScrapeEndpoint(t *testing.T) { + r := prometheus.NewRegistry() + prometheus.DefaultGatherer = r + prometheus.DefaultRegisterer = r - chM := make(chan prometheus.Metric) - go func() { - e.Collect(chM) - close(chM) - }() + e, _ := NewRedisExporter("", Options{Namespace: "test"}) - want := map[string]float64{"test_exporter_last_scrape_error": 1.0, "test_exporter_scrapes_total": 1.0} + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) + prometheus.Register(e) - for m := range chM { + ts := httptest.NewServer(http.HandlerFunc(e.ScrapeHandler)) + defer ts.Close() - descString := m.Desc().String() + u := fmt.Sprintf(ts.URL+"/?target=%s", os.Getenv("TEST_REDIS_URI")) + body := downloadURL(t, u) - switch m.(type) { - case prometheus.Counter: - - for k := range want { - if strings.Contains(descString, k) { - - g := &dto.Metric{} - m.Write(g) - - val := 0.0 - - if g.GetGauge() != nil { - val = *g.GetGauge().Value - } else if g.GetCounter() != nil { - val = *g.GetCounter().Value - } else { - continue - } - if val == want[k] { - want[k] = -1.0 - } - } - } + wants := []string{ + // metrics + `test_connected_clients`, + `test_commands_processed_total`, + `test_instance_info`, - default: - log.Printf("default: m: %#v", m) - } + "db_keys", + "db_avg_ttl_seconds", + "used_cpu_sys", + "loading_dump_file", // testing renames + "config_maxmemory", // testing config extraction + "config_maxclients", // testing config extraction + "slowlog_length", + "slowlog_last_id", + "start_time_seconds", + "uptime_in_seconds", + + // labels and label values + `redis_mode`, + `standalone`, + `cmd="get"`, + `test_db_keys{db="db11"} 11`, + `test_db_keys_expiring{db="db11"} `, } - for k, v := range want { - if v > 0 { - t.Errorf("didn't find %s", k) + for _, want := range wants { + if !strings.Contains(body, want) { + t.Errorf("want metrics to include %q, have:\n%s", want, body) } } } -func TestMoreThanOneHost(t *testing.T) { - firstHost := defaultRedisHost.Addrs[0] - secondHostURI := os.Getenv("TEST_SECOND_REDIS_URI") - if secondHostURI == "" { - log.Printf("TEST_SECOND_REDIS_URI not set - skipping test") - t.SkipNow() - return - } - - c, err := redis.DialURL(secondHostURI) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - defer c.Close() - - _, err = c.Do("PING") - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - defer c.Close() - - setupDBKeys(t, firstHost) - defer deleteKeysFromDB(t, firstHost) - - setupDBKeys(t, secondHostURI) - defer deleteKeysFromDB(t, secondHostURI) - - _, err = c.Do("SELECT", dbNumStr) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - - secondHostValue := float64(5678.9) - _, err = c.Do("SET", keys[0], secondHostValue) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - - twoHostCfg := RedisHost{Addrs: []string{firstHost, secondHostURI}, Aliases: []string{"", ""}} - checkKey := dbNumStrFull + "=" + url.QueryEscape(keys[0]) - e, _ := NewRedisExporter(twoHostCfg, "test", checkKey, "") +func TestNonExistingHost(t *testing.T) { + e, _ := NewRedisExporter("unix:///tmp/doesnt.exist", Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -1155,37 +979,33 @@ func TestMoreThanOneHost(t *testing.T) { close(chM) }() - want := map[string]float64{ - firstHost: TestValue, - secondHostURI: secondHostValue, - } + want := map[string]float64{"test_exporter_last_scrape_error": 1.0, "test_exporter_scrapes_total": 1.0} for m := range chM { + descString := m.Desc().String() + for k := range want { + if strings.Contains(descString, k) { + g := &dto.Metric{} + m.Write(g) + val := 0.0 + + if g.GetGauge() != nil { + val = *g.GetGauge().Value + } else if g.GetCounter() != nil { + val = *g.GetCounter().Value + } else { + continue + } - switch m.(type) { - case prometheus.Gauge: - pb := &dto.Metric{} - m.Write(pb) - - if !strings.Contains(m.Desc().String(), "test_key_value") { - continue - } - - for _, l := range pb.GetLabel() { - for lbl, val := range want { - if l.GetName() == "addr" && l.GetValue() == lbl && pb.GetGauge().GetValue() == val { - want[lbl] = -1 - } + if val == want[k] { + want[k] = -1.0 } } - default: - log.Printf("default: m: %#v", m) } } - - for lbl, val := range want { - if val > 0 { - t.Errorf("Never found value for: %s", lbl) + for k, v := range want { + if v > 0 { + t.Errorf("didn't find %s", k) } } } @@ -1211,10 +1031,10 @@ func TestKeysReset(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+keys[0], "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + keys[0]}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) @@ -1224,14 +1044,14 @@ func TestKeysReset(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") if !strings.Contains(body, keys[0]) { t.Errorf("Did not found key %q\n%s", keys[0], body) } - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) - body = downloadUrl(t, ts.URL+"/metrics") + body = downloadURL(t, ts.URL+"/metrics") if strings.Contains(body, keys[0]) { t.Errorf("Metric is present in metrics list %q\n%s", keys[0], body) } @@ -1239,9 +1059,7 @@ func TestKeysReset(t *testing.T) { func TestClusterMaster(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") } r := prometheus.NewRegistry() @@ -1251,10 +1069,8 @@ func TestClusterMaster(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"master"}} - log.Printf("master - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") + e, _ := NewRedisExporter(addr, Options{Namespace: "test"}) prometheus.Register(e) @@ -1264,10 +1080,10 @@ func TestClusterMaster(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("master - body: %s", body) + body := downloadURL(t, ts.URL+"/metrics") + log.Debugf("master - body: %s", body) for _, want := range []string{ - "test_instance_info{addr=\"redis://redis-cluster:7000\",alias=\"master\"", + "test_instance_info{", "test_master_repl_offset", } { if !strings.Contains(body, want) { @@ -1277,39 +1093,16 @@ func TestClusterMaster(t *testing.T) { } func TestPasswordProtectedInstance(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{testPwd} - - setupDBKeys(t, host.Addrs[0]) - - // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) - if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) - return - } - defer c.Close() - - if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil { - t.Fatalf("error setting password, err: %s", err) - } - c.Flush() - - defer func() { - if _, err = c.Do("auth", testPwd); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - deleteKeysFromDB(t, host.Addrs[0]) - }() - e, _ := NewRedisExporter(host, "test", "", "") + uri := os.Getenv("TEST_PWD_REDIS_URI") + setupDBKeys(t, uri) + e, _ := NewRedisExporter(uri, Options{Namespace: "test"}) prometheus.Register(e) chM := make(chan prometheus.Metric, 10000) @@ -1318,7 +1111,7 @@ func TestPasswordProtectedInstance(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") if !strings.Contains(body, "test_up") { t.Errorf("error, missing test_up") @@ -1326,6 +1119,9 @@ func TestPasswordProtectedInstance(t *testing.T) { } func TestPasswordInvalid(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } r := prometheus.NewRegistry() prometheus.DefaultGatherer = r prometheus.DefaultRegisterer = r @@ -1333,35 +1129,10 @@ func TestPasswordInvalid(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{"wrong_password"} + testPwd := "redis-password" + uri := strings.Replace(os.Getenv("TEST_PWD_REDIS_URI"), testPwd, "wrong-pwd", -1) - setupDBKeys(t, host.Addrs[0]) - - // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) - if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) - return - } - defer c.Close() - - if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil { - t.Fatalf("error setting password, err: %s", err) - } - c.Flush() - - defer func() { - if _, err = c.Do("auth", testPwd); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - deleteKeysFromDB(t, host.Addrs[0]) - }() - e, _ := NewRedisExporter(host, "test", "", "") + e, _ := NewRedisExporter(uri, Options{Namespace: "test"}) prometheus.Register(e) @@ -1371,17 +1142,16 @@ func TestPasswordInvalid(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - if !strings.Contains(body, "test_exporter_last_scrape_error 1") { - t.Errorf(`error, expected string "test_exporter_last_scrape_error 1" in body`) + want := `test_exporter_last_scrape_error{err="dial redis: unknown network redis"} 1` + body := downloadURL(t, ts.URL+"/metrics") + if !strings.Contains(body, want) { + t.Errorf(`error, expected string "%s" in body, got body: \n\n%s`, want, body) } } func TestClusterSlave(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") } r := prometheus.NewRegistry() @@ -1391,10 +1161,8 @@ func TestClusterSlave(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"slave"}} - log.Printf("slave - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") + e, _ := NewRedisExporter(addr, Options{Namespace: "test"}) prometheus.Register(e) @@ -1404,12 +1172,12 @@ func TestClusterSlave(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("slave - body: %s", body) + body := downloadURL(t, ts.URL+"/metrics") + log.Debugf("slave - body: %s", body) for _, want := range []string{ "test_instance_info", "test_master_last_io_seconds", - "test_slave_info{addr=\"redis://redis-cluster:7005\",alias=\"slave\",", + "test_slave_info", } { if !strings.Contains(body, want) { t.Errorf("Did not find key [%s] \nbody: %s", want, body) @@ -1430,8 +1198,7 @@ func TestCheckKeys(t *testing.T) { {"wrong=wrong=1", "", false}, {"", "wrong=wrong=2", false}, } { - - _, err := NewRedisExporter(defaultRedisHost, "test", tst.SingleCheckKey, tst.CheckKeys) + _, err := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: tst.SingleCheckKey, CheckKeys: tst.CheckKeys}) if tst.ExpectSuccess && err != nil { t.Errorf("Expected success for test: %#v, got err: %s", tst, err) return @@ -1445,8 +1212,6 @@ func TestCheckKeys(t *testing.T) { } func init() { - flag.Parse() - ll := strings.ToLower(os.Getenv("LOG_LEVEL")) if pl, err := log.ParseLevel(ll); err == nil { log.Printf("Setting log level to: %s", ll) @@ -1466,17 +1231,4 @@ func init() { key := fmt.Sprintf("key_exp_%s_%d", n, ts) keysExpiring = append(keysExpiring, key) } - - addrs := strings.Split(*redisAddr, *separator) - if len(addrs) == 0 || len(addrs[0]) == 0 { - log.Fatal("Invalid parameter --redis.addr") - } - - aliases := strings.Split(*redisAlias, *separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) - } - - log.Printf("Using redis addrs: %#v", addrs) - defaultRedisHost = RedisHost{Addrs: []string{"redis://" + *redisAddr}, Aliases: aliases} } diff --git a/main.go b/main.go index 86a83343..b86ac096 100644 --- a/main.go +++ b/main.go @@ -37,23 +37,19 @@ func getEnvBool(key string) (envValBool bool) { func main() { var ( - redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of one or more redis nodes, separated by separator") - redisFile = flag.String("redis.file", getEnv("REDIS_FILE", ""), "Path to file containing one or more redis nodes, separated by newline. NOTE: mutually exclusive with redis.addr") - redisPassword = flag.String("redis.password", getEnv("REDIS_PASSWORD", ""), "Password for one or more redis nodes, separated by separator") - redisPasswordFile = flag.String("redis.password-file", getEnv("REDIS_PASSWORD_FILE", ""), "File containing the password for one or more redis nodes, separated by separator. NOTE: mutually exclusive with redis.password") - redisAlias = flag.String("redis.alias", getEnv("REDIS_ALIAS", ""), "Redis instance alias for one or more redis nodes, separated by separator") - namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") - checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") - checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") - scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") - separator = flag.String("separator", getEnv("REDIS_EXPORTER_SEPARATOR", ","), "separator used to split redis.addr, redis.password and redis.alias into several elements.") - listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") - metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") - logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") - isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") - showVersion = flag.Bool("version", false, "Show version information and exit") - useCfBindings = flag.Bool("use-cf-bindings", getEnvBool("REDIS_EXPORTER_USE-CF-BINDINGS"), "Use Cloud Foundry service bindings") - redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to export go runtime metrics also") + redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of the redis instsance to scrape") + namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") + checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") + checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") + scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") + listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") + metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") + logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") + configCommand = flag.String("config-command", getEnv("REDIS_EXPORTER_CONFIG_COMMAND", "CONFIG"), "What to use for the CONFIG command") + isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") + showVersion = flag.Bool("version", false, "Show version information and exit") + redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to export go runtime metrics also") + inclVerbotenMetrics = flag.Bool("incl-verboten-metrics", getEnvBool("REDIS_EXPORTER_INCL_VERBOTEN_METRICS"), "Whether to include metrics that are not considered ok by Prometheus standards") ) flag.Parse() @@ -78,44 +74,15 @@ func main() { return } - if *redisFile != "" && *redisAddr != "" { - log.Fatal("Cannot specify both redis.addr and redis.file") - } - - var parsedRedisPassword string - - if *redisPasswordFile != "" { - if *redisPassword != "" { - log.Fatal("Cannot specify both redis.password and redis.password-file") - } - b, err := ioutil.ReadFile(*redisPasswordFile) - if err != nil { - log.Fatal(err) - } - parsedRedisPassword = string(b) - } else { - parsedRedisPassword = *redisPassword - } - - var addrs, passwords, aliases []string - - switch { - case *redisFile != "": - var err error - if addrs, passwords, aliases, err = exporter.LoadRedisFile(*redisFile); err != nil { - log.Fatal(err) - } - case *useCfBindings: - addrs, passwords, aliases = exporter.GetCloudFoundryRedisBindings() - default: - addrs, passwords, aliases = exporter.LoadRedisArgs(*redisAddr, parsedRedisPassword, *redisAlias, *separator) - } - exp, err := exporter.NewRedisExporter( - exporter.RedisHost{Addrs: addrs, Passwords: passwords, Aliases: aliases}, - *namespace, - *checkSingleKeys, - *checkKeys, + *redisAddr, + exporter.Options{ + Namespace: *namespace, + ConfigCommandName: *configCommand, + CheckKeys: *checkKeys, + CheckSingleKeys: *checkSingleKeys, + IncludeVerbotenMetrics: *inclVerbotenMetrics, + }, ) if err != nil { log.Fatal(err) @@ -144,9 +111,10 @@ func main() { http.Handle(*metricPath, promhttp.Handler()) } + http.HandleFunc("/scrape", exp.ScrapeHandler) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - + w.Write([]byte(` Redis Exporter v` + BuildVersion + `

Redis Exporter ` + BuildVersion + `

@@ -157,7 +125,6 @@ func main() { }) log.Printf("Providing metrics at %s%s", *listenAddress, *metricPath) - log.Printf("Connecting to redis hosts: %#v", addrs) - log.Printf("Using alias: %#v", aliases) + log.Printf("Connecting to redis hosts: %#v", *redisAddr) log.Fatal(http.ListenAndServe(*listenAddress, nil)) }