From bfd9d4bf57a57a102df517ab07b7322aa21a4d60 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 7 May 2019 09:17:34 -0400 Subject: [PATCH] wip --- exporter/redis.go | 210 +++++++++++++++++++++++------------------ exporter/redis_test.go | 17 ++-- 2 files changed, 126 insertions(+), 101 deletions(-) diff --git a/exporter/redis.go b/exporter/redis.go index 5ebe19aa..9e078ef9 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -172,10 +172,22 @@ func (e *Exporter) ScrapeHandler(w http.ResponseWriter, r *http.Request) { return } - // todo: allow passing check-keys? - start := time.Now() - exp, _ := NewRedisExporter(target, e.options) + + // todo: this needs a test + checkKeys := r.URL.Query().Get("check-keys") + checkSingleKey := r.URL.Query().Get("check-single-keys") + + opts := e.options + opts.CheckKeys = checkKeys + opts.CheckSingleKeys = checkSingleKey + + exp, err := NewRedisExporter(target, opts) + if err != nil { + http.Error(w, "NewRedisExporter() err: err", 400) + e.targetScrapeRequestErrors.Inc() + return + } registry := prometheus.NewRegistry() registry.MustRegister(exp) h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) @@ -268,11 +280,10 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { metricMapGauges["total_system_memory"] = "total_system_memory_bytes" } - l := []string{} e.metricDescriptions = map[string]*prometheus.Desc{} - e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", l) + e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", nil) e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"role", "redis_version", "redis_build_id", "redis_mode", "os"}) - e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", l) + e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", nil) e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"err"}) e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"key"}) @@ -281,17 +292,17 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"cmd"}) e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"cmd"}) - e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, l) - e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, l) - e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, l) + e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, nil) + e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, nil) + e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, nil) e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"event_name"}) e.metricDescriptions["latency_spike_seconds"] = newMetricDescr(opts.Namespace, "latency_spike_seconds", `Length of the last latency spike in seconds`, []string{"event_name"}) e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"master_host", "master_port", "read_only"}) - e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", l) - e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", l) + e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", nil) + e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", nil) e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) @@ -309,11 +320,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } for _, v := range metricMapGauges { - ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{}) + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) } for _, v := range metricMapCounters { - ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{}) + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) } ch <- e.totalScrapes.Desc() @@ -485,16 +496,16 @@ func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels...) } -func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labels ...string) { +func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labelValues ...string) { descr := e.metricDescriptions[metric] if descr == nil { - descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", []string{}) + descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", nil) } - ch <- prometheus.MustNewConstMetric(descr, valType, val, labels...) + ch <- prometheus.MustNewConstMetric(descr, valType, val, labelValues...) } -func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) error { +func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) { for i := 0; i < len(info); i += 2 { log.Debugf("tile38: %s:%s", info[i], info[i+1]) @@ -507,8 +518,6 @@ func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []stri e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } - - return nil } func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { @@ -699,6 +708,89 @@ func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, info s return nil } +func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + log.Debugf("e.singleKeys: %#v", e.singleKeys) + allKeys := append([]dbKeyPair{}, e.singleKeys...) + + log.Debugf("e.keys: %#v", e.keys) + scannedKeys, err := getKeysFromPatterns(c, e.keys) + if err != nil { + log.Errorf("Error expanding key patterns: %#v", err) + } else { + allKeys = append(allKeys, scannedKeys...) + } + + log.Debugf("allKeys: %#v", allKeys) + for _, k := range allKeys { + if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { + log.Debugf("Couldn't select database %#v when getting key info.", k.db) + continue + } + + info, err := getKeyInfo(c, k.key) + if err != nil { + switch err { + case errNotFound: + log.Debugf("Key '%s' not found when trying to get type and size.", k.key) + default: + log.Error(err) + } + continue + } + dbLabel := "db" + k.db + e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key) + + // Only record value metric if value is float-y + if val, err := redis.Float64(c.Do("GET", k.key)); err == nil { + e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key) + } + } +} + +func (e *Exporter) extractLuaScriptMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if e.LuaScript == nil || len(e.LuaScript) == 0 { + return + } + + log.Debug("Evaluating e.LuaScript") + kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) + if err != nil { + log.Errorf("LuaScript error: %v", err) + return + } + + if kv != nil { + for key, stringVal := range kv { + if val, err := strconv.ParseFloat(stringVal, 64); err == nil { + e.registerGaugeValue(ch, "script_values", val, key) + } + } + } +} + +func (e *Exporter) extractSlowLogMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { + e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64))) + } + + if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { + var slowlogLastID int64 + var lastSlowExecutionDurationSeconds float64 + + if len(values) > 0 { + if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { + slowlogLastID = values[0].(int64) + if len(values) > 2 { + lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 + } + } + } + + e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID)) + e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds) + } +} + func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, fieldKey, fieldValue string) error { orgMetricName := sanitizeMetricName(fieldKey) metricName := orgMetricName @@ -762,7 +854,7 @@ func getKeyInfo(c redis.Conn, key string) (info keyInfo, err error) { return info, errNotFound case "string": if size, err := redis.Int64(c.Do("PFCOUNT", key)); err == nil { - info.keyType = "hyperloglog" + // hyperloglog info.size = float64(size) } else if size, err := redis.Int64(c.Do("STRLEN", key)); err == nil { info.size = float64(size) @@ -880,12 +972,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { log.Debugf("aborting for addr: %s - redis err: %s", e.redisAddr, err) return err } - defer c.Close() + log.Debugf("connected to: %s", e.redisAddr) dbCount := 0 - if config, err := redis.Strings(c.Do(e.options.ConfigCommandName, "GET", "*")); err == nil { dbCount, err = e.extractConfigMetrics(ch, config) if err != nil { @@ -904,9 +995,8 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { return err } } - isClusterEnabled := strings.Contains(infoAll, "cluster_enabled:1") - if isClusterEnabled { + if strings.Contains(infoAll, "cluster_enabled:1") { if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil { e.extractClusterInfoMetrics(ch, clusterInfo) @@ -946,77 +1036,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { } } - log.Debugf("e.singleKeys: %#v", e.singleKeys) - allKeys := append([]dbKeyPair{}, e.singleKeys...) + e.extractCheckKeyMetrics(ch, c) - log.Debugf("e.keys: %#v", e.keys) - scannedKeys, err := getKeysFromPatterns(c, e.keys) - if err != nil { - log.Errorf("Error expanding key patterns: %#v", err) - } else { - allKeys = append(allKeys, scannedKeys...) - } - - log.Debugf("allKeys: %#v", allKeys) - for _, k := range allKeys { - if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { - log.Debugf("Couldn't select database %#v when getting key info.", k.db) - continue - } + e.extractLuaScriptMetrics(ch, c) - info, err := getKeyInfo(c, k.key) - if err != nil { - switch err { - case errNotFound: - log.Debugf("Key '%s' not found when trying to get type and size.", k.key) - default: - log.Error(err) - } - continue - } - dbLabel := "db" + k.db - e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key) - - // Only record value metric if value is float-y - if val, err := redis.Float64(c.Do("GET", k.key)); err == nil { - e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key) - } - } - - if e.LuaScript != nil && len(e.LuaScript) > 0 { - log.Debug("e.script") - kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) - if err != nil { - log.Errorf("Collect script error: %v", err) - } else if kv != nil { - for key, stringVal := range kv { - if val, err := strconv.ParseFloat(stringVal, 64); err == nil { - e.registerGaugeValue(ch, "script_values", val, key) - } - } - } - } - - if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { - e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64))) - } - - if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { - var slowlogLastID int64 - var lastSlowExecutionDurationSeconds float64 - - if len(values) > 0 { - if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { - slowlogLastID = values[0].(int64) - if len(values) > 2 { - lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 - } - } - } - - e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID)) - e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds) - } + e.extractSlowLogMetrics(ch, c) log.Debugf("scrapeRedisHost() done") return nil diff --git a/exporter/redis_test.go b/exporter/redis_test.go index 22eef0f9..cd3e3b65 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -209,7 +209,7 @@ func TestLatencySpike(t *testing.T) { func TestTile38(t *testing.T) { if os.Getenv("TEST_TILE38_URI") == "" { - t.SkipNow() + t.Skipf("TEST_TILE38_URI not set - skipping") } e, _ := NewRedisExporter(os.Getenv("TEST_TILE38_URI"), Options{Namespace: "test"}) @@ -776,7 +776,6 @@ func TestKeySizeList(t *testing.T) { }() found := false - for m := range chM { if strings.Contains(m.Desc().String(), "test_key_size") { found = true @@ -1060,9 +1059,7 @@ func TestKeysReset(t *testing.T) { func TestClusterMaster(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") } r := prometheus.NewRegistry() @@ -1096,6 +1093,9 @@ func TestClusterMaster(t *testing.T) { } func TestPasswordProtectedInstance(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() @@ -1119,6 +1119,9 @@ func TestPasswordProtectedInstance(t *testing.T) { } func TestPasswordInvalid(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } r := prometheus.NewRegistry() prometheus.DefaultGatherer = r prometheus.DefaultRegisterer = r @@ -1148,9 +1151,7 @@ func TestPasswordInvalid(t *testing.T) { func TestClusterSlave(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") } r := prometheus.NewRegistry()