Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@oliver006
oliver006 committed May 7, 2019
1 parent 6288963 commit bfd9d4b
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 101 deletions.
210 changes: 117 additions & 93 deletions exporter/redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,22 @@ func (e *Exporter) ScrapeHandler(w http.ResponseWriter, r *http.Request) {
return
}

// todo: allow passing check-keys?

start := time.Now()
exp, _ := NewRedisExporter(target, e.options)

// todo: this needs a test
checkKeys := r.URL.Query().Get("check-keys")
checkSingleKey := r.URL.Query().Get("check-single-keys")

opts := e.options
opts.CheckKeys = checkKeys
opts.CheckSingleKeys = checkSingleKey

exp, err := NewRedisExporter(target, opts)
if err != nil {
http.Error(w, "NewRedisExporter() err: err", 400)
e.targetScrapeRequestErrors.Inc()
return
}
registry := prometheus.NewRegistry()
registry.MustRegister(exp)
h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
Expand Down Expand Up @@ -268,11 +280,10 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
metricMapGauges["total_system_memory"] = "total_system_memory_bytes"
}

l := []string{}
e.metricDescriptions = map[string]*prometheus.Desc{}
e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", l)
e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", nil)
e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"role", "redis_version", "redis_build_id", "redis_mode", "os"})
e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", l)
e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", nil)
e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"err"})

e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"key"})
Expand All @@ -281,17 +292,17 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {

e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"cmd"})
e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"cmd"})
e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, l)
e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, l)
e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, l)
e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, nil)
e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, nil)
e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, nil)

e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"event_name"})
e.metricDescriptions["latency_spike_seconds"] = newMetricDescr(opts.Namespace, "latency_spike_seconds", `Length of the last latency spike in seconds`, []string{"event_name"})

e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"master_host", "master_port", "read_only"})

e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", l)
e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", l)
e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", nil)
e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", nil)
e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"slave_ip", "slave_port", "slave_state"})
e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"slave_ip", "slave_port", "slave_state"})

Expand All @@ -309,11 +320,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
}

for _, v := range metricMapGauges {
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{})
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil)
}

for _, v := range metricMapCounters {
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{})
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil)
}

ch <- e.totalScrapes.Desc()
Expand Down Expand Up @@ -485,16 +496,16 @@ func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string
e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels...)
}

func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labels ...string) {
func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labelValues ...string) {
descr := e.metricDescriptions[metric]
if descr == nil {
descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", []string{})
descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", nil)
}

ch <- prometheus.MustNewConstMetric(descr, valType, val, labels...)
ch <- prometheus.MustNewConstMetric(descr, valType, val, labelValues...)
}

func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) error {
func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) {
for i := 0; i < len(info); i += 2 {
log.Debugf("tile38: %s:%s", info[i], info[i+1])

Expand All @@ -507,8 +518,6 @@ func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []stri

e.parseAndRegisterMetric(ch, fieldKey, fieldValue)
}

return nil
}

func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) {
Expand Down Expand Up @@ -699,6 +708,89 @@ func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, info s
return nil
}

func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
log.Debugf("e.singleKeys: %#v", e.singleKeys)
allKeys := append([]dbKeyPair{}, e.singleKeys...)

log.Debugf("e.keys: %#v", e.keys)
scannedKeys, err := getKeysFromPatterns(c, e.keys)
if err != nil {
log.Errorf("Error expanding key patterns: %#v", err)
} else {
allKeys = append(allKeys, scannedKeys...)
}

log.Debugf("allKeys: %#v", allKeys)
for _, k := range allKeys {
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
log.Debugf("Couldn't select database %#v when getting key info.", k.db)
continue
}

info, err := getKeyInfo(c, k.key)
if err != nil {
switch err {
case errNotFound:
log.Debugf("Key '%s' not found when trying to get type and size.", k.key)
default:
log.Error(err)
}
continue
}
dbLabel := "db" + k.db
e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key)

// Only record value metric if value is float-y
if val, err := redis.Float64(c.Do("GET", k.key)); err == nil {
e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key)
}
}
}

func (e *Exporter) extractLuaScriptMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
if e.LuaScript == nil || len(e.LuaScript) == 0 {
return
}

log.Debug("Evaluating e.LuaScript")
kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0))
if err != nil {
log.Errorf("LuaScript error: %v", err)
return
}

if kv != nil {
for key, stringVal := range kv {
if val, err := strconv.ParseFloat(stringVal, 64); err == nil {
e.registerGaugeValue(ch, "script_values", val, key)
}
}
}
}

func (e *Exporter) extractSlowLogMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
if reply, err := c.Do("SLOWLOG", "LEN"); err == nil {
e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64)))
}

if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil {
var slowlogLastID int64
var lastSlowExecutionDurationSeconds float64

if len(values) > 0 {
if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 {
slowlogLastID = values[0].(int64)
if len(values) > 2 {
lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6
}
}
}

e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID))
e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds)
}
}

func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, fieldKey, fieldValue string) error {
orgMetricName := sanitizeMetricName(fieldKey)
metricName := orgMetricName
Expand Down Expand Up @@ -762,7 +854,7 @@ func getKeyInfo(c redis.Conn, key string) (info keyInfo, err error) {
return info, errNotFound
case "string":
if size, err := redis.Int64(c.Do("PFCOUNT", key)); err == nil {
info.keyType = "hyperloglog"
// hyperloglog
info.size = float64(size)
} else if size, err := redis.Int64(c.Do("STRLEN", key)); err == nil {
info.size = float64(size)
Expand Down Expand Up @@ -880,12 +972,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
log.Debugf("aborting for addr: %s - redis err: %s", e.redisAddr, err)
return err
}

defer c.Close()

log.Debugf("connected to: %s", e.redisAddr)

dbCount := 0

if config, err := redis.Strings(c.Do(e.options.ConfigCommandName, "GET", "*")); err == nil {
dbCount, err = e.extractConfigMetrics(ch, config)
if err != nil {
Expand All @@ -904,9 +995,8 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
return err
}
}
isClusterEnabled := strings.Contains(infoAll, "cluster_enabled:1")

if isClusterEnabled {
if strings.Contains(infoAll, "cluster_enabled:1") {
if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil {
e.extractClusterInfoMetrics(ch, clusterInfo)

Expand Down Expand Up @@ -946,77 +1036,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
}
}

log.Debugf("e.singleKeys: %#v", e.singleKeys)
allKeys := append([]dbKeyPair{}, e.singleKeys...)
e.extractCheckKeyMetrics(ch, c)

log.Debugf("e.keys: %#v", e.keys)
scannedKeys, err := getKeysFromPatterns(c, e.keys)
if err != nil {
log.Errorf("Error expanding key patterns: %#v", err)
} else {
allKeys = append(allKeys, scannedKeys...)
}

log.Debugf("allKeys: %#v", allKeys)
for _, k := range allKeys {
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
log.Debugf("Couldn't select database %#v when getting key info.", k.db)
continue
}
e.extractLuaScriptMetrics(ch, c)

info, err := getKeyInfo(c, k.key)
if err != nil {
switch err {
case errNotFound:
log.Debugf("Key '%s' not found when trying to get type and size.", k.key)
default:
log.Error(err)
}
continue
}
dbLabel := "db" + k.db
e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key)

// Only record value metric if value is float-y
if val, err := redis.Float64(c.Do("GET", k.key)); err == nil {
e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key)
}
}

if e.LuaScript != nil && len(e.LuaScript) > 0 {
log.Debug("e.script")
kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0))
if err != nil {
log.Errorf("Collect script error: %v", err)
} else if kv != nil {
for key, stringVal := range kv {
if val, err := strconv.ParseFloat(stringVal, 64); err == nil {
e.registerGaugeValue(ch, "script_values", val, key)
}
}
}
}

if reply, err := c.Do("SLOWLOG", "LEN"); err == nil {
e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64)))
}

if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil {
var slowlogLastID int64
var lastSlowExecutionDurationSeconds float64

if len(values) > 0 {
if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 {
slowlogLastID = values[0].(int64)
if len(values) > 2 {
lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6
}
}
}

e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID))
e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds)
}
e.extractSlowLogMetrics(ch, c)

log.Debugf("scrapeRedisHost() done")
return nil
Expand Down
17 changes: 9 additions & 8 deletions exporter/redis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func TestLatencySpike(t *testing.T) {

func TestTile38(t *testing.T) {
if os.Getenv("TEST_TILE38_URI") == "" {
t.SkipNow()
t.Skipf("TEST_TILE38_URI not set - skipping")
}

e, _ := NewRedisExporter(os.Getenv("TEST_TILE38_URI"), Options{Namespace: "test"})
Expand Down Expand Up @@ -776,7 +776,6 @@ func TestKeySizeList(t *testing.T) {
}()

found := false

for m := range chM {
if strings.Contains(m.Desc().String(), "test_key_size") {
found = true
Expand Down Expand Up @@ -1060,9 +1059,7 @@ func TestKeysReset(t *testing.T) {

func TestClusterMaster(t *testing.T) {
if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" {
log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
t.SkipNow()
return
t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
}

r := prometheus.NewRegistry()
Expand Down Expand Up @@ -1096,6 +1093,9 @@ func TestClusterMaster(t *testing.T) {
}

func TestPasswordProtectedInstance(t *testing.T) {
if os.Getenv("TEST_PWD_REDIS_URI") == "" {
t.Skipf("TEST_PWD_REDIS_URI not set - skipping")
}
ts := httptest.NewServer(promhttp.Handler())
defer ts.Close()

Expand All @@ -1119,6 +1119,9 @@ func TestPasswordProtectedInstance(t *testing.T) {
}

func TestPasswordInvalid(t *testing.T) {
if os.Getenv("TEST_PWD_REDIS_URI") == "" {
t.Skipf("TEST_PWD_REDIS_URI not set - skipping")
}
r := prometheus.NewRegistry()
prometheus.DefaultGatherer = r
prometheus.DefaultRegisterer = r
Expand Down Expand Up @@ -1148,9 +1151,7 @@ func TestPasswordInvalid(t *testing.T) {

func TestClusterSlave(t *testing.T) {
if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" {
log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping")
t.SkipNow()
return
t.Skipf("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping")
}

r := prometheus.NewRegistry()
Expand Down

0 comments on commit bfd9d4b

Please sign in to comment.