From 36cdf48962147cc5891397d27c5f139cd81ce6de Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 29 Apr 2019 12:19:40 -0400 Subject: [PATCH] partial rewrite for version v1.0.0 with lots of changes --- .drone.yml | 17 +- README.md | 167 +- build-github-binaries.sh | 13 +- contrib/docker-compose-for-tests.yml | 27 + .../grafana_prometheus_redis_dashboard.json | 2097 +++++++++-------- ...edis_dashboard_exporter_version_0.3x.json} | 349 ++- exporter/discovery.go | 111 - exporter/discovery_test.go | 103 - exporter/redis.go | 762 +++--- exporter/redis_test.go | 657 ++---- go.mod | 5 - go.sum | 39 - main.go | 86 +- .../go-cfenv/.gitignore | 24 - .../go-cfenv/.travis.yml | 15 - .../cloudfoundry-community/go-cfenv/LICENSE | 202 -- .../cloudfoundry-community/go-cfenv/README.md | 40 - .../go-cfenv/application.go | 30 - .../cloudfoundry-community/go-cfenv/cfenv.go | 59 - .../go-cfenv/environment.go | 16 - .../cloudfoundry-community/go-cfenv/envmap.go | 20 - .../cloudfoundry-community/go-cfenv/go.mod | 11 - .../cloudfoundry-community/go-cfenv/go.sum | 33 - .../go-cfenv/service.go | 127 - .../mitchellh/mapstructure/.travis.yml | 8 - .../mitchellh/mapstructure/CHANGELOG.md | 21 - .../github.com/mitchellh/mapstructure/LICENSE | 21 - .../mitchellh/mapstructure/README.md | 46 - .../mitchellh/mapstructure/decode_hooks.go | 217 -- .../mitchellh/mapstructure/error.go | 50 - .../github.com/mitchellh/mapstructure/go.mod | 1 - .../mitchellh/mapstructure/mapstructure.go | 1149 --------- vendor/modules.txt | 4 - 33 files changed, 1994 insertions(+), 4533 deletions(-) create mode 100644 contrib/docker-compose-for-tests.yml rename contrib/{grafana_prometheus_redis_dashboard_alias.json => grafana_prometheus_redis_dashboard_exporter_version_0.3x.json} (80%) delete mode 100644 exporter/discovery.go delete mode 100644 exporter/discovery_test.go delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/.gitignore delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/.travis.yml delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/LICENSE delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/README.md delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/application.go delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/cfenv.go delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/environment.go delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/envmap.go delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/go.mod delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/go.sum delete mode 100644 vendor/github.com/cloudfoundry-community/go-cfenv/service.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/.travis.yml delete mode 100644 vendor/github.com/mitchellh/mapstructure/CHANGELOG.md delete mode 100644 vendor/github.com/mitchellh/mapstructure/LICENSE delete mode 100644 vendor/github.com/mitchellh/mapstructure/README.md delete mode 100644 vendor/github.com/mitchellh/mapstructure/decode_hooks.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/error.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/go.mod delete mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure.go diff --git a/.drone.yml b/.drone.yml index 494df11d..fc7ba9ec 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,10 +11,10 @@ services: image: redis:5 ports: - 6379 - - name: moar-redis + - name: pwd-redis image: redis:5 - commands: - - /usr/local/bin/redis-server --port 6380 + commands: + - "/usr/local/bin/redis-server --port 6380 --requirepass redis-password" ports: - 6380 - name: redis-cluster @@ -38,16 +38,17 @@ steps: environment: GO111MODULE: on LOG_LEVEL: "info" - TEST_TILE38_URI: "tile38:9851" - TEST_SECOND_REDIS_URI: "redis://moar-redis:6380" - TEST_REDIS_CLUSTER_MASTER_URI: "redis-cluster:7000" - TEST_REDIS_CLUSTER_SLAVE_URI: "redis-cluster:7005" + TEST_TILE38_URI: "redis://tile38:9851" + TEST_REDIS_URI: "redis://redis:6379" + TEST_PWD_REDIS_URI: "redis://h:redis-password@pwd-redis:6380" + TEST_REDIS_CLUSTER_MASTER_URI: "redis://redis-cluster:7000" + TEST_REDIS_CLUSTER_SLAVE_URI: "redis://redis-cluster:7005" COVERALLS_TOKEN: from_secret: coveralls-token commands: - 'go build -mod=vendor' - "sleep 10" # let the redis test instances all come up first - - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/... --redis.addr=redis' + - 'go test -mod=vendor -v -covermode=atomic -cover -race -coverprofile=coverage.txt ./exporter/...' - 'echo "checking gofmt"' - 'echo " ! gofmt -d main.go 2>&1 | read " | bash' - 'echo " ! gofmt -d exporter/*.go 2>&1 | read " | bash' diff --git a/README.md b/README.md index c9520b44..be63b7f5 100644 --- a/README.md +++ b/README.md @@ -19,23 +19,83 @@ Supports Redis 2.x, 3.x, 4.x, and 5.x ``` -### Prometheus Configuration +### Basic Prometheus Configuration Add a block to the `scrape_configs` of your prometheus.yml config file: ```yaml scrape_configs: + - job_name: redis_exporter + static_configs: + - targets: ['<>:9121'] +``` + +and adjust the host name accordingly. -... +### Prometheus Configuration to Scrape Several Hosts -- job_name: redis_exporter - static_configs: - - targets: ['localhost:9121'] +Run the exporter with the command line flag `--redis.addr=` so it won't try to scrape +the local instance when the `/metrics` endpoint is hit. + +```yaml -... +scrape_configs: + ## config for the multiple redis targets that the exporter will scrape + - job_name: 'redis_exporter_targets' + static_configs: + - targets: + - redis://first-redis-host:6379 + - redis://h:password@second-redis-host:6379 + metrics_path: /scrape + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: <>:9121 + + ## config for scraping the exporter itself + - job_name: 'redis_exporter' + static_configs: + - targets: + - <>:9121 ``` -and adjust the host name accordingly. +The redis instances are listed under `targets`, the redis exporter hostname is configured via the last relabel_config rule.\ + +You can also use a file supply multiple targets by using `file_sd_configs` like so: + +```yaml + +scrape_configs: + - job_name: 'redis_exporter' + file_sd_configs: + - files: + - targets-redis-instances.json + metrics_path: /scrape + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: <>:9121 +``` + +The `targets-redis-instances.json` should look something like this: + +```json +[ + { + "targets": [ "redis-host-01:6379", "redis-host-02:6379"], + "labels": { } + } +] +``` + +Prometheus uses file watches and all changes to the json file are applied immediately. + ### Run via Docker: @@ -44,7 +104,6 @@ The latest release is automatically published to the [Docker registry](https://h You can run it like this: ```sh - $ docker pull oliver006/redis_exporter $ docker run -d --name redis_exporter -p 9121:9121 oliver006/redis_exporter ``` @@ -64,88 +123,34 @@ redis_exporter container can access it: ### Run on Kubernetes -[Here](contrib/k8s-redis-and-exporter-deployment.yaml) is an example Kubernetes deployment configuration for how to deploy the redis_exporter as a sidecar with a Redis instance. - +[Here](contrib/k8s-redis-and-exporter-deployment.yaml) is an example Kubernetes deployment configuration for how to deploy the redis_exporter as a sidecar to a Redis instance. -### Run on Openshift -In order to deploy the exporter on Openshift environment. -```sh -oc project - -oc process -f https://raw.githubusercontent.com/oliver006/redis_exporter/master/contrib/openshift-template.yaml \ - -p REDIS_ADDR=: \ - -p REDIS_PASSWORD= \ - -p REDIS_ALIAS= \ - -p REDIS_FILE= \ - | oc create -f - -``` - -*NOTE*: Some of the parameters can be omitted if no authentication is used or the default redis config is applied. - -```sh -oc process -f https://raw.githubusercontent.com/oliver006/redis_exporter/master/contrib/openshift-template.yaml \ - -p REDIS_ADDR=: \ - | oc create -f - -``` - -If you are running Prometheus on Openshift on the same cluster, **target** in `prometheus.yml` should point to the correct service name of the exporter - -```yaml -scrape_configs: - -... +### Flags -- job_name: redis_exporter - static_configs: - - targets: [':9121'] +Name | Environment Variable Name | Description +--------------------|------------------------------------|----------------- +redis.addr | REDIS_ADDR | Address of the redis instance, defaults to `redis://localhost:6379`. +debug | REDIS_EXPORTER_DEBUG | Verbose debug output +log-format | REDIS_EXPORTER_LOG_FORMAT | Log format, valid options are `txt` (default) and `json`. +check-keys | REDIS_EXPORTER_CHECK_KEYS | Comma separated list of key patterns to export value and length/size, eg: `db3=user_count` will export key `user_count` from db `3`. db defaults to `0` if omitted. The key patterns specified with this flag will be found using [SCAN](https://redis.io/commands/scan). Use this option if you need glob pattern matching; `check-single-keys` is faster for non-pattern keys. +check-single-keys | REDIS_EXPORTER_CHECK_SINGLE_KEYS | Comma separated list of keys to export value and length/size, eg: `db3=user_count` will export key `user_count` from db `3`. db defaults to `0` if omitted. The keys specified with this flag will be looked up directly without any glob pattern matching. Use this option if you don't need glob pattern matching; it is faster than `check-keys`. +script | REDIS_EXPORTER_SCRIPT | Path to Redis Lua script for gathering extra metrics. +namespace | REDIS_EXPORTER_NAMESPACE | Namespace for the metrics, defaults to `redis`. +web.listen-address | REDIS_EXPORTER_WEB_LISTEN_ADDRESS | Address to listen on for web interface and telemetry, defaults to `0.0.0.0:9121`. +web.telemetry-path | REDIS_EXPORTER_WEB_TELEMETRY_PATH | Path under which to expose metrics, defaults to `/metrics`. +redis-only-metrics | REDIS_EXPORTER_REDIS_ONLY_METRICS | Whether to also export go runtime metrics -... -``` -### Run on Cloud Foundry +Redis instance addresses can be tcp addresses: `redis://localhost:6379`, `redis.example.com:6379` or e.g. unix sockets: `unix:///tmp/redis.sock`.\ +SSL is supported by using the `rediss://` schema, for example: `rediss://azure-ssl-enabled-host.redis.cache.windows.net:6380` (note that the port is required when connecting to a non-standard 6379 port, e.g. with Azure Redis instances).\ +Password-protected instances can be accessed by using the URI format including a password: `redis://h:<>@<>:<>` -```sh -cf push -f contrib/manifest.yml -``` +Command line settings take precedence over any configurations provided by the environment variables. -### Flags - -Name | Description ---------------------|------------ -debug | Verbose debug output -log-format | Log format, valid options are `txt` (default) and `json`. -check-keys | Comma separated list of key patterns to export value and length/size, eg: `db3=user_count` will export key `user_count` from db `3`. db defaults to `0` if omitted. The key patterns specified with this flag will be found using [SCAN](https://redis.io/commands/scan). Use this option if you need glob pattern matching; `check-single-keys` is faster for non-pattern keys. -check-single-keys | Comma separated list of keys to export value and length/size, eg: `db3=user_count` will export key `user_count` from db `3`. db defaults to `0` if omitted. The keys specified with this flag will be looked up directly without any glob pattern matching. Use this option if you don't need glob pattern matching; it is faster than `check-keys`. -script | Path to Redis Lua script for gathering extra metrics. -redis.addr | Address of one or more redis nodes, comma separated, defaults to `redis://localhost:6379`. -redis.password | Password to use when authenticating to Redis -redis.password-file | Path to a file containing the password to use when authenticating to Redis (note: this is mutually exclusive with `redis.password`) -redis.alias | Alias for redis node addr, comma separated. -redis.file | Path to file containing one or more redis nodes, separated by newline. This option is mutually exclusive with redis.addr. Each line can optionally be comma-separated with the fields `,,`. See [here](./contrib/sample_redis_hosts_file.txt) for an example file. -namespace | Namespace for the metrics, defaults to `redis`. -web.listen-address | Address to listen on for web interface and telemetry, defaults to `0.0.0.0:9121`. -web.telemetry-path | Path under which to expose metrics, defaults to `metrics`. -use-cf-bindings | Enable usage of Cloud Foundry service bindings. Defaults to `false` -separator | Separator used to split redis.addr, redis.password and redis.alias into several elements. Defaults to `,` - -Redis node addresses can be tcp addresses like `redis://localhost:6379`, `redis.example.com:6379` or unix socket addresses like `unix:///tmp/redis.sock`.\ -SSL is supported by using the `rediss://` schema, for example: `rediss://azure-ssl-enabled-host.redis.cache.windows.net:6380` (note that the port is required when connecting to a non-standard 6379 port, e.g. with Azure Redis instances). - -These settings take precedence over any configurations provided by [environment variables](#environment-variables). - -### Environment Variables - -Name | Description --------------------|------------ -REDIS_ADDR | Address of Redis node(s) -REDIS_PASSWORD | Password to use when authenticating to Redis -REDIS_ALIAS | Alias name of Redis node(s) -REDIS_FILE | Path to file containing Redis node(s) - ### What's exported? Most items from the INFO command are exported, @@ -163,8 +168,6 @@ Example [Grafana](http://grafana.org/) screenshots:\ Grafana dashboard is available on [grafana.net](https://grafana.net/dashboards/763) and/or [github.com](contrib/grafana_prometheus_redis_dashboard.json). -Grafana dashboard with host & alias selector is available on [github.com](contrib/grafana_prometheus_redis_dashboard_alias.json). - ### What else? Open an issue or PR if you have more suggestions or ideas about what to add. diff --git a/build-github-binaries.sh b/build-github-binaries.sh index 4d884bf5..8a14cceb 100755 --- a/build-github-binaries.sh +++ b/build-github-binaries.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash + set -u -e -o pipefail if [[ -z "${DRONE_TAG}" ]] ; then @@ -19,17 +20,21 @@ if [[ -f 'go.mod' ]] ; then go mod tidy fi -gox -verbose -os="darwin linux freebsd windows netbsd" -arch="386 amd64" -rebuild -ldflags "${GO_LDFLAGS}" -output '.build/{{.OS}}-{{.Arch}}/{{.Dir}}' +gox -verbose -os="darwin linux" -arch="386 amd64" -rebuild -ldflags "${GO_LDFLAGS}" -output ".build/redis_exporter-${DRONE_TAG}.{{.OS}}-{{.Arch}}/{{.Dir}}" mkdir -p dist for build in $(ls .build); do echo "Creating archive for ${build}" - if [[ "${build}" =~ ^windows-.*$ ]] ; then + + cp LICENSE README.md ".build/${build}/" + + if [[ "${build}" =~ windows-.*$ ]] ; then + # Make sure to clear out zip files to prevent zip from appending to the archive. rm "dist/redis_exporter-${DRONE_TAG}.${build}.zip" || true - cd ".build/${build}" && zip --quiet -9 "../../dist/redis_exporter-${DRONE_TAG}.${build}.zip" 'redis_exporter.exe' && cd ../../ + cd ".build/" && zip -r --quiet -9 "../dist/${build}.zip" "${build}" && cd ../ else - tar -C ".build/${build}" -czf "dist/redis_exporter-${DRONE_TAG}.${build}.tar.gz" 'redis_exporter' + tar -C ".build/" -czf "dist/${build}.tar.gz" "${build}" fi done diff --git a/contrib/docker-compose-for-tests.yml b/contrib/docker-compose-for-tests.yml new file mode 100644 index 00000000..bb50654c --- /dev/null +++ b/contrib/docker-compose-for-tests.yml @@ -0,0 +1,27 @@ +version: '3' +services: + redis: + image: 'redis:5-alpine' + ports: + - '6379:6379' + + pwd-redis: + image: 'redis:5-alpine' + command: '/usr/local/bin/redis-server --port 6380 --requirepass redis-password' + ports: + - '6380:6380' + + redis-cluster: + image: 'grokzen/redis-cluster' + ports: + - 7000 + - 7001 + - 7002 + - 7003 + - 7004 + - 7005 + + tile38: + image: "tile38/tile38:latest" + ports: + - 9851 diff --git a/contrib/grafana_prometheus_redis_dashboard.json b/contrib/grafana_prometheus_redis_dashboard.json index 36a680cb..f4200356 100644 --- a/contrib/grafana_prometheus_redis_dashboard.json +++ b/contrib/grafana_prometheus_redis_dashboard.json @@ -1,1018 +1,1142 @@ { - "__inputs": [ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus dashboard for Redis servers", + "editable": true, + "gnetId": 763, + "graphTooltip": 0, + "id": 5, + "iteration": 1557495413494, + "links": [], + "panels": [ { - "name": "DS_PROM", - "label": "prom", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(max_over_time(redis_uptime_in_seconds{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom", + "decimals": 0, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 2, + "y": 0 + }, + "hideTimeOverride": true, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "redis_connected_clients{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": "", + "timeFrom": "1m", + "timeShift": null, + "title": "Clients", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom", + "decimals": 0, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "hideTimeOverride": true, + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 * (redis_memory_used_bytes{instance=~\"$instance\"} / redis_memory_max_bytes{instance=~\"$instance\"} )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": "80,95", + "timeFrom": "1m", + "timeShift": null, + "title": "Memory Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "3.1.1" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(redis_commands_processed_total{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "A", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Commands Executed / sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "id": null, - "title": "Prometheus Redis", - "description": "Prometheus dashboard for Redis servers", - "tags": [ - "prometheus", - "redis" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(redis_keyspace_hits_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hits", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "irate(redis_keyspace_misses_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "misses", + "metric": "", + "refId": "B", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hits / Misses per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { - "collapse": false, + "aliasColors": { + "max": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", "editable": true, - "height": "250px", - "panels": [ + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROM}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 9, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "targets": [ - { - "expr": "max(max_over_time(redis_uptime_in_seconds{addr=\"$addr\"}[$__interval]))", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Uptime", - "type": "singlestat", - "valueFontSize": "70%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "expr": "redis_memory_used_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "metric": "", + "refId": "A", + "step": 240, + "target": "" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_PROM}", - "decimals": 0, - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": true, - "id": 12, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 1, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "targets": [ - { - "expr": "redis_connected_clients{addr=\"$addr\"}", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": "", - "timeFrom": "1m", - "timeShift": null, - "title": "Clients", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "expr": "redis_memory_max_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "max", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "${DS_PROM}", - "decimals": 0, - "editable": true, - "error": false, - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "hideTimeOverride": true, - "id": 11, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "targets": [ - { - "expr": "100 * (redis_memory_used_bytes{addr=~\"$addr\"} / redis_memory_max_bytes{addr=~\"$addr\"} )", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 2 - } - ], - "thresholds": "80,95", - "timeFrom": "1m", - "timeShift": null, - "title": "Memory Usage", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(redis_net_input_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ input }}", + "refId": "A", + "step": 240 }, { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 2, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(redis_commands_processed_total{addr=~\"$addr\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "A", - "refId": "A", - "step": 240, - "target": "" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commands Executed / sec", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "rate(redis_net_output_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ output }}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "decimals": 2, - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": true, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(redis_keyspace_hits_total{addr=\"$addr\"}[5m])", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "hits", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "irate(redis_keyspace_misses_total{addr=\"$addr\"}[5m])", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "misses", - "metric": "", - "refId": "B", - "step": 240, - "target": "" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Hits / Misses per Sec", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "title": "Row" + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "collapse": false, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", "editable": true, - "height": "250px", - "panels": [ + "error": false, + "fill": 7, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ { - "aliasColors": { - "max": "#BF1B00" - }, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 7, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "redis_memory_used_bytes{addr=~\"$addr\"} ", - "intervalFactor": 2, - "legendFormat": "used", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "redis_memory_max_bytes{addr=~\"$addr\"} ", - "hide": false, - "intervalFactor": 2, - "legendFormat": "max", - "refId": "B", - "step": 240 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Memory Usage", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ db }} ", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Items per DB", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(redis_net_input_bytes_total{addr=\"$addr\"}[5m])", - "intervalFactor": 2, - "legendFormat": "{{ input }}", - "refId": "A", - "step": 240 - }, - { - "expr": "rate(redis_net_output_bytes_total{addr=\"$addr\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ output }}", - "refId": "B", - "step": 240 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Network I/O", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "title": "New row" + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "collapse": false, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", "editable": true, - "height": "250px", - "panels": [ + "error": false, + "fill": 7, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) - sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "not expiring", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum (redis_db_keys_expiring{instance=~\"$instance\"}) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "expiring", + "metric": "", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expiring vs Not-Expiring Keys", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 7, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 5, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{addr=~\"$addr\"}) by (db)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ db }} ", - "refId": "A", - "step": 240, - "target": "" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Items per DB", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 7, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 13, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum (redis_db_keys{addr=~\"$addr\"}) - sum (redis_db_keys_expiring{addr=~\"$addr\"}) ", - "interval": "", - "intervalFactor": 2, - "legendFormat": "not expiring", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum (redis_db_keys_expiring{addr=~\"$addr\"}) ", - "interval": "", - "intervalFactor": 2, - "legendFormat": "expiring", - "metric": "", - "refId": "B", - "step": 240 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Expiring vs Not-Expiring Keys", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(redis_expired_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "expired", + "metric": "", + "refId": "A", + "step": 240, + "target": "" }, { - "aliasColors": { - "evicts": "#890F02", - "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", - "reclaims": "#3F6833" - }, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 1, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "reclaims", - "yaxis": 2 - } - ], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(redis_expired_keys_total{addr=~\"$addr\"}[5m])) by (addr)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "expired", - "metric": "", - "refId": "A", - "step": 240, - "target": "" - }, - { - "expr": "sum(rate(redis_evicted_keys_total{addr=~\"$addr\"}[5m])) by (addr)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "evicted", - "refId": "B", - "step": 240 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Expired / Evicted", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum(rate(redis_evicted_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "evicted", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expired / Evicted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_PROM}", - "editable": true, - "error": false, - "fill": 8, - "grid": { - "threshold1": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2": null, - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "id": 14, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "topk(5, irate(redis_commands_total{addr=~\"$addr\"} [1m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ cmd }}", - "metric": "redis_command_calls_total", - "refId": "A", - "step": 240 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Command Calls / sec", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "show": true - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "title": "New row" + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom", + "editable": true, + "error": false, + "fill": 8, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 14, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "topk(5, irate(redis_commands_total{instance=~\"$instance\"} [1m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ cmd }}", + "metric": "redis_command_calls_total", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Command Calls / sec", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "prometheus", + "redis" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "fbrss.nbg1.ex.ohardt.net:9121", + "value": "fbrss.nbg1.ex.ohardt.net:9121" + }, + "datasource": "prom", + "definition": "label_values(redis_up, instance)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(redis_up, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, "time": { "from": "now-24h", "to": "now" @@ -1042,29 +1166,8 @@ "30d" ] }, - "templating": { - "list": [ - { - "current": {}, - "datasource": "${DS_PROM}", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "addr", - "options": [], - "query": "label_values(redis_up, addr)", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "annotations": { - "list": [] - }, - "refresh": "30s", - "schemaVersion": 12, - "version": 52, - "links": [], - "gnetId": 37 -} + "timezone": "browser", + "title": "Prometheus Redis for Redis Exporter 1.0.0", + "uid": "tRX1IOmWz", + "version": 11 +} \ No newline at end of file diff --git a/contrib/grafana_prometheus_redis_dashboard_alias.json b/contrib/grafana_prometheus_redis_dashboard_exporter_version_0.3x.json similarity index 80% rename from contrib/grafana_prometheus_redis_dashboard_alias.json rename to contrib/grafana_prometheus_redis_dashboard_exporter_version_0.3x.json index c428986b..36a680cb 100644 --- a/contrib/grafana_prometheus_redis_dashboard_alias.json +++ b/contrib/grafana_prometheus_redis_dashboard_exporter_version_0.3x.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_PROMETHEUS", - "label": "Prometheus", + "name": "DS_PROM", + "label": "prom", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -11,10 +11,10 @@ ], "__requires": [ { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.1.2" + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" }, { "type": "panel", @@ -22,32 +22,35 @@ "name": "Graph", "version": "" }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" } ], - "annotations": { - "list": [] - }, - "description": "Prometheus dashboard for Redis servers (by host and alias)", + "id": null, + "title": "Prometheus Redis", + "description": "Prometheus dashboard for Redis servers", + "tags": [ + "prometheus", + "redis" + ], + "style": "dark", + "timezone": "browser", "editable": true, - "gnetId": 763, - "graphTooltip": 0, "hideControls": false, - "id": null, - "links": [], + "sharedCrosshair": false, "rows": [ { "collapse": false, + "editable": true, "height": "250px", "panels": [ { @@ -59,7 +62,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "decimals": 0, "editable": true, "error": false, @@ -73,6 +76,7 @@ }, "id": 9, "interval": null, + "isNew": true, "links": [], "mappingType": 1, "mappingTypes": [ @@ -108,12 +112,12 @@ }, "targets": [ { - "expr": "max(max_over_time(redis_uptime_in_seconds{instance=\"$host\",alias=\"$alias\"}[$__interval]))", + "expr": "max(max_over_time(redis_uptime_in_seconds{addr=\"$addr\"}[$__interval]))", "intervalFactor": 2, "legendFormat": "", "metric": "", "refId": "A", - "step": 4 + "step": 1800 } ], "thresholds": "", @@ -138,7 +142,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "decimals": 0, "editable": true, "error": false, @@ -153,6 +157,7 @@ "hideTimeOverride": true, "id": 12, "interval": null, + "isNew": true, "links": [], "mappingType": 1, "mappingTypes": [ @@ -188,7 +193,7 @@ }, "targets": [ { - "expr": "redis_connected_clients{instance=\"$host\",alias=\"$alias\"}", + "expr": "redis_connected_clients{addr=\"$addr\"}", "intervalFactor": 2, "legendFormat": "", "metric": "", @@ -220,7 +225,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "decimals": 0, "editable": true, "error": false, @@ -235,6 +240,7 @@ "hideTimeOverride": true, "id": 11, "interval": null, + "isNew": true, "links": [], "mappingType": 1, "mappingTypes": [ @@ -270,7 +276,7 @@ }, "targets": [ { - "expr": "100 * (redis_memory_used_bytes{instance=\"$host\",alias=\"$alias\"} / redis_memory_max_bytes{instance=\"$host\",alias=\"$alias\"} )", + "expr": "100 * (redis_memory_used_bytes{addr=~\"$addr\"} / redis_memory_max_bytes{addr=~\"$addr\"} )", "intervalFactor": 2, "legendFormat": "", "metric": "", @@ -296,12 +302,18 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 1, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 2, + "isNew": true, "legend": { "avg": false, "current": false, @@ -325,17 +337,16 @@ "steppedLine": false, "targets": [ { - "expr": "rate(redis_commands_processed_total{instance=\"$host\",alias=\"$alias\"}[5m])", + "expr": "rate(redis_commands_processed_total{addr=~\"$addr\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "", - "metric": "", + "metric": "A", "refId": "A", - "step": 2, + "step": 240, "target": "" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Commands Executed / sec", @@ -347,10 +358,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -374,13 +382,19 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "decimals": 2, "editable": true, "error": false, "fill": 1, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 1, + "isNew": true, "legend": { "avg": false, "current": false, @@ -404,29 +418,28 @@ "steppedLine": false, "targets": [ { - "expr": "irate(redis_keyspace_hits_total{instance=\"$host\",alias=\"$alias\"}[5m])", + "expr": "irate(redis_keyspace_hits_total{addr=\"$addr\"}[5m])", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "hits", "metric": "", "refId": "A", - "step": 2, + "step": 240, "target": "" }, { - "expr": "irate(redis_keyspace_misses_total{instance=\"$host\",alias=\"$alias\"}[5m])", + "expr": "irate(redis_keyspace_misses_total{addr=\"$addr\"}[5m])", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "misses", "metric": "", "refId": "B", - "step": 2, + "step": 240, "target": "" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Hits / Misses per Sec", @@ -438,10 +451,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -463,15 +473,11 @@ ] } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Row", - "titleSize": "h6" + "title": "Row" }, { "collapse": false, + "editable": true, "height": "250px", "panels": [ { @@ -479,12 +485,18 @@ "max": "#BF1B00" }, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 1, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 7, + "isNew": true, "legend": { "avg": false, "current": false, @@ -510,24 +522,23 @@ "steppedLine": false, "targets": [ { - "expr": "redis_memory_used_bytes{instance=\"$host\",alias=\"$alias\"} ", + "expr": "redis_memory_used_bytes{addr=~\"$addr\"} ", "intervalFactor": 2, "legendFormat": "used", "metric": "", "refId": "A", - "step": 2, + "step": 240, "target": "" }, { - "expr": "redis_memory_max_bytes{instance=\"$host\",alias=\"$alias\"} ", + "expr": "redis_memory_max_bytes{addr=~\"$addr\"} ", "hide": false, "intervalFactor": 2, "legendFormat": "max", "refId": "B", - "step": 2 + "step": 240 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Memory Usage", @@ -539,10 +550,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -566,12 +574,18 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 1, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 10, + "isNew": true, "legend": { "avg": false, "current": false, @@ -595,22 +609,21 @@ "steppedLine": false, "targets": [ { - "expr": "rate(redis_net_input_bytes_total{instance=\"$host\",alias=\"$alias\"}[5m])", + "expr": "rate(redis_net_input_bytes_total{addr=\"$addr\"}[5m])", "intervalFactor": 2, "legendFormat": "{{ input }}", "refId": "A", - "step": 2 + "step": 240 }, { - "expr": "rate(redis_net_output_bytes_total{instance=\"$host\",alias=\"$alias\"}[5m])", + "expr": "rate(redis_net_output_bytes_total{addr=\"$addr\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{ output }}", "refId": "B", - "step": 2 + "step": 240 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Network I/O", @@ -622,10 +635,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -647,26 +657,28 @@ ] } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "New row", - "titleSize": "h6" + "title": "New row" }, { "collapse": false, + "editable": true, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 7, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 5, + "isNew": true, "legend": { "alignAsTable": true, "avg": false, @@ -692,16 +704,15 @@ "steppedLine": false, "targets": [ { - "expr": "sum (redis_db_keys{instance=\"$host\",alias=\"$alias\"}) by (db)", + "expr": "sum (redis_db_keys{addr=~\"$addr\"}) by (db)", "interval": "", "intervalFactor": 2, "legendFormat": "{{ db }} ", "refId": "A", - "step": 2, + "step": 240, "target": "" } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Items per DB", @@ -713,10 +724,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -740,12 +748,18 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 7, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 13, + "isNew": true, "legend": { "avg": false, "current": false, @@ -769,25 +783,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum (redis_db_keys{instance=\"$host\",alias=\"$alias\"}) - sum (redis_db_keys_expiring{instance=\"$host\",alias=\"$alias\"}) ", + "expr": "sum (redis_db_keys{addr=~\"$addr\"}) - sum (redis_db_keys_expiring{addr=~\"$addr\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "not expiring", "refId": "A", - "step": 2, + "step": 240, "target": "" }, { - "expr": "sum (redis_db_keys_expiring{instance=\"$host\",alias=\"$alias\"}) ", + "expr": "sum (redis_db_keys_expiring{addr=~\"$addr\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "expiring", "metric": "", "refId": "B", - "step": 2 + "step": 240 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Expiring vs Not-Expiring Keys", @@ -799,10 +812,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -825,15 +835,23 @@ }, { "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", "reclaims": "#3F6833" }, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 1, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 8, + "isNew": true, "legend": { "avg": false, "current": false, @@ -862,25 +880,24 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(redis_expired_keys_total{instance=\"$host\",alias=\"$alias\"}[5m])) by (addr)", + "expr": "sum(rate(redis_expired_keys_total{addr=~\"$addr\"}[5m])) by (addr)", "interval": "", "intervalFactor": 2, "legendFormat": "expired", "metric": "", "refId": "A", - "step": 2, + "step": 240, "target": "" }, { - "expr": "sum(rate(redis_evicted_keys_total{instance=\"$host\",alias=\"$alias\"}[5m])) by (addr)", + "expr": "sum(rate(redis_evicted_keys_total{addr=~\"$addr\"}[5m])) by (addr)", "interval": "", "intervalFactor": 2, "legendFormat": "evicted", "refId": "B", - "step": 2 + "step": 240 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Expired / Evicted", @@ -892,10 +909,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -919,12 +933,18 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "${DS_PROM}", "editable": true, "error": false, "fill": 8, - "grid": {}, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, "id": 14, + "isNew": true, "legend": { "avg": false, "current": false, @@ -948,16 +968,15 @@ "steppedLine": false, "targets": [ { - "expr": "topk(5, irate(redis_commands_total{instance=\"$host\",alias=\"$alias\"} [1m]))", + "expr": "topk(5, irate(redis_commands_total{addr=~\"$addr\"} [1m]))", "interval": "", "intervalFactor": 2, "legendFormat": "{{ cmd }}", - "metric": "", + "metric": "redis_command_calls_total", "refId": "A", - "step": 2 + "step": 240 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Command Calls / sec", @@ -969,10 +988,7 @@ }, "type": "graph", "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] + "show": true }, "yaxes": [ { @@ -994,66 +1010,11 @@ ] } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "New row", - "titleSize": "h6" + "title": "New row" } ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "prometheus", - "redis" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "host", - "options": [], - "query": "label_values(redis_connected_clients, instance)", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "alias", - "options": [], - "query": "label_values(redis_connected_clients{instance=\"$host\"}, alias)", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, "time": { - "from": "now-5m", + "from": "now-24h", "to": "now" }, "timepicker": { @@ -1081,7 +1042,29 @@ "30d" ] }, - "timezone": "browser", - "title": "Prometheus Redis (host & alias)", - "version": 7 + "templating": { + "list": [ + { + "current": {}, + "datasource": "${DS_PROM}", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "addr", + "options": [], + "query": "label_values(redis_up, addr)", + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "annotations": { + "list": [] + }, + "refresh": "30s", + "schemaVersion": 12, + "version": 52, + "links": [], + "gnetId": 37 } diff --git a/exporter/discovery.go b/exporter/discovery.go deleted file mode 100644 index 962776c6..00000000 --- a/exporter/discovery.go +++ /dev/null @@ -1,111 +0,0 @@ -package exporter - -import ( - "encoding/csv" - "os" - "strings" - - "github.com/cloudfoundry-community/go-cfenv" - log "github.com/sirupsen/logrus" -) - -// loadRedisArgs loads the configuration for which redis hosts to monitor from either -// the environment or as passed from program arguments. Returns the list of host addrs, -// passwords, and their aliases. -func LoadRedisArgs(addr, password, alias, separator string) ([]string, []string, []string) { - if addr == "" { - addr = "redis://localhost:6379" - } - addrs := strings.Split(addr, separator) - passwords := strings.Split(password, separator) - for len(passwords) < len(addrs) { - passwords = append(passwords, passwords[0]) - } - aliases := strings.Split(alias, separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) - } - return addrs, passwords, aliases -} - -// loadRedisFile opens the specified file and loads the configuration for which redis -// hosts to monitor. Returns the list of hosts addrs, passwords, and their aliases. -func LoadRedisFile(fileName string) ([]string, []string, []string, error) { - var addrs []string - var passwords []string - var aliases []string - file, err := os.Open(fileName) - if err != nil { - return nil, nil, nil, err - } - r := csv.NewReader(file) - r.FieldsPerRecord = -1 - records, err := r.ReadAll() - if err != nil { - return nil, nil, nil, err - } - file.Close() - // For each line, test if it contains an optional password and alias and provide them, - // else give them empty strings - for _, record := range records { - length := len(record) - switch length { - case 3: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, record[2]) - case 2: - addrs = append(addrs, record[0]) - passwords = append(passwords, record[1]) - aliases = append(aliases, "") - case 1: - addrs = append(addrs, record[0]) - passwords = append(passwords, "") - aliases = append(aliases, "") - } - } - return addrs, passwords, aliases, nil -} - -func GetCloudFoundryRedisBindings() (addrs, passwords, aliases []string) { - if !cfenv.IsRunningOnCF() { - return - } - - appEnv, err := cfenv.Current() - if err != nil { - log.Warnln("Unable to get current CF environment", err) - return - } - - redisServices, err := appEnv.Services.WithTag("redis") - if err != nil { - log.Warnln("Error while getting redis services", err) - return - } - - for _, redisService := range redisServices { - credentials := redisService.Credentials - host := getAlternative(credentials, "host", "hostname") - port := getAlternative(credentials, "port") - password := getAlternative(credentials, "password") - - addr := host + ":" + port - alias := redisService.Name - - addrs = append(addrs, addr) - passwords = append(passwords, password) - aliases = append(aliases, alias) - } - - return -} - -func getAlternative(credentials map[string]interface{}, alternatives ...string) string { - for _, key := range alternatives { - if value, ok := credentials[key]; ok { - return value.(string) - } - } - return "" -} diff --git a/exporter/discovery_test.go b/exporter/discovery_test.go deleted file mode 100644 index 4a09f2e5..00000000 --- a/exporter/discovery_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package exporter - -import ( - "log" - "testing" -) - -func cmpStringArrays(a1, a2 []string) bool { - if len(a1) != len(a2) { - return false - } - for n := range a1 { - if a1[n] != a2[n] { - return false - } - } - return true -} - -func TestLoadRedisArgs(t *testing.T) { - log.Println("TestLoadRedisArgs()") - tests := []struct { - addr, pwd, alias, sep string - wantAddr, wantPwds, wantAliases []string - }{ - { - addr: "", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{""}, - }, - { - addr: "redis://localhost:6379", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{""}, - }, - { - addr: "redis://localhost:6379,redis://localhost:7000", - sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000"}, - wantPwds: []string{"", ""}, - wantAliases: []string{"", ""}, - }, - { - addr: "redis://localhost:6379,redis://localhost:7000,redis://localhost:7001", - sep: ",", - wantAddr: []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7001"}, - wantPwds: []string{"", "", ""}, - wantAliases: []string{"", "", ""}, - }, - { - alias: "host-1", - sep: ",", - wantAddr: []string{"redis://localhost:6379"}, - wantPwds: []string{""}, - wantAliases: []string{"host-1"}, - }, - } - - for _, test := range tests { - sep := test.sep - addrs, pwds, aliases := LoadRedisArgs(test.addr, test.pwd, test.alias, sep) - if !cmpStringArrays(addrs, test.wantAddr) { - t.Errorf("addrs not matching wantAliases, got: %v want: %v", addrs, test.wantAddr) - } - if !cmpStringArrays(pwds, test.wantPwds) { - t.Errorf("pwds not matching wantAliases, got: %v want: %v", pwds, test.wantPwds) - } - if !cmpStringArrays(aliases, test.wantAliases) { - t.Errorf("aliases not matching wantAliases, got: %v want: %v", aliases, test.wantAliases) - } - } -} - -func TestLoadRedisFile(t *testing.T) { - if _, _, _, err := LoadRedisFile("doesnt-exist.txt"); err == nil { - t.Errorf("should have failed opening non existing file") - return - } - - addrs, pwds, aliases, err := LoadRedisFile("../contrib/sample_redis_hosts_file.txt") - if err != nil { - t.Errorf("LoadRedisFile() failed, err: %s", err) - return - } - log.Printf("aliases: %v \n", aliases) - if !cmpStringArrays(addrs, []string{"redis://localhost:6379", "redis://localhost:7000", "redis://localhost:7000"}) { - t.Errorf("addrs not matching want") - } - if !cmpStringArrays(pwds, []string{"", "password", "second-pwd"}) { - t.Errorf("pwds not matching want") - } - if !cmpStringArrays(aliases, []string{"", "alias", ""}) { - t.Errorf("aliases not matching want") - } -} - -func TestGetCloudFoundryRedisBindings(t *testing.T) { - GetCloudFoundryRedisBindings() -} diff --git a/exporter/redis.go b/exporter/redis.go index 6e1ea6bd..8e9131a0 100644 --- a/exporter/redis.go +++ b/exporter/redis.go @@ -1,8 +1,10 @@ package exporter import ( + "crypto/tls" "errors" "fmt" + "net/http" "net/url" "regexp" "strconv" @@ -12,17 +14,11 @@ import ( "github.com/gomodule/redigo/redis" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" prom_strutil "github.com/prometheus/prometheus/util/strutil" log "github.com/sirupsen/logrus" ) -// RedisHost represents a set of Redis Hosts to health check. -type RedisHost struct { - Addrs []string - Passwords []string - Aliases []string -} - type dbKeyPair struct { db, key string } @@ -34,34 +30,33 @@ type keyInfo struct { // Exporter implements the prometheus.Exporter interface, and exports Redis metrics. type Exporter struct { - redis RedisHost - namespace string - keys []dbKeyPair - singleKeys []dbKeyPair - keyValues *prometheus.GaugeVec - keySizes *prometheus.GaugeVec - scriptValues *prometheus.GaugeVec - duration prometheus.Gauge - scrapeErrors prometheus.Gauge - totalScrapes prometheus.Counter - metrics map[string]*prometheus.GaugeVec + sync.Mutex + redisAddr string + namespace string + keys []dbKeyPair + singleKeys []dbKeyPair - LuaScript []byte + totalScrapes prometheus.Counter + targetScrapeDuration prometheus.Summary + targetScrapeRequestErrors prometheus.Counter + + metricDescriptions map[string]*prometheus.Desc - metricsMtx sync.RWMutex - sync.RWMutex + options Options + LuaScript []byte } -type scrapeResult struct { - Name string - Value float64 - Addr string - Alias string - DB string +type Options struct { + Namespace string + ConfigCommandName string + CheckSingleKeys string + CheckKeys string + IncludeMetricTotalSysMemory bool + SkipTLSVerification bool } var ( - metricMap = map[string]string{ + metricMapGauges = map[string]string{ // # Server "uptime_in_seconds": "uptime_in_seconds", "process_id": "process_id", @@ -80,7 +75,6 @@ var ( "used_memory_rss": "memory_used_rss_bytes", "used_memory_peak": "memory_used_peak_bytes", "used_memory_lua": "memory_used_lua_bytes", - "total_system_memory": "total_system_memory_bytes", "maxmemory": "memory_max_bytes", // # Persistence @@ -108,21 +102,12 @@ var ( "aof_last_write_status": "aof_last_write_status", // # Stats - "total_connections_received": "connections_received_total", - "total_commands_processed": "commands_processed_total", - "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", - "total_net_input_bytes": "net_input_bytes_total", - "total_net_output_bytes": "net_output_bytes_total", - "instantaneous_input_kbps": "instantaneous_input_kbps", - "instantaneous_output_kbps": "instantaneous_output_kbps", - "rejected_connections": "rejected_connections_total", - "expired_keys": "expired_keys_total", - "evicted_keys": "evicted_keys_total", - "keyspace_hits": "keyspace_hits_total", - "keyspace_misses": "keyspace_misses_total", - "pubsub_channels": "pubsub_channels", - "pubsub_patterns": "pubsub_patterns", - "latest_fork_usec": "latest_fork_usec", + "instantaneous_ops_per_sec": "instantaneous_ops_per_sec", + "instantaneous_input_kbps": "instantaneous_input_kbps", + "instantaneous_output_kbps": "instantaneous_output_kbps", + "pubsub_channels": "pubsub_channels", + "pubsub_patterns": "pubsub_patterns", + "latest_fork_usec": "latest_fork_usec", // # Replication "loading": "loading_dump_file", @@ -131,12 +116,6 @@ var ( "master_last_io_seconds_ago": "master_last_io_seconds", "master_repl_offset": "master_repl_offset", - // # CPU - "used_cpu_sys": "used_cpu_sys", - "used_cpu_user": "used_cpu_user", - "used_cpu_sys_children": "used_cpu_sys_children", - "used_cpu_user_children": "used_cpu_user_children", - // # Cluster "cluster_stats_messages_sent": "cluster_messages_sent_total", "cluster_stats_messages_received": "cluster_messages_received_total", @@ -162,96 +141,62 @@ var ( "version": "version", // since tile38 version 1.14.1 } + metricMapCounters = map[string]string{ + "total_connections_received": "connections_received_total", + "total_commands_processed": "commands_processed_total", + + "rejected_connections": "rejected_connections_total", + "total_net_input_bytes": "net_input_bytes_total", + "total_net_output_bytes": "net_output_bytes_total", + + "expired_keys": "expired_keys_total", + "evicted_keys": "evicted_keys_total", + "keyspace_hits": "keyspace_hits_total", + "keyspace_misses": "keyspace_misses_total", + + "used_cpu_sys": "used_cpu_sys_seconds_total", + "used_cpu_user": "used_cpu_user_seconds_total", + "used_cpu_sys_children": "used_cpu_sys_children_seconds_total", + "used_cpu_user_children": "used_cpu_user_children_seconds_total", + } + instanceInfoFields = map[string]bool{"role": true, "redis_version": true, "redis_build_id": true, "redis_mode": true, "os": true} slaveInfoFields = map[string]bool{"master_host": true, "master_port": true, "slave_read_only": true} ) -func (e *Exporter) initGauges() { - - e.metrics = map[string]*prometheus.GaugeVec{} - e.metrics["instance_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "instance_info", - Help: "Information about the Redis instance", - }, []string{"addr", "alias", "role", "redis_version", "redis_build_id", "redis_mode", "os"}) - e.metrics["slave_info"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slave_info", - Help: "Information about the Redis slave", - }, []string{"addr", "alias", "master_host", "master_port", "read_only"}) - e.metrics["start_time_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "start_time_seconds", - Help: "Start time of the Redis instance since unix epoch in seconds.", - }, []string{"addr", "alias"}) - e.metrics["master_link_up"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "master_link_up", - Help: "Master link status on Redis slave", - }, []string{"addr", "alias"}) - e.metrics["connected_slave_offset"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_offset", - Help: "Offset of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["connected_slave_lag_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "connected_slave_lag_seconds", - Help: "Lag of connected slave", - }, []string{"addr", "alias", "slave_ip", "slave_port", "slave_state"}) - e.metrics["db_keys"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys", - Help: "Total number of keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_keys_expiring"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_keys_expiring", - Help: "Total number of expiring keys by DB", - }, []string{"addr", "alias", "db"}) - e.metrics["db_avg_ttl_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "db_avg_ttl_seconds", - Help: "Avg TTL in seconds", - }, []string{"addr", "alias", "db"}) - - // Latency info - e.metrics["latency_spike_last"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_last", - Help: "When the latency spike last occurred", - }, []string{"addr", "alias", "event_name"}) - e.metrics["latency_spike_milliseconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "latency_spike_milliseconds", - Help: "Length of the last latency spike in milliseconds", - }, []string{"addr", "alias", "event_name"}) - - e.metrics["commands_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_total", - Help: "Total number of calls per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["commands_duration_seconds_total"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "commands_duration_seconds_total", - Help: "Total amount of time in seconds spent per command", - }, []string{"addr", "alias", "cmd"}) - e.metrics["slowlog_length"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_length", - Help: "Total slowlog", - }, []string{"addr", "alias"}) - e.metrics["slowlog_last_id"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "slowlog_last_id", - Help: "Last id of slowlog", - }, []string{"addr", "alias"}) - e.metrics["last_slow_execution_duration_seconds"] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: "last_slow_execution_duration_seconds", - Help: "The amount of time needed for last slow execution, in seconds", - }, []string{"addr", "alias"}) +func (e *Exporter) ScrapeHandler(w http.ResponseWriter, r *http.Request) { + target := r.URL.Query().Get("target") + if target == "" { + http.Error(w, "'target' parameter must be specified", 400) + e.targetScrapeRequestErrors.Inc() + return + } + + start := time.Now() + + // todo: this needs a test + checkKeys := r.URL.Query().Get("check-keys") + checkSingleKey := r.URL.Query().Get("check-single-keys") + + opts := e.options + opts.CheckKeys = checkKeys + opts.CheckSingleKeys = checkSingleKey + + exp, err := NewRedisExporter(target, opts) + if err != nil { + http.Error(w, "NewRedisExporter() err: err", 400) + e.targetScrapeRequestErrors.Inc() + return + } + registry := prometheus.NewRegistry() + registry.MustRegister(exp) + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + + h.ServeHTTP(w, r) + + duration := time.Since(start).Seconds() + e.targetScrapeDuration.Observe(duration) + log.Debugf("Scrape of target '%s' took %f seconds", target, duration) } // splitKeyArgs splits a command-line supplied argument into a slice of dbKeyPairs. @@ -282,104 +227,146 @@ func parseKeyArg(keysArgString string) (keys []dbKeyPair, err error) { return keys, err } -// NewRedisExporter returns a new exporter of Redis metrics. -// note to self: next time we add an argument, instead add a RedisExporter struct -func NewRedisExporter(host RedisHost, namespace, checkSingleKeys, checkKeys string) (*Exporter, error) { +func newMetricDescr(namespace string, metricName string, docString string, labels []string) *prometheus.Desc { + return prometheus.NewDesc(prometheus.BuildFQName(namespace, "", metricName), docString, labels, nil) +} +// NewRedisExporter returns a new exporter of Redis metrics. +func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { e := Exporter{ - redis: host, - namespace: namespace, - keyValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_value", - Help: "The value of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - keySizes: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "key_size", - Help: "The length or size of \"key\"", - }, []string{"addr", "alias", "db", "key"}), - scriptValues: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "script_value", - Help: "Values returned by the collect script", - }, []string{"addr", "alias", "key"}), - duration: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_duration_seconds", - Help: "The last scrape duration.", - }), + redisAddr: redisURI, + options: opts, + namespace: opts.Namespace, + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, + Namespace: opts.Namespace, Name: "exporter_scrapes_total", Help: "Current total redis scrapes.", }), - scrapeErrors: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "exporter_last_scrape_error", - Help: "The last scrape error status.", - }), + + targetScrapeDuration: prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Name: "target_scrape_collection_duration_seconds", + Help: "Duration of collections by the SNMP exporter", + }, + ), + targetScrapeRequestErrors: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: opts.Namespace, + Name: "target_scrape_request_errors_total", + Help: "Errors in requests to the SNMP exporter", + }, + ), + } + + if e.options.ConfigCommandName == "" { + e.options.ConfigCommandName = "CONFIG" } var err error - if e.keys, err = parseKeyArg(checkKeys); err != nil { + if e.keys, err = parseKeyArg(opts.CheckKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-keys: %#v", err) } log.Debugf("keys: %#v", e.keys) - if e.singleKeys, err = parseKeyArg(checkSingleKeys); err != nil { + if e.singleKeys, err = parseKeyArg(opts.CheckSingleKeys); err != nil { return &e, fmt.Errorf("Couldn't parse check-single-keys: %#v", err) } log.Debugf("singleKeys: %#v", e.singleKeys) - e.initGauges() + if opts.IncludeMetricTotalSysMemory { + metricMapGauges["total_system_memory"] = "total_system_memory_bytes" + } + + e.metricDescriptions = map[string]*prometheus.Desc{} + e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", nil) + e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"role", "redis_version", "redis_build_id", "redis_mode", "os"}) + e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", nil) + e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"err"}) + + e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"key"}) + e.metricDescriptions["key_values"] = newMetricDescr(opts.Namespace, "key_value", `The value of "key"`, []string{"db", "key"}) + e.metricDescriptions["key_sizes"] = newMetricDescr(opts.Namespace, "key_size", `The length or size of "key"`, []string{"db", "key"}) + + e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"cmd"}) + e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"cmd"}) + e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, nil) + e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, nil) + e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, nil) + + e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"event_name"}) + e.metricDescriptions["latency_spike_seconds"] = newMetricDescr(opts.Namespace, "latency_spike_seconds", `Length of the last latency spike in seconds`, []string{"event_name"}) + + e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"master_host", "master_port", "read_only"}) + + e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", nil) + e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", nil) + e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) + e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"slave_ip", "slave_port", "slave_state"}) + + e.metricDescriptions["db_keys"] = newMetricDescr(opts.Namespace, "db_keys", "Total number of keys by DB", []string{"db"}) + e.metricDescriptions["db_keys_expiring"] = newMetricDescr(opts.Namespace, "db_keys_expiring", "Total number of expiring keys by DB", []string{"db"}) + e.metricDescriptions["db_avg_ttl_seconds"] = newMetricDescr(opts.Namespace, "db_avg_ttl_seconds", "Avg TTL in seconds", []string{"db"}) + return &e, nil } // Describe outputs Redis metric descriptions. func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { + for _, desc := range e.metricDescriptions { + ch <- desc + } - for _, m := range e.metrics { - m.Describe(ch) + for _, v := range metricMapGauges { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) + } + + for _, v := range metricMapCounters { + ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil) } - e.keySizes.Describe(ch) - e.keyValues.Describe(ch) - ch <- e.duration.Desc() ch <- e.totalScrapes.Desc() - ch <- e.scrapeErrors.Desc() + ch <- e.targetScrapeDuration.Desc() + ch <- e.targetScrapeRequestErrors.Desc() } // Collect fetches new metrics from the RedisHost and updates the appropriate metrics. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - scrapes := make(chan scrapeResult) - e.Lock() defer e.Unlock() + e.totalScrapes.Inc() - e.keySizes.Reset() - e.keyValues.Reset() + if e.redisAddr != "" { + start := time.Now().UnixNano() + var up float64 = 1 + if err := e.scrapeRedisHost(ch); err != nil { + up = 0 + e.registerGaugeValue(ch, "scrape_error", 1.0, fmt.Sprintf("%s", err)) + } else { + e.registerGaugeValue(ch, "scrape_error", 0, "") + } - e.initGauges() - go e.scrape(scrapes) - e.setMetrics(scrapes) + e.registerGaugeValue(ch, "up", up) + e.registerGaugeValue(ch, "last_scrape_duration", float64(time.Now().UnixNano()-start)/1000000000) - e.keySizes.Collect(ch) - e.keyValues.Collect(ch) - e.scriptValues.Collect(ch) + } - ch <- e.duration ch <- e.totalScrapes - ch <- e.scrapeErrors - e.collectMetrics(ch) + ch <- e.targetScrapeDuration + ch <- e.targetScrapeRequestErrors } func includeMetric(s string) bool { if strings.HasPrefix(s, "db") || strings.HasPrefix(s, "cmdstat_") || strings.HasPrefix(s, "cluster_") { return true } - _, ok := metricMap[s] + if _, ok := metricMapGauges[s]; ok { + return true + } + + _, ok := metricMapCounters[s] return ok } @@ -478,7 +465,7 @@ func parseConnectedSlaveString(slaveName string, slaveInfo string) (offset float return } -func extractConfigMetrics(config []string, addr string, alias string, scrapes chan<- scrapeResult) (dbCount int, err error) { +func (e *Exporter) extractConfigMetrics(ch chan<- prometheus.Metric, config []string) (dbCount int, err error) { if len(config)%2 != 0 { return 0, fmt.Errorf("invalid config: %#v", config) } @@ -502,13 +489,26 @@ func extractConfigMetrics(config []string, addr string, alias string, scrapes ch } if val, err := strconv.ParseFloat(strVal, 64); err == nil { - scrapes <- scrapeResult{Name: fmt.Sprintf("config_%s", config[pos*2]), Addr: addr, Alias: alias, Value: val} + e.registerGaugeValue(ch, fmt.Sprintf("config_%s", config[pos*2]), val) } } return } -func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string, val float64, labels ...string) { + e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels...) +} + +func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labelValues ...string) { + descr := e.metricDescriptions[metric] + if descr == nil { + descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", nil) + } + + ch <- prometheus.MustNewConstMetric(descr, valType, val, labelValues...) +} + +func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) { for i := 0; i < len(info); i += 2 { log.Debugf("tile38: %s:%s", info[i], info[i+1]) @@ -519,13 +519,11 @@ func (e *Exporter) extractTile38Metrics(info []string, addr string, alias string continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } - - return nil } -func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { /* Format: cmdstat_get:calls=21,usec=175,usec_per_call=8.33 @@ -552,46 +550,36 @@ func (e *Exporter) handleMetricsCommandStats(addr string, alias string, fieldKey return } - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - cmd := splitKey[1] - e.metrics["commands_total"].WithLabelValues(addr, alias, cmd).Set(calls) - e.metrics["commands_duration_seconds_total"].WithLabelValues(addr, alias, cmd).Set(usecTotal / 1e6) + e.registerMetricValue(ch, "commands_total", calls, prometheus.CounterValue, cmd) + e.registerMetricValue(ch, "commands_duration_seconds_total", usecTotal/1e6, prometheus.CounterValue, cmd) } -func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey string, fieldValue string) bool { - e.metricsMtx.RLock() - defer e.metricsMtx.RUnlock() - +func (e *Exporter) handleMetricsReplication(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) bool { // only slaves have this field if fieldKey == "master_link_status" { if fieldValue == "up" { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(1) + e.registerGaugeValue(ch, "master_link_up", 1) } else { - e.metrics["master_link_up"].WithLabelValues(addr, alias).Set(0) + e.registerGaugeValue(ch, "master_link_up", 0) } return true } // not a slave, try extracting master metrics - if slaveOffset, slaveIp, slavePort, slaveState, lag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { - e.metrics["connected_slave_offset"].WithLabelValues( - addr, - alias, - slaveIp, - slavePort, - slaveState, - ).Set(slaveOffset) - - if lag > -1 { - e.metrics["connected_slave_lag_seconds"].WithLabelValues( - addr, - alias, - slaveIp, - slavePort, - slaveState, - ).Set(lag) + if slaveOffset, slaveIP, slavePort, slaveState, slaveLag, ok := parseConnectedSlaveString(fieldKey, fieldValue); ok { + e.registerGaugeValue(ch, + "connected_slave_offset", + slaveOffset, + slaveIP, slavePort, slaveState, + ) + + if slaveLag > -1 { + e.registerGaugeValue(ch, + "connected_slave_lag_seconds", + slaveLag, + slaveIP, slavePort, slaveState, + ) } return true } @@ -599,17 +587,15 @@ func (e *Exporter) handleMetricsReplication(addr string, alias string, fieldKey return false } -func (e *Exporter) handleMetricsServer(addr string, alias string, fieldKey string, fieldValue string) { +func (e *Exporter) handleMetricsServer(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) { if fieldKey == "uptime_in_seconds" { if uptime, err := strconv.ParseFloat(fieldValue, 64); err == nil { - e.metricsMtx.RLock() - e.metrics["start_time_seconds"].WithLabelValues(addr, alias).Set(float64(time.Now().Unix()) - uptime) - e.metricsMtx.RUnlock() + e.registerGaugeValue(ch, "start_time_seconds", float64(time.Now().Unix())-uptime) } } } -func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes chan<- scrapeResult, dbCount int) error { +func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string, dbCount int) error { instanceInfo := map[string]string{} slaveInfo := map[string]string{} handledDBs := map[string]bool{} @@ -644,24 +630,26 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c switch fieldClass { case "Replication": - if ok := e.handleMetricsReplication(addr, alias, fieldKey, fieldValue); ok { + if ok := e.handleMetricsReplication(ch, fieldKey, fieldValue); ok { continue } case "Server": - e.handleMetricsServer(addr, alias, fieldKey, fieldValue) + e.handleMetricsServer(ch, fieldKey, fieldValue) case "Commandstats": - e.handleMetricsCommandStats(addr, alias, fieldKey, fieldValue) + e.handleMetricsCommandStats(ch, fieldKey, fieldValue) continue case "Keyspace": if keysTotal, keysEx, avgTTL, ok := parseDBKeyspaceString(fieldKey, fieldValue); ok { dbName := fieldKey - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: keysTotal} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: keysEx} + + e.registerGaugeValue(ch, "db_keys", keysTotal, dbName) + e.registerGaugeValue(ch, "db_keys_expiring", keysEx, dbName) + if avgTTL > -1 { - scrapes <- scrapeResult{Name: "db_avg_ttl_seconds", Addr: addr, Alias: alias, DB: dbName, Value: avgTTL} + e.registerGaugeValue(ch, "db_avg_ttl_seconds", avgTTL, dbName) } handledDBs[dbName] = true continue @@ -672,40 +660,35 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } for dbIndex := 0; dbIndex < dbCount; dbIndex++ { dbName := "db" + strconv.Itoa(dbIndex) if _, exists := handledDBs[dbName]; !exists { - scrapes <- scrapeResult{Name: "db_keys", Addr: addr, Alias: alias, DB: dbName, Value: 0} - scrapes <- scrapeResult{Name: "db_keys_expiring", Addr: addr, Alias: alias, DB: dbName, Value: 0} + e.registerGaugeValue(ch, "db_keys", 0, dbName) + e.registerGaugeValue(ch, "db_keys_expiring", 0, dbName) } } - e.metricsMtx.RLock() - e.metrics["instance_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "instance_info", 1, instanceInfo["role"], instanceInfo["redis_version"], instanceInfo["redis_build_id"], instanceInfo["redis_mode"], - instanceInfo["os"], - ).Set(1) + instanceInfo["os"]) + if instanceInfo["role"] == "slave" { - e.metrics["slave_info"].WithLabelValues( - addr, alias, + e.registerGaugeValue(ch, "slave_info", 1, slaveInfo["master_host"], slaveInfo["master_port"], - slaveInfo["slave_read_only"], - ).Set(1) + slaveInfo["slave_read_only"]) } - e.metricsMtx.RUnlock() return nil } -func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes chan<- scrapeResult) error { +func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, info string) error { lines := strings.Split(info, "\r\n") for _, line := range lines { @@ -722,16 +705,104 @@ func (e *Exporter) extractClusterInfoMetrics(info, addr, alias string, scrapes c continue } - registerMetric(addr, alias, fieldKey, fieldValue, scrapes) + e.parseAndRegisterMetric(ch, fieldKey, fieldValue) } return nil } -func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scrapeResult) error { - metricName := sanitizeMetricName(fieldKey) - if newName, ok := metricMap[metricName]; ok { +func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + log.Debugf("e.singleKeys: %#v", e.singleKeys) + allKeys := append([]dbKeyPair{}, e.singleKeys...) + + log.Debugf("e.keys: %#v", e.keys) + scannedKeys, err := getKeysFromPatterns(c, e.keys) + if err != nil { + log.Errorf("Error expanding key patterns: %#v", err) + } else { + allKeys = append(allKeys, scannedKeys...) + } + + log.Debugf("allKeys: %#v", allKeys) + for _, k := range allKeys { + if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { + log.Debugf("Couldn't select database %#v when getting key info.", k.db) + continue + } + + info, err := getKeyInfo(c, k.key) + if err != nil { + switch err { + case errNotFound: + log.Debugf("Key '%s' not found when trying to get type and size.", k.key) + default: + log.Error(err) + } + continue + } + dbLabel := "db" + k.db + e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key) + + // Only record value metric if value is float-y + if val, err := redis.Float64(c.Do("GET", k.key)); err == nil { + e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key) + } + } +} + +func (e *Exporter) extractLuaScriptMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if e.LuaScript == nil || len(e.LuaScript) == 0 { + return + } + + log.Debug("Evaluating e.LuaScript") + kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) + if err != nil { + log.Errorf("LuaScript error: %v", err) + return + } + + if kv != nil { + for key, stringVal := range kv { + if val, err := strconv.ParseFloat(stringVal, 64); err == nil { + e.registerGaugeValue(ch, "script_values", val, key) + } + } + } +} + +func (e *Exporter) extractSlowLogMetrics(ch chan<- prometheus.Metric, c redis.Conn) { + if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { + e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64))) + } + + if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { + var slowlogLastID int64 + var lastSlowExecutionDurationSeconds float64 + + if len(values) > 0 { + if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { + slowlogLastID = values[0].(int64) + if len(values) > 2 { + lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 + } + } + } + + e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID)) + e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds) + } +} + +func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, fieldKey, fieldValue string) error { + orgMetricName := sanitizeMetricName(fieldKey) + metricName := orgMetricName + if newName, ok := metricMapGauges[metricName]; ok { metricName = newName + } else { + if newName, ok := metricMapCounters[metricName]; ok { + metricName = newName + } } var err error @@ -753,7 +824,11 @@ func registerMetric(addr, alias, fieldKey, fieldValue string, scrapes chan<- scr log.Debugf("couldn't parse %s, err: %s", fieldValue, err) } - scrapes <- scrapeResult{Name: metricName, Addr: addr, Alias: alias, Value: val} + t := prometheus.GaugeValue + if metricMapCounters[orgMetricName] != "" { + t = prometheus.CounterValue + } + e.registerMetricValue(ch, metricName, val, t) return nil } @@ -782,7 +857,7 @@ func getKeyInfo(c redis.Conn, key string) (info keyInfo, err error) { return info, errNotFound case "string": if size, err := redis.Int64(c.Do("PFCOUNT", key)); err == nil { - info.keyType = "hyperloglog" + // hyperloglog info.size = float64(size) } else if size, err := redis.Int64(c.Do("STRLEN", key)); err == nil { info.size = float64(size) @@ -864,20 +939,23 @@ func getKeysFromPatterns(c redis.Conn, keys []dbKeyPair) (expandedKeys []dbKeyPa return expandedKeys, err } -func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error { +func connectToRedis(addr string, skipTLSVerification bool) (redis.Conn, error) { options := []redis.DialOption{ redis.DialConnectTimeout(5 * time.Second), redis.DialReadTimeout(5 * time.Second), redis.DialWriteTimeout(5 * time.Second), - } - if len(e.redis.Passwords) > idx && e.redis.Passwords[idx] != "" { - options = append(options, redis.DialPassword(e.redis.Passwords[idx])) + redis.DialTLSConfig(&tls.Config{ + InsecureSkipVerify: skipTLSVerification, + }), } - log.Debugf("Trying DialURL(): %s", addr) - c, err := redis.DialURL(addr, options...) - + uri := addr + if !strings.Contains(uri, "://") { + uri = "redis://" + uri + } + log.Debugf("Trying DialURL(): %s", uri) + c, err := redis.DialURL(uri, options...) if err != nil { log.Debugf("DialURL() failed, err: %s", err) if frags := strings.Split(addr, "://"); len(frags) == 2 { @@ -888,19 +966,23 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx c, err = redis.Dial("tcp", addr, options...) } } + return c, err +} +func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error { + c, err := connectToRedis(e.redisAddr, e.options.SkipTLSVerification) if err != nil { - log.Errorf("aborting for addr: %s - redis err: %s", addr, err) + log.Errorf("Couldn't connect to redis instance") + log.Debugf("connectToRedis() err: %s", err) return err } - defer c.Close() - log.Debugf("connected to: %s", addr) - dbCount := 0 + log.Debugf("connected to: %s", e.redisAddr) - if config, err := redis.Strings(c.Do("CONFIG", "GET", "*")); err == nil { - dbCount, err = extractConfigMetrics(config, addr, e.redis.Aliases[idx], scrapes) + dbCount := 0 + if config, err := redis.Strings(c.Do(e.options.ConfigCommandName, "GET", "*")); err == nil { + dbCount, err = e.extractConfigMetrics(ch, config) if err != nil { log.Errorf("Redis CONFIG err: %s", err) return err @@ -917,11 +999,10 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx return err } } - isClusterEnabled := strings.Contains(infoAll, "cluster_enabled:1") - if isClusterEnabled { + if strings.Contains(infoAll, "cluster_enabled:1") { if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil { - e.extractClusterInfoMetrics(clusterInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractClusterInfoMetrics(ch, clusterInfo) // in cluster mode Redis only supports one database so no extra padding beyond that needed dbCount = 1 @@ -936,152 +1017,35 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx } } - e.extractInfoMetrics(infoAll, addr, e.redis.Aliases[idx], scrapes, dbCount) + e.extractInfoMetrics(ch, infoAll, dbCount) // SERVER command only works on tile38 database. check the following link to // find out more: https://tile38.com/ if serverInfo, err := redis.Strings(doRedisCmd(c, "SERVER")); err == nil { - e.extractTile38Metrics(serverInfo, addr, e.redis.Aliases[idx], scrapes) + e.extractTile38Metrics(ch, serverInfo) } else { log.Debugf("Tile38 SERVER err: %s", err) } if reply, err := doRedisCmd(c, "LATENCY", "LATEST"); err == nil { var eventName string - var spikeLast, milliseconds, max int64 if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 { latencyResult := tempVal[0].([]interface{}) - if _, err := redis.Scan(latencyResult, &eventName, &spikeLast, &milliseconds, &max); err == nil { - e.metricsMtx.RLock() - e.metrics["latency_spike_last"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(spikeLast)) - e.metrics["latency_spike_milliseconds"].WithLabelValues(addr, e.redis.Aliases[idx], eventName).Set(float64(milliseconds)) - e.metricsMtx.RUnlock() + var spikeLast, spikeDuration, max int64 + if _, err := redis.Scan(latencyResult, &eventName, &spikeLast, &spikeDuration, &max); err == nil { + spikeDuration = spikeDuration / 1e6 + e.registerGaugeValue(ch, "latency_spike_last", float64(spikeLast), eventName) + e.registerGaugeValue(ch, "latency_spike_seconds", float64(spikeDuration), eventName) } } } - log.Debugf("e.singleKeys: %#v", e.singleKeys) - allKeys := append([]dbKeyPair{}, e.singleKeys...) + e.extractCheckKeyMetrics(ch, c) - log.Debugf("e.keys: %#v", e.keys) - scannedKeys, err := getKeysFromPatterns(c, e.keys) - if err != nil { - log.Errorf("Error expanding key patterns: %#v", err) - } else { - allKeys = append(allKeys, scannedKeys...) - } + e.extractLuaScriptMetrics(ch, c) - log.Debugf("allKeys: %#v", allKeys) - for _, k := range allKeys { - if _, err := doRedisCmd(c, "SELECT", k.db); err != nil { - log.Debugf("Couldn't select database %#v when getting key info.", k.db) - continue - } - - info, err := getKeyInfo(c, k.key) - if err != nil { - switch err { - case errNotFound: - log.Debugf("Key '%s' not found when trying to get type and size.", k.key) - default: - log.Error(err) - } - continue - } - dbLabel := "db" + k.db - e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(info.size) - - // Only record value metric if value is float-y - if value, err := redis.Float64(c.Do("GET", k.key)); err == nil { - e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, k.key).Set(value) - } - } - - if e.LuaScript != nil && len(e.LuaScript) > 0 { - log.Debug("e.script") - kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0)) - if err != nil { - log.Errorf("Collect script error: %v", err) - } else if kv != nil { - for key, stringVal := range kv { - if val, err := strconv.ParseFloat(stringVal, 64); err == nil { - e.scriptValues.WithLabelValues(addr, e.redis.Aliases[idx], key).Set(val) - } - } - } - } - - if reply, err := c.Do("SLOWLOG", "LEN"); err == nil { - e.metricsMtx.RLock() - e.metrics["slowlog_length"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(reply.(int64))) - e.metricsMtx.RUnlock() - } - - if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil { - var slowlogLastId int64 = 0 - var lastSlowExecutionDurationSeconds float64 = 0 - - if len(values) > 0 { - if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 { - slowlogLastId = values[0].(int64) - if len(values) > 2 { - lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6 - } - } - } - - e.metricsMtx.RLock() - e.metrics["slowlog_last_id"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(float64(slowlogLastId)) - e.metrics["last_slow_execution_duration_seconds"].WithLabelValues(addr, e.redis.Aliases[idx]).Set(lastSlowExecutionDurationSeconds) - e.metricsMtx.RUnlock() - } + e.extractSlowLogMetrics(ch, c) log.Debugf("scrapeRedisHost() done") return nil } - -func (e *Exporter) scrape(scrapes chan<- scrapeResult) { - defer close(scrapes) - - now := time.Now().UnixNano() - e.totalScrapes.Inc() - - errorCount := 0 - for idx, addr := range e.redis.Addrs { - var up float64 = 1 - if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil { - errorCount++ - up = 0 - } - scrapes <- scrapeResult{Name: "up", Addr: addr, Alias: e.redis.Aliases[idx], Value: up} - } - - e.scrapeErrors.Set(float64(errorCount)) - e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000) -} - -func (e *Exporter) setMetrics(scrapes <-chan scrapeResult) { - for scr := range scrapes { - name := scr.Name - if _, ok := e.metrics[name]; !ok { - e.metricsMtx.Lock() - e.metrics[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: e.namespace, - Name: name, - Help: name + "metric", // needs to be set for prometheus >= 2.3.1 - }, []string{"addr", "alias"}) - e.metricsMtx.Unlock() - } - var labels prometheus.Labels = map[string]string{"addr": scr.Addr, "alias": scr.Alias} - if len(scr.DB) > 0 { - labels["db"] = scr.DB - } - e.metrics[name].With(labels).Set(scr.Value) - } -} - -func (e *Exporter) collectMetrics(metrics chan<- prometheus.Metric) { - for _, m := range e.metrics { - m.Collect(metrics) - } -} diff --git a/exporter/redis_test.go b/exporter/redis_test.go index dcd8bdd3..61864df1 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -9,7 +9,6 @@ package exporter */ import ( - "flag" "fmt" "io/ioutil" "net/http" @@ -35,27 +34,25 @@ const ( ) var ( - redisAddr = flag.String("redis.addr", ":6379", "Address of the test instance, without `redis://`") - redisAlias = flag.String("redis.alias", "foo", "Alias of the test instance") - separator = flag.String("separator", ",", "separator used to split redis.addr, redis.password and redis.alias into several elements.") - - keys = []string{} - keysExpiring = []string{} - listKeys = []string{} - ts = int32(time.Now().Unix()) - defaultRedisHost = RedisHost{} + keys = []string{} + keysExpiring = []string{} + listKeys = []string{} + ts = int32(time.Now().Unix()) dbNumStr = "11" altDBNumStr = "12" dbNumStrFull = fmt.Sprintf("db%s", dbNumStr) - - TestServerURL = "" ) const ( TestSetName = "test-set" ) +func getTestExporter() *Exporter { + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test"}) + return e +} + func setupLatency(t *testing.T, addr string) error { c, err := redis.DialURL(addr) @@ -151,7 +148,7 @@ func resetSlowLog(t *testing.T, addr string) error { return nil } -func downloadUrl(t *testing.T, url string) string { +func downloadURL(t *testing.T, url string) string { log.Debugf("downloadURL() %s", url) resp, err := http.Get(url) if err != nil { @@ -166,10 +163,10 @@ func downloadUrl(t *testing.T, url string) string { } func TestLatencySpike(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() - setupLatency(t, defaultRedisHost.Addrs[0]) - defer resetLatency(t, defaultRedisHost.Addrs[0]) + setupLatency(t, os.Getenv("TEST_REDIS_URI")) + defer resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -178,24 +175,21 @@ func TestLatencySpike(t *testing.T) { }() for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { - got := &dto.Metric{} - m.Write(got) - - val := got.GetGauge().GetValue() - // Because we're dealing with latency, there might be a slight delay - // even after sleeping for a specific amount of time so checking - // to see if we're between +-5 of our expected value - if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { - t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) - } + if strings.Contains(m.Desc().String(), "latency_spike_milliseconds") { + got := &dto.Metric{} + m.Write(got) + + val := got.GetGauge().GetValue() + // Because we're dealing with latency, there might be a slight delay + // even after sleeping for a specific amount of time so checking + // to see if we're between +-5 of our expected value + if val > float64(TimeToSleep)-5 && val < float64(TimeToSleep) { + t.Errorf("values not matching, %f != %f", float64(TimeToSleep), val) } } } - resetLatency(t, defaultRedisHost.Addrs[0]) + resetLatency(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -215,13 +209,10 @@ func TestLatencySpike(t *testing.T) { func TestTile38(t *testing.T) { if os.Getenv("TEST_TILE38_URI") == "" { - t.SkipNow() + t.Skipf("TEST_TILE38_URI not set - skipping") } - e, _ := NewRedisExporter( - RedisHost{Addrs: []string{os.Getenv("TEST_TILE38_URI")}, Aliases: []string{"tile"}}, - "test", "", "", - ) + e, _ := NewRedisExporter(os.Getenv("TEST_TILE38_URI"), Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -229,22 +220,21 @@ func TestTile38(t *testing.T) { close(chM) }() - find := false + found := false for m := range chM { - switch m := m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "cpus_total") { - find = true - } + if strings.Contains(m.Desc().String(), "cpus_total") { + found = true + log.Debugf("type: %T", m) } + } - if !find { + if !found { t.Errorf("cpus_total was not found in tile38 metrics") } } func TestSlowLog(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() chM := make(chan prometheus.Metric) go func() { @@ -252,7 +242,7 @@ func TestSlowLog(t *testing.T) { close(chM) }() - oldSlowLogId := float64(0) + oldSlowLogID := float64(0) for m := range chM { switch m := m.(type) { @@ -261,13 +251,13 @@ func TestSlowLog(t *testing.T) { got := &dto.Metric{} m.Write(got) - oldSlowLogId = got.GetGauge().GetValue() + oldSlowLogID = got.GetGauge().GetValue() } } } - setupSlowLog(t, defaultRedisHost.Addrs[0]) - defer resetSlowLog(t, defaultRedisHost.Addrs[0]) + setupSlowLog(t, os.Getenv("TEST_REDIS_URI")) + defer resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -284,7 +274,7 @@ func TestSlowLog(t *testing.T) { val := got.GetGauge().GetValue() - if oldSlowLogId > val { + if oldSlowLogID > val { t.Errorf("no new slowlogs found") } } @@ -300,7 +290,7 @@ func TestSlowLog(t *testing.T) { } } - resetSlowLog(t, defaultRedisHost.Addrs[0]) + resetSlowLog(t, os.Getenv("TEST_REDIS_URI")) chM = make(chan prometheus.Metric) go func() { @@ -324,11 +314,10 @@ func TestSlowLog(t *testing.T) { } } -func setupDBKeys(t *testing.T, addr string) error { - - c, err := redis.DialURL(addr) +func setupDBKeys(t *testing.T, uri string) error { + c, err := redis.DialURL(uri) if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) + t.Errorf("couldn't setup redis for uri %s, err: %s ", uri, err) return err } defer c.Close() @@ -404,145 +393,31 @@ func deleteKeysFromDB(t *testing.T, addr string) error { } func TestHostVariations(t *testing.T) { - for _, prefix := range []string{"", "redis://", "tcp://"} { - addr := prefix + *redisAddr - host := RedisHost{Addrs: []string{addr}, Aliases: []string{""}} - e, _ := NewRedisExporter(host, "test", "", "") - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - found := 0 - for range scrapes { - found++ - } - - if found == 0 { - t.Errorf("didn't find any scrapes for host: %s", addr) - } - } -} - -func TestCountingKeys(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) + host := strings.ReplaceAll(os.Getenv("TEST_REDIS_URI"), "redis://", "") - var keysTestDB float64 - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - keysTestDB = s.Value - break - } - } - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - - scrapes = make(chan scrapeResult, 1000) - e.scrape(scrapes) - - // +1 for the one SET key - want := keysTestDB + float64(len(keys)) + float64(len(keysExpiring)) + 1 + float64(len(listKeys)) - - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - if want != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break - } - } - - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - scrapes = make(chan scrapeResult, 10000) - e.scrape(scrapes) - - for s := range scrapes { - if s.Name == "db_keys" && s.DB == dbNumStrFull { - if keysTestDB != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break - } - if s.Name == "db_avg_ttl_seconds" && s.DB == dbNumStrFull { - if keysTestDB != s.Value { - t.Errorf("values not matching, %f != %f", keysTestDB, s.Value) - } - break + for _, prefix := range []string{"", "redis://", "tcp://", ""} { + addr := prefix + host + c, err := connectToRedis(addr, true) + if err != nil { + t.Errorf("connectToRedis() err: %s", err) + continue } - } -} - -func TestExporterMetrics(t *testing.T) { - - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - - e.setMetrics(scrapes) - - want := 25 - if len(e.metrics) < want { - t.Errorf("need moar metrics, found: %d, want: %d", len(e.metrics), want) - } - - wantKeys := []string{ - "db_keys", - "db_avg_ttl_seconds", - "used_cpu_sys", - "loading_dump_file", // testing renames - "config_maxmemory", // testing config extraction - "config_maxclients", // testing config extraction - "slowlog_length", - "slowlog_last_id", - "start_time_seconds", - "uptime_in_seconds", - } - for _, k := range wantKeys { - if _, ok := e.metrics[k]; !ok { - t.Errorf("missing metrics key: %s", k) + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) } - } -} - -func TestExporterValues(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") - - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) - scrapes := make(chan scrapeResult, 10000) - e.scrape(scrapes) - - wantValues := map[string]float64{ - "db_keys_total": float64(len(keys)+len(keysExpiring)) + 1, // + 1 for the SET key - "db_expiring_keys_total": float64(len(keysExpiring)), - } - - for s := range scrapes { - if wantVal, ok := wantValues[s.Name]; ok { - if dbNumStrFull == s.DB && wantVal != s.Value { - t.Errorf("values not matching, %f != %f", wantVal, s.Value) - } - } + c.Close() } } -type tstData struct { - db string - stats string - keysTotal, keysEx, avgTTL float64 - ok bool -} - func TestKeyspaceStringParser(t *testing.T) { - tsts := []tstData{ + tsts := []struct { + db string + stats string + keysTotal, keysEx, avgTTL float64 + ok bool + }{ {db: "xxx", stats: "", ok: false}, {db: "xxx", stats: "keys=1,expires=0,avg_ttl=0", ok: false}, {db: "db0", stats: "xxx", ok: false}, @@ -609,10 +484,13 @@ func TestParseConnectedSlaveString(t *testing.T) { } func TestKeyValuesAndSizes(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter( + os.Getenv("TEST_REDIS_URI"), + Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}, + ) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -623,15 +501,10 @@ func TestKeyValuesAndSizes(t *testing.T) { want := map[string]bool{"test_key_size": false, "test_key_value": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -702,7 +575,7 @@ func TestScanForKeys(t *testing.T) { fixtures = append(fixtures, newKeyFixture("SET", key, "Rats!")) } - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -739,7 +612,7 @@ func TestScanForKeys(t *testing.T) { } func TestGetKeysFromPatterns(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") dbMain := dbNumStr dbAlt := altDBNumStr @@ -823,7 +696,7 @@ func TestGetKeysFromPatterns(t *testing.T) { } func TestGetKeyInfo(t *testing.T) { - addr := defaultRedisHost.Addrs[0] + addr := os.Getenv("TEST_REDIS_URI") db := dbNumStr c, err := redis.DialURL(addr) @@ -891,10 +764,10 @@ func TestGetKeyInfo(t *testing.T) { func TestKeySizeList(t *testing.T) { s := dbNumStrFull + "=" + listKeys[0] - e, _ := NewRedisExporter(defaultRedisHost, "test", s, "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: s}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -903,16 +776,10 @@ func TestKeySizeList(t *testing.T) { }() found := false - for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_key_size") { - found = true - break - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_key_size") { + found = true + break } } @@ -922,12 +789,12 @@ func TestKeySizeList(t *testing.T) { } func TestScript(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "", "") + e := getTestExporter() e.LuaScript = []byte(`return {"a", "11", "b", "12", "c", "13"}`) nKeys := 3 - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -936,13 +803,8 @@ func TestScript(t *testing.T) { }() for m := range chM { - switch m.(type) { - case prometheus.Gauge: - if strings.Contains(m.Desc().String(), "test_script_value") { - nKeys-- - } - default: - log.Printf("default: m: %#v", m) + if strings.Contains(m.Desc().String(), "test_script_value") { + nKeys-- } } if nKeys != 0 { @@ -951,7 +813,7 @@ func TestScript(t *testing.T) { } func TestKeyValueInvalidDB(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", "999="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: "999=" + url.QueryEscape(keys[0])}) chM := make(chan prometheus.Metric) go func() { @@ -970,7 +832,7 @@ func TestKeyValueInvalidDB(t *testing.T) { } } default: - log.Printf("default: m: %#v", m) + log.Debugf("default: m: %#v", m) } } for k, found := range dontWant { @@ -982,10 +844,10 @@ func TestKeyValueInvalidDB(t *testing.T) { } func TestCommandStats(t *testing.T) { - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e := getTestExporter() - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) chM := make(chan prometheus.Metric) go func() { @@ -996,15 +858,10 @@ func TestCommandStats(t *testing.T) { want := map[string]bool{"test_commands_duration_seconds_total": false, "test_commands_total": false} for m := range chM { - switch m.(type) { - case prometheus.Gauge: - for k := range want { - if strings.Contains(m.Desc().String(), k) { - want[k] = true - } + for k := range want { + if strings.Contains(m.Desc().String(), k) { + want[k] = true } - default: - log.Printf("default: m: %#v", m) } } for k, found := range want { @@ -1023,13 +880,13 @@ func TestHTTPEndpoint(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]), "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") tests := []string{ // metrics @@ -1038,11 +895,24 @@ func TestHTTPEndpoint(t *testing.T) { `test_key_size`, `test_instance_info`, + "db_keys", + "db_avg_ttl_seconds", + "used_cpu_sys", + "loading_dump_file", // testing renames + "config_maxmemory", // testing config extraction + "config_maxclients", // testing config extraction + "slowlog_length", + "slowlog_last_id", + "start_time_seconds", + "uptime_in_seconds", + // labels and label values - `addr="redis://` + *redisAddr, `redis_mode`, `standalone`, `cmd="get"`, + + `test_db_keys{db="db11"} 11`, + `test_db_keys_expiring{db="db11"} `, } for _, test := range tests { if !strings.Contains(body, test) { @@ -1051,103 +921,66 @@ func TestHTTPEndpoint(t *testing.T) { } } -func TestNonExistingHost(t *testing.T) { - rr := RedisHost{Addrs: []string{"unix:///tmp/doesnt.exist"}, Aliases: []string{""}} - e, _ := NewRedisExporter(rr, "test", "", "") - - chM := make(chan prometheus.Metric) - go func() { - e.Collect(chM) - close(chM) - }() - - want := map[string]float64{"test_exporter_last_scrape_error": 1.0, "test_exporter_scrapes_total": 1.0} +func TestHTTPScrapeEndpoint(t *testing.T) { + r := prometheus.NewRegistry() + prometheus.DefaultGatherer = r + prometheus.DefaultRegisterer = r - for m := range chM { + e, _ := NewRedisExporter("", Options{Namespace: "test"}) - descString := m.Desc().String() + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) + prometheus.Register(e) - switch m.(type) { - case prometheus.Counter: - - for k := range want { - if strings.Contains(descString, k) { - - g := &dto.Metric{} - m.Write(g) - - val := 0.0 - - if g.GetGauge() != nil { - val = *g.GetGauge().Value - } else if g.GetCounter() != nil { - val = *g.GetCounter().Value - } else { - continue - } - if val == want[k] { - want[k] = -1.0 - } - } - } + ts := httptest.NewServer(http.HandlerFunc(e.ScrapeHandler)) + defer ts.Close() - default: - log.Printf("default: m: %#v", m) - } + v := url.Values{} + v.Add("target", os.Getenv("TEST_REDIS_URI")) + v.Add("check-single-keys", dbNumStrFull+"="+url.QueryEscape(keys[0])) - } - for k, v := range want { - if v > 0 { - t.Errorf("didn't find %s", k) - } - } -} + u, _ := url.Parse(ts.URL) + u.RawQuery = v.Encode() -func TestMoreThanOneHost(t *testing.T) { - firstHost := defaultRedisHost.Addrs[0] - secondHostURI := os.Getenv("TEST_SECOND_REDIS_URI") - if secondHostURI == "" { - log.Printf("TEST_SECOND_REDIS_URI not set - skipping test") - t.SkipNow() - return - } + body := downloadURL(t, u.String()) - c, err := redis.DialURL(secondHostURI) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - defer c.Close() + wants := []string{ + // metrics + `test_connected_clients`, + `test_commands_processed_total`, + `test_instance_info`, - _, err = c.Do("PING") - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return - } - defer c.Close() + "db_keys", + "db_avg_ttl_seconds", + "used_cpu_sys", + "loading_dump_file", // testing renames + "config_maxmemory", // testing config extraction + "config_maxclients", // testing config extraction + "slowlog_length", + "slowlog_last_id", + "start_time_seconds", + "uptime_in_seconds", - setupDBKeys(t, firstHost) - defer deleteKeysFromDB(t, firstHost) + // labels and label values + `redis_mode`, + `standalone`, + `cmd="get"`, - setupDBKeys(t, secondHostURI) - defer deleteKeysFromDB(t, secondHostURI) + `test_key_size{db="db11",key="` + keys[0] + `"} 7`, + `test_key_value{db="db11",key="` + keys[0] + `"} 1234.56`, - _, err = c.Do("SELECT", dbNumStr) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return + `test_db_keys{db="db11"} 11`, + `test_db_keys_expiring{db="db11"} `, } - - secondHostValue := float64(5678.9) - _, err = c.Do("SET", keys[0], secondHostValue) - if err != nil { - t.Errorf("couldn't connect to second redis host, err: %s - skipping test \n", err) - return + for _, want := range wants { + if !strings.Contains(body, want) { + t.Errorf("want metrics to include %q, have:\n%s", want, body) + } } +} - twoHostCfg := RedisHost{Addrs: []string{firstHost, secondHostURI}, Aliases: []string{"", ""}} - checkKey := dbNumStrFull + "=" + url.QueryEscape(keys[0]) - e, _ := NewRedisExporter(twoHostCfg, "test", checkKey, "") +func TestNonExistingHost(t *testing.T) { + e, _ := NewRedisExporter("unix:///tmp/doesnt.exist", Options{Namespace: "test"}) chM := make(chan prometheus.Metric) go func() { @@ -1155,37 +988,33 @@ func TestMoreThanOneHost(t *testing.T) { close(chM) }() - want := map[string]float64{ - firstHost: TestValue, - secondHostURI: secondHostValue, - } + want := map[string]float64{"test_exporter_last_scrape_error": 1.0, "test_exporter_scrapes_total": 1.0} for m := range chM { + descString := m.Desc().String() + for k := range want { + if strings.Contains(descString, k) { + g := &dto.Metric{} + m.Write(g) + val := 0.0 + + if g.GetGauge() != nil { + val = *g.GetGauge().Value + } else if g.GetCounter() != nil { + val = *g.GetCounter().Value + } else { + continue + } - switch m.(type) { - case prometheus.Gauge: - pb := &dto.Metric{} - m.Write(pb) - - if !strings.Contains(m.Desc().String(), "test_key_value") { - continue - } - - for _, l := range pb.GetLabel() { - for lbl, val := range want { - if l.GetName() == "addr" && l.GetValue() == lbl && pb.GetGauge().GetValue() == val { - want[lbl] = -1 - } + if val == want[k] { + want[k] = -1.0 } } - default: - log.Printf("default: m: %#v", m) } } - - for lbl, val := range want { - if val > 0 { - t.Errorf("Never found value for: %s", lbl) + for k, v := range want { + if v > 0 { + t.Errorf("didn't find %s", k) } } } @@ -1211,10 +1040,10 @@ func TestKeysReset(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+keys[0], "") + e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + keys[0]}) - setupDBKeys(t, defaultRedisHost.Addrs[0]) - defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + setupDBKeys(t, os.Getenv("TEST_REDIS_URI")) + defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) prometheus.Register(e) @@ -1224,14 +1053,14 @@ func TestKeysReset(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") if !strings.Contains(body, keys[0]) { t.Errorf("Did not found key %q\n%s", keys[0], body) } - deleteKeysFromDB(t, defaultRedisHost.Addrs[0]) + deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI")) - body = downloadUrl(t, ts.URL+"/metrics") + body = downloadURL(t, ts.URL+"/metrics") if strings.Contains(body, keys[0]) { t.Errorf("Metric is present in metrics list %q\n%s", keys[0], body) } @@ -1239,9 +1068,7 @@ func TestKeysReset(t *testing.T) { func TestClusterMaster(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping") } r := prometheus.NewRegistry() @@ -1251,10 +1078,8 @@ func TestClusterMaster(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"master"}} - log.Printf("master - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") + e, _ := NewRedisExporter(addr, Options{Namespace: "test"}) prometheus.Register(e) @@ -1264,10 +1089,10 @@ func TestClusterMaster(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("master - body: %s", body) + body := downloadURL(t, ts.URL+"/metrics") + log.Debugf("master - body: %s", body) for _, want := range []string{ - "test_instance_info{addr=\"redis://redis-cluster:7000\",alias=\"master\"", + "test_instance_info{", "test_master_repl_offset", } { if !strings.Contains(body, want) { @@ -1277,39 +1102,16 @@ func TestClusterMaster(t *testing.T) { } func TestPasswordProtectedInstance(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{testPwd} - - setupDBKeys(t, host.Addrs[0]) - - // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) - if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) - return - } - defer c.Close() - - if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil { - t.Fatalf("error setting password, err: %s", err) - } - c.Flush() - - defer func() { - if _, err = c.Do("auth", testPwd); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - deleteKeysFromDB(t, host.Addrs[0]) - }() - e, _ := NewRedisExporter(host, "test", "", "") + uri := os.Getenv("TEST_PWD_REDIS_URI") + setupDBKeys(t, uri) + e, _ := NewRedisExporter(uri, Options{Namespace: "test"}) prometheus.Register(e) chM := make(chan prometheus.Metric, 10000) @@ -1318,7 +1120,7 @@ func TestPasswordProtectedInstance(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") + body := downloadURL(t, ts.URL+"/metrics") if !strings.Contains(body, "test_up") { t.Errorf("error, missing test_up") @@ -1326,6 +1128,9 @@ func TestPasswordProtectedInstance(t *testing.T) { } func TestPasswordInvalid(t *testing.T) { + if os.Getenv("TEST_PWD_REDIS_URI") == "" { + t.Skipf("TEST_PWD_REDIS_URI not set - skipping") + } r := prometheus.NewRegistry() prometheus.DefaultGatherer = r prometheus.DefaultRegisterer = r @@ -1333,35 +1138,10 @@ func TestPasswordInvalid(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - testPwd := "p4$$w0rd" - host := defaultRedisHost - host.Passwords = []string{"wrong_password"} + testPwd := "redis-password" + uri := strings.Replace(os.Getenv("TEST_PWD_REDIS_URI"), testPwd, "wrong-pwd", -1) - setupDBKeys(t, host.Addrs[0]) - - // set password for redis instance - c, err := redis.DialURL(host.Addrs[0]) - if err != nil { - t.Errorf("couldn't setup redis, err: %s ", err) - return - } - defer c.Close() - - if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil { - t.Fatalf("error setting password, err: %s", err) - } - c.Flush() - - defer func() { - if _, err = c.Do("auth", testPwd); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil { - t.Fatalf("error unsetting password, err: %s", err) - } - deleteKeysFromDB(t, host.Addrs[0]) - }() - e, _ := NewRedisExporter(host, "test", "", "") + e, _ := NewRedisExporter(uri, Options{Namespace: "test"}) prometheus.Register(e) @@ -1371,17 +1151,16 @@ func TestPasswordInvalid(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - if !strings.Contains(body, "test_exporter_last_scrape_error 1") { - t.Errorf(`error, expected string "test_exporter_last_scrape_error 1" in body`) + want := `test_exporter_last_scrape_error{err="dial redis: unknown network redis"} 1` + body := downloadURL(t, ts.URL+"/metrics") + if !strings.Contains(body, want) { + t.Errorf(`error, expected string "%s" in body, got body: \n\n%s`, want, body) } } func TestClusterSlave(t *testing.T) { if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" { - log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") - t.SkipNow() - return + t.Skipf("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping") } r := prometheus.NewRegistry() @@ -1391,10 +1170,8 @@ func TestClusterSlave(t *testing.T) { ts := httptest.NewServer(promhttp.Handler()) defer ts.Close() - addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") - host := RedisHost{Addrs: []string{addr}, Aliases: []string{"slave"}} - log.Printf("slave - using host cfg: %#v", host) - e, _ := NewRedisExporter(host, "test", "", "") + addr := os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") + e, _ := NewRedisExporter(addr, Options{Namespace: "test"}) prometheus.Register(e) @@ -1404,12 +1181,12 @@ func TestClusterSlave(t *testing.T) { close(chM) }() - body := downloadUrl(t, ts.URL+"/metrics") - // log.Printf("slave - body: %s", body) + body := downloadURL(t, ts.URL+"/metrics") + log.Debugf("slave - body: %s", body) for _, want := range []string{ "test_instance_info", "test_master_last_io_seconds", - "test_slave_info{addr=\"redis://redis-cluster:7005\",alias=\"slave\",", + "test_slave_info", } { if !strings.Contains(body, want) { t.Errorf("Did not find key [%s] \nbody: %s", want, body) @@ -1430,8 +1207,7 @@ func TestCheckKeys(t *testing.T) { {"wrong=wrong=1", "", false}, {"", "wrong=wrong=2", false}, } { - - _, err := NewRedisExporter(defaultRedisHost, "test", tst.SingleCheckKey, tst.CheckKeys) + _, err := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", CheckSingleKeys: tst.SingleCheckKey, CheckKeys: tst.CheckKeys}) if tst.ExpectSuccess && err != nil { t.Errorf("Expected success for test: %#v, got err: %s", tst, err) return @@ -1445,8 +1221,6 @@ func TestCheckKeys(t *testing.T) { } func init() { - flag.Parse() - ll := strings.ToLower(os.Getenv("LOG_LEVEL")) if pl, err := log.ParseLevel(ll); err == nil { log.Printf("Setting log level to: %s", ll) @@ -1466,17 +1240,4 @@ func init() { key := fmt.Sprintf("key_exp_%s_%d", n, ts) keysExpiring = append(keysExpiring, key) } - - addrs := strings.Split(*redisAddr, *separator) - if len(addrs) == 0 || len(addrs[0]) == 0 { - log.Fatal("Invalid parameter --redis.addr") - } - - aliases := strings.Split(*redisAlias, *separator) - for len(aliases) < len(addrs) { - aliases = append(aliases, aliases[0]) - } - - log.Printf("Using redis addrs: %#v", addrs) - defaultRedisHost = RedisHost{Addrs: []string{"redis://" + *redisAddr}, Aliases: aliases} } diff --git a/go.mod b/go.mod index a58ef6a2..65fa675b 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,9 @@ module github.com/oliver006/redis_exporter go 1.12 require ( - github.com/cloudfoundry-community/go-cfenv v1.18.0 github.com/golang/protobuf v1.3.1 // indirect github.com/gomodule/redigo v2.0.0+incompatible github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/prometheus/common v0.2.0 // indirect @@ -15,8 +13,5 @@ require ( github.com/prometheus/prometheus v2.5.0+incompatible github.com/sirupsen/logrus v1.4.1 github.com/stretchr/testify v1.3.0 // indirect - golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index b7ae790c..c4bca3d1 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,9 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/cloudfoundry-community/go-cfenv v1.18.0 h1:dOIRSHUSaj4r6Q9Cx+nzz2OytHt+QNKqtOuKTQsa+zw= -github.com/cloudfoundry-community/go-cfenv v1.18.0/go.mod h1:qGMSI6lygPzqugFs9M1NFjJBtEPgl0MgT6drMFZGUoU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -18,30 +14,14 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -60,8 +40,6 @@ github.com/prometheus/procfs v0.0.0-20190403104016-ea9eea638872 h1:0aNv3xC7DmQoy github.com/prometheus/procfs v0.0.0-20190403104016-ea9eea638872/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= -github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -71,31 +49,14 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 5e645085..c55341ff 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,6 @@ import ( "os" "runtime" "strconv" - "strings" "github.com/oliver006/redis_exporter/exporter" "github.com/prometheus/client_golang/prometheus" @@ -38,23 +37,19 @@ func getEnvBool(key string) (envValBool bool) { func main() { var ( - redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of one or more redis nodes, separated by separator") - redisFile = flag.String("redis.file", getEnv("REDIS_FILE", ""), "Path to file containing one or more redis nodes, separated by newline. NOTE: mutually exclusive with redis.addr") - redisPassword = flag.String("redis.password", getEnv("REDIS_PASSWORD", ""), "Password for one or more redis nodes, separated by separator") - redisPasswordFile = flag.String("redis.password-file", getEnv("REDIS_PASSWORD_FILE", ""), "File containing the password for one or more redis nodes, separated by separator. NOTE: mutually exclusive with redis.password") - redisAlias = flag.String("redis.alias", getEnv("REDIS_ALIAS", ""), "Redis instance alias for one or more redis nodes, separated by separator") - namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") - checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") - checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") - scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") - separator = flag.String("separator", getEnv("REDIS_EXPORTER_SEPARATOR", ","), "separator used to split redis.addr, redis.password and redis.alias into several elements.") - listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") - metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") - logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") - isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") - showVersion = flag.Bool("version", false, "Show version information and exit") - useCfBindings = flag.Bool("use-cf-bindings", getEnvBool("REDIS_EXPORTER_USE-CF-BINDINGS"), "Use Cloud Foundry service bindings") - redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to export go runtime metrics also") + redisAddr = flag.String("redis.addr", getEnv("REDIS_ADDR", ""), "Address of the redis instsance to scrape") + namespace = flag.String("namespace", getEnv("REDIS_EXPORTER_NAMESPACE", "redis"), "Namespace for metrics") + checkKeys = flag.String("check-keys", getEnv("REDIS_EXPORTER_CHECK_KEYS", ""), "Comma separated list of key-patterns to export value and length/size, searched for with SCAN") + checkSingleKeys = flag.String("check-single-keys", getEnv("REDIS_EXPORTER_CHECK_SINGLE_KEYS", ""), "Comma separated list of single keys to export value and length/size") + scriptPath = flag.String("script", getEnv("REDIS_EXPORTER_SCRIPT", ""), "Path to Lua Redis script for collecting extra metrics") + listenAddress = flag.String("web.listen-address", getEnv("REDIS_EXPORTER_WEB_LISTEN_ADDRESS", ":9121"), "Address to listen on for web interface and telemetry.") + metricPath = flag.String("web.telemetry-path", getEnv("REDIS_EXPORTER_WEB_TELEMETRY_PATH", "/metrics"), "Path under which to expose metrics.") + logFormat = flag.String("log-format", getEnv("REDIS_EXPORTER_LOG_FORMAT", "txt"), "Log format, valid options are txt and json") + configCommand = flag.String("config-command", getEnv("REDIS_EXPORTER_CONFIG_COMMAND", "CONFIG"), "What to use for the CONFIG command") + isDebug = flag.Bool("debug", getEnvBool("REDIS_EXPORTER_DEBUG"), "Output verbose debug information") + showVersion = flag.Bool("version", false, "Show version information and exit") + redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS"), "Whether to also export go runtime metrics") + inclMetricTotalSysMemory = flag.Bool("incl-metric-total-system-memory", getEnvBool("REDIS_EXPORTER_INCL_METRIC_TOTAL_SYS_MEM"), "Whether to include the redis_total_system_memory_bytes metric") ) flag.Parse() @@ -79,44 +74,15 @@ func main() { return } - if *redisFile != "" && *redisAddr != "" { - log.Fatal("Cannot specify both redis.addr and redis.file") - } - - var parsedRedisPassword string - - if *redisPasswordFile != "" { - if *redisPassword != "" { - log.Fatal("Cannot specify both redis.password and redis.password-file") - } - b, err := ioutil.ReadFile(*redisPasswordFile) - if err != nil { - log.Fatal(err) - } - parsedRedisPassword = strings.TrimSpace(string(b)) - } else { - parsedRedisPassword = *redisPassword - } - - var addrs, passwords, aliases []string - - switch { - case *redisFile != "": - var err error - if addrs, passwords, aliases, err = exporter.LoadRedisFile(*redisFile); err != nil { - log.Fatal(err) - } - case *useCfBindings: - addrs, passwords, aliases = exporter.GetCloudFoundryRedisBindings() - default: - addrs, passwords, aliases = exporter.LoadRedisArgs(*redisAddr, parsedRedisPassword, *redisAlias, *separator) - } - exp, err := exporter.NewRedisExporter( - exporter.RedisHost{Addrs: addrs, Passwords: passwords, Aliases: aliases}, - *namespace, - *checkSingleKeys, - *checkKeys, + *redisAddr, + exporter.Options{ + Namespace: *namespace, + ConfigCommandName: *configCommand, + CheckKeys: *checkKeys, + CheckSingleKeys: *checkSingleKeys, + IncludeMetricTotalSysMemory: *inclMetricTotalSysMemory, + }, ) if err != nil { log.Fatal(err) @@ -145,9 +111,10 @@ func main() { http.Handle(*metricPath, promhttp.Handler()) } + http.HandleFunc("/scrape", exp.ScrapeHandler) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(` - + w.Write([]byte(` Redis Exporter v` + BuildVersion + `

Redis Exporter ` + BuildVersion + `

@@ -157,8 +124,7 @@ func main() { `)) }) - log.Printf("Providing metrics at %s%s", *listenAddress, *metricPath) - log.Printf("Connecting to redis hosts: %#v", addrs) - log.Printf("Using alias: %#v", aliases) + log.Infof("Providing metrics at %s%s", *listenAddress, *metricPath) + log.Debugf("Connecting to redis hosts: %#v", *redisAddr) log.Fatal(http.ListenAndServe(*listenAddress, nil)) } diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/.gitignore b/vendor/github.com/cloudfoundry-community/go-cfenv/.gitignore deleted file mode 100644 index 33cdeddb..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -vendor -Godeps diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/.travis.yml b/vendor/github.com/cloudfoundry-community/go-cfenv/.travis.yml deleted file mode 100644 index fbf9ebd1..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go -sudo: false - -env: - - GO111MODULE=on - -go: - - 1.11.x - - 1.12.x - - tip - -script: - - go build ./... - - go test -race ./... - - go test -cover ./... diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/LICENSE b/vendor/github.com/cloudfoundry-community/go-cfenv/LICENSE deleted file mode 100644 index e14591c2..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014-Present Joe Fitzgerald - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/README.md b/vendor/github.com/cloudfoundry-community/go-cfenv/README.md deleted file mode 100644 index 5d0ac32b..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Go CF Environment Package [![Build Status - Master](https://travis-ci.org/cloudfoundry-community/go-cfenv.svg?branch=master)](https://travis-ci.org/cloudfoundry-community/go-cfenv) - -### Overview - -[![GoDoc](https://godoc.org/github.com/cloudfoundry-community/go-cfenv?status.png)](https://godoc.org/github.com/cloudfoundry-community/go-cfenv) - -`cfenv` is a package to assist you in writing Go apps that run on [Cloud Foundry](http://cloudfoundry.org). It provides convenience functions and structures that map to Cloud Foundry environment variable primitives (http://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html). - -### Usage - -`go get github.com/cloudfoundry-community/go-cfenv` - -```go -package main - -import ( - "github.com/cloudfoundry-community/go-cfenv" -) - -func main() { - appEnv, _ := cfenv.Current() - - fmt.Println("ID:", appEnv.ID) - fmt.Println("Index:", appEnv.Index) - fmt.Println("Name:", appEnv.Name) - fmt.Println("Host:", appEnv.Host) - fmt.Println("Port:", appEnv.Port) - fmt.Println("Version:", appEnv.Version) - fmt.Println("Home:", appEnv.Home) - fmt.Println("MemoryLimit:", appEnv.MemoryLimit) - fmt.Println("WorkingDir:", appEnv.WorkingDir) - fmt.Println("TempDir:", appEnv.TempDir) - fmt.Println("User:", appEnv.User) - fmt.Println("Services:", appEnv.Services) -} -``` - -### Contributing - -Pull requests welcomed. diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/application.go b/vendor/github.com/cloudfoundry-community/go-cfenv/application.go deleted file mode 100644 index de5888ff..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/application.go +++ /dev/null @@ -1,30 +0,0 @@ -package cfenv - -// An App holds information about the current app running on Cloud Foundry -type App struct { - ID string `json:"-"` // DEPRECATED id of the instance - InstanceID string `json:"instance_id"` // id of the instance - AppID string `json:"application_id"` // id of the application - Index int `json:"instance_index"` // index of the app - Name string `json:"name"` // name of the app - Host string `json:"host"` // host of the app - Port int `json:"port"` // port of the app - Version string `json:"version"` // version of the app - ApplicationURIs []string `json:"application_uris"` // application uri of the app - SpaceID string `json:"space_id"` // id of the space - SpaceName string `json:"space_name"` // name of the space - Home string // root folder for the deployed app - MemoryLimit string // maximum amount of memory that each instance of the application can consume - WorkingDir string // present working directory, where the buildpack that processed the application ran - TempDir string // directory location where temporary and staging files are stored - User string // user account under which the DEA runs - Services Services // services bound to the app - CFAPI string `json:"cf_api"` // URL for the Cloud Foundry API endpoint - Limits *Limits `json:"limits"` // limits imposed on this process -} - -type Limits struct { - Disk int `json:"disk"` // disk limit - FDs int `json:"fds"` // file descriptors limit - Mem int `json:"mem"` // memory limit -} diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/cfenv.go b/vendor/github.com/cloudfoundry-community/go-cfenv/cfenv.go deleted file mode 100644 index d4808b37..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/cfenv.go +++ /dev/null @@ -1,59 +0,0 @@ -// Package cfenv provides information about the current app deployed on Cloud Foundry, including any bound service(s). -package cfenv - -import ( - "encoding/json" - "os" - "strconv" - "strings" - - "github.com/mitchellh/mapstructure" -) - -// New creates a new App with the provided environment. -func New(env map[string]string) (*App, error) { - var app App - appVar := env["VCAP_APPLICATION"] - if err := json.Unmarshal([]byte(appVar), &app); err != nil { - return nil, err - } - // duplicate the InstanceID to the previously named ID field for backwards - // compatibility - app.ID = app.InstanceID - - app.Home = env["HOME"] - app.MemoryLimit = env["MEMORY_LIMIT"] - if port, err := strconv.Atoi(env["PORT"]); err == nil { - app.Port = port - } - app.WorkingDir = env["PWD"] - app.TempDir = env["TMPDIR"] - app.User = env["USER"] - - var rawServices map[string]interface{} - servicesVar := env["VCAP_SERVICES"] - if err := json.Unmarshal([]byte(servicesVar), &rawServices); err != nil { - return nil, err - } - - services := make(map[string][]Service) - for k, v := range rawServices { - var serviceInstances []Service - if err := mapstructure.WeakDecode(v, &serviceInstances); err != nil { - return nil, err - } - services[k] = serviceInstances - } - app.Services = services - return &app, nil -} - -// Current creates a new App with the current environment; returns an error if the current environment is not a Cloud Foundry environment -func Current() (*App, error) { - return New(CurrentEnv()) -} - -// IsRunningOnCF returns true if the current environment is Cloud Foundry and false if it is not Cloud Foundry -func IsRunningOnCF() bool { - return strings.TrimSpace(os.Getenv("VCAP_APPLICATION")) != "" -} diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/environment.go b/vendor/github.com/cloudfoundry-community/go-cfenv/environment.go deleted file mode 100644 index 18180e55..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/environment.go +++ /dev/null @@ -1,16 +0,0 @@ -package cfenv - -import ( - "os" -) - -// CurrentEnv translates the current environment to a map[string]string. -func CurrentEnv() map[string]string { - return Env(os.Environ()) -} - -// Env translates the provided environment to a map[string]string. -func Env(env []string) map[string]string { - vars := mapEnv(env, splitEnv) - return vars -} diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/envmap.go b/vendor/github.com/cloudfoundry-community/go-cfenv/envmap.go deleted file mode 100644 index a3d88eb8..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/envmap.go +++ /dev/null @@ -1,20 +0,0 @@ -package cfenv - -import "strings" - -// splitEnv splits item, a key=value string, into its key and value components. -func splitEnv(item string) (key, val string) { - splits := strings.Split(item, "=") - key = splits[0] - val = strings.Join(splits[1:], "=") - return -} - -func mapEnv(data []string, keyFunc func(item string) (key, val string)) map[string]string { - items := make(map[string]string) - for _, item := range data { - key, val := keyFunc(item) - items[key] = val - } - return items -} diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/go.mod b/vendor/github.com/cloudfoundry-community/go-cfenv/go.mod deleted file mode 100644 index dbf78c42..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/go.mod +++ /dev/null @@ -1,11 +0,0 @@ -module github.com/cloudfoundry-community/go-cfenv - -go 1.11 - -require ( - github.com/joefitzgerald/rainbow-reporter v0.1.0 - github.com/mitchellh/mapstructure v1.1.2 - github.com/onsi/ginkgo v1.8.0 // indirect - github.com/onsi/gomega v1.5.0 - github.com/sclevine/spec v1.2.0 -) diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/go.sum b/vendor/github.com/cloudfoundry-community/go-cfenv/go.sum deleted file mode 100644 index 9c59bb2b..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/go.sum +++ /dev/null @@ -1,33 +0,0 @@ -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/cloudfoundry-community/go-cfenv/service.go b/vendor/github.com/cloudfoundry-community/go-cfenv/service.go deleted file mode 100644 index b1ce3bef..00000000 --- a/vendor/github.com/cloudfoundry-community/go-cfenv/service.go +++ /dev/null @@ -1,127 +0,0 @@ -package cfenv - -import ( - "fmt" - "regexp" - "strings" -) - -// Service describes a bound service. For bindable services Cloud Foundry will -// add connection details to the VCAP_SERVICES environment variable when you -// restart your application, after binding a service instance to your -// application. -// -// The results are returned as a JSON document that contains an object for each -// service for which one or more instances are bound to the application. The -// service object contains a child object for each service instance of that -// service that is bound to the application. -type Service struct { - Name string // name of the service - Label string // label of the service - Tags []string // tags for the service - Plan string // plan of the service - Credentials map[string]interface{} // credentials for the service - VolumeMounts []map[string]string `mapstructure:"volume_mounts"` // volume mount info as provided by the nfsbroker -} - -func (s *Service) CredentialString(key string) (string, bool) { - credential, ok := s.Credentials[key].(string) - return credential, ok -} - -// Services is an association of service labels to a slice of services with that -// label. -type Services map[string][]Service - -// WithTag finds services with the specified tag. -func (s *Services) WithTag(tag string) ([]Service, error) { - result := []Service{} - for _, services := range *s { - for i := range services { - service := services[i] - for _, t := range service.Tags { - if strings.EqualFold(tag, t) { - result = append(result, service) - break - } - } - } - } - - if len(result) > 0 { - return result, nil - } - - return nil, fmt.Errorf("no services with tag %s", tag) -} - -// WithTag finds services with a tag pattern. -func (s *Services) WithTagUsingPattern(tagPattern string) ([]Service, error) { - result := []Service{} - for _, services := range *s { - for i := range services { - service := services[i] - for _, t := range service.Tags { - if s.match(tagPattern, t) { - result = append(result, service) - break - } - } - } - } - - if len(result) > 0 { - return result, nil - } - - return nil, fmt.Errorf("no services with tag pattern %s", tagPattern) -} - -// WithLabel finds the service with the specified label. -func (s *Services) WithLabel(label string) ([]Service, error) { - for l, services := range *s { - if strings.EqualFold(label, l) { - return services, nil - } - } - - return nil, fmt.Errorf("no services with label %s", label) -} -func (s *Services) match(matcher, content string) bool { - regex, err := regexp.Compile("(?i)^" + matcher + "$") - if err != nil { - return false - } - return regex.MatchString(content) -} - -// WithName finds the service with a name pattern. -func (s *Services) WithNameUsingPattern(namePattern string) ([]Service, error) { - result := []Service{} - for _, services := range *s { - for i := range services { - service := services[i] - if s.match(namePattern, service.Name) { - result = append(result, service) - } - } - } - if len(result) > 0 { - return result, nil - } - return nil, fmt.Errorf("no service with name pattern %s", namePattern) -} - -// WithName finds the service with the specified name. -func (s *Services) WithName(name string) (*Service, error) { - for _, services := range *s { - for i := range services { - service := services[i] - if strings.EqualFold(name, service.Name) { - return &service, nil - } - } - } - - return nil, fmt.Errorf("no service with name %s", name) -} diff --git a/vendor/github.com/mitchellh/mapstructure/.travis.yml b/vendor/github.com/mitchellh/mapstructure/.travis.yml deleted file mode 100644 index 1689c7d7..00000000 --- a/vendor/github.com/mitchellh/mapstructure/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: go - -go: - - "1.11.x" - - tip - -script: - - go test diff --git a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md deleted file mode 100644 index 3b3cb723..00000000 --- a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md +++ /dev/null @@ -1,21 +0,0 @@ -## 1.1.2 - -* Fix error when decode hook decodes interface implementation into interface - type. [GH-140] - -## 1.1.1 - -* Fix panic that can happen in `decodePtr` - -## 1.1.0 - -* Added `StringToIPHookFunc` to convert `string` to `net.IP` and `net.IPNet` [GH-133] -* Support struct to struct decoding [GH-137] -* If source map value is nil, then destination map value is nil (instead of empty) -* If source slice value is nil, then destination slice value is nil (instead of empty) -* If source pointer is nil, then destination pointer is set to nil (instead of - allocated zero value of type) - -## 1.0.0 - -* Initial tagged stable release. diff --git a/vendor/github.com/mitchellh/mapstructure/LICENSE b/vendor/github.com/mitchellh/mapstructure/LICENSE deleted file mode 100644 index f9c841a5..00000000 --- a/vendor/github.com/mitchellh/mapstructure/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/mapstructure/README.md b/vendor/github.com/mitchellh/mapstructure/README.md deleted file mode 100644 index 0018dc7d..00000000 --- a/vendor/github.com/mitchellh/mapstructure/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure) - -mapstructure is a Go library for decoding generic map values to structures -and vice versa, while providing helpful error handling. - -This library is most useful when decoding values from some data stream (JSON, -Gob, etc.) where you don't _quite_ know the structure of the underlying data -until you read a part of it. You can therefore read a `map[string]interface{}` -and use this library to decode it into the proper underlying native Go -structure. - -## Installation - -Standard `go get`: - -``` -$ go get github.com/mitchellh/mapstructure -``` - -## Usage & Example - -For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure). - -The `Decode` function has examples associated with it there. - -## But Why?! - -Go offers fantastic standard libraries for decoding formats such as JSON. -The standard method is to have a struct pre-created, and populate that struct -from the bytes of the encoded format. This is great, but the problem is if -you have configuration or an encoding that changes slightly depending on -specific fields. For example, consider this JSON: - -```json -{ - "type": "person", - "name": "Mitchell" -} -``` - -Perhaps we can't populate a specific structure without first reading -the "type" field from the JSON. We could always do two passes over the -decoding of the JSON (reading the "type" first, and the rest later). -However, it is much simpler to just decode this into a `map[string]interface{}` -structure, read the "type" key, then use something like this library -to decode it into the proper structure. diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go deleted file mode 100644 index 1f0abc65..00000000 --- a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go +++ /dev/null @@ -1,217 +0,0 @@ -package mapstructure - -import ( - "errors" - "fmt" - "net" - "reflect" - "strconv" - "strings" - "time" -) - -// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns -// it into the proper DecodeHookFunc type, such as DecodeHookFuncType. -func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc { - // Create variables here so we can reference them with the reflect pkg - var f1 DecodeHookFuncType - var f2 DecodeHookFuncKind - - // Fill in the variables into this interface and the rest is done - // automatically using the reflect package. - potential := []interface{}{f1, f2} - - v := reflect.ValueOf(h) - vt := v.Type() - for _, raw := range potential { - pt := reflect.ValueOf(raw).Type() - if vt.ConvertibleTo(pt) { - return v.Convert(pt).Interface() - } - } - - return nil -} - -// DecodeHookExec executes the given decode hook. This should be used -// since it'll naturally degrade to the older backwards compatible DecodeHookFunc -// that took reflect.Kind instead of reflect.Type. -func DecodeHookExec( - raw DecodeHookFunc, - from reflect.Type, to reflect.Type, - data interface{}) (interface{}, error) { - switch f := typedDecodeHook(raw).(type) { - case DecodeHookFuncType: - return f(from, to, data) - case DecodeHookFuncKind: - return f(from.Kind(), to.Kind(), data) - default: - return nil, errors.New("invalid decode hook signature") - } -} - -// ComposeDecodeHookFunc creates a single DecodeHookFunc that -// automatically composes multiple DecodeHookFuncs. -// -// The composed funcs are called in order, with the result of the -// previous transformation. -func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data interface{}) (interface{}, error) { - var err error - for _, f1 := range fs { - data, err = DecodeHookExec(f1, f, t, data) - if err != nil { - return nil, err - } - - // Modify the from kind to be correct with the new data - f = nil - if val := reflect.ValueOf(data); val.IsValid() { - f = val.Type() - } - } - - return data, nil - } -} - -// StringToSliceHookFunc returns a DecodeHookFunc that converts -// string to []string by splitting on the given sep. -func StringToSliceHookFunc(sep string) DecodeHookFunc { - return func( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - if f != reflect.String || t != reflect.Slice { - return data, nil - } - - raw := data.(string) - if raw == "" { - return []string{}, nil - } - - return strings.Split(raw, sep), nil - } -} - -// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts -// strings to time.Duration. -func StringToTimeDurationHookFunc() DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data interface{}) (interface{}, error) { - if f.Kind() != reflect.String { - return data, nil - } - if t != reflect.TypeOf(time.Duration(5)) { - return data, nil - } - - // Convert it by parsing - return time.ParseDuration(data.(string)) - } -} - -// StringToIPHookFunc returns a DecodeHookFunc that converts -// strings to net.IP -func StringToIPHookFunc() DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data interface{}) (interface{}, error) { - if f.Kind() != reflect.String { - return data, nil - } - if t != reflect.TypeOf(net.IP{}) { - return data, nil - } - - // Convert it by parsing - ip := net.ParseIP(data.(string)) - if ip == nil { - return net.IP{}, fmt.Errorf("failed parsing ip %v", data) - } - - return ip, nil - } -} - -// StringToIPNetHookFunc returns a DecodeHookFunc that converts -// strings to net.IPNet -func StringToIPNetHookFunc() DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data interface{}) (interface{}, error) { - if f.Kind() != reflect.String { - return data, nil - } - if t != reflect.TypeOf(net.IPNet{}) { - return data, nil - } - - // Convert it by parsing - _, net, err := net.ParseCIDR(data.(string)) - return net, err - } -} - -// StringToTimeHookFunc returns a DecodeHookFunc that converts -// strings to time.Time. -func StringToTimeHookFunc(layout string) DecodeHookFunc { - return func( - f reflect.Type, - t reflect.Type, - data interface{}) (interface{}, error) { - if f.Kind() != reflect.String { - return data, nil - } - if t != reflect.TypeOf(time.Time{}) { - return data, nil - } - - // Convert it by parsing - return time.Parse(layout, data.(string)) - } -} - -// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to -// the decoder. -// -// Note that this is significantly different from the WeaklyTypedInput option -// of the DecoderConfig. -func WeaklyTypedHook( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - dataVal := reflect.ValueOf(data) - switch t { - case reflect.String: - switch f { - case reflect.Bool: - if dataVal.Bool() { - return "1", nil - } - return "0", nil - case reflect.Float32: - return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil - case reflect.Int: - return strconv.FormatInt(dataVal.Int(), 10), nil - case reflect.Slice: - dataType := dataVal.Type() - elemKind := dataType.Elem().Kind() - if elemKind == reflect.Uint8 { - return string(dataVal.Interface().([]uint8)), nil - } - case reflect.Uint: - return strconv.FormatUint(dataVal.Uint(), 10), nil - } - } - - return data, nil -} diff --git a/vendor/github.com/mitchellh/mapstructure/error.go b/vendor/github.com/mitchellh/mapstructure/error.go deleted file mode 100644 index 47a99e5a..00000000 --- a/vendor/github.com/mitchellh/mapstructure/error.go +++ /dev/null @@ -1,50 +0,0 @@ -package mapstructure - -import ( - "errors" - "fmt" - "sort" - "strings" -) - -// Error implements the error interface and can represents multiple -// errors that occur in the course of a single decode. -type Error struct { - Errors []string -} - -func (e *Error) Error() string { - points := make([]string, len(e.Errors)) - for i, err := range e.Errors { - points[i] = fmt.Sprintf("* %s", err) - } - - sort.Strings(points) - return fmt.Sprintf( - "%d error(s) decoding:\n\n%s", - len(e.Errors), strings.Join(points, "\n")) -} - -// WrappedErrors implements the errwrap.Wrapper interface to make this -// return value more useful with the errwrap and go-multierror libraries. -func (e *Error) WrappedErrors() []error { - if e == nil { - return nil - } - - result := make([]error, len(e.Errors)) - for i, e := range e.Errors { - result[i] = errors.New(e) - } - - return result -} - -func appendErrors(errors []string, err error) []string { - switch e := err.(type) { - case *Error: - return append(errors, e.Errors...) - default: - return append(errors, e.Error()) - } -} diff --git a/vendor/github.com/mitchellh/mapstructure/go.mod b/vendor/github.com/mitchellh/mapstructure/go.mod deleted file mode 100644 index d2a71256..00000000 --- a/vendor/github.com/mitchellh/mapstructure/go.mod +++ /dev/null @@ -1 +0,0 @@ -module github.com/mitchellh/mapstructure diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go deleted file mode 100644 index 256ee63f..00000000 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure.go +++ /dev/null @@ -1,1149 +0,0 @@ -// Package mapstructure exposes functionality to convert an arbitrary -// map[string]interface{} into a native Go structure. -// -// The Go structure can be arbitrarily complex, containing slices, -// other structs, etc. and the decoder will properly decode nested -// maps and so on into the proper structures in the native Go struct. -// See the examples to see what the decoder is capable of. -package mapstructure - -import ( - "encoding/json" - "errors" - "fmt" - "reflect" - "sort" - "strconv" - "strings" -) - -// DecodeHookFunc is the callback function that can be used for -// data transformations. See "DecodeHook" in the DecoderConfig -// struct. -// -// The type should be DecodeHookFuncType or DecodeHookFuncKind. -// Either is accepted. Types are a superset of Kinds (Types can return -// Kinds) and are generally a richer thing to use, but Kinds are simpler -// if you only need those. -// -// The reason DecodeHookFunc is multi-typed is for backwards compatibility: -// we started with Kinds and then realized Types were the better solution, -// but have a promise to not break backwards compat so we now support -// both. -type DecodeHookFunc interface{} - -// DecodeHookFuncType is a DecodeHookFunc which has complete information about -// the source and target types. -type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface{}, error) - -// DecodeHookFuncKind is a DecodeHookFunc which knows only the Kinds of the -// source and target types. -type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) - -// DecoderConfig is the configuration that is used to create a new decoder -// and allows customization of various aspects of decoding. -type DecoderConfig struct { - // DecodeHook, if set, will be called before any decoding and any - // type conversion (if WeaklyTypedInput is on). This lets you modify - // the values before they're set down onto the resulting struct. - // - // If an error is returned, the entire decode will fail with that - // error. - DecodeHook DecodeHookFunc - - // If ErrorUnused is true, then it is an error for there to exist - // keys in the original map that were unused in the decoding process - // (extra keys). - ErrorUnused bool - - // ZeroFields, if set to true, will zero fields before writing them. - // For example, a map will be emptied before decoded values are put in - // it. If this is false, a map will be merged. - ZeroFields bool - - // If WeaklyTypedInput is true, the decoder will make the following - // "weak" conversions: - // - // - bools to string (true = "1", false = "0") - // - numbers to string (base 10) - // - bools to int/uint (true = 1, false = 0) - // - strings to int/uint (base implied by prefix) - // - int to bool (true if value != 0) - // - string to bool (accepts: 1, t, T, TRUE, true, True, 0, f, F, - // FALSE, false, False. Anything else is an error) - // - empty array = empty map and vice versa - // - negative numbers to overflowed uint values (base 10) - // - slice of maps to a merged map - // - single values are converted to slices if required. Each - // element is weakly decoded. For example: "4" can become []int{4} - // if the target type is an int slice. - // - WeaklyTypedInput bool - - // Metadata is the struct that will contain extra metadata about - // the decoding. If this is nil, then no metadata will be tracked. - Metadata *Metadata - - // Result is a pointer to the struct that will contain the decoded - // value. - Result interface{} - - // The tag name that mapstructure reads for field names. This - // defaults to "mapstructure" - TagName string -} - -// A Decoder takes a raw interface value and turns it into structured -// data, keeping track of rich error information along the way in case -// anything goes wrong. Unlike the basic top-level Decode method, you can -// more finely control how the Decoder behaves using the DecoderConfig -// structure. The top-level Decode method is just a convenience that sets -// up the most basic Decoder. -type Decoder struct { - config *DecoderConfig -} - -// Metadata contains information about decoding a structure that -// is tedious or difficult to get otherwise. -type Metadata struct { - // Keys are the keys of the structure which were successfully decoded - Keys []string - - // Unused is a slice of keys that were found in the raw value but - // weren't decoded since there was no matching field in the result interface - Unused []string -} - -// Decode takes an input structure and uses reflection to translate it to -// the output structure. output must be a pointer to a map or struct. -func Decode(input interface{}, output interface{}) error { - config := &DecoderConfig{ - Metadata: nil, - Result: output, - } - - decoder, err := NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(input) -} - -// WeakDecode is the same as Decode but is shorthand to enable -// WeaklyTypedInput. See DecoderConfig for more info. -func WeakDecode(input, output interface{}) error { - config := &DecoderConfig{ - Metadata: nil, - Result: output, - WeaklyTypedInput: true, - } - - decoder, err := NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(input) -} - -// DecodeMetadata is the same as Decode, but is shorthand to -// enable metadata collection. See DecoderConfig for more info. -func DecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { - config := &DecoderConfig{ - Metadata: metadata, - Result: output, - } - - decoder, err := NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(input) -} - -// WeakDecodeMetadata is the same as Decode, but is shorthand to -// enable both WeaklyTypedInput and metadata collection. See -// DecoderConfig for more info. -func WeakDecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { - config := &DecoderConfig{ - Metadata: metadata, - Result: output, - WeaklyTypedInput: true, - } - - decoder, err := NewDecoder(config) - if err != nil { - return err - } - - return decoder.Decode(input) -} - -// NewDecoder returns a new decoder for the given configuration. Once -// a decoder has been returned, the same configuration must not be used -// again. -func NewDecoder(config *DecoderConfig) (*Decoder, error) { - val := reflect.ValueOf(config.Result) - if val.Kind() != reflect.Ptr { - return nil, errors.New("result must be a pointer") - } - - val = val.Elem() - if !val.CanAddr() { - return nil, errors.New("result must be addressable (a pointer)") - } - - if config.Metadata != nil { - if config.Metadata.Keys == nil { - config.Metadata.Keys = make([]string, 0) - } - - if config.Metadata.Unused == nil { - config.Metadata.Unused = make([]string, 0) - } - } - - if config.TagName == "" { - config.TagName = "mapstructure" - } - - result := &Decoder{ - config: config, - } - - return result, nil -} - -// Decode decodes the given raw interface to the target pointer specified -// by the configuration. -func (d *Decoder) Decode(input interface{}) error { - return d.decode("", input, reflect.ValueOf(d.config.Result).Elem()) -} - -// Decodes an unknown data type into a specific reflection value. -func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) error { - var inputVal reflect.Value - if input != nil { - inputVal = reflect.ValueOf(input) - - // We need to check here if input is a typed nil. Typed nils won't - // match the "input == nil" below so we check that here. - if inputVal.Kind() == reflect.Ptr && inputVal.IsNil() { - input = nil - } - } - - if input == nil { - // If the data is nil, then we don't set anything, unless ZeroFields is set - // to true. - if d.config.ZeroFields { - outVal.Set(reflect.Zero(outVal.Type())) - - if d.config.Metadata != nil && name != "" { - d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) - } - } - return nil - } - - if !inputVal.IsValid() { - // If the input value is invalid, then we just set the value - // to be the zero value. - outVal.Set(reflect.Zero(outVal.Type())) - if d.config.Metadata != nil && name != "" { - d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) - } - return nil - } - - if d.config.DecodeHook != nil { - // We have a DecodeHook, so let's pre-process the input. - var err error - input, err = DecodeHookExec( - d.config.DecodeHook, - inputVal.Type(), outVal.Type(), input) - if err != nil { - return fmt.Errorf("error decoding '%s': %s", name, err) - } - } - - var err error - outputKind := getKind(outVal) - switch outputKind { - case reflect.Bool: - err = d.decodeBool(name, input, outVal) - case reflect.Interface: - err = d.decodeBasic(name, input, outVal) - case reflect.String: - err = d.decodeString(name, input, outVal) - case reflect.Int: - err = d.decodeInt(name, input, outVal) - case reflect.Uint: - err = d.decodeUint(name, input, outVal) - case reflect.Float32: - err = d.decodeFloat(name, input, outVal) - case reflect.Struct: - err = d.decodeStruct(name, input, outVal) - case reflect.Map: - err = d.decodeMap(name, input, outVal) - case reflect.Ptr: - err = d.decodePtr(name, input, outVal) - case reflect.Slice: - err = d.decodeSlice(name, input, outVal) - case reflect.Array: - err = d.decodeArray(name, input, outVal) - case reflect.Func: - err = d.decodeFunc(name, input, outVal) - default: - // If we reached this point then we weren't able to decode it - return fmt.Errorf("%s: unsupported type: %s", name, outputKind) - } - - // If we reached here, then we successfully decoded SOMETHING, so - // mark the key as used if we're tracking metainput. - if d.config.Metadata != nil && name != "" { - d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) - } - - return err -} - -// This decodes a basic type (bool, int, string, etc.) and sets the -// value to "data" of that type. -func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error { - if val.IsValid() && val.Elem().IsValid() { - return d.decode(name, data, val.Elem()) - } - - dataVal := reflect.ValueOf(data) - - // If the input data is a pointer, and the assigned type is the dereference - // of that exact pointer, then indirect it so that we can assign it. - // Example: *string to string - if dataVal.Kind() == reflect.Ptr && dataVal.Type().Elem() == val.Type() { - dataVal = reflect.Indirect(dataVal) - } - - if !dataVal.IsValid() { - dataVal = reflect.Zero(val.Type()) - } - - dataValType := dataVal.Type() - if !dataValType.AssignableTo(val.Type()) { - return fmt.Errorf( - "'%s' expected type '%s', got '%s'", - name, val.Type(), dataValType) - } - - val.Set(dataVal) - return nil -} - -func (d *Decoder) decodeString(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataKind := getKind(dataVal) - - converted := true - switch { - case dataKind == reflect.String: - val.SetString(dataVal.String()) - case dataKind == reflect.Bool && d.config.WeaklyTypedInput: - if dataVal.Bool() { - val.SetString("1") - } else { - val.SetString("0") - } - case dataKind == reflect.Int && d.config.WeaklyTypedInput: - val.SetString(strconv.FormatInt(dataVal.Int(), 10)) - case dataKind == reflect.Uint && d.config.WeaklyTypedInput: - val.SetString(strconv.FormatUint(dataVal.Uint(), 10)) - case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: - val.SetString(strconv.FormatFloat(dataVal.Float(), 'f', -1, 64)) - case dataKind == reflect.Slice && d.config.WeaklyTypedInput, - dataKind == reflect.Array && d.config.WeaklyTypedInput: - dataType := dataVal.Type() - elemKind := dataType.Elem().Kind() - switch elemKind { - case reflect.Uint8: - var uints []uint8 - if dataKind == reflect.Array { - uints = make([]uint8, dataVal.Len(), dataVal.Len()) - for i := range uints { - uints[i] = dataVal.Index(i).Interface().(uint8) - } - } else { - uints = dataVal.Interface().([]uint8) - } - val.SetString(string(uints)) - default: - converted = false - } - default: - converted = false - } - - if !converted { - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - - return nil -} - -func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataKind := getKind(dataVal) - dataType := dataVal.Type() - - switch { - case dataKind == reflect.Int: - val.SetInt(dataVal.Int()) - case dataKind == reflect.Uint: - val.SetInt(int64(dataVal.Uint())) - case dataKind == reflect.Float32: - val.SetInt(int64(dataVal.Float())) - case dataKind == reflect.Bool && d.config.WeaklyTypedInput: - if dataVal.Bool() { - val.SetInt(1) - } else { - val.SetInt(0) - } - case dataKind == reflect.String && d.config.WeaklyTypedInput: - i, err := strconv.ParseInt(dataVal.String(), 0, val.Type().Bits()) - if err == nil { - val.SetInt(i) - } else { - return fmt.Errorf("cannot parse '%s' as int: %s", name, err) - } - case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": - jn := data.(json.Number) - i, err := jn.Int64() - if err != nil { - return fmt.Errorf( - "error decoding json.Number into %s: %s", name, err) - } - val.SetInt(i) - default: - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - - return nil -} - -func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataKind := getKind(dataVal) - - switch { - case dataKind == reflect.Int: - i := dataVal.Int() - if i < 0 && !d.config.WeaklyTypedInput { - return fmt.Errorf("cannot parse '%s', %d overflows uint", - name, i) - } - val.SetUint(uint64(i)) - case dataKind == reflect.Uint: - val.SetUint(dataVal.Uint()) - case dataKind == reflect.Float32: - f := dataVal.Float() - if f < 0 && !d.config.WeaklyTypedInput { - return fmt.Errorf("cannot parse '%s', %f overflows uint", - name, f) - } - val.SetUint(uint64(f)) - case dataKind == reflect.Bool && d.config.WeaklyTypedInput: - if dataVal.Bool() { - val.SetUint(1) - } else { - val.SetUint(0) - } - case dataKind == reflect.String && d.config.WeaklyTypedInput: - i, err := strconv.ParseUint(dataVal.String(), 0, val.Type().Bits()) - if err == nil { - val.SetUint(i) - } else { - return fmt.Errorf("cannot parse '%s' as uint: %s", name, err) - } - default: - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - - return nil -} - -func (d *Decoder) decodeBool(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataKind := getKind(dataVal) - - switch { - case dataKind == reflect.Bool: - val.SetBool(dataVal.Bool()) - case dataKind == reflect.Int && d.config.WeaklyTypedInput: - val.SetBool(dataVal.Int() != 0) - case dataKind == reflect.Uint && d.config.WeaklyTypedInput: - val.SetBool(dataVal.Uint() != 0) - case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: - val.SetBool(dataVal.Float() != 0) - case dataKind == reflect.String && d.config.WeaklyTypedInput: - b, err := strconv.ParseBool(dataVal.String()) - if err == nil { - val.SetBool(b) - } else if dataVal.String() == "" { - val.SetBool(false) - } else { - return fmt.Errorf("cannot parse '%s' as bool: %s", name, err) - } - default: - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - - return nil -} - -func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataKind := getKind(dataVal) - dataType := dataVal.Type() - - switch { - case dataKind == reflect.Int: - val.SetFloat(float64(dataVal.Int())) - case dataKind == reflect.Uint: - val.SetFloat(float64(dataVal.Uint())) - case dataKind == reflect.Float32: - val.SetFloat(dataVal.Float()) - case dataKind == reflect.Bool && d.config.WeaklyTypedInput: - if dataVal.Bool() { - val.SetFloat(1) - } else { - val.SetFloat(0) - } - case dataKind == reflect.String && d.config.WeaklyTypedInput: - f, err := strconv.ParseFloat(dataVal.String(), val.Type().Bits()) - if err == nil { - val.SetFloat(f) - } else { - return fmt.Errorf("cannot parse '%s' as float: %s", name, err) - } - case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": - jn := data.(json.Number) - i, err := jn.Float64() - if err != nil { - return fmt.Errorf( - "error decoding json.Number into %s: %s", name, err) - } - val.SetFloat(i) - default: - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - - return nil -} - -func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) error { - valType := val.Type() - valKeyType := valType.Key() - valElemType := valType.Elem() - - // By default we overwrite keys in the current map - valMap := val - - // If the map is nil or we're purposely zeroing fields, make a new map - if valMap.IsNil() || d.config.ZeroFields { - // Make a new map to hold our result - mapType := reflect.MapOf(valKeyType, valElemType) - valMap = reflect.MakeMap(mapType) - } - - // Check input type and based on the input type jump to the proper func - dataVal := reflect.Indirect(reflect.ValueOf(data)) - switch dataVal.Kind() { - case reflect.Map: - return d.decodeMapFromMap(name, dataVal, val, valMap) - - case reflect.Struct: - return d.decodeMapFromStruct(name, dataVal, val, valMap) - - case reflect.Array, reflect.Slice: - if d.config.WeaklyTypedInput { - return d.decodeMapFromSlice(name, dataVal, val, valMap) - } - - fallthrough - - default: - return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) - } -} - -func (d *Decoder) decodeMapFromSlice(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { - // Special case for BC reasons (covered by tests) - if dataVal.Len() == 0 { - val.Set(valMap) - return nil - } - - for i := 0; i < dataVal.Len(); i++ { - err := d.decode( - fmt.Sprintf("%s[%d]", name, i), - dataVal.Index(i).Interface(), val) - if err != nil { - return err - } - } - - return nil -} - -func (d *Decoder) decodeMapFromMap(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { - valType := val.Type() - valKeyType := valType.Key() - valElemType := valType.Elem() - - // Accumulate errors - errors := make([]string, 0) - - // If the input data is empty, then we just match what the input data is. - if dataVal.Len() == 0 { - if dataVal.IsNil() { - if !val.IsNil() { - val.Set(dataVal) - } - } else { - // Set to empty allocated value - val.Set(valMap) - } - - return nil - } - - for _, k := range dataVal.MapKeys() { - fieldName := fmt.Sprintf("%s[%s]", name, k) - - // First decode the key into the proper type - currentKey := reflect.Indirect(reflect.New(valKeyType)) - if err := d.decode(fieldName, k.Interface(), currentKey); err != nil { - errors = appendErrors(errors, err) - continue - } - - // Next decode the data into the proper type - v := dataVal.MapIndex(k).Interface() - currentVal := reflect.Indirect(reflect.New(valElemType)) - if err := d.decode(fieldName, v, currentVal); err != nil { - errors = appendErrors(errors, err) - continue - } - - valMap.SetMapIndex(currentKey, currentVal) - } - - // Set the built up map to the value - val.Set(valMap) - - // If we had errors, return those - if len(errors) > 0 { - return &Error{errors} - } - - return nil -} - -func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { - typ := dataVal.Type() - for i := 0; i < typ.NumField(); i++ { - // Get the StructField first since this is a cheap operation. If the - // field is unexported, then ignore it. - f := typ.Field(i) - if f.PkgPath != "" { - continue - } - - // Next get the actual value of this field and verify it is assignable - // to the map value. - v := dataVal.Field(i) - if !v.Type().AssignableTo(valMap.Type().Elem()) { - return fmt.Errorf("cannot assign type '%s' to map value field of type '%s'", v.Type(), valMap.Type().Elem()) - } - - tagValue := f.Tag.Get(d.config.TagName) - tagParts := strings.Split(tagValue, ",") - - // Determine the name of the key in the map - keyName := f.Name - if tagParts[0] != "" { - if tagParts[0] == "-" { - continue - } - keyName = tagParts[0] - } - - // If "squash" is specified in the tag, we squash the field down. - squash := false - for _, tag := range tagParts[1:] { - if tag == "squash" { - squash = true - break - } - } - if squash && v.Kind() != reflect.Struct { - return fmt.Errorf("cannot squash non-struct type '%s'", v.Type()) - } - - switch v.Kind() { - // this is an embedded struct, so handle it differently - case reflect.Struct: - x := reflect.New(v.Type()) - x.Elem().Set(v) - - vType := valMap.Type() - vKeyType := vType.Key() - vElemType := vType.Elem() - mType := reflect.MapOf(vKeyType, vElemType) - vMap := reflect.MakeMap(mType) - - err := d.decode(keyName, x.Interface(), vMap) - if err != nil { - return err - } - - if squash { - for _, k := range vMap.MapKeys() { - valMap.SetMapIndex(k, vMap.MapIndex(k)) - } - } else { - valMap.SetMapIndex(reflect.ValueOf(keyName), vMap) - } - - default: - valMap.SetMapIndex(reflect.ValueOf(keyName), v) - } - } - - if val.CanAddr() { - val.Set(valMap) - } - - return nil -} - -func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) error { - // If the input data is nil, then we want to just set the output - // pointer to be nil as well. - isNil := data == nil - if !isNil { - switch v := reflect.Indirect(reflect.ValueOf(data)); v.Kind() { - case reflect.Chan, - reflect.Func, - reflect.Interface, - reflect.Map, - reflect.Ptr, - reflect.Slice: - isNil = v.IsNil() - } - } - if isNil { - if !val.IsNil() && val.CanSet() { - nilValue := reflect.New(val.Type()).Elem() - val.Set(nilValue) - } - - return nil - } - - // Create an element of the concrete (non pointer) type and decode - // into that. Then set the value of the pointer to this type. - valType := val.Type() - valElemType := valType.Elem() - if val.CanSet() { - realVal := val - if realVal.IsNil() || d.config.ZeroFields { - realVal = reflect.New(valElemType) - } - - if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { - return err - } - - val.Set(realVal) - } else { - if err := d.decode(name, data, reflect.Indirect(val)); err != nil { - return err - } - } - return nil -} - -func (d *Decoder) decodeFunc(name string, data interface{}, val reflect.Value) error { - // Create an element of the concrete (non pointer) type and decode - // into that. Then set the value of the pointer to this type. - dataVal := reflect.Indirect(reflect.ValueOf(data)) - if val.Type() != dataVal.Type() { - return fmt.Errorf( - "'%s' expected type '%s', got unconvertible type '%s'", - name, val.Type(), dataVal.Type()) - } - val.Set(dataVal) - return nil -} - -func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataValKind := dataVal.Kind() - valType := val.Type() - valElemType := valType.Elem() - sliceType := reflect.SliceOf(valElemType) - - valSlice := val - if valSlice.IsNil() || d.config.ZeroFields { - if d.config.WeaklyTypedInput { - switch { - // Slice and array we use the normal logic - case dataValKind == reflect.Slice, dataValKind == reflect.Array: - break - - // Empty maps turn into empty slices - case dataValKind == reflect.Map: - if dataVal.Len() == 0 { - val.Set(reflect.MakeSlice(sliceType, 0, 0)) - return nil - } - // Create slice of maps of other sizes - return d.decodeSlice(name, []interface{}{data}, val) - - case dataValKind == reflect.String && valElemType.Kind() == reflect.Uint8: - return d.decodeSlice(name, []byte(dataVal.String()), val) - - // All other types we try to convert to the slice type - // and "lift" it into it. i.e. a string becomes a string slice. - default: - // Just re-try this function with data as a slice. - return d.decodeSlice(name, []interface{}{data}, val) - } - } - - // Check input type - if dataValKind != reflect.Array && dataValKind != reflect.Slice { - return fmt.Errorf( - "'%s': source data must be an array or slice, got %s", name, dataValKind) - - } - - // If the input value is empty, then don't allocate since non-nil != nil - if dataVal.Len() == 0 { - return nil - } - - // Make a new slice to hold our result, same size as the original data. - valSlice = reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) - } - - // Accumulate any errors - errors := make([]string, 0) - - for i := 0; i < dataVal.Len(); i++ { - currentData := dataVal.Index(i).Interface() - for valSlice.Len() <= i { - valSlice = reflect.Append(valSlice, reflect.Zero(valElemType)) - } - currentField := valSlice.Index(i) - - fieldName := fmt.Sprintf("%s[%d]", name, i) - if err := d.decode(fieldName, currentData, currentField); err != nil { - errors = appendErrors(errors, err) - } - } - - // Finally, set the value to the slice we built up - val.Set(valSlice) - - // If there were errors, we return those - if len(errors) > 0 { - return &Error{errors} - } - - return nil -} - -func (d *Decoder) decodeArray(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - dataValKind := dataVal.Kind() - valType := val.Type() - valElemType := valType.Elem() - arrayType := reflect.ArrayOf(valType.Len(), valElemType) - - valArray := val - - if valArray.Interface() == reflect.Zero(valArray.Type()).Interface() || d.config.ZeroFields { - // Check input type - if dataValKind != reflect.Array && dataValKind != reflect.Slice { - if d.config.WeaklyTypedInput { - switch { - // Empty maps turn into empty arrays - case dataValKind == reflect.Map: - if dataVal.Len() == 0 { - val.Set(reflect.Zero(arrayType)) - return nil - } - - // All other types we try to convert to the array type - // and "lift" it into it. i.e. a string becomes a string array. - default: - // Just re-try this function with data as a slice. - return d.decodeArray(name, []interface{}{data}, val) - } - } - - return fmt.Errorf( - "'%s': source data must be an array or slice, got %s", name, dataValKind) - - } - if dataVal.Len() > arrayType.Len() { - return fmt.Errorf( - "'%s': expected source data to have length less or equal to %d, got %d", name, arrayType.Len(), dataVal.Len()) - - } - - // Make a new array to hold our result, same size as the original data. - valArray = reflect.New(arrayType).Elem() - } - - // Accumulate any errors - errors := make([]string, 0) - - for i := 0; i < dataVal.Len(); i++ { - currentData := dataVal.Index(i).Interface() - currentField := valArray.Index(i) - - fieldName := fmt.Sprintf("%s[%d]", name, i) - if err := d.decode(fieldName, currentData, currentField); err != nil { - errors = appendErrors(errors, err) - } - } - - // Finally, set the value to the array we built up - val.Set(valArray) - - // If there were errors, we return those - if len(errors) > 0 { - return &Error{errors} - } - - return nil -} - -func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) error { - dataVal := reflect.Indirect(reflect.ValueOf(data)) - - // If the type of the value to write to and the data match directly, - // then we just set it directly instead of recursing into the structure. - if dataVal.Type() == val.Type() { - val.Set(dataVal) - return nil - } - - dataValKind := dataVal.Kind() - switch dataValKind { - case reflect.Map: - return d.decodeStructFromMap(name, dataVal, val) - - case reflect.Struct: - // Not the most efficient way to do this but we can optimize later if - // we want to. To convert from struct to struct we go to map first - // as an intermediary. - m := make(map[string]interface{}) - mval := reflect.Indirect(reflect.ValueOf(&m)) - if err := d.decodeMapFromStruct(name, dataVal, mval, mval); err != nil { - return err - } - - result := d.decodeStructFromMap(name, mval, val) - return result - - default: - return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) - } -} - -func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) error { - dataValType := dataVal.Type() - if kind := dataValType.Key().Kind(); kind != reflect.String && kind != reflect.Interface { - return fmt.Errorf( - "'%s' needs a map with string keys, has '%s' keys", - name, dataValType.Key().Kind()) - } - - dataValKeys := make(map[reflect.Value]struct{}) - dataValKeysUnused := make(map[interface{}]struct{}) - for _, dataValKey := range dataVal.MapKeys() { - dataValKeys[dataValKey] = struct{}{} - dataValKeysUnused[dataValKey.Interface()] = struct{}{} - } - - errors := make([]string, 0) - - // This slice will keep track of all the structs we'll be decoding. - // There can be more than one struct if there are embedded structs - // that are squashed. - structs := make([]reflect.Value, 1, 5) - structs[0] = val - - // Compile the list of all the fields that we're going to be decoding - // from all the structs. - type field struct { - field reflect.StructField - val reflect.Value - } - fields := []field{} - for len(structs) > 0 { - structVal := structs[0] - structs = structs[1:] - - structType := structVal.Type() - - for i := 0; i < structType.NumField(); i++ { - fieldType := structType.Field(i) - fieldKind := fieldType.Type.Kind() - - // If "squash" is specified in the tag, we squash the field down. - squash := false - tagParts := strings.Split(fieldType.Tag.Get(d.config.TagName), ",") - for _, tag := range tagParts[1:] { - if tag == "squash" { - squash = true - break - } - } - - if squash { - if fieldKind != reflect.Struct { - errors = appendErrors(errors, - fmt.Errorf("%s: unsupported type for squash: %s", fieldType.Name, fieldKind)) - } else { - structs = append(structs, structVal.FieldByName(fieldType.Name)) - } - continue - } - - // Normal struct field, store it away - fields = append(fields, field{fieldType, structVal.Field(i)}) - } - } - - // for fieldType, field := range fields { - for _, f := range fields { - field, fieldValue := f.field, f.val - fieldName := field.Name - - tagValue := field.Tag.Get(d.config.TagName) - tagValue = strings.SplitN(tagValue, ",", 2)[0] - if tagValue != "" { - fieldName = tagValue - } - - rawMapKey := reflect.ValueOf(fieldName) - rawMapVal := dataVal.MapIndex(rawMapKey) - if !rawMapVal.IsValid() { - // Do a slower search by iterating over each key and - // doing case-insensitive search. - for dataValKey := range dataValKeys { - mK, ok := dataValKey.Interface().(string) - if !ok { - // Not a string key - continue - } - - if strings.EqualFold(mK, fieldName) { - rawMapKey = dataValKey - rawMapVal = dataVal.MapIndex(dataValKey) - break - } - } - - if !rawMapVal.IsValid() { - // There was no matching key in the map for the value in - // the struct. Just ignore. - continue - } - } - - // Delete the key we're using from the unused map so we stop tracking - delete(dataValKeysUnused, rawMapKey.Interface()) - - if !fieldValue.IsValid() { - // This should never happen - panic("field is not valid") - } - - // If we can't set the field, then it is unexported or something, - // and we just continue onwards. - if !fieldValue.CanSet() { - continue - } - - // If the name is empty string, then we're at the root, and we - // don't dot-join the fields. - if name != "" { - fieldName = fmt.Sprintf("%s.%s", name, fieldName) - } - - if err := d.decode(fieldName, rawMapVal.Interface(), fieldValue); err != nil { - errors = appendErrors(errors, err) - } - } - - if d.config.ErrorUnused && len(dataValKeysUnused) > 0 { - keys := make([]string, 0, len(dataValKeysUnused)) - for rawKey := range dataValKeysUnused { - keys = append(keys, rawKey.(string)) - } - sort.Strings(keys) - - err := fmt.Errorf("'%s' has invalid keys: %s", name, strings.Join(keys, ", ")) - errors = appendErrors(errors, err) - } - - if len(errors) > 0 { - return &Error{errors} - } - - // Add the unused keys to the list of unused keys if we're tracking metadata - if d.config.Metadata != nil { - for rawKey := range dataValKeysUnused { - key := rawKey.(string) - if name != "" { - key = fmt.Sprintf("%s.%s", name, key) - } - - d.config.Metadata.Unused = append(d.config.Metadata.Unused, key) - } - } - - return nil -} - -func getKind(val reflect.Value) reflect.Kind { - kind := val.Kind() - - switch { - case kind >= reflect.Int && kind <= reflect.Int64: - return reflect.Int - case kind >= reflect.Uint && kind <= reflect.Uint64: - return reflect.Uint - case kind >= reflect.Float32 && kind <= reflect.Float64: - return reflect.Float32 - default: - return kind - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 685d281a..8f3e6cf5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,7 +1,5 @@ # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/beorn7/perks/quantile -# github.com/cloudfoundry-community/go-cfenv v1.18.0 -github.com/cloudfoundry-community/go-cfenv # github.com/golang/protobuf v1.3.1 github.com/golang/protobuf/proto # github.com/gomodule/redigo v2.0.0+incompatible @@ -11,8 +9,6 @@ github.com/gomodule/redigo/internal github.com/konsorten/go-windows-terminal-sequences # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/mitchellh/mapstructure v1.1.2 -github.com/mitchellh/mapstructure # github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/promhttp