Added optional use of new v1 StatusCake API (#10)

* Added optional use of v1 StatusCake API

* Resolved linting errors

README.md

@@ -22,13 +22,15 @@ http://status-cake-exporter.default.svc:8000
 
 ## Usage
 
-| Setting  | Required | Default |
-|----------|----------|---------|
-| USERNAME | Yes      | Null    |
-| API_KEY  | Yes      | Null    |
-| TAGS     | No       | Null    |
-| LOG_LEVEL| No       | info    |
-| PORT     | No       | 8000    |
+| Setting                              | Required | Default |
+|--------------------------------------|----------|---------|
+| USE_V1_UPTIME_ENDPOINTS              | No       | False   | 
+| USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS | No       | False   | 
+| USERNAME                             | Yes      | Null    | 
+| API_KEY                              | Yes      | Null    |
+| TAGS                                 | No       | Null    |
+| LOG_LEVEL                            | No       | info    | 
+| PORT                                 | No       | 8000    |
 
 ### Docker
 
@@ -57,13 +59,27 @@ override environment variables which override defaults.
 
 optional arguments:
   -h, --help            show this help message and exit
+  --use_v1_uptime_endpoints true                Boolean for using v1 uptime endpoints [env var: USE_V1_UPTIME_ENDPOINTS]
+  --use_v1_maintenance_windows_endpoints true   Boolean for using v1 maintenance windows endpoints [env var: USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS]
   --username USERNAME   Username for the account [env var: USERNAME]
   --api-key API_KEY     API key for the account [env var: API_KEY]
   --tests.tags TAGS     A comma separated list of tags used to filter tests returned from the api [env var: TAGS]
   --logging.level       {debug,info,warn,error} Set a log level for the application [env var: LOG_LEVEL]
   --port                The TCP port to start the web server on [env var: PORT]
 ```
 
+## V1 API
+StatusCake have a new v1 API with documentation available at https://www.statuscake.com/api/v1/, deprecating the legacy API https://www.statuscake.com/api/.
+
+The new `Get all uptime tests` endpoint https://www.statuscake.com/api/v1/#operation/list-uptime-tests provides paged responses to get all tests, overcoming the limit of only 100 tests in the response from the legacy API https://www.statuscake.com/api/Tests/Get%20All%20Tests.md
+
+Environment variables `USE_V1_UPTIME_ENDPOINTS` and `USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS` are used to enable use of the v1 API.
+
+### Maintenance Windows endpoints
+Endpoints of the new v1 API are available to be used by all accounts with the exception of the maintenance windows endpoints, from https://www.statuscake.com/api/v1/#tag/uptime:
+>NOTE: the API endpoints concerned with maintenance windows will only work with accounts registed to use the newer version of maintenance windows. This version of maintenance windows is incompatible with the original version and all existing windows will require migrating to be further used. Presently a tool to automate the migration of maintenance windows is under development.
+Similarly, if an account is registered to use the newer version of maintenance windows, the legacy API's maintenance windows endpoints cannot be used.
+
 ## Metrics
 
 | Name| Type | Description |
@@ -94,7 +110,7 @@ Data collected by Prometheus can be easily surfaced in Grafana.
 
 Using the [Statusmap panel](https://grafana.com/grafana/plugins/flant-statusmap-panel) by [flant](https://github.com/flant/grafana-statusmap) you can create a basic status visualization based on uptime percentage:
 
-![grafan](examples/grafana.png)
+![grafana](examples/grafana.png)
 
 ### PromQL
 

chart/status-cake-exporter/Tiltfile

@@ -1,7 +1,7 @@
 docker_build('status-cake-exporter:dev', '../../')
 # If not using a standard local dev name, specify your k8s context here
 #allow_k8s_contexts('microk8s')
-k8s_yaml(helm('.', values='values.yaml', set=['statuscake.logLevel=debug', 'image.repository=status-cake-exporter', 'image.tag=dev', 'statuscake.username=', 'statuscake.apiKey=', 'statuscake.tags=firstTag,secondTag']))
+k8s_yaml(helm('.', values='values.yaml', set=['statuscake.logLevel=debug', 'image.repository=status-cake-exporter', 'image.tag=dev', 'statuscake.useV1UptimeEndpoints=', 'statuscake.useV1MaintenanceWindowsEndpoints=', 'statuscake.username=', 'statuscake.apiKey=', 'statuscake.tags=firstTag,secondTag']))
 watch_file('.')
 watch_file('../../Dockerfile')
 watch_file('../../exporter')

chart/status-cake-exporter/templates/deployment.yaml

@@ -24,6 +24,14 @@ spec:
         ports:
         - containerPort: {{ .Values.service.port }}
         env:
+{{- if .Values.statuscake.useV1UptimeEndpoints }}
+        - name: USE_V1_UPTIME_ENDPOINTS
+          value: {{ .Values.statuscake.useV1UptimeEndpoints }}
+{{- end }}
+{{- if .Values.statuscake.useV1MaintenanceWindowsEndpoints }}
+        - name: USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS
+          value: {{ .Values.statuscake.useV1MaintenanceWindowsEndpoints }}
+{{- end }}
         - name: USERNAME
           valueFrom:
             secretKeyRef:

chart/status-cake-exporter/values.yaml

@@ -5,6 +5,10 @@ image:
   pullSecrets: []
 
 statuscake:
+  # optional: a boolean format string for using the uptime endpoints of the v1 API
+  # useV1UptimeEndpoints:
+  # optional: a boolean format string for using the maintenance windows endpoints of the v1 API
+  # useV1MaintenanceWindowsEndpoints:
   # REQUIRED: username to use when connecting to statuscake
   username: ""
   # REQUIRED: apikey to use when connecting to statuscake

exporter/app.py

@@ -23,7 +23,11 @@
 
         logger.info("Registering collectors.")
         REGISTRY.register(test_collector.TestCollector(
-            args.username, args.api_key, args.tags))
+            args.use_v1_uptime_endpoints,
+            args.use_v1_maintenance_windows_endpoints,
+            args.username,
+            args.api_key,
+            args.tags))
 
         while True:
             time.sleep(1)

exporter/collectors/test_collector.py

@@ -3,57 +3,56 @@
 import sys
 import logging
 from prometheus_client.core import GaugeMetricFamily
+from setuptools import distutils
 from status_cake_client import tests as t
 from status_cake_client import maintenance as m
 
 logger = logging.getLogger("test_collector")
 
 
-def parse_test_response(r, m):
+def parse_test_response(use_v1_uptime_endpoints, tests, m_test_id_flat_list):
     t = []
     try:
-        tests = r.json()
+        if use_v1_uptime_endpoints:
+            for i in tests:
+                t.append(
+                    {
+                        "test_id": str(i['id']),
+                        "test_type": i['test_type'],
+                        "test_name": i['name'],
+                        "test_url": i['website_url'],
+                        "test_status_int": str(1 if (i["status"] == "up") else 0),
+                        "test_uptime_percent": str(i['uptime']),
+                        "maintenance_status_int": str(1 if (str(i["id"])) in m_test_id_flat_list else 0)
+                    }
+                )
+        else:
+            for i in tests:
+                t.append(
+                    {
+                        "test_id": str(i['TestID']),
+                        "test_type": i['TestType'],
+                        "test_name": i['WebsiteName'],
+                        "test_url": i['WebsiteURL'],
+                        "test_status_int": str(1 if (i["Status"] == "Up") else 0),
+                        "test_uptime_percent": str(i['Uptime']),
+                        "maintenance_status_int": str(1 if (str(i["TestID"])) in m_test_id_flat_list else 0)
+                    }
+                )
+
+        return t
+
     except Exception as e:
         logger.error(f"Could not parse test data, exception: {e}")
-        logger.error(f"Test data was:\n{r}")
+        logger.error(f"Test data was:\n{tests}")
         sys.exit(1)
-    for i in tests:
-        t.append(
-            {
-                "test_id": str(i['TestID']),
-                "test_type": i['TestType'],
-                "test_name": i['WebsiteName'],
-                "test_url": i['WebsiteURL'],
-                "test_status_int": str(1 if (i["Status"] == "Up") else 0),
-                "test_uptime_percent": str(i['Uptime']),
-                "maintenance_status_int": str(1 if (str(i["TestID"])) in m else 0)
-            }
-        )
-
-    return t
-
-
-def parse_test_details_response(r):
-    t = []
-    for i in r:
-        t.append(
-            {
-                "test_id": str(i['TestID']),
-                "test_status_string": i['Status'],
-                "test_status_int": str(1 if (i["Status"] == "Up") else 0),
-                "test_uptime_percent": str(i['Uptime']),
-                "test_last_tested": i['LastTested'],
-                "test_processing": i['Processing'],
-                "test_down_times": str(i['DownTimes'])
-            }
-        )
-
-    return t
 
 
 class TestCollector(object):
 
-    def __init__(self, username, api_key, tags):
+    def __init__(self, use_v1_uptime_endpoints, use_v1_maintenance_windows_endpoints, username, api_key, tags):
+        self.use_v1_uptime_endpoints = bool(distutils.util.strtobool(use_v1_uptime_endpoints))
+        self.use_v1_maintenance_windows_endpoints = bool(distutils.util.strtobool(use_v1_maintenance_windows_endpoints))
         self.username = username
         self.api_key = api_key
         self.tags = tags
@@ -64,7 +63,7 @@ def collect(self):
 
         try:
 
-            maintenance = m.get_maintenance(self.api_key, self.username)
+            maintenance = m.get_maintenance(self.use_v1_maintenance_windows_endpoints, self.api_key, self.username)
             try:
                 maintenance_data = maintenance.json()['data']
             except Exception as e:
@@ -74,14 +73,17 @@ def collect(self):
             logger.debug(f"Maintenance response:\n{maintenance_data}")
 
             # Grab the test_ids from the response
-            m_test_id_list = [i['all_tests'] for i in maintenance_data]
+            if self.use_v1_maintenance_windows_endpoints:
+                m_test_id_list = [i['tests'] for i in maintenance_data]
+            else:
+                m_test_id_list = [i['all_tests'] for i in maintenance_data]
 
             # Flatten the test_ids into a list
             m_test_id_flat_list = [item for sublist in m_test_id_list for item in sublist]
             logger.info(f"Found {len(m_test_id_flat_list)} tests that are in maintenance.")
 
-            tests = t.get_tests(self.api_key, self.username, self.tags)
-            parsed_tests = parse_test_response(tests, m_test_id_flat_list)
+            tests = t.get_tests(self.use_v1_uptime_endpoints, self.api_key, self.username, self.tags)
+            parsed_tests = parse_test_response(self.use_v1_uptime_endpoints, tests, m_test_id_flat_list)
             logger.info(f"Publishing {len(parsed_tests)} tests.")
 
             # status_cake_test_info - gauge

exporter/status_cake_client/base.py

@@ -4,20 +4,28 @@
 import requests
 
 STATUS_CAKE_BASE_URL = "https://app.statuscake.com/API/"
+V1_STATUS_CAKE_BASE_URL = "https://api.statuscake.com/v1/"
 
 logger = logging.getLogger(__name__)
 
 
-def get(apikey, username, endpoint, params={}):
+def get(use_v1_api, apikey, username, endpoint, params={}):
 
-    request_url = f"{STATUS_CAKE_BASE_URL}{endpoint}"
+    if use_v1_api:
+        headers = {
+            "Authorization": "Bearer %s" % apikey
+        }
+        BASE_URL = V1_STATUS_CAKE_BASE_URL
+    else:
+        headers = {
+            "API": apikey,
+            "Username": username
+        }
+        BASE_URL = STATUS_CAKE_BASE_URL
 
-    logger.debug(f"Starting request: {request_url} {endpoint} {params}")
+    request_url = f"{BASE_URL}{endpoint}"
 
-    headers = {
-        "API": apikey,
-        "Username": username
-    }
+    logger.debug(f"Starting request: {request_url} {endpoint} {params}")
 
     response = requests.get(url=request_url, params=params, headers=headers)
     response.raise_for_status()

exporter/status_cake_client/maintenance.py

@@ -9,16 +9,22 @@
 logger = logging.getLogger(__name__)
 
 
-def get_maintenance(apikey, username, state="ACT"):
-    endpoint = "Maintenance"
-    params = {
-        "state": state
-    }
+def get_maintenance(use_v1_maintenance_windows_endpoints, apikey, username):
+    if use_v1_maintenance_windows_endpoints:
+        endpoint = "maintenance-windows"
+        params = {
+            "state": "active"
+        }
+    else:
+        endpoint = "Maintenance"
+        params = {
+            "state": "ACT"
+        }
 
     try:
-        response = get(apikey, username, endpoint, params)
+        response = get(use_v1_maintenance_windows_endpoints, apikey, username, endpoint, params)
     except requests.exceptions.HTTPError as e:
-        if e.response.status_code == 404:
+        if not(use_v1_maintenance_windows_endpoints) and e.response.status_code == 404:
             logger.info("Currently no active maintenance.")
             response = e.response
         else:

exporter/status_cake_client/tests.py

@@ -6,24 +6,28 @@
 logger = logging.getLogger(__name__)
 
 
-def get_tests(apikey, username, tags=""):
-    endpoint = "Tests"
-    params = {
-        "tags": tags
-    }
-    response = get(apikey, username, endpoint, params)
+def get_tests(use_v1_uptime_endpoints, apikey, username, tags=""):
+    if use_v1_uptime_endpoints:
+        page = 1
+        endpoint = "uptime"
+        params = {
+            "tags": tags,
+            "page": page
+        }
+        response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params)
+        tests = response.json()['data']
+        while (page < (response.json()['metadata']['page_count'])):
+            page += 1
+            params["page"] = page
+            response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params)
+            tests += response.json()['data']
+    else:
+        endpoint = "Tests"
+        params = {
+            "tags": tags
+        }
+        response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params)
+        tests = response.json()['data']
     logger.debug(f"Request response:\n{response.content}")
 
-    return response
-
-
-def get_test_details(apikey, username, test_id):
-    endpoint = "Tests/Details/"
-    params = {
-        "TestID": test_id
-    }
-
-    response = get(apikey, username, endpoint, params)
-    logger.debug(f"Request response:\n{response.content}")
-
-    return response
+    return tests

exporter/utilities/arguments.py

@@ -7,6 +7,24 @@
 def get_args():
 
     parser = configargparse.ArgParser()
+    parser.add("--use_v1_uptime_endpoints",
+               dest="use_v1_uptime_endpoints",
+               env_var="USE_V1_UPTIME_ENDPOINTS",
+               default="false",
+               type=str.lower,
+               choices={'false', 'f', '0', 'off', 'no', 'n', 'off',
+                        'true', 't', '1', 'on', 'yes', 'y', 't', 'true', 'on'},
+               help='Boolean format string for using the uptime endpoints of the v1 API')
+
+    parser.add("--use_v1_maintenance_windows_endpoints",
+               dest="use_v1_maintenance_windows_endpoints",
+               env_var="USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS",
+               default="false",
+               type=str.lower,
+               choices={'false', 'f', '0', 'off', 'no', 'n', 'off',
+                        'true', 't', '1', 'on', 'yes', 'y', 't', 'true', 'on'},
+               help='Boolean format string for using the maintenance windows endpoints of the v1 API')
+
     parser.add("--username",
                dest="username",
                env_var="USERNAME",
@@ -44,7 +62,7 @@ def get_args():
         sys.exit(1)
 
     if args.api_key is None:
-        print("Required argument --username is missing")
+        print("Required argument --api_key is missing")
         print(parser.print_help())
         sys.exit(1)
 

requirements.txt

@@ -1,5 +1,4 @@
 prometheus-client==0.7.1
 requests==2.22.0
 ConfigArgParse==0.14.0
-
-
+setuptools==58.3.0