TYPE stsanalyzerg_services gauge
November 10, 2018 ยท View on GitHub
servicecheckerdb2prometheus.py
This project also provides the servicecheckerdb2prometheus.py module which will monitor a directory for *servicecheckerdb*.json files generated by servicechecker.py and expose them as /metrics consumable by Prometheus for analysis in Grafana through a few provided dashboards (see the /grafana directory for dashboards or click here to view samples etc)
./servicecheckerdb2prometheus.py --help
usage: servicecheckerdb2prometheus.py [-h] [-i INPUT_DIR] [-p LISTEN_PORT]
[-a LISTEN_ADDR] [-t METRIC_TTL_SECONDS]
[-l LOG_FILE] [-x LOG_LEVEL]
[-d THREADS]
optional arguments:
-h, --help show this help message and exit
-i INPUT_DIR, --input-dir INPUT_DIR
Directory path to recursively monitor for new
'*servicecheckerdb*' json output files. Default
'./output'
-p LISTEN_PORT, --listen-port LISTEN_PORT
HTTP port to expose /metrics at, default 8000
-a LISTEN_ADDR, --listen-addr LISTEN_ADDR
Address to expost metrics http server on, default
127.0.0.1
-t METRIC_TTL_SECONDS, --metric-ttl-seconds METRIC_TTL_SECONDS
TTL for generated metrics that will be exposed. This
value should be > than the interval that new
*servicecheckerdb*.json are created. Default 300
-l LOG_FILE, --log-file LOG_FILE
Path to log file, default None, STDOUT
-x LOG_LEVEL, --log-level LOG_LEVEL
log level, default DEBUG
-d THREADS, --threads THREADS
max threads for watchdog file processing, default 1
Once run, it will automatically update all exposed metrics as new *servicecheckerdb*.json are created within the --input-dir. The metrics produced are several Prometheus Counters and Gauges appropriately labeled for very broad or granular slices.
Metrics exposed:
# HELP python_info Python platform information
# TYPE python_info gauge
python_info{implementation="CPython",major="3",minor="6",patchlevel="4",version="3.6.4"} 1.0
# HELP sts_analyzer_g_services Current total number of services with 1+ replicas
# TYPE sts_analyzer_g_services gauge
sts_analyzer_g_services{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 1.0
...
# HELP sts_analyzer_g_health_rating Most recent % OK checks for
# TYPE sts_analyzer_g_health_rating gauge
sts_analyzer_g_health_rating{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 100.0
...
# HELP sts_analyzer_g_retry_percentage Most recent % of checks that had to be retried
# TYPE sts_analyzer_g_retry_percentage gauge
sts_analyzer_g_retry_percentage{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 0.0
....
# HELP sts_analyzer_g_fail_percentage Most recent % of checks that have failed
# TYPE sts_analyzer_g_fail_percentage gauge
sts_analyzer_g_fail_percentage{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 0.0
...
# HELP sts_analyzer_g_ok Most recent total OK checks
# TYPE sts_analyzer_g_ok gauge
sts_analyzer_g_ok{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 14.0
...
# HELP sts_analyzer_g_failures Most recent total FAILED checks
# TYPE sts_analyzer_g_failures gauge
sts_analyzer_g_failures{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 0.0
...
# HELP sts_analyzer_g_total_checks Most recent total checks executed
# TYPE sts_analyzer_g_total_checks gauge
sts_analyzer_g_total_checks{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 14.0
...
# HELP sts_analyzer_g_attempts Most recent total ATTEMPTS checks
# TYPE sts_analyzer_g_attempts gauge
sts_analyzer_g_attempts{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 14.0
...
# HELP sts_analyzer_g_avg_resp_time_seconds Average response time for checks in seconds
# TYPE sts_analyzer_g_avg_resp_time_seconds gauge
sts_analyzer_g_avg_resp_time_seconds{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 3.9597857142857142
...
# HELP sts_analyzer_g_total_resp_time_seconds Total response time for checks in seconds
# TYPE sts_analyzer_g_total_resp_time_seconds gauge
sts_analyzer_g_total_resp_time_seconds{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 55.437
...
# HELP sts_analyzer_g_replicas Most recent total replicas
# TYPE sts_analyzer_g_replicas gauge
sts_analyzer_g_replicas{classifier="none",context="pre_prod",docker_service_name="my_app_pre_prod_beta_12_1__1_app",formal_name="my_app",layer="layer0",swarm="myswarm9",tags="previous",version="beta-12.1--1"} 1.0
...
# HELP sts_analyzer_g_attempt_errors Most recent total attempt errors
# TYPE sts_analyzer_g_attempt_errors gauge
sts_analyzer_g_attempt_errors{classifier="none",context="pre_prod",dns="192.168.22.177",docker_service_name="my_app_pre_prod_beta_12_1__1_app",error="timeout",formal_name="my_app",host="mydockerhost2.test.com",layer="layer1",path="/healthcheck",port="10001",swarm="myswarm9",tags="previous",url="https://mydockerhost2.test.com:45001/healthcheck",version="beta-12.1--1"} 1.0
...