From 6dfb1de5c6af0b4d737b574cda9612c965edb62c Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 1 Jul 2020 20:01:47 +0530 Subject: [PATCH 1/3] feat: ping additional services using doctor doctor --ping-service postgres:5432 -p mariadb:3306 --- build/common/commands/check_connection.py | 26 +++++++++++++---- build/common/commands/doctor.py | 34 +++++++++++++++++++++-- build/common/worker/docker-entrypoint.sh | 2 +- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index cd87e44b..2b47c753 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -53,18 +53,32 @@ def get_config(): return config -# Check mariadb -def check_mariadb(retry=10, delay=3, print_attempt=True): +# Check service +def check_service( + retry=10, + delay=3, + print_attempt=True, + service_name=None, + service_port=None): + config = get_config() + if not service_name: + service_name = config.get(DB_HOST_KEY, 'mariadb') + if not service_port: + service_port = config.get(DB_PORT_KEY, DB_PORT) + is_db_connected = False is_db_connected = check_host( - config.get(DB_HOST_KEY, 'mariadb'), - config.get(DB_PORT_KEY, DB_PORT), + service_name, + service_port, retry, delay, print_attempt) if not is_db_connected: - print("Connection to MariaDB timed out") + print("Connection to {service_name}:{service_port} timed out".format( + service_name=service_name, + service_port=service_port, + )) exit(1) @@ -128,7 +142,7 @@ def get_site_config(site_name): def main(): - check_mariadb() + check_service() check_redis_queue() check_redis_cache() check_redis_socketio() diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index c317f291..5354e476 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -1,20 +1,48 @@ +import argparse + from check_connection import ( - check_mariadb, + check_service, check_redis_cache, check_redis_queue, check_redis_socketio, ) +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', + '--ping-service', + dest='ping_services', + action='append', + type=str, + help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"', + ) + args = parser.parse_args() + return args + + def main(): - check_mariadb(retry=1, delay=0, print_attempt=False) - print("MariaDB Connected") + args = parse_args() + check_service(retry=1, delay=0, print_attempt=False) + print("Bench database Connected") check_redis_cache(retry=1, delay=0, print_attempt=False) print("Redis Cache Connected") check_redis_queue(retry=1, delay=0, print_attempt=False) print("Redis Queue Connected") check_redis_socketio(retry=1, delay=0, print_attempt=False) print("Redis SocketIO Connected") + if(args.ping_services): + for service in args.ping_services: + service_name, service_port = service.split(':') + check_service( + retry=1, + delay=0, + print_attempt=False, + service_name=service_name, + service_port=service_port, + ) + print("{0}:{1} Connected".format(service_name, service_port)) print("Health check successful") exit(0) diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index 0a8fc0f6..5b68eec4 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -154,7 +154,7 @@ elif [ "$1" = 'migrate' ]; then elif [ "$1" = 'doctor' ]; then su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/doctor.py" + && python /home/frappe/frappe-bench/commands/doctor.py ${@:2}" exit elif [ "$1" = 'backup' ]; then From 5648e3a6c4363a5d33ebc7d327ecea2cf92ed7af Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 2 Jul 2020 13:40:23 +0530 Subject: [PATCH 2/3] feat: ping additional services using doctor handle error docs entry for health check command --- build/common/commands/doctor.py | 11 ++++++++++- docs/site-operations.md | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index 5354e476..1cb9b200 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -32,9 +32,18 @@ def main(): print("Redis Queue Connected") check_redis_socketio(retry=1, delay=0, print_attempt=False) print("Redis SocketIO Connected") + if(args.ping_services): for service in args.ping_services: - service_name, service_port = service.split(':') + service_name = None + service_port = None + + try: + service_name, service_port = service.split(':') + except ValueError: + print('Service should be in format host:port, e.g postgres:5432') + exit(1) + check_service( retry=1, delay=0, diff --git a/docs/site-operations.md b/docs/site-operations.md index f6dab293..6211e928 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -189,3 +189,15 @@ docker run \ ``` Instead of `alpine` use any image of your choice. + +## Health check + +For socketio and gunicorn service ping the hostname:port and that will be sufficient. For workers and scheduler, there is a command that needs to be executed. + +```shell +docker exec -it _erpnext-worker-d \ + docker-entrypoint.sh doctor -p postgresql-master.pgsql.svc.cluster.local:5432 +``` + +This check ensures that given service should be connected along with services in common_site_config.json. +If connection to service(s) fails, the command fails with exit code 1. From 36c161d2be8b6ad7b938ce5df489b642dbee19ba Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 2 Jul 2020 15:52:06 +0530 Subject: [PATCH 3/3] docs: health check [skip travis] --- docs/site-operations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/site-operations.md b/docs/site-operations.md index 6211e928..1a05525f 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -196,8 +196,10 @@ For socketio and gunicorn service ping the hostname:port and that will be suffic ```shell docker exec -it _erpnext-worker-d \ - docker-entrypoint.sh doctor -p postgresql-master.pgsql.svc.cluster.local:5432 +docker-entrypoint.sh doctor -p postgresql:5432 --ping-service mongodb:27017 ``` +Additional services can be pinged as part of health check with option `-p` or `--ping-service`. + This check ensures that given service should be connected along with services in common_site_config.json. If connection to service(s) fails, the command fails with exit code 1.