From 884a82d81454d2d97b4a03631085b7caf13970fd Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 29 Apr 2020 01:45:59 +0530 Subject: [PATCH 1/2] fix: make semantic changes to commands * add missing __main__ call to commands.py * remove unnecessary imports * fix backup WITH_FILES logic * follow python semantics (?) Signed-off-by: Chinmay D. Pai --- build/common/commands/auto_migrate.py | 22 +++++++++------ build/common/commands/background.py | 3 ++- build/common/commands/backup.py | 11 +++++--- build/common/commands/check_connection.py | 26 +++++++++++++----- build/common/commands/console.py | 5 ++++ build/common/commands/doctor.py | 6 ++--- build/common/commands/migrate.py | 17 ++++++++---- build/common/commands/new.py | 11 +++++--- build/common/commands/push_backup.py | 20 +++++++++----- build/common/commands/restore_backup.py | 33 ++++++++++++----------- build/common/commands/worker.py | 4 ++- 11 files changed, 103 insertions(+), 55 deletions(-) diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py index 1c096ef7..3c53db8d 100644 --- a/build/common/commands/auto_migrate.py +++ b/build/common/commands/auto_migrate.py @@ -9,10 +9,12 @@ from check_connection import get_config APP_VERSIONS_JSON_FILE = 'app_versions.json' APPS_TXT_FILE = 'apps.txt' + def save_version_file(versions): with open(APP_VERSIONS_JSON_FILE, 'w') as f: return json.dump(versions, f, indent=1, sort_keys=True) + def get_apps(): apps = [] try: @@ -24,40 +26,43 @@ def get_apps(): except FileNotFoundError as exception: print(exception) exit(1) - except: + except Exception: print(APPS_TXT_FILE+" is not valid") exit(1) return apps + def get_container_versions(apps): versions = {} for app in apps: try: version = __import__(app).__version__ - versions.update({app:version}) - except: + versions.update({app: version}) + except Exception: pass try: - path = os.path.join('..','apps', app) + path = os.path.join('..', 'apps', app) repo = git.Repo(path) commit_hash = repo.head.object.hexsha - versions.update({app+'_git_hash':commit_hash}) - except: + versions.update({app+'_git_hash': commit_hash}) + except Exception: pass return versions + def get_version_file(): versions = None try: with open(APP_VERSIONS_JSON_FILE) as versions_file: versions = json.load(versions_file) - except: + except Exception: pass return versions + def main(): is_ready = False apps = get_apps() @@ -76,7 +81,7 @@ def main(): version_file_hash = None container_hash = None - repo = git.Repo(os.path.join('..','apps',app)) + repo = git.Repo(os.path.join('..', 'apps', app)) branch = repo.active_branch.name if branch == 'develop': @@ -105,5 +110,6 @@ def main(): version_file = container_versions save_version_file(version_file) + if __name__ == "__main__": main() diff --git a/build/common/commands/background.py b/build/common/commands/background.py index fb008ba9..a26b7934 100644 --- a/build/common/commands/background.py +++ b/build/common/commands/background.py @@ -1,10 +1,11 @@ -import frappe from frappe.utils.scheduler import start_scheduler + def main(): print("Starting background scheduler . . .") start_scheduler() exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/backup.py b/build/common/commands/backup.py index 9fa01cfd..c26468dd 100644 --- a/build/common/commands/backup.py +++ b/build/common/commands/backup.py @@ -1,7 +1,8 @@ -import os, frappe, compileall, re +import os +import frappe from frappe.utils.backups import scheduled_backup -from frappe.utils import now -from frappe.utils import get_sites +from frappe.utils import cint, get_sites, now + def backup(sites, with_files=False): for site in sites: @@ -20,13 +21,15 @@ def backup(sites, with_files=False): print("private files backup taken -", odb.backup_path_private_files, "- on", now()) frappe.destroy() + def main(): installed_sites = ":".join(get_sites()) sites = os.environ.get("SITES", installed_sites).split(":") - with_files=True if os.environ.get("WITH_FILES") else False + with_files = cint(os.environ.get("WITH_FILES")) backup(sites, with_files) exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index b44ea12b..35fe93ff 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -1,4 +1,6 @@ -import socket, os, json, time +import socket +import json +import time from six.moves.urllib.parse import urlparse COMMON_SITE_CONFIG_FILE = 'common_site_config.json' @@ -8,6 +10,7 @@ REDIS_SOCKETIO_KEY = 'redis_socketio' DB_HOST_KEY = 'db_host' DB_PORT = 3306 + def is_open(ip, port, timeout=30): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout) @@ -15,16 +18,17 @@ def is_open(ip, port, timeout=30): s.connect((ip, int(port))) s.shutdown(socket.SHUT_RDWR) return True - except: + except Exception: return False finally: s.close() + def check_host(ip, port, retry=10, delay=3, print_attempt=True): ipup = False for i in range(retry): if print_attempt: - print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1)) if is_open(ip, port): ipup = True break @@ -32,6 +36,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True): time.sleep(delay) return ipup + # Check connection to servers def get_config(): config = None @@ -41,11 +46,12 @@ def get_config(): except FileNotFoundError as exception: print(exception) exit(1) - except: + except Exception: print(COMMON_SITE_CONFIG_FILE+" is not valid") exit(1) return config + # Check mariadb def check_mariadb(retry=10, delay=3, print_attempt=True): config = get_config() @@ -60,11 +66,12 @@ def check_mariadb(retry=10, delay=3, print_attempt=True): print("Connection to MariaDB timed out") exit(1) + # Check redis queue def check_redis_queue(retry=10, delay=3, print_attempt=True): check_redis_queue = False config = get_config() - redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY,"redis://redis-queue:6379")).netloc + redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc redis_queue, redis_queue_port = redis_queue_url.split(":") check_redis_queue = check_host( redis_queue, @@ -76,11 +83,12 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True): print("Connection to redis queue timed out") exit(1) + # Check redis cache def check_redis_cache(retry=10, delay=3, print_attempt=True): check_redis_cache = False config = get_config() - redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY,"redis://redis-cache:6379")).netloc + redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc redis_cache, redis_cache_port = redis_cache_url.split(":") check_redis_cache = check_host( redis_cache, @@ -92,11 +100,12 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True): print("Connection to redis cache timed out") exit(1) + # Check redis socketio def check_redis_socketio(retry=10, delay=3, print_attempt=True): check_redis_socketio = False config = get_config() - redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY,"redis://redis-socketio:6379")).netloc + redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc redis_socketio, redis_socketio_port = redis_socketio_url.split(":") check_redis_socketio = check_host( redis_socketio, @@ -108,6 +117,7 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True): print("Connection to redis socketio timed out") exit(1) + # Get site_config.json def get_site_config(site_name): site_config = None @@ -115,6 +125,7 @@ def get_site_config(site_name): site_config = json.load(site_config_file) return site_config + def main(): check_mariadb() check_redis_queue() @@ -122,5 +133,6 @@ def main(): check_redis_socketio() print('Connections OK') + if __name__ == "__main__": main() diff --git a/build/common/commands/console.py b/build/common/commands/console.py index a9ade863..369f049a 100644 --- a/build/common/commands/console.py +++ b/build/common/commands/console.py @@ -20,6 +20,11 @@ def console(site): print("Apps in this namespace:\n{}".format(", ".join(all_apps))) IPython.embed(display_banner="", header="") + def main(): site = sys.argv[-1] console(site) + + +if __name__ == "__main__": + main() diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index 98f2509f..c317f291 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -1,7 +1,3 @@ -import frappe -import json -import redis -from rq import Worker from check_connection import ( check_mariadb, check_redis_cache, @@ -9,6 +5,7 @@ from check_connection import ( check_redis_socketio, ) + def main(): check_mariadb(retry=1, delay=0, print_attempt=False) print("MariaDB Connected") @@ -22,5 +19,6 @@ def main(): print("Health check successful") exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index 7f8a1602..70471981 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -1,29 +1,34 @@ -import os, frappe, compileall, re, json +import os +import frappe +import json from frappe.migrate import migrate -from frappe.utils import get_sites +from frappe.utils import cint, get_sites from check_connection import get_config + def save_config(config): with open('common_site_config.json', 'w') as f: return json.dump(config, f, indent=1, sort_keys=True) + def set_maintenance_mode(enable=True): conf = get_config() if enable: - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) save_config(conf) if not enable: - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) save_config(conf) + def migrate_sites(maintenance_mode=False): installed_sites = ":".join(get_sites()) sites = os.environ.get("SITES", installed_sites).split(":") if not maintenance_mode: - maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE")) if maintenance_mode: set_maintenance_mode(True) @@ -40,9 +45,11 @@ def migrate_sites(maintenance_mode=False): if maintenance_mode: set_maintenance_mode(False) + def main(): migrate_sites() exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 48ea44fb..0db5616c 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -1,17 +1,19 @@ -import os, frappe, json +import os +import frappe from frappe.commands.site import _new_site from check_connection import get_config, get_site_config + def get_password(env_var, default=None): return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default def _get_password_from_secret(env_var): - """Fetches the secret value from the docker secret file + """Fetches the secret value from the docker secret file usually located inside /run/secrets/ Arguments: - env_var {str} -- Name of the environment variable + env_var {str} -- Name of the environment variable containing the path to the secret file. Returns: [str] -- Secret value @@ -71,11 +73,12 @@ def main(): os.system(command) # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index 26b5c464..8cf2b484 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -8,6 +8,7 @@ from frappe.utils import get_sites DATE_FORMAT = "%Y%m%d_%H%M%S" + def get_file_ext(): return { "database": "-database.sql.gz", @@ -15,6 +16,7 @@ def get_file_ext(): "public_files": "-files.tar" } + def get_backup_details(sitename): backup_details = dict() file_ext = get_file_ext() @@ -41,6 +43,7 @@ def get_backup_details(sitename): return backup_details + def get_s3_config(): check_environment_variables() bucket = os.environ.get('BUCKET_NAME') @@ -55,31 +58,33 @@ def get_s3_config(): return conn, bucket + def check_environment_variables(): - if not 'BUCKET_NAME' in os.environ: + if 'BUCKET_NAME' not in os.environ: print('Variable BUCKET_NAME not set') exit(1) - if not 'ACCESS_KEY_ID' in os.environ: + if 'ACCESS_KEY_ID' not in os.environ: print('Variable ACCESS_KEY_ID not set') exit(1) - if not 'SECRET_ACCESS_KEY' in os.environ: + if 'SECRET_ACCESS_KEY' not in os.environ: print('Variable SECRET_ACCESS_KEY not set') exit(1) - if not 'ENDPOINT_URL' in os.environ: + if 'ENDPOINT_URL' not in os.environ: print('Variable ENDPOINT_URL not set') exit(1) - if not 'BUCKET_DIR' in os.environ: + if 'BUCKET_DIR' not in os.environ: print('Variable BUCKET_DIR not set') exit(1) - if not 'REGION' in os.environ: + if 'REGION' not in os.environ: print('Variable REGION not set') exit(1) + def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) @@ -91,6 +96,7 @@ def upload_file_to_s3(filename, folder, conn, bucket): print("Error uploading: %s" % (e)) exit(1) + def delete_old_backups(limit, bucket, site_name): all_backups = list() all_backup_dates = list() @@ -156,6 +162,7 @@ def delete_old_backups(limit, bucket, site_name): print('Deleteing ' + obj.key) s3.Object(bucket.name, obj.key).delete() + def main(): details = dict() sites = get_sites() @@ -184,5 +191,6 @@ def main(): print('push-backup complete') exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 38c1494b..341849cd 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -8,10 +8,10 @@ import boto3 from new import get_password from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string -from frappe.commands.site import _new_site from frappe.installer import make_conf, get_conf_params, make_site_dirs from check_connection import get_site_config, get_config + def list_directories(path): directories = [] for name in os.listdir(path): @@ -19,25 +19,25 @@ def list_directories(path): directories.append(name) return directories + def get_backup_dir(): return os.path.join( os.path.expanduser('~'), 'backups' ) + def decompress_db(files_base, site): database_file = files_base + '-database.sql.gz' - config = get_config() - site_config = get_site_config(site) - db_root_user = os.environ.get('DB_ROOT_USER', 'root') command = 'gunzip -c {database_file} > {database_extract}'.format( database_file=database_file, - database_extract=database_file.replace('.gz','') + database_extract=database_file.replace('.gz', '') ) print('Extract Database GZip for site {}'.format(site)) os.system(command) + def restore_database(files_base, site): db_root_password = get_password('MYSQL_ROOT_PASSWORD') if not db_root_password: @@ -60,13 +60,13 @@ def restore_database(files_base, site): ) # drop db if exists for clean restore - drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( + drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format( db_name=site_config.get('db_name') ) os.system(drop_database) # create db - create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( + create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format( db_name=site_config.get('db_name') ) os.system(create_database) @@ -86,7 +86,7 @@ def restore_database(files_base, site): os.system(set_user_password) # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(grant_privileges) @@ -96,12 +96,13 @@ def restore_database(files_base, site): db_host=config.get('db_host'), db_password=db_root_password, db_name=site_config.get('db_name'), - database_file=database_file.replace('.gz',''), + database_file=database_file.replace('.gz', ''), ) print('Restoring database for site: {}'.format(site)) os.system(command) + def restore_files(files_base): public_files = files_base + '-files.tar' # extract tar @@ -109,12 +110,14 @@ def restore_files(files_base): print('Extracting {}'.format(public_files)) public_tar.extractall() + def restore_private_files(files_base): private_files = files_base + '-private-files.tar' private_tar = tarfile.open(private_files) print('Extracting {}'.format(private_files)) private_tar.extractall() + def pull_backup_from_s3(): check_environment_variables() @@ -134,8 +137,8 @@ def pull_backup_from_s3(): # Change directory to /home/frappe/backups os.chdir(get_backup_dir()) - for obj in bucket.objects.filter(Prefix = bucket_dir): - backup_file = obj.key.replace(os.path.join(bucket_dir,''),'') + for obj in bucket.objects.filter(Prefix=bucket_dir): + backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '') if not os.path.exists(os.path.dirname(backup_file)): os.makedirs(os.path.dirname(backup_file)) print('Downloading {}'.format(backup_file)) @@ -143,6 +146,7 @@ def pull_backup_from_s3(): os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) + def main(): backup_dir = get_backup_dir() @@ -150,8 +154,8 @@ def main(): pull_backup_from_s3() for site in list_directories(backup_dir): - site_slug = site.replace('.','_') - backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))] + site_slug = site.replace('.', '_') + backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))] latest_backup = max(backups).strftime(DATE_FORMAT) files_base = os.path.join(backup_dir, site, latest_backup, '') files_base += latest_backup + '-' + site_slug @@ -164,8 +168,6 @@ def main(): if not mariadb_root_password: print('Variable MYSQL_ROOT_PASSWORD not set') exit(1) - mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root') - database_file = files_base + '-database.sql.gz' site_config = get_conf_params( db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], @@ -186,5 +188,6 @@ def main(): exit(0) + if __name__ == "__main__": main() diff --git a/build/common/commands/worker.py b/build/common/commands/worker.py index c810adf9..ba107220 100644 --- a/build/common/commands/worker.py +++ b/build/common/commands/worker.py @@ -1,10 +1,12 @@ -import os, frappe +import os from frappe.utils.background_jobs import start_worker + def main(): queue = os.environ.get("WORKER_TYPE", "default") start_worker(queue, False) exit(0) + if __name__ == "__main__": main() From 8f20e5e00d6071ce138e0dacb1437b55e3175965 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 29 Apr 2020 11:44:58 +0530 Subject: [PATCH 2/2] chore: escape backticks on db commands Signed-off-by: Chinmay D. Pai --- build/common/commands/new.py | 2 +- build/common/commands/restore_backup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 0db5616c..82c0ab7b 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -73,7 +73,7 @@ def main(): os.system(command) # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 341849cd..1c098157 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -60,13 +60,13 @@ def restore_database(files_base, site): ) # drop db if exists for clean restore - drop_database = mysql_command + "\"DROP DATABASE IF EXISTS `{db_name}`;\"".format( + drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( db_name=site_config.get('db_name') ) os.system(drop_database) # create db - create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS `{db_name}`;\"".format( + create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( db_name=site_config.get('db_name') ) os.system(create_database) @@ -86,7 +86,7 @@ def restore_database(files_base, site): os.system(set_user_password) # grant db privileges to user - grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(grant_privileges)