From 9f0d602c72134c4d300398315937e61e45072e27 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:50:39 +0300 Subject: [PATCH 1/9] chore(lint): Fix trailing-whitespace --- .vscode/extensions.json | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- build/frappe-socketio/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index fbf7d101..e1e75845 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,6 +8,6 @@ ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [ - + ] } \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83715779..fbe1bafb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ Before publishing a PR, please test builds locally: On each PR that contains changes relevant to Docker builds, images are being built and tested in our CI (GitHub Actions). > :evergreen_tree: Please be considerate when pushing commits and opening PR for multiple branches, as the process of building images uses energy and contributes to global warming. -> +> ## Lint diff --git a/README.md b/README.md index 9b4520e5..6fefaac3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Stable](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml) +[![Build Stable](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml) [![Build Develop](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml) ## Getting Started diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 3294bb4f..f9d5f817 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -3,7 +3,7 @@ FROM alpine/git as builder ARG GIT_REPO=https://github.com/frappe/frappe.git ARG GIT_BRANCH=develop -RUN git clone --depth 1 -b ${GIT_BRANCH} ${GIT_REPO} /opt/frappe +RUN git clone --depth 1 -b ${GIT_BRANCH} ${GIT_REPO} /opt/frappe FROM node:bullseye-slim From 3e4e66769b61f8fa055d923d2d50ba362b384462 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:51:11 +0300 Subject: [PATCH 2/9] chore(lint): Run end-of-file-fixer --- .github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md | 2 +- .vscode/extensions.json | 2 +- build/bench/Dockerfile | 2 +- build/frappe-worker/commands/backup.py | 2 +- build/frappe-worker/commands/constants.py | 2 +- development/vscode-example/settings.json | 2 +- docs/docker-swarm.md | 1 - docs/multi-bench.md | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md b/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md index 02fdd694..6b4b9cbf 100644 --- a/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md +++ b/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md @@ -9,4 +9,4 @@ Welcome to the frappe_docker issue tracker! Before creating an issue, please hee 1. Use the search function before creating a new issue. Duplicates will be closed and directed to the original discussion. 2. Please write extensively, clearly and in detail. ---> \ No newline at end of file +--> diff --git a/.vscode/extensions.json b/.vscode/extensions.json index e1e75845..61a2d874 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -10,4 +10,4 @@ "unwantedRecommendations": [ ] -} \ No newline at end of file +} diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index 7e138a96..3947e488 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -135,4 +135,4 @@ RUN node --version \ RUN bash -c "node --version" \ && bash -c "npm --version" \ && bash -c "yarn --version" \ - && bash -c "bench --help" \ No newline at end of file + && bash -c "bench --help" diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py index b6aa60a2..10ed6a92 100644 --- a/build/frappe-worker/commands/backup.py +++ b/build/frappe-worker/commands/backup.py @@ -36,4 +36,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/build/frappe-worker/commands/constants.py b/build/frappe-worker/commands/constants.py index 9f888099..da59e9f9 100644 --- a/build/frappe-worker/commands/constants.py +++ b/build/frappe-worker/commands/constants.py @@ -10,4 +10,4 @@ COMMON_SITE_CONFIG_FILE = 'common_site_config.json' DATE_FORMAT = "%Y%m%d_%H%M%S" RDS_DB = 'rds_db' RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" -ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites' \ No newline at end of file +ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites' diff --git a/development/vscode-example/settings.json b/development/vscode-example/settings.json index b0f14919..8b133090 100644 --- a/development/vscode-example/settings.json +++ b/development/vscode-example/settings.json @@ -1,3 +1,3 @@ { "debug.node.autoAttach": "disabled" -} \ No newline at end of file +} diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md index 692d5118..210eecae 100644 --- a/docs/docker-swarm.md +++ b/docs/docker-swarm.md @@ -292,4 +292,3 @@ SITES=`site1.example.com`,`site2.example.com` 6. Env variables: - MAINTENANCE_MODE=1 7. Start container - diff --git a/docs/multi-bench.md b/docs/multi-bench.md index ebd64bd9..11a8b931 100644 --- a/docs/multi-bench.md +++ b/docs/multi-bench.md @@ -191,4 +191,4 @@ docker run \ -v _sites-vol:/home/frappe/frappe-bench/sites \ --network _default \ frappe/erpnext-worker:$VERSION migrate -``` \ No newline at end of file +``` From c4772bc5b558b2443c4b4769715d2cfa4f2b8bfd Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:51:47 +0300 Subject: [PATCH 3/9] chore(lint): Run pyupgrade --- build/frappe-worker/commands/check_connection.py | 2 +- build/frappe-worker/commands/doctor.py | 2 +- build/frappe-worker/commands/restore_backup.py | 16 ++++++++-------- build/frappe-worker/commands/utils.py | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py index 359cc58f..946f41e7 100644 --- a/build/frappe-worker/commands/check_connection.py +++ b/build/frappe-worker/commands/check_connection.py @@ -29,7 +29,7 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True): ipup = False for i in range(retry): if print_attempt: - print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1)) + print(f"Attempt {i+1} to connect to {ip}:{port}") if is_open(ip, port): ipup = True break diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py index 1cb9b200..37c69e2c 100644 --- a/build/frappe-worker/commands/doctor.py +++ b/build/frappe-worker/commands/doctor.py @@ -51,7 +51,7 @@ def main(): service_name=service_name, service_port=service_port, ) - print("{0}:{1} Connected".format(service_name, service_port)) + print(f"{service_name}:{service_port} Connected") print("Health check successful") exit(0) diff --git a/build/frappe-worker/commands/restore_backup.py b/build/frappe-worker/commands/restore_backup.py index 9989d18d..39ddeb47 100644 --- a/build/frappe-worker/commands/restore_backup.py +++ b/build/frappe-worker/commands/restore_backup.py @@ -34,7 +34,7 @@ def get_backup_dir(): def decompress_db(database_file, site): command = ["gunzip", "-c", database_file] with open(database_file.replace(".gz", ""), "w") as db_file: - print('Extract Database GZip for site {}'.format(site)) + print(f'Extract Database GZip for site {site}') run_command(command, stdout=db_file) @@ -80,14 +80,14 @@ def restore_files(files_base): public_files = files_base + '-files.tar' # extract tar public_tar = tarfile.open(public_files) - print('Extracting {}'.format(public_files)) + print(f'Extracting {public_files}') public_tar.extractall() def restore_private_files(files_base): private_files = files_base + '-private-files.tar' private_tar = tarfile.open(private_files) - print('Extracting {}'.format(private_files)) + print(f'Extracting {private_files}') private_tar.extractall() @@ -143,7 +143,7 @@ def pull_backup_from_s3(): if backup in backup_file: if not os.path.exists(os.path.dirname(backup_file)): os.makedirs(os.path.dirname(backup_file)) - print('Downloading {}'.format(backup_file)) + print(f'Downloading {backup_file}') bucket.download_file(bucket_dir + '/' + backup_file, backup_file) os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) @@ -196,7 +196,7 @@ def restore_postgres(config, site_config, database_file): run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""]) run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]) run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"]) - with open(database_file.replace('.gz', ''), 'r') as db_file: + with open(database_file.replace('.gz', '')) as db_file: run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) @@ -240,7 +240,7 @@ def restore_mariadb(config, site_config, database_file): run_command(grant_privileges_command) print('Restoring MariaDB') - with open(database_file.replace('.gz', ''), 'r') as db_file: + with open(database_file.replace('.gz', '')) as db_file: run_command(mysql_command + [f"{db_name}"], stdin=db_file) @@ -260,7 +260,7 @@ def main(): if not os.path.exists(site_config_path): site_config_path = os.path.join(backup_dir, site, 'site_config.json') if site in get_sites(): - print('Overwrite site {}'.format(site)) + print(f'Overwrite site {site}') restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) @@ -279,7 +279,7 @@ def main(): ) make_site_dirs() - print('Create site {}'.format(site)) + print(f'Create site {site}') restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) diff --git a/build/frappe-worker/commands/utils.py b/build/frappe-worker/commands/utils.py index eaa9a2cf..0fdd6005 100644 --- a/build/frappe-worker/commands/utils.py +++ b/build/frappe-worker/commands/utils.py @@ -94,7 +94,7 @@ def get_config(): def get_site_config(site_name): site_config = None - with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + with open(f'{site_name}/site_config.json') as site_config_file: site_config = json.load(site_config_file) return site_config @@ -164,7 +164,7 @@ def list_directories(path): def get_site_config_from_path(site_config_path): site_config = dict() if os.path.exists(site_config_path): - with open(site_config_path, 'r') as sc: + with open(site_config_path) as sc: site_config = json.load(sc) return site_config @@ -173,7 +173,7 @@ def set_key_in_site_config(key, site, site_config_path): site_config = get_site_config_from_path(site_config_path) value = site_config.get(key) if value: - print('Set {key} in site config for site: {site}'.format(key=key, site=site)) + print(f'Set {key} in site config for site: {site}') update_site_config(key, value, site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) From f8b7b7af05d2a5985745e2273349b3444f445783 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:52:40 +0300 Subject: [PATCH 4/9] chore(lint): Run black --- build/frappe-worker/bench | 14 +- build/frappe-worker/commands/auto_migrate.py | 12 +- build/frappe-worker/commands/backup.py | 9 +- .../commands/check_connection.py | 63 +++--- build/frappe-worker/commands/constants.py | 20 +- build/frappe-worker/commands/doctor.py | 14 +- build/frappe-worker/commands/gevent_patch.py | 1 + build/frappe-worker/commands/migrate.py | 3 +- build/frappe-worker/commands/new.py | 63 ++++-- build/frappe-worker/commands/push_backup.py | 95 ++++++--- .../frappe-worker/commands/restore_backup.py | 193 ++++++++++-------- build/frappe-worker/commands/utils.py | 68 +++--- 12 files changed, 316 insertions(+), 239 deletions(-) diff --git a/build/frappe-worker/bench b/build/frappe-worker/bench index e9c931c2..b7d9280b 100755 --- a/build/frappe-worker/bench +++ b/build/frappe-worker/bench @@ -6,14 +6,18 @@ import os if __name__ == "__main__": - bench_dir = os.path.join(os.sep, 'home', 'frappe', 'frappe-bench') - sites_dir = os.path.join(bench_dir, 'sites') + bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench") + sites_dir = os.path.join(bench_dir, "sites") bench_helper = os.path.join( - bench_dir, 'apps', 'frappe', - 'frappe', 'utils', 'bench_helper.py', + bench_dir, + "apps", + "frappe", + "frappe", + "utils", + "bench_helper.py", ) cwd = os.getcwd() os.chdir(sites_dir) subprocess.check_call( - [sys.executable, bench_helper, 'frappe'] + sys.argv[1:], + [sys.executable, bench_helper, "frappe"] + sys.argv[1:], ) diff --git a/build/frappe-worker/commands/auto_migrate.py b/build/frappe-worker/commands/auto_migrate.py index ce5f3c2f..095830a3 100644 --- a/build/frappe-worker/commands/auto_migrate.py +++ b/build/frappe-worker/commands/auto_migrate.py @@ -8,7 +8,7 @@ from utils import ( get_apps, get_container_versions, get_version_file, - get_config + get_config, ) @@ -30,12 +30,12 @@ def main(): version_file_hash = None container_hash = None - repo = git.Repo(os.path.join('..', 'apps', app)) + repo = git.Repo(os.path.join("..", "apps", app)) branch = repo.active_branch.name - if branch == 'develop': - version_file_hash = version_file.get(app+'_git_hash') - container_hash = container_versions.get(app+'_git_hash') + if branch == "develop": + version_file_hash = version_file.get(app + "_git_hash") + container_hash = container_versions.get(app + "_git_hash") if container_hash and version_file_hash: if container_hash != version_file_hash: is_ready = True @@ -54,7 +54,7 @@ def main(): config = get_config() - if is_ready and config.get('maintenance_mode') != 1: + if is_ready and config.get("maintenance_mode") != 1: migrate_sites(maintenance_mode=True) version_file = container_versions save_version_file(version_file) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py index 10ed6a92..9a599d87 100644 --- a/build/frappe-worker/commands/backup.py +++ b/build/frappe-worker/commands/backup.py @@ -13,12 +13,17 @@ def backup(sites, with_files=False): backup_path_db=None, backup_path_files=None, backup_path_private_files=None, - force=True + force=True, ) print("database backup taken -", odb.backup_path_db, "- on", now()) if with_files: print("files backup taken -", odb.backup_path_files, "- on", now()) - print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + print( + "private files backup taken -", + odb.backup_path_private_files, + "- on", + now(), + ) frappe.destroy() diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py index 946f41e7..c05dfb9a 100644 --- a/build/frappe-worker/commands/check_connection.py +++ b/build/frappe-worker/commands/check_connection.py @@ -8,7 +8,7 @@ from constants import ( REDIS_SOCKETIO_KEY, DB_HOST_KEY, DB_PORT_KEY, - DB_PORT + DB_PORT, ) @@ -40,30 +40,26 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True): # Check service def check_service( - retry=10, - delay=3, - print_attempt=True, - service_name=None, - service_port=None): + retry=10, delay=3, print_attempt=True, service_name=None, service_port=None +): config = get_config() if not service_name: - service_name = config.get(DB_HOST_KEY, 'mariadb') + service_name = config.get(DB_HOST_KEY, "mariadb") if not service_port: service_port = config.get(DB_PORT_KEY, DB_PORT) is_db_connected = False is_db_connected = check_host( - service_name, - service_port, - retry, - delay, - print_attempt) + service_name, service_port, retry, delay, print_attempt + ) if not is_db_connected: - print("Connection to {service_name}:{service_port} timed out".format( - service_name=service_name, - service_port=service_port, - )) + print( + "Connection to {service_name}:{service_port} timed out".format( + service_name=service_name, + service_port=service_port, + ) + ) exit(1) @@ -71,14 +67,13 @@ def check_service( def check_redis_queue(retry=10, delay=3, print_attempt=True): check_redis_queue = False config = get_config() - redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc + redis_queue_url = urlparse( + config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379") + ).netloc redis_queue, redis_queue_port = redis_queue_url.split(":") check_redis_queue = check_host( - redis_queue, - redis_queue_port, - retry, - delay, - print_attempt) + redis_queue, redis_queue_port, retry, delay, print_attempt + ) if not check_redis_queue: print("Connection to redis queue timed out") exit(1) @@ -88,14 +83,13 @@ def check_redis_queue(retry=10, delay=3, print_attempt=True): def check_redis_cache(retry=10, delay=3, print_attempt=True): check_redis_cache = False config = get_config() - redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc + redis_cache_url = urlparse( + config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379") + ).netloc redis_cache, redis_cache_port = redis_cache_url.split(":") check_redis_cache = check_host( - redis_cache, - redis_cache_port, - retry, - delay, - print_attempt) + redis_cache, redis_cache_port, retry, delay, print_attempt + ) if not check_redis_cache: print("Connection to redis cache timed out") exit(1) @@ -105,14 +99,13 @@ def check_redis_cache(retry=10, delay=3, print_attempt=True): def check_redis_socketio(retry=10, delay=3, print_attempt=True): check_redis_socketio = False config = get_config() - redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc + redis_socketio_url = urlparse( + config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379") + ).netloc redis_socketio, redis_socketio_port = redis_socketio_url.split(":") check_redis_socketio = check_host( - redis_socketio, - redis_socketio_port, - retry, - delay, - print_attempt) + redis_socketio, redis_socketio_port, retry, delay, print_attempt + ) if not check_redis_socketio: print("Connection to redis socketio timed out") exit(1) @@ -123,7 +116,7 @@ def main(): check_redis_queue() check_redis_cache() check_redis_socketio() - print('Connections OK') + print("Connections OK") if __name__ == "__main__": diff --git a/build/frappe-worker/commands/constants.py b/build/frappe-worker/commands/constants.py index da59e9f9..fd70bbdc 100644 --- a/build/frappe-worker/commands/constants.py +++ b/build/frappe-worker/commands/constants.py @@ -1,13 +1,13 @@ -REDIS_QUEUE_KEY = 'redis_queue' -REDIS_CACHE_KEY = 'redis_cache' -REDIS_SOCKETIO_KEY = 'redis_socketio' -DB_HOST_KEY = 'db_host' -DB_PORT_KEY = 'db_port' +REDIS_QUEUE_KEY = "redis_queue" +REDIS_CACHE_KEY = "redis_cache" +REDIS_SOCKETIO_KEY = "redis_socketio" +DB_HOST_KEY = "db_host" +DB_PORT_KEY = "db_port" DB_PORT = 3306 -APP_VERSIONS_JSON_FILE = 'app_versions.json' -APPS_TXT_FILE = 'apps.txt' -COMMON_SITE_CONFIG_FILE = 'common_site_config.json' +APP_VERSIONS_JSON_FILE = "app_versions.json" +APPS_TXT_FILE = "apps.txt" +COMMON_SITE_CONFIG_FILE = "common_site_config.json" DATE_FORMAT = "%Y%m%d_%H%M%S" -RDS_DB = 'rds_db' +RDS_DB = "rds_db" RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" -ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites' +ARCHIVE_SITES_PATH = "/home/frappe/frappe-bench/sites/archive_sites" diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py index 37c69e2c..2874fc57 100644 --- a/build/frappe-worker/commands/doctor.py +++ b/build/frappe-worker/commands/doctor.py @@ -11,10 +11,10 @@ from check_connection import ( def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( - '-p', - '--ping-service', - dest='ping_services', - action='append', + "-p", + "--ping-service", + dest="ping_services", + action="append", type=str, help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"', ) @@ -33,15 +33,15 @@ def main(): check_redis_socketio(retry=1, delay=0, print_attempt=False) print("Redis SocketIO Connected") - if(args.ping_services): + if args.ping_services: for service in args.ping_services: service_name = None service_port = None try: - service_name, service_port = service.split(':') + service_name, service_port = service.split(":") except ValueError: - print('Service should be in format host:port, e.g postgres:5432') + print("Service should be in format host:port, e.g postgres:5432") exit(1) check_service( diff --git a/build/frappe-worker/commands/gevent_patch.py b/build/frappe-worker/commands/gevent_patch.py index 9d7cc3c3..09f65ce7 100644 --- a/build/frappe-worker/commands/gevent_patch.py +++ b/build/frappe-worker/commands/gevent_patch.py @@ -1,2 +1,3 @@ import gevent.monkey + gevent.monkey.patch_all() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py index 6e2846b0..9c9402d2 100644 --- a/build/frappe-worker/commands/migrate.py +++ b/build/frappe-worker/commands/migrate.py @@ -27,11 +27,12 @@ def migrate_sites(maintenance_mode=False): set_maintenance_mode(True) for site in sites: - print('Migrating', site) + print("Migrating", site) frappe.init(site=site) frappe.connect() try: from frappe.migrate import migrate + migrate() finally: frappe.destroy() diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py index b54bfc18..5f3c0ae6 100644 --- a/build/frappe-worker/commands/new.py +++ b/build/frappe-worker/commands/new.py @@ -24,33 +24,43 @@ except ImportError: def main(): config = get_config() - db_type = 'mariadb' - db_port = config.get('db_port', 3306) - db_host = config.get('db_host') - site_name = os.environ.get("SITE_NAME", 'site1.localhost') - db_root_username = os.environ.get("DB_ROOT_USER", 'root') - mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') + db_type = "mariadb" + db_port = config.get("db_port", 3306) + db_host = config.get("db_host") + site_name = os.environ.get("SITE_NAME", "site1.localhost") + db_root_username = os.environ.get("DB_ROOT_USER", "root") + mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", "admin") postgres_root_password = get_password("POSTGRES_PASSWORD") db_root_password = mariadb_root_password if postgres_root_password: - db_type = 'postgres' + db_type = "postgres" db_host = os.environ.get("POSTGRES_HOST") db_port = 5432 db_root_password = postgres_root_password if not db_host: - db_host = config.get('db_host') - print('Environment variable POSTGRES_HOST not found.') - print('Using db_host from common_site_config.json') + db_host = config.get("db_host") + print("Environment variable POSTGRES_HOST not found.") + print("Using db_host from common_site_config.json") sites_path = os.getcwd() common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE) - update_site_config("root_login", db_root_username, validate = False, site_config_path = common_site_config_path) - update_site_config("root_password", db_root_password, validate = False, site_config_path = common_site_config_path) + update_site_config( + "root_login", + db_root_username, + validate=False, + site_config_path=common_site_config_path, + ) + update_site_config( + "root_password", + db_root_password, + validate=False, + site_config_path=common_site_config_path, + ) force = True if os.environ.get("FORCE", None) else False install_apps = os.environ.get("INSTALL_APPS", None) - install_apps = install_apps.split(',') if install_apps else [] + install_apps = install_apps.split(",") if install_apps else [] frappe.init(site_name, new_site=True) if semantic_version.Version(frappe.__version__).major > 11: @@ -59,7 +69,7 @@ def main(): site_name, mariadb_root_username=db_root_username, mariadb_root_password=db_root_password, - admin_password=get_password("ADMIN_PASSWORD", 'admin'), + admin_password=get_password("ADMIN_PASSWORD", "admin"), verbose=True, install_apps=install_apps, source_sql=None, @@ -75,7 +85,7 @@ def main(): site_name, mariadb_root_username=db_root_username, mariadb_root_password=db_root_password, - admin_password=get_password("ADMIN_PASSWORD", 'admin'), + admin_password=get_password("ADMIN_PASSWORD", "admin"), verbose=True, install_apps=install_apps, source_sql=None, @@ -83,16 +93,23 @@ def main(): reinstall=False, ) - if db_type == "mariadb": site_config = get_site_config(site_name) - db_name = site_config.get('db_name') - db_password = site_config.get('db_password') + db_name = site_config.get("db_name") + db_password = site_config.get("db_password") - mysql_command = ["mysql", f"-h{db_host}", f"-u{db_root_username}", f"-p{mariadb_root_password}", "-e"] + mysql_command = [ + "mysql", + f"-h{db_host}", + f"-u{db_root_username}", + f"-p{mariadb_root_password}", + "-e", + ] # Drop User if exists - command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;"] + command = mysql_command + [ + f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;" + ] run_command(command) # Grant permission to database and set password @@ -102,10 +119,12 @@ def main(): if config.get(RDS_DB) or site_config.get(RDS_DB): grant_privileges = RDS_PRIVILEGES - command = mysql_command + [f"\ + command = mysql_command + [ + f"\ CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \ GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \ - FLUSH PRIVILEGES;"] + FLUSH PRIVILEGES;" + ] run_command(command) if frappe.redis_server: diff --git a/build/frappe-worker/commands/push_backup.py b/build/frappe-worker/commands/push_backup.py index 246eb362..cb466f5f 100644 --- a/build/frappe-worker/commands/push_backup.py +++ b/build/frappe-worker/commands/push_backup.py @@ -18,7 +18,7 @@ def get_file_ext(): "database": "-database.sql.gz", "private_files": "-private-files.tar", "public_files": "-files.tar", - "site_config": "-site_config_backup.json" + "site_config": "-site_config_backup.json", } @@ -31,19 +31,26 @@ def get_backup_details(sitename): if os.path.exists(site_backup_path): for filetype, ext in file_ext.items(): - site_slug = sitename.replace('.', '_') - pattern = site_backup_path + '*-' + site_slug + ext + site_slug = sitename.replace(".", "_") + pattern = site_backup_path + "*-" + site_slug + ext backup_files = list(filter(os.path.isfile, glob(pattern))) if len(backup_files) > 0: - backup_files.sort(key=lambda file: os.stat(os.path.join(site_backup_path, file)).st_ctime) - backup_date = datetime.datetime.strptime(time.ctime(os.path.getmtime(backup_files[0])), "%a %b %d %H:%M:%S %Y") + backup_files.sort( + key=lambda file: os.stat( + os.path.join(site_backup_path, file) + ).st_ctime + ) + backup_date = datetime.datetime.strptime( + time.ctime(os.path.getmtime(backup_files[0])), + "%a %b %d %H:%M:%S %Y", + ) backup_details[filetype] = { "sitename": sitename, "file_size_in_bytes": os.stat(backup_files[-1]).st_size, "file_path": os.path.abspath(backup_files[-1]), "filename": os.path.basename(backup_files[-1]), - "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S") + "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S"), } return backup_details @@ -54,31 +61,34 @@ def delete_old_backups(limit, bucket, site_name): all_backup_dates = list() backup_limit = int(limit) check_s3_environment_variables() - bucket_dir = os.environ.get('BUCKET_DIR') + bucket_dir = os.environ.get("BUCKET_DIR") oldest_backup_date = None s3 = boto3.resource( - 's3', - region_name=os.environ.get('REGION'), - aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), - endpoint_url=os.environ.get('ENDPOINT_URL') + "s3", + region_name=os.environ.get("REGION"), + aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), + endpoint_url=os.environ.get("ENDPOINT_URL"), ) bucket = s3.Bucket(bucket) - objects = bucket.meta.client.list_objects_v2( - Bucket=bucket.name, - Delimiter='/') + objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter="/") if objects: - for obj in objects.get('CommonPrefixes'): - if obj.get('Prefix') == bucket_dir + '/': - for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')): + for obj in objects.get("CommonPrefixes"): + if obj.get("Prefix") == bucket_dir + "/": + for backup_obj in bucket.objects.filter(Prefix=obj.get("Prefix")): if backup_obj.get()["ContentType"] == "application/x-directory": continue try: # backup_obj.key is bucket_dir/site/date_time/backupfile.extension - bucket_dir, site_slug, date_time, backupfile = backup_obj.key.split('/') + ( + bucket_dir, + site_slug, + date_time, + backupfile, + ) = backup_obj.key.split("/") date_time_object = datetime.datetime.strptime( date_time, DATE_FORMAT ) @@ -98,7 +108,7 @@ def delete_old_backups(limit, bucket, site_name): for backup in all_backups: try: # backup is bucket_dir/site/date_time/backupfile.extension - backup_dir, site_slug, backup_dt_string, filename = backup.split('/') + backup_dir, site_slug, backup_dt_string, filename = backup.split("/") backup_datetime = datetime.datetime.strptime( backup_dt_string, DATE_FORMAT ) @@ -113,7 +123,7 @@ def delete_old_backups(limit, bucket, site_name): for obj in bucket.objects.filter(Prefix=oldest_backup): # delete all keys that are inside the oldest_backup if bucket_dir in obj.key: - print('Deleteing ' + obj.key) + print("Deleteing " + obj.key) s3.Object(bucket.name, obj.key).delete() @@ -124,31 +134,52 @@ def main(): for site in sites: details = get_backup_details(site) - db_file = details.get('database', {}).get('file_path') - folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + db_file = details.get("database", {}).get("file_path") + folder = os.environ.get("BUCKET_DIR") + "/" + site + "/" if db_file: - folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(db_file)[:15] + '/' + folder = ( + os.environ.get("BUCKET_DIR") + + "/" + + site + + "/" + + os.path.basename(db_file)[:15] + + "/" + ) upload_file_to_s3(db_file, folder, conn, bucket) # Archive site_config.json - site_config_file = details.get('site_config', {}).get('file_path') + site_config_file = details.get("site_config", {}).get("file_path") if not site_config_file: - site_config_file = os.path.join(os.getcwd(), site, 'site_config.json') + site_config_file = os.path.join(os.getcwd(), site, "site_config.json") upload_file_to_s3(site_config_file, folder, conn, bucket) - public_files = details.get('public_files', {}).get('file_path') + public_files = details.get("public_files", {}).get("file_path") if public_files: - folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(public_files)[:15] + '/' + folder = ( + os.environ.get("BUCKET_DIR") + + "/" + + site + + "/" + + os.path.basename(public_files)[:15] + + "/" + ) upload_file_to_s3(public_files, folder, conn, bucket) - private_files = details.get('private_files', {}).get('file_path') + private_files = details.get("private_files", {}).get("file_path") if private_files: - folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(private_files)[:15] + '/' + folder = ( + os.environ.get("BUCKET_DIR") + + "/" + + site + + "/" + + os.path.basename(private_files)[:15] + + "/" + ) upload_file_to_s3(private_files, folder, conn, bucket) - delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, site) + delete_old_backups(os.environ.get("BACKUP_LIMIT", "3"), bucket, site) - print('push-backup complete') + print("push-backup complete") exit(0) diff --git a/build/frappe-worker/commands/restore_backup.py b/build/frappe-worker/commands/restore_backup.py index 39ddeb47..6e19c2b9 100644 --- a/build/frappe-worker/commands/restore_backup.py +++ b/build/frappe-worker/commands/restore_backup.py @@ -10,7 +10,7 @@ from frappe.installer import ( make_conf, get_conf_params, make_site_dirs, - update_site_config + update_site_config, ) from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES from utils import ( @@ -25,69 +25,64 @@ from utils import ( def get_backup_dir(): - return os.path.join( - os.path.expanduser('~'), - 'backups' - ) + return os.path.join(os.path.expanduser("~"), "backups") def decompress_db(database_file, site): command = ["gunzip", "-c", database_file] with open(database_file.replace(".gz", ""), "w") as db_file: - print(f'Extract Database GZip for site {site}') + print(f"Extract Database GZip for site {site}") run_command(command, stdout=db_file) def restore_database(files_base, site_config_path, site): # restore database - database_file = files_base + '-database.sql.gz' + database_file = files_base + "-database.sql.gz" decompress_db(database_file, site) config = get_config() # Set db_type if it exists in backup site_config.json - set_key_in_site_config('db_type', site, site_config_path) + set_key_in_site_config("db_type", site, site_config_path) # Set db_host if it exists in backup site_config.json - set_key_in_site_config('db_host', site, site_config_path) + set_key_in_site_config("db_host", site, site_config_path) # Set db_port if it exists in backup site_config.json - set_key_in_site_config('db_port', site, site_config_path) + set_key_in_site_config("db_port", site, site_config_path) # get updated site_config site_config = get_site_config(site) # if no db_type exists, default to mariadb - db_type = site_config.get('db_type', 'mariadb') + db_type = site_config.get("db_type", "mariadb") is_database_restored = False - if db_type == 'mariadb': + if db_type == "mariadb": restore_mariadb( - config=config, - site_config=site_config, - database_file=database_file) + config=config, site_config=site_config, database_file=database_file + ) is_database_restored = True - elif db_type == 'postgres': + elif db_type == "postgres": restore_postgres( - config=config, - site_config=site_config, - database_file=database_file) + config=config, site_config=site_config, database_file=database_file + ) is_database_restored = True if is_database_restored: # Set encryption_key if it exists in backup site_config.json - set_key_in_site_config('encryption_key', site, site_config_path) + set_key_in_site_config("encryption_key", site, site_config_path) def restore_files(files_base): - public_files = files_base + '-files.tar' + public_files = files_base + "-files.tar" # extract tar public_tar = tarfile.open(public_files) - print(f'Extracting {public_files}') + print(f"Extracting {public_files}") public_tar.extractall() def restore_private_files(files_base): - private_files = files_base + '-private-files.tar' + private_files = files_base + "-private-files.tar" private_tar = tarfile.open(private_files) - print(f'Extracting {private_files}') + print(f"Extracting {private_files}") private_tar.extractall() @@ -96,15 +91,15 @@ def pull_backup_from_s3(): # https://stackoverflow.com/a/54672690 s3 = boto3.resource( - 's3', - region_name=os.environ.get('REGION'), - aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), - endpoint_url=os.environ.get('ENDPOINT_URL') + "s3", + region_name=os.environ.get("REGION"), + aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), + endpoint_url=os.environ.get("ENDPOINT_URL"), ) - bucket_dir = os.environ.get('BUCKET_DIR') - bucket_name = os.environ.get('BUCKET_NAME') + bucket_dir = os.environ.get("BUCKET_DIR") + bucket_name = os.environ.get("BUCKET_NAME") bucket = s3.Bucket(bucket_name) # Change directory to /home/frappe/backups @@ -118,10 +113,10 @@ def pull_backup_from_s3(): for obj in bucket.objects.filter(Prefix=bucket_dir): if obj.get()["ContentType"] == "application/x-directory": continue - backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '') + backup_file = obj.key.replace(os.path.join(bucket_dir, ""), "") backup_files.append(backup_file) - site_name, timestamp, backup_type = backup_file.split('/') - site_timestamp = site_name + '/' + timestamp + site_name, timestamp, backup_type = backup_file.split("/") + site_timestamp = site_name + "/" + timestamp sites.add(site_name) site_timestamps.add(site_timestamp) @@ -129,13 +124,11 @@ def pull_backup_from_s3(): for site in sites: backup_timestamps = [] for site_timestamp in site_timestamps: - site_name, timestamp = site_timestamp.split('/') + site_name, timestamp = site_timestamp.split("/") if site == site_name: - timestamp_datetime = datetime.datetime.strptime( - timestamp, DATE_FORMAT - ) + timestamp_datetime = datetime.datetime.strptime(timestamp, DATE_FORMAT) backup_timestamps.append(timestamp) - download_backups.append(site + '/' + max(backup_timestamps)) + download_backups.append(site + "/" + max(backup_timestamps)) # Only download latest backups for backup_file in backup_files: @@ -143,21 +136,21 @@ def pull_backup_from_s3(): if backup in backup_file: if not os.path.exists(os.path.dirname(backup_file)): os.makedirs(os.path.dirname(backup_file)) - print(f'Downloading {backup_file}') - bucket.download_file(bucket_dir + '/' + backup_file, backup_file) + print(f"Downloading {backup_file}") + bucket.download_file(bucket_dir + "/" + backup_file, backup_file) - os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) + os.chdir(os.path.join(os.path.expanduser("~"), "frappe-bench", "sites")) def restore_postgres(config, site_config, database_file): # common config common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE) - db_root_user = config.get('root_login') + db_root_user = config.get("root_login") if not db_root_user: - postgres_user = os.environ.get('DB_ROOT_USER') + postgres_user = os.environ.get("DB_ROOT_USER") if not postgres_user: - print('Variable DB_ROOT_USER not set') + print("Variable DB_ROOT_USER not set") exit(1) db_root_user = postgres_user @@ -165,13 +158,14 @@ def restore_postgres(config, site_config, database_file): "root_login", db_root_user, validate=False, - site_config_path=common_site_config_path) + site_config_path=common_site_config_path, + ) - db_root_password = config.get('root_password') + db_root_password = config.get("root_password") if not db_root_password: - root_password = get_password('POSTGRES_PASSWORD') + root_password = get_password("POSTGRES_PASSWORD") if not root_password: - print('Variable POSTGRES_PASSWORD not set') + print("Variable POSTGRES_PASSWORD not set") exit(1) db_root_password = root_password @@ -179,53 +173,72 @@ def restore_postgres(config, site_config, database_file): "root_password", db_root_password, validate=False, - site_config_path=common_site_config_path) + site_config_path=common_site_config_path, + ) # site config - db_host = site_config.get('db_host') - db_port = site_config.get('db_port', 5432) - db_name = site_config.get('db_name') - db_password = site_config.get('db_password') + db_host = site_config.get("db_host") + db_port = site_config.get("db_port", 5432) + db_name = site_config.get("db_name") + db_password = site_config.get("db_password") psql_command = ["psql"] psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}" - print('Restoring PostgreSQL') - run_command(psql_command + [psql_uri, "-c", f"DROP DATABASE IF EXISTS \"{db_name}\""]) + print("Restoring PostgreSQL") + run_command(psql_command + [psql_uri, "-c", f'DROP DATABASE IF EXISTS "{db_name}"']) run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"]) - run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""]) - run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]) - run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"]) - with open(database_file.replace('.gz', '')) as db_file: + run_command(psql_command + [psql_uri, "-c", f'CREATE DATABASE "{db_name}"']) + run_command( + psql_command + + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"] + ) + run_command( + psql_command + + [psql_uri, "-c", f'GRANT ALL PRIVILEGES ON DATABASE "{db_name}" TO {db_name}'] + ) + with open(database_file.replace(".gz", "")) as db_file: run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) def restore_mariadb(config, site_config, database_file): - db_root_password = get_password('MYSQL_ROOT_PASSWORD') + db_root_password = get_password("MYSQL_ROOT_PASSWORD") if not db_root_password: - print('Variable MYSQL_ROOT_PASSWORD not set') + print("Variable MYSQL_ROOT_PASSWORD not set") exit(1) - db_root_user = os.environ.get("DB_ROOT_USER", 'root') + db_root_user = os.environ.get("DB_ROOT_USER", "root") - db_host = site_config.get('db_host', config.get('db_host')) - db_port = site_config.get('db_port', config.get('db_port', 3306)) - db_name = site_config.get('db_name') - db_password = site_config.get('db_password') + db_host = site_config.get("db_host", config.get("db_host")) + db_port = site_config.get("db_port", config.get("db_port", 3306)) + db_name = site_config.get("db_name") + db_password = site_config.get("db_password") # mysql command prefix - mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"] + mysql_command = [ + "mysql", + f"-u{db_root_user}", + f"-h{db_host}", + f"-p{db_root_password}", + f"-P{db_port}", + ] # drop db if exists for clean restore - drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] + drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] run_command(drop_database) # create db - create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS `{db_name}`;"] + create_database = mysql_command + [ + "-e", + f"CREATE DATABASE IF NOT EXISTS `{db_name}`;", + ] run_command(create_database) # create user - create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] + create_user = mysql_command + [ + "-e", + f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", + ] run_command(create_user) # grant db privileges to user @@ -236,11 +249,14 @@ def restore_mariadb(config, site_config, database_file): if config.get(RDS_DB) or site_config.get(RDS_DB): grant_privileges = RDS_PRIVILEGES - grant_privileges_command = mysql_command + ["-e", f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] + grant_privileges_command = mysql_command + [ + "-e", + f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;", + ] run_command(grant_privileges_command) - print('Restoring MariaDB') - with open(database_file.replace('.gz', '')) as db_file: + print("Restoring MariaDB") + with open(database_file.replace(".gz", "")) as db_file: run_command(mysql_command + [f"{db_name}"], stdin=db_file) @@ -251,35 +267,38 @@ def main(): pull_backup_from_s3() for site in list_directories(backup_dir): - site_slug = site.replace('.', '_') - backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))] + site_slug = site.replace(".", "_") + backups = [ + datetime.datetime.strptime(backup, DATE_FORMAT) + for backup in list_directories(os.path.join(backup_dir, site)) + ] latest_backup = max(backups).strftime(DATE_FORMAT) - files_base = os.path.join(backup_dir, site, latest_backup, '') - files_base += latest_backup + '-' + site_slug - site_config_path = files_base + '-site_config_backup.json' + files_base = os.path.join(backup_dir, site, latest_backup, "") + files_base += latest_backup + "-" + site_slug + site_config_path = files_base + "-site_config_backup.json" if not os.path.exists(site_config_path): - site_config_path = os.path.join(backup_dir, site, 'site_config.json') + site_config_path = os.path.join(backup_dir, site, "site_config.json") if site in get_sites(): - print(f'Overwrite site {site}') + print(f"Overwrite site {site}") restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) else: site_config = get_conf_params( - db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], - db_password=random_string(16) + db_name="_" + hashlib.sha1(site.encode()).hexdigest()[:16], + db_password=random_string(16), ) frappe.local.site = site frappe.local.sites_path = os.getcwd() - frappe.local.site_path = os.getcwd() + '/' + site + frappe.local.site_path = os.getcwd() + "/" + site make_conf( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password'), + db_name=site_config.get("db_name"), + db_password=site_config.get("db_password"), ) make_site_dirs() - print(f'Create site {site}') + print(f"Create site {site}") restore_database(files_base, site_config_path, site) restore_private_files(files_base) restore_files(files_base) diff --git a/build/frappe-worker/commands/utils.py b/build/frappe-worker/commands/utils.py index 0fdd6005..5f81fdc6 100644 --- a/build/frappe-worker/commands/utils.py +++ b/build/frappe-worker/commands/utils.py @@ -5,11 +5,8 @@ import boto3 import git from frappe.installer import update_site_config -from constants import ( - APP_VERSIONS_JSON_FILE, - APPS_TXT_FILE, - COMMON_SITE_CONFIG_FILE -) +from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE + def run_command(command, stdout=None, stdin=None, stderr=None): stdout = stdout or subprocess.PIPE @@ -26,7 +23,7 @@ def run_command(command, stdout=None, stdin=None, stderr=None): def save_version_file(versions): - with open(APP_VERSIONS_JSON_FILE, 'w') as f: + with open(APP_VERSIONS_JSON_FILE, "w") as f: return json.dump(versions, f, indent=1, sort_keys=True) @@ -58,10 +55,10 @@ def get_container_versions(apps): pass try: - path = os.path.join('..', 'apps', app) + path = os.path.join("..", "apps", app) repo = git.Repo(path) commit_hash = repo.head.object.hexsha - versions.update({app+'_git_hash': commit_hash}) + versions.update({app + "_git_hash": commit_hash}) except Exception: pass @@ -94,18 +91,22 @@ def get_config(): def get_site_config(site_name): site_config = None - with open(f'{site_name}/site_config.json') as site_config_file: + with open(f"{site_name}/site_config.json") as site_config_file: site_config = json.load(site_config_file) return site_config def save_config(config): - with open(COMMON_SITE_CONFIG_FILE, 'w') as f: + with open(COMMON_SITE_CONFIG_FILE, "w") as f: return json.dump(config, f, indent=1, sort_keys=True) def get_password(env_var, default=None): - return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default + return ( + os.environ.get(env_var) + or get_password_from_secret(f"{env_var}_FILE") + or default + ) def get_password_from_secret(env_var): @@ -128,14 +129,14 @@ def get_password_from_secret(env_var): def get_s3_config(): check_s3_environment_variables() - bucket = os.environ.get('BUCKET_NAME') + bucket = os.environ.get("BUCKET_NAME") conn = boto3.client( - 's3', - region_name=os.environ.get('REGION'), - aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), - endpoint_url=os.environ.get('ENDPOINT_URL') + "s3", + region_name=os.environ.get("REGION"), + aws_access_key_id=os.environ.get("ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"), + endpoint_url=os.environ.get("ENDPOINT_URL"), ) return conn, bucket @@ -173,32 +174,35 @@ def set_key_in_site_config(key, site, site_config_path): site_config = get_site_config_from_path(site_config_path) value = site_config.get(key) if value: - print(f'Set {key} in site config for site: {site}') - update_site_config(key, value, - site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) + print(f"Set {key} in site config for site: {site}") + update_site_config( + key, + value, + site_config_path=os.path.join(os.getcwd(), site, "site_config.json"), + ) def check_s3_environment_variables(): - if 'BUCKET_NAME' not in os.environ: - print('Variable BUCKET_NAME not set') + if "BUCKET_NAME" not in os.environ: + print("Variable BUCKET_NAME not set") exit(1) - if 'ACCESS_KEY_ID' not in os.environ: - print('Variable ACCESS_KEY_ID not set') + if "ACCESS_KEY_ID" not in os.environ: + print("Variable ACCESS_KEY_ID not set") exit(1) - if 'SECRET_ACCESS_KEY' not in os.environ: - print('Variable SECRET_ACCESS_KEY not set') + if "SECRET_ACCESS_KEY" not in os.environ: + print("Variable SECRET_ACCESS_KEY not set") exit(1) - if 'ENDPOINT_URL' not in os.environ: - print('Variable ENDPOINT_URL not set') + if "ENDPOINT_URL" not in os.environ: + print("Variable ENDPOINT_URL not set") exit(1) - if 'BUCKET_DIR' not in os.environ: - print('Variable BUCKET_DIR not set') + if "BUCKET_DIR" not in os.environ: + print("Variable BUCKET_DIR not set") exit(1) - if 'REGION' not in os.environ: - print('Variable REGION not set') + if "REGION" not in os.environ: + print("Variable REGION not set") exit(1) From 8b4dfce4f04a5128a281a672304707237f2ede4d Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:53:00 +0300 Subject: [PATCH 5/9] chore(lint): Run isort --- build/frappe-worker/bench | 3 +-- build/frappe-worker/commands/auto_migrate.py | 8 +++---- build/frappe-worker/commands/backup.py | 3 ++- .../commands/check_connection.py | 17 ++++++------- build/frappe-worker/commands/doctor.py | 2 +- build/frappe-worker/commands/migrate.py | 2 +- build/frappe-worker/commands/new.py | 11 +++------ build/frappe-worker/commands/push_backup.py | 14 ++++------- .../frappe-worker/commands/restore_backup.py | 24 +++++++++---------- build/frappe-worker/commands/utils.py | 4 ++-- 10 files changed, 40 insertions(+), 48 deletions(-) diff --git a/build/frappe-worker/bench b/build/frappe-worker/bench index b7d9280b..f702d0d7 100755 --- a/build/frappe-worker/bench +++ b/build/frappe-worker/bench @@ -1,9 +1,8 @@ #!/home/frappe/frappe-bench/env/bin/python +import os import subprocess import sys -import os - if __name__ == "__main__": bench_dir = os.path.join(os.sep, "home", "frappe", "frappe-bench") diff --git a/build/frappe-worker/commands/auto_migrate.py b/build/frappe-worker/commands/auto_migrate.py index 095830a3..a9f84743 100644 --- a/build/frappe-worker/commands/auto_migrate.py +++ b/build/frappe-worker/commands/auto_migrate.py @@ -1,14 +1,14 @@ import os -import semantic_version -import git +import git +import semantic_version from migrate import migrate_sites from utils import ( - save_version_file, get_apps, + get_config, get_container_versions, get_version_file, - get_config, + save_version_file, ) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py index 9a599d87..9338d207 100644 --- a/build/frappe-worker/commands/backup.py +++ b/build/frappe-worker/commands/backup.py @@ -1,7 +1,8 @@ import os + import frappe -from frappe.utils.backups import scheduled_backup from frappe.utils import cint, get_sites, now +from frappe.utils.backups import scheduled_backup def backup(sites, with_files=False): diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py index c05dfb9a..d2bae21f 100644 --- a/build/frappe-worker/commands/check_connection.py +++ b/build/frappe-worker/commands/check_connection.py @@ -1,15 +1,16 @@ import socket import time + +from constants import ( + DB_HOST_KEY, + DB_PORT, + DB_PORT_KEY, + REDIS_CACHE_KEY, + REDIS_QUEUE_KEY, + REDIS_SOCKETIO_KEY, +) from six.moves.urllib.parse import urlparse from utils import get_config -from constants import ( - REDIS_QUEUE_KEY, - REDIS_CACHE_KEY, - REDIS_SOCKETIO_KEY, - DB_HOST_KEY, - DB_PORT_KEY, - DB_PORT, -) def is_open(ip, port, timeout=30): diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py index 2874fc57..c7eac75f 100644 --- a/build/frappe-worker/commands/doctor.py +++ b/build/frappe-worker/commands/doctor.py @@ -1,10 +1,10 @@ import argparse from check_connection import ( - check_service, check_redis_cache, check_redis_queue, check_redis_socketio, + check_service, ) diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py index 9c9402d2..2e99ed54 100644 --- a/build/frappe-worker/commands/migrate.py +++ b/build/frappe-worker/commands/migrate.py @@ -1,6 +1,6 @@ import os -import frappe +import frappe from frappe.utils import cint, get_sites from utils import get_config, save_config diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py index 5f3c0ae6..546ef9d8 100644 --- a/build/frappe-worker/commands/new.py +++ b/build/frappe-worker/commands/new.py @@ -1,15 +1,10 @@ import os + import frappe import semantic_version - -from frappe.installer import update_site_config from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES -from utils import ( - run_command, - get_config, - get_site_config, - get_password, -) +from frappe.installer import update_site_config +from utils import get_config, get_password, get_site_config, run_command # try to import _new_site from frappe, which could possibly # exist in either commands.py or installer.py, and so we need diff --git a/build/frappe-worker/commands/push_backup.py b/build/frappe-worker/commands/push_backup.py index cb466f5f..2a49933a 100644 --- a/build/frappe-worker/commands/push_backup.py +++ b/build/frappe-worker/commands/push_backup.py @@ -1,16 +1,12 @@ +import datetime import os import time -import boto3 - -import datetime from glob import glob -from frappe.utils import get_sites + +import boto3 from constants import DATE_FORMAT -from utils import ( - get_s3_config, - upload_file_to_s3, - check_s3_environment_variables, -) +from frappe.utils import get_sites +from utils import check_s3_environment_variables, get_s3_config, upload_file_to_s3 def get_file_ext(): diff --git a/build/frappe-worker/commands/restore_backup.py b/build/frappe-worker/commands/restore_backup.py index 6e19c2b9..1462cd27 100644 --- a/build/frappe-worker/commands/restore_backup.py +++ b/build/frappe-worker/commands/restore_backup.py @@ -1,26 +1,26 @@ -import os import datetime -import tarfile import hashlib -import frappe -import boto3 +import os +import tarfile -from frappe.utils import get_sites, random_string +import boto3 +import frappe +from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES from frappe.installer import ( - make_conf, get_conf_params, + make_conf, make_site_dirs, update_site_config, ) -from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES +from frappe.utils import get_sites, random_string from utils import ( - run_command, - list_directories, - set_key_in_site_config, - get_site_config, + check_s3_environment_variables, get_config, get_password, - check_s3_environment_variables, + get_site_config, + list_directories, + run_command, + set_key_in_site_config, ) diff --git a/build/frappe-worker/commands/utils.py b/build/frappe-worker/commands/utils.py index 5f81fdc6..7ea48225 100644 --- a/build/frappe-worker/commands/utils.py +++ b/build/frappe-worker/commands/utils.py @@ -1,11 +1,11 @@ import json import os import subprocess + import boto3 import git - -from frappe.installer import update_site_config from constants import APP_VERSIONS_JSON_FILE, APPS_TXT_FILE, COMMON_SITE_CONFIG_FILE +from frappe.installer import update_site_config def run_command(command, stdout=None, stdin=None, stderr=None): From fb07581595f9ee8d7b470b112c9f62db361f0bb0 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:53:43 +0300 Subject: [PATCH 6/9] chore(lint): Run prettier --- .github/workflows/greetings.yml | 14 +-- .vscode/extensions.json | 16 +-- CODE_OF_CONDUCT.md | 26 ++-- CONTRIBUTING.md | 9 +- README.md | 18 +-- development/README.md | 12 +- development/vscode-example/launch.json | 153 ++++++++++++----------- development/vscode-example/settings.json | 2 +- docs/docker-swarm.md | 17 +-- docs/multi-bench.md | 89 ++++++------- docs/single-bench.md | 94 +++++++------- docs/site-operations.md | 23 ++-- docs/tips-for-moving-deployments.md | 16 +-- installation/docker-compose-common.yml | 2 +- installation/docker-compose-erpnext.yml | 2 +- installation/docker-compose-frappe.yml | 2 +- installation/docker-compose-networks.yml | 2 +- 17 files changed, 253 insertions(+), 244 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 669221c0..15d554e8 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -6,10 +6,10 @@ jobs: greeting: runs-on: ubuntu-latest steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: | - Hello! We're very happy to see your first issue. If your issue is about a problem, go back and check you have copy-pasted all the debug logs you can so we can help you as fast as possible! - pr-message: | - Hello! Thank you about this PR. Since this is your first PR, please make sure you have described the improvements and your code is well documented. + - uses: actions/first-interaction@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + issue-message: | + Hello! We're very happy to see your first issue. If your issue is about a problem, go back and check you have copy-pasted all the debug logs you can so we can help you as fast as possible! + pr-message: | + Hello! Thank you about this PR. Since this is your first PR, please make sure you have described the improvements and your code is well documented. diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 61a2d874..6151dbb7 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,13 +1,9 @@ { - // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. - // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp - // List of extensions which should be recommended for users of this workspace. - "recommendations": [ - "ms-vscode-remote.remote-containers" - ], - // List of extensions recommended by VS Code that should not be recommended for users of this workspace. - "unwantedRecommendations": [ - - ] + // List of extensions which should be recommended for users of this workspace. + "recommendations": ["ms-vscode-remote.remote-containers"], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] } diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 18e5ca1d..55c27d81 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -14,22 +14,22 @@ appearance, race, religion, or sexual identity and orientation. Examples of behavior that contributes to creating a positive environment include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting ## Our Responsibilities diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fbe1bafb..17733fd6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,9 +7,8 @@ Before publishing a PR, please test builds locally: - with VSCode for testing environments (only for frappe/bench image). On each PR that contains changes relevant to Docker builds, images are being built and tested in our CI (GitHub Actions). -> :evergreen_tree: Please be considerate when pushing commits and opening PR for multiple branches, as the process of building images uses energy and contributes to global warming. -> +> :evergreen_tree: Please be considerate when pushing commits and opening PR for multiple branches, as the process of building images uses energy and contributes to global warming. ## Lint @@ -38,7 +37,6 @@ To run all the files in repository, run: pre-commit run --all-files ``` - ## Build ```shell @@ -52,20 +50,25 @@ docker buildx bake -f docker-bake.hcl *...* ## Test ### Ping site + Lightweight test that just checks if site will be available after creation. Frappe: + ```shell ./tests/test-frappe.sh ``` ERPNext: + ```shell ./tests/test-erpnext.sh ``` ### Integration test + Tests frappe-bench-like commands, for example, `backup` and `restore`. + ```shell ./tests/integration-test.sh ``` diff --git a/README.md b/README.md index 6fefaac3..9580cda2 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,21 @@ cd frappe_docker It takes care of the following: -* Setting up the desired version of Frappe/ERPNext. -* Setting up all the system requirements: eg. MariaDB, Node, Redis. -* Configure networking for remote access and setting up LetsEncrypt. +- Setting up the desired version of Frappe/ERPNext. +- Setting up all the system requirements: eg. MariaDB, Node, Redis. +- Configure networking for remote access and setting up LetsEncrypt. It doesn't take care of the following: -* Cron Job to backup sites is not created by default. -* Use `CronJob` on k8s or refer wiki for alternatives. +- Cron Job to backup sites is not created by default. +- Use `CronJob` on k8s or refer wiki for alternatives. 1. Single Server Installs - 1. [Single bench](docs/single-bench.md). Easiest Install! - 2. [Multi bench](docs/multi-bench.md) + 1. [Single bench](docs/single-bench.md). Easiest Install! + 2. [Multi bench](docs/multi-bench.md) 2. Multi Server Installs - 1. [Docker Swarm](docs/docker-swarm.md) - 2. [Kubernetes](https://helm.erpnext.com) + 1. [Docker Swarm](docs/docker-swarm.md) + 2. [Kubernetes](https://helm.erpnext.com) 3. [Site Operations](docs/site-operations.md) 4. [Environment Variables](docs/environment-variables.md) 5. [Custom apps for production](docs/custom-apps-for-production.md) diff --git a/development/README.md b/development/README.md index 48760cf9..162df9ca 100644 --- a/development/README.md +++ b/development/README.md @@ -44,14 +44,14 @@ If you want to use PostgreSQL instead, edit `.devcontainer/docker-compose.yml` a VSCode should automatically inquire you to install the required extensions, that can also be installed manually as follows: - Install Remote - Containers for VSCode - - through command line `code --install-extension ms-vscode-remote.remote-containers` - - clicking on the Install button in the Vistual Studio Marketplace: [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) - - View: Extensions command in VSCode (Windows: Ctrl+Shift+X; macOS: Cmd+Shift+X) then search for extension `ms-vscode-remote.remote-containers` + - through command line `code --install-extension ms-vscode-remote.remote-containers` + - clicking on the Install button in the Vistual Studio Marketplace: [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + - View: Extensions command in VSCode (Windows: Ctrl+Shift+X; macOS: Cmd+Shift+X) then search for extension `ms-vscode-remote.remote-containers` After the extensions are installed, you can: - Open frappe_docker folder in VS Code. - - `code .` + - `code .` - Launch the command, from Command Palette (Ctrl + Shift + P) `Execute Remote Containers : Reopen in Container`. You can also click in the bottom left corner to access the remote container menu. Notes: @@ -94,6 +94,7 @@ code Procfile ``` Or running the following command: + ```shell sed -i '/redis/d' ./Procfile ``` @@ -105,6 +106,7 @@ You can create a new site with the following command: ```shell bench new-site sitename --no-mariadb-socket ``` + sitename MUST end with .localhost for trying deployments locally. for example: @@ -259,7 +261,6 @@ Example shows the queries to be executed for site `localhost` Open sites/localhost/site_config.json: - ```shell code sites/localhost/site_config.json ``` @@ -286,6 +287,7 @@ EXIT; In case you don't use VSCode, you may start the containers manually with the following command: ### Running the containers + ```shell docker-compose -f .devcontainer/docker-compose.yml up -d ``` diff --git a/development/vscode-example/launch.json b/development/vscode-example/launch.json index 0c8046bb..68fa9309 100644 --- a/development/vscode-example/launch.json +++ b/development/vscode-example/launch.json @@ -1,76 +1,81 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Bench Web", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", - "args": [ - "frappe", "serve", "--port", "8000", "--noreload", "--nothreading" - ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", - "cwd": "${workspaceFolder}/frappe-bench/sites", - "env": { - "DEV_SERVER": "1" - } - }, - { - "name": "Bench Default Worker", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", - "args": [ - "frappe", "worker", "--queue", "default" - ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", - "cwd": "${workspaceFolder}/frappe-bench/sites", - "env": { - "DEV_SERVER": "1" - } - }, - { - "name": "Bench Short Worker", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", - "args": [ - "frappe", "worker", "--queue", "short" - ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", - "cwd": "${workspaceFolder}/frappe-bench/sites", - "env": { - "DEV_SERVER": "1" - } - }, - { - "name": "Bench Long Worker", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", - "args": [ - "frappe", "worker", "--queue", "long" - ], - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", - "cwd": "${workspaceFolder}/frappe-bench/sites", - "env": { - "DEV_SERVER": "1" - } - }, - { - "name": "Honcho SocketIO Watch Schedule Worker", - "type": "python", - "request": "launch", - "program": "/home/frappe/.local/bin/honcho", - "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", - "cwd": "${workspaceFolder}/frappe-bench", - "console": "internalConsole", - "args": [ - "start", "socketio", "watch", "schedule", "worker_short", "worker_long", "worker_default" - ] - } - ] + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Bench Web", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": [ + "frappe", + "serve", + "--port", + "8000", + "--noreload", + "--nothreading" + ], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } + }, + { + "name": "Bench Default Worker", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": ["frappe", "worker", "--queue", "default"], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } + }, + { + "name": "Bench Short Worker", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": ["frappe", "worker", "--queue", "short"], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } + }, + { + "name": "Bench Long Worker", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": ["frappe", "worker", "--queue", "long"], + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench/sites", + "env": { + "DEV_SERVER": "1" + } + }, + { + "name": "Honcho SocketIO Watch Schedule Worker", + "type": "python", + "request": "launch", + "program": "/home/frappe/.local/bin/honcho", + "pythonPath": "${workspaceFolder}/frappe-bench/env/bin/python", + "cwd": "${workspaceFolder}/frappe-bench", + "console": "internalConsole", + "args": [ + "start", + "socketio", + "watch", + "schedule", + "worker_short", + "worker_long", + "worker_default" + ] + } + ] } diff --git a/development/vscode-example/settings.json b/development/vscode-example/settings.json index 8b133090..e5d9441c 100644 --- a/development/vscode-example/settings.json +++ b/development/vscode-example/settings.json @@ -1,3 +1,3 @@ { - "debug.node.autoAttach": "disabled" + "debug.node.autoAttach": "disabled" } diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md index 210eecae..7bf2a412 100644 --- a/docs/docker-swarm.md +++ b/docs/docker-swarm.md @@ -42,7 +42,7 @@ version: "3.7" services: mariadb-master: - image: 'bitnami/mariadb:10.3' + image: "bitnami/mariadb:10.3" deploy: restart_policy: condition: on-failure @@ -54,7 +54,7 @@ services: secrets: - frappe-mariadb-root-password volumes: - - 'mariadb_master_data:/bitnami/mariadb' + - "mariadb_master_data:/bitnami/mariadb" environment: - MARIADB_REPLICATION_MODE=master - MARIADB_REPLICATION_USER=repl_user @@ -62,7 +62,7 @@ services: - MARIADB_ROOT_PASSWORD_FILE=/run/secrets/frappe-mariadb-root-password mariadb-slave: - image: 'bitnami/mariadb:10.3' + image: "bitnami/mariadb:10.3" deploy: restart_policy: condition: on-failure @@ -74,7 +74,7 @@ services: secrets: - frappe-mariadb-root-password volumes: - - 'mariadb_slave_data:/bitnami/mariadb' + - "mariadb_slave_data:/bitnami/mariadb" environment: - MARIADB_REPLICATION_MODE=slave - MARIADB_REPLICATION_USER=repl_user @@ -265,6 +265,7 @@ Use environment variables: - `FRAPPE_VERSION` variable to be set to desired version of Frappe Framework. e.g. 12.7.0 - `MARIADB_HOST=frappe-mariadb_mariadb-master` - `SITES` variable is list of sites in back tick and separated by comma + ``` SITES=`site1.example.com`,`site2.example.com` ``` @@ -277,9 +278,9 @@ SITES=`site1.example.com`,`site2.example.com` 4. Select network `frappe-network` 5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` 6. Env variables: - - MYSQL_ROOT_PASSWORD=longsecretpassword - - SITE_NAME=site1.example.com - - INSTALL_APPS=erpnext + - MYSQL_ROOT_PASSWORD=longsecretpassword + - SITE_NAME=site1.example.com + - INSTALL_APPS=erpnext 7. Start container ### Migrate Sites job @@ -290,5 +291,5 @@ SITES=`site1.example.com`,`site2.example.com` 4. Select network `frappe-network` 5. Select Volume `frappe-bench-v13_sites-vol` and mount in container `/home/frappe/frappe-bench/sites` 6. Env variables: - - MAINTENANCE_MODE=1 + - MAINTENANCE_MODE=1 7. Start container diff --git a/docs/multi-bench.md b/docs/multi-bench.md index 11a8b931..831ec71c 100644 --- a/docs/multi-bench.md +++ b/docs/multi-bench.md @@ -17,17 +17,17 @@ cp env-example .env To get started, copy the existing `env-example` file to `.env`. By default, the file will contain the following variables: - `VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11) + - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11) - `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. + - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. - `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. + - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. + - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. - `SITES=site1.domain.com,site2.domain.com` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,). - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. + - List of sites that are part of the deployment "bench" Each site is separated by a comma(,). + - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. - `LETSENCRYPT_EMAIL=your.email@your.domain.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. + - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. Notes: @@ -123,46 +123,49 @@ Notes: ## Docker containers This repository contains the following docker-compose files, each one containing the described images: -* docker-compose-common.yml - * redis-cache - * volume: redis-cache-vol - * redis-queue - * volume: redis-queue-vol - * redis-socketio - * volume: redis-socketio-vol - * mariadb: main database - * volume: mariadb-vol -* docker-compose-erpnext.yml - * erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - * volume: assets-vol - * erpnext-python: main application code - * frappe-socketio: enables realtime communication to the user interface through websockets - * frappe-worker-default: background runner - * frappe-worker-short: background runner for short-running jobs - * frappe-worker-long: background runner for long-running jobs - * frappe-schedule -* docker-compose-frappe.yml - * frappe-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - * volume: assets-vol, sites-vol - * erpnext-python: main application code - * volume: sites-vol - * frappe-socketio: enables realtime communication to the user interface through websockets - * volume: sites-vol - * frappe-worker-default: background runner - * volume: sites-vol - * frappe-worker-short: background runner for short-running jobs - * volume: sites-vol - * frappe-worker-long: background runner for long-running jobs - * volume: sites-vol - * frappe-schedule - * volume: sites-vol +- docker-compose-common.yml + - redis-cache + - volume: redis-cache-vol + - redis-queue + - volume: redis-queue-vol + - redis-socketio + - volume: redis-socketio-vol + - mariadb: main database + - volume: mariadb-vol +- docker-compose-erpnext.yml -* docker-compose-networks.yml: this yaml define the network to communicate with *Letsencrypt Nginx Proxy Companion*. + - erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. + - volume: assets-vol + - erpnext-python: main application code + - frappe-socketio: enables realtime communication to the user interface through websockets + - frappe-worker-default: background runner + - frappe-worker-short: background runner for short-running jobs + - frappe-worker-long: background runner for long-running jobs + - frappe-schedule -* erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml +- docker-compose-frappe.yml -* frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml + - frappe-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. + - volume: assets-vol, sites-vol + - erpnext-python: main application code + - volume: sites-vol + - frappe-socketio: enables realtime communication to the user interface through websockets + - volume: sites-vol + - frappe-worker-default: background runner + - volume: sites-vol + - frappe-worker-short: background runner for short-running jobs + - volume: sites-vol + - frappe-worker-long: background runner for long-running jobs + - volume: sites-vol + - frappe-schedule + - volume: sites-vol + +- docker-compose-networks.yml: this yaml define the network to communicate with _Letsencrypt Nginx Proxy Companion_. + +- erpnext-publish.yml: this yml extends erpnext-nginx service to publish port 80, can only be used with docker-compose-erpnext.yml + +- frappe-publish.yml: this yml extends frappe-nginx service to publish port 80, can only be used with docker-compose-frappe.yml ## Updating and Migrating Sites diff --git a/docs/single-bench.md b/docs/single-bench.md index a84c4862..44949691 100644 --- a/docs/single-bench.md +++ b/docs/single-bench.md @@ -26,39 +26,39 @@ cp env-production .env To get started, copy the existing `env-local` or `env-production` file to `.env`. By default, the file will contain the following variables: - `ERPNEXT_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). + - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). - `FRAPPE_VERSION=edge` - - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). + - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. v13.0.0, version-12, v11.1.15, v11). - `MARIADB_HOST=mariadb` - - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. + - Sets the hostname to `mariadb`. This is required if the database is managed by the containerized MariaDB instance. - `MYSQL_ROOT_PASSWORD=admin` - - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. - - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. + - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is used, there is no need to set the password here. + - In case of a separately managed database setups, set the value to the database's hostname/IP/domain. - `SITE_NAME=erp.example.com` - - Creates this site after starting all services and installs ERPNext. Site name must be resolvable by users machines and the ERPNext components. e.g. `erp.example.com` or `mysite.localhost`. -- ``SITES=`erp.example.com` `` - - List of sites that are part of the deployment "bench" Each site is separated by a comma(,) and quoted in backtick (`). By default site created by ``SITE_NAME`` variable is added here. - - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. + - Creates this site after starting all services and installs ERPNext. Site name must be resolvable by users machines and the ERPNext components. e.g. `erp.example.com` or `mysite.localhost`. +- `` SITES=`erp.example.com` `` + - List of sites that are part of the deployment "bench" Each site is separated by a comma(,) and quoted in backtick (`). By default site created by `SITE_NAME` variable is added here. + - If LetsEncrypt is being setup, make sure that the DNS for all the site's domains correctly point to the current instance. - `DB_ROOT_USER=root` - - MariaDB root username + - MariaDB root username - `ADMIN_PASSWORD=admin` - - Password for the `Administrator` user, credentials after install `Administrator:$ADMIN_PASSWORD`. + - Password for the `Administrator` user, credentials after install `Administrator:$ADMIN_PASSWORD`. - `INSTALL_APPS=erpnext` - - Apps to install, the app must be already in the container image, to install other application read the [instructions on installing custom apps](./custom-apps-for-production.md). + - Apps to install, the app must be already in the container image, to install other application read the [instructions on installing custom apps](./custom-apps-for-production.md). - `LETSENCRYPT_EMAIL=email@example.com` - - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. + - Email for LetsEncrypt expiry notification. This is only required if you are setting up LetsEncrypt. - `ENTRYPOINT_LABEL=traefik.http.routers.erpnext-nginx.entrypoints=websecure` - - Related to the traefik configuration, says all traffic from outside should come from HTTP or HTTPS, for local development should be web, for production websecure. if redirection is needed, read below. + - Related to the traefik configuration, says all traffic from outside should come from HTTP or HTTPS, for local development should be web, for production websecure. if redirection is needed, read below. - `CERT_RESOLVER_LABEL=traefik.http.routers.erpnext-nginx.tls.certresolver=myresolver` - - Which traefik resolver to use to get TLS certificate, sets `erpnext.local.no-cert-resolver` for local setup. -- ``HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) `` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-redirect-rule` for local setup. + - Which traefik resolver to use to get TLS certificate, sets `erpnext.local.no-cert-resolver` for local setup. +- `` HTTPS_REDIRECT_RULE_LABEL=traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`) `` + - Related to the traefik https redirection configuration, sets `erpnext.local.no-redirect-rule` for local setup. - `HTTPS_REDIRECT_ENTRYPOINT_LABEL=traefik.http.routers.http-catchall.entrypoints=web` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-entrypoint` for local setup. + - Related to the traefik https redirection configuration, sets `erpnext.local.no-entrypoint` for local setup. - `HTTPS_REDIRECT_MIDDLEWARE_LABEL=traefik.http.routers.http-catchall.middlewares=redirect-to-https` - - Related to the traefik https redirection configuration, sets `erpnext.local.no-middleware` for local setup. + - Related to the traefik https redirection configuration, sets `erpnext.local.no-middleware` for local setup. - `HTTPS_USE_REDIRECT_MIDDLEWARE_LABEL=traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https` - - Related to the traefik https redirection configuration, sets `erpnext.local-no-redirect-middleware` for local setup. + - Related to the traefik https redirection configuration, sets `erpnext.local-no-redirect-middleware` for local setup. Notes: @@ -77,7 +77,7 @@ Make sure to replace `` with the desired name you wish to set for Notes: -- If it is the first time running and site is being initialized, *it can take multiple minutes for the site to be up*. Monitor `site-creator` container logs to check progress. Use command `docker logs _site-creator_1 -f` +- If it is the first time running and site is being initialized, _it can take multiple minutes for the site to be up_. Monitor `site-creator` container logs to check progress. Use command `docker logs _site-creator_1 -f` - After the site is ready the username is `Administrator` and the password is `$ADMIN_PASSWORD` - The local deployment is for testing and REST API development purpose only - A complete development environment is available [here](../development) @@ -86,32 +86,32 @@ Notes: The docker-compose file contains following services: -* traefik: manages letsencrypt - * volume: cert-vol -* redis-cache: cache store - * volume: redis-cache-vol -* redis-queue: used by workers - * volume: redis-queue-vol -* redis-socketio: used by socketio service - * volume: redis-socketio-vol -* mariadb: main database - * volume: mariadb-vol -* erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. - * volume: assets-vol and sites-vol -* erpnext-python: main application code - * volume: sites-vol and sites-vol -* frappe-socketio: enables realtime communication to the user interface through websockets - * volume: sites-vol -* erpnext-worker-default: background runner - * volume: sites-vol -* erpnext-worker-short: background runner for short-running jobs - * volume: sites-vol -* erpnext-worker-long: background runner for long-running jobs - * volume: sites-vol -* erpnext-schedule - * volume: sites-vol -* site-creator: run once container to create new site. - * volume: sites-vol +- traefik: manages letsencrypt + - volume: cert-vol +- redis-cache: cache store + - volume: redis-cache-vol +- redis-queue: used by workers + - volume: redis-queue-vol +- redis-socketio: used by socketio service + - volume: redis-socketio-vol +- mariadb: main database + - volume: mariadb-vol +- erpnext-nginx: serves static assets and proxies web request to the appropriate container, allowing to offer all services on the same port. + - volume: assets-vol and sites-vol +- erpnext-python: main application code + - volume: sites-vol and sites-vol +- frappe-socketio: enables realtime communication to the user interface through websockets + - volume: sites-vol +- erpnext-worker-default: background runner + - volume: sites-vol +- erpnext-worker-short: background runner for short-running jobs + - volume: sites-vol +- erpnext-worker-long: background runner for long-running jobs + - volume: sites-vol +- erpnext-schedule + - volume: sites-vol +- site-creator: run once container to create new site. + - volume: sites-vol ## Updating and Migrating Sites diff --git a/docs/site-operations.md b/docs/site-operations.md index 5f93f832..bf7ac0c3 100644 --- a/docs/site-operations.md +++ b/docs/site-operations.md @@ -13,8 +13,8 @@ Or specify environment variables instead of passing secrets as command arguments Note: - Wait for the database service to start before trying to create a new site. - - If new site creation fails, retry after the MariaDB container is up and running. - - If you're using a managed database instance, make sure that the database is running before setting up a new site. + - If new site creation fails, retry after the MariaDB container is up and running. + - If you're using a managed database instance, make sure that the database is running before setting up a new site. #### MariaDB Site @@ -72,7 +72,7 @@ Notes: ## Add sites to proxy -Change `SITES` variable to the list of sites created encapsulated in backtick and separated by comma with no space. e.g. ``SITES=`site1.example.com`,`site2.example.com` ``. +Change `SITES` variable to the list of sites created encapsulated in backtick and separated by comma with no space. e.g. `` SITES=`site1.example.com`,`site2.example.com` ``. Reload variables with following command. @@ -165,19 +165,19 @@ Note: - Volume must be mounted at location `/home/frappe/backups` for restoring sites - If no backup files are found in volume, it will use s3 credentials to pull backups - Backup structure for mounted volume or downloaded from s3: - - /home/frappe/backups - - site1.domain.com - - 20200420_162000 - - 20200420_162000-site1_domain_com-* - - site2.domain.com - - 20200420_162000 - - 20200420_162000-site2_domain_com-* + - /home/frappe/backups + - site1.domain.com + - 20200420_162000 + - 20200420_162000-site1_domain_com-\* + - site2.domain.com + - 20200420_162000 + - 20200420_162000-site2_domain_com-\* ## Edit configs Editing config manually might be required in some cases, one such case is to use Amazon RDS (or any other DBaaS). -For full instructions, refer to the [wiki](https://github.com/frappe/frappe/wiki/Using-Frappe-with-Amazon-RDS-(or-any-other-DBaaS)). Common question can be found in Issues and on forum. +For full instructions, refer to the [wiki](). Common question can be found in Issues and on forum. `common_site_config.json` or `site_config.json` from `sites-vol` volume has to be edited using following command: @@ -231,7 +231,6 @@ Notes: - Use it to install/uninstall custom apps, add system manager user, etc. - To run the command as non root user add the command option `--user frappe`. - ## Delete/Drop Site #### MariaDB Site diff --git a/docs/tips-for-moving-deployments.md b/docs/tips-for-moving-deployments.md index d1a25e68..8687c5a4 100644 --- a/docs/tips-for-moving-deployments.md +++ b/docs/tips-for-moving-deployments.md @@ -1,15 +1,15 @@ # Tips for moving deployments - Take regular automatic backups and push the files to S3 compatible cloud. Setup backup and push with cronjobs - - Use regular cron for single machine installs - - Use [swarm-cronjob](https://github.com/crazy-max/swarm-cronjob) for docker swarm - - Use Kubernetes CronJob + - Use regular cron for single machine installs + - Use [swarm-cronjob](https://github.com/crazy-max/swarm-cronjob) for docker swarm + - Use Kubernetes CronJob - It makes it easy to transfer data from cloud to any new deployment. - They are just [site operations](site-operations.md) that can be manually pipelined as per need. - Remember to restore encryption keys and other custom configuration from `site_config.json`. - Steps to move deployment: - - [Take backup](site-operations.md#backup-sites) - - [Push backup to cloud](site-operations.md#push-backup-to-s3-compatible-storage) - - Create new deployment type anywhere - - [Restore backup from cloud](site-operations.md#restore-backups) - - [Restore `site_config.json` from cloud](site-operations.md#edit-configs) + - [Take backup](site-operations.md#backup-sites) + - [Push backup to cloud](site-operations.md#push-backup-to-s3-compatible-storage) + - Create new deployment type anywhere + - [Restore backup from cloud](site-operations.md#restore-backups) + - [Restore `site_config.json` from cloud](site-operations.md#edit-configs) diff --git a/installation/docker-compose-common.yml b/installation/docker-compose-common.yml index 5699ad7f..3786a32e 100644 --- a/installation/docker-compose-common.yml +++ b/installation/docker-compose-common.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: redis-cache: diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index 1f769dd6..6593e18c 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: erpnext-nginx: diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml index 5c64424e..0ffacbeb 100644 --- a/installation/docker-compose-frappe.yml +++ b/installation/docker-compose-frappe.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: frappe-nginx: diff --git a/installation/docker-compose-networks.yml b/installation/docker-compose-networks.yml index 377f01d2..4daa6684 100644 --- a/installation/docker-compose-networks.yml +++ b/installation/docker-compose-networks.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" networks: default: From 1c9c4bd7809ad96095a5bcff03b2c120bd3f3a73 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:54:27 +0300 Subject: [PATCH 7/9] chore(lint): Change `exclude` in codespell hook --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c769e86..5fb06d93 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: rev: v2.1.0 hooks: - id: codespell - exclude: ".*Dockerfile.*" + exclude: "build/bench/Dockerfile" - repo: local hooks: From be7c6254f81e5d5738b5a3808e1285744d820c63 Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:54:51 +0300 Subject: [PATCH 8/9] chore(lint): Run codespell --- build/frappe-nginx/nginx-default.conf.template | 2 +- build/frappe-worker/commands/push_backup.py | 2 +- development/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/frappe-nginx/nginx-default.conf.template b/build/frappe-nginx/nginx-default.conf.template index c3ce9b9f..fd7c9719 100644 --- a/build/frappe-nginx/nginx-default.conf.template +++ b/build/frappe-nginx/nginx-default.conf.template @@ -89,7 +89,7 @@ server { client_body_buffer_size 16K; client_header_buffer_size 1k; - # enable gzip compresion + # enable gzip compression # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge gzip on; gzip_http_version 1.1; diff --git a/build/frappe-worker/commands/push_backup.py b/build/frappe-worker/commands/push_backup.py index 2a49933a..78337ade 100644 --- a/build/frappe-worker/commands/push_backup.py +++ b/build/frappe-worker/commands/push_backup.py @@ -119,7 +119,7 @@ def delete_old_backups(limit, bucket, site_name): for obj in bucket.objects.filter(Prefix=oldest_backup): # delete all keys that are inside the oldest_backup if bucket_dir in obj.key: - print("Deleteing " + obj.key) + print("Deleting " + obj.key) s3.Object(bucket.name, obj.key).delete() diff --git a/development/README.md b/development/README.md index 162df9ca..9e073d7a 100644 --- a/development/README.md +++ b/development/README.md @@ -236,7 +236,7 @@ The first step is installing and updating the required software. Usually the fra /workspace/development/frappe-bench/env/bin/python -m pip install --upgrade jupyter ipykernel ipython ``` -Then, run the commmand `Python: Show Python interactive window` from the VSCode command palette. +Then, run the command `Python: Show Python interactive window` from the VSCode command palette. Replace `mysite.localhost` with your site and run the following code in a Jupyter cell: From 0b861de11752b80666c63146f88d0d9e463f04cd Mon Sep 17 00:00:00 2001 From: Lev Vereshchagin Date: Fri, 10 Dec 2021 11:56:18 +0300 Subject: [PATCH 9/9] chore(lint): Run shfmt --- .github/scripts/get-latest-tags.sh | 6 +- tests/functions.sh | 34 ++-- tests/integration-test.sh | 304 ++++++++++++++--------------- tests/test-erpnext.sh | 34 ++-- tests/test-frappe.sh | 24 +-- 5 files changed, 201 insertions(+), 201 deletions(-) diff --git a/.github/scripts/get-latest-tags.sh b/.github/scripts/get-latest-tags.sh index cfcf85e9..46784ba6 100755 --- a/.github/scripts/get-latest-tags.sh +++ b/.github/scripts/get-latest-tags.sh @@ -4,9 +4,9 @@ set -e set -x get_tag() { - tags=$(git ls-remote --refs --tags --sort='v:refname' "https://github.com/$1" "v$2.*") - tag=$(echo "$tags" | tail -n1 | sed 's/.*\///') - echo "$tag" + tags=$(git ls-remote --refs --tags --sort='v:refname' "https://github.com/$1" "v$2.*") + tag=$(echo "$tags" | tail -n1 | sed 's/.*\///') + echo "$tag" } FRAPPE_VERSION=$(get_tag frappe/frappe "$VERSION") diff --git a/tests/functions.sh b/tests/functions.sh index 8453c499..e077dbd9 100644 --- a/tests/functions.sh +++ b/tests/functions.sh @@ -1,26 +1,26 @@ #!/bin/bash print_group() { - echo ::endgroup:: - echo "::group::$*" + echo ::endgroup:: + echo "::group::$*" } ping_site() { - print_group "Ping site $SITE_NAME" + print_group "Ping site $SITE_NAME" - echo Ping version - ping_res=$(curl -sS "http://$SITE_NAME/api/method/version") - echo "$ping_res" - if [[ -z $(echo "$ping_res" | grep "message" || echo "") ]]; then - echo "Ping failed" - exit 1 - fi + echo Ping version + ping_res=$(curl -sS "http://$SITE_NAME/api/method/version") + echo "$ping_res" + if [[ -z $(echo "$ping_res" | grep "message" || echo "") ]]; then + echo "Ping failed" + exit 1 + fi - echo Check index - index_res=$(curl -sS "http://$SITE_NAME") - if [[ -n $(echo "$index_res" | grep "Internal Server Error" || echo "") ]]; then - echo "Index check failed" - echo "$index_res" - exit 1 - fi + echo Check index + index_res=$(curl -sS "http://$SITE_NAME") + if [[ -n $(echo "$index_res" | grep "Internal Server Error" || echo "") ]]; then + echo "Index check failed" + echo "$index_res" + exit 1 + fi } diff --git a/tests/integration-test.sh b/tests/integration-test.sh index af7a8831..a76a5479 100755 --- a/tests/integration-test.sh +++ b/tests/integration-test.sh @@ -7,63 +7,63 @@ source tests/functions.sh project_name=frappe_bench_00 docker_compose_with_args() { - # shellcheck disable=SC2068 - docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - $@ + # shellcheck disable=SC2068 + docker-compose \ + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-frappe.yml \ + -f installation/frappe-publish.yml \ + $@ } check_migration_complete() { - print_group Check migration + print_group Check migration - container_id=$(docker_compose_with_args ps -q frappe-python) - cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''" + container_id=$(docker_compose_with_args ps -q frappe-python) + cmd="docker logs ${container_id} 2>&1 | grep 'Starting gunicorn' || echo ''" + worker_log=$(eval "$cmd") + INCREMENT=0 + + while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do + sleep 3 + ((INCREMENT = INCREMENT + 1)) + echo "Wait for migration to complete..." worker_log=$(eval "$cmd") - INCREMENT=0 + if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then + echo Migration timeout + docker logs "${container_id}" + exit 1 + fi + done - while [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -lt 120 ]]; do - sleep 3 - ((INCREMENT = INCREMENT + 1)) - echo "Wait for migration to complete..." - worker_log=$(eval "$cmd") - if [[ ${worker_log} != *"Starting gunicorn"* && ${INCREMENT} -eq 120 ]]; then - echo Migration timeout - docker logs "${container_id}" - exit 1 - fi - done - - echo Migration Log - docker logs "${container_id}" + echo Migration Log + docker logs "${container_id}" } check_health() { - print_group Loop health check + print_group Loop health check - docker run --name frappe_doctor \ - -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \ - --network "${project_name}_default" \ - frappe/frappe-worker:edge doctor || true + docker run --name frappe_doctor \ + -v "${project_name}_sites-vol:/home/frappe/frappe-bench/sites" \ + --network "${project_name}_default" \ + frappe/frappe-worker:edge doctor || true - cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""' + cmd='docker logs frappe_doctor | grep "Health check successful" || echo ""' + doctor_log=$(eval "$cmd") + INCREMENT=0 + + while [[ -z "${doctor_log}" && ${INCREMENT} -lt 60 ]]; do + sleep 1 + ((INCREMENT = INCREMENT + 1)) + container=$(docker start frappe_doctor) + echo "Restarting ${container}..." doctor_log=$(eval "$cmd") - INCREMENT=0 - while [[ -z "${doctor_log}" && ${INCREMENT} -lt 60 ]]; do - sleep 1 - ((INCREMENT = INCREMENT + 1)) - container=$(docker start frappe_doctor) - echo "Restarting ${container}..." - doctor_log=$(eval "$cmd") - - if [[ ${INCREMENT} -eq 60 ]]; then - docker logs "${container}" - exit 1 - fi - done + if [[ ${INCREMENT} -eq 60 ]]; then + docker logs "${container}" + exit 1 + fi + done } # Initial group @@ -81,23 +81,23 @@ docker_compose_with_args up -d --quiet-pull echo Start postgres docker pull postgres:11.8 -q docker run \ - --name postgresql \ - -d \ - -e POSTGRES_PASSWORD=admin \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - postgres:11.8 + --name postgresql \ + -d \ + -e POSTGRES_PASSWORD=admin \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + postgres:11.8 check_health print_group "Create new site " SITE_NAME=test.localhost docker run \ - --rm \ - -e SITE_NAME=$SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:v13 new + --rm \ + -e SITE_NAME=$SITE_NAME \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:v13 new ping_site @@ -118,25 +118,25 @@ ping_site PG_SITE_NAME=pgsql.localhost print_group "Create new site (Postgres)" docker run \ - --rm \ - -e SITE_NAME=$PG_SITE_NAME \ - -e POSTGRES_HOST=postgresql \ - -e DB_ROOT_USER=postgres \ - -e POSTGRES_PASSWORD=admin \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge new + --rm \ + -e SITE_NAME=$PG_SITE_NAME \ + -e POSTGRES_HOST=postgresql \ + -e DB_ROOT_USER=postgres \ + -e POSTGRES_PASSWORD=admin \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge new check_migration_complete SITE_NAME=$PG_SITE_NAME ping_site print_group Backup site docker run \ - --rm \ - -e WITH_FILES=1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge backup + --rm \ + -e WITH_FILES=1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge backup MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE" MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @@ -144,39 +144,39 @@ MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" print_group Prepare S3 server echo Start S3 server docker run \ - --name minio \ - -d \ - -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ - -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ - --network ${project_name}_default \ - minio/minio server /data + --name minio \ + -d \ + -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \ + -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \ + --network ${project_name}_default \ + minio/minio server /data echo Create bucket docker run \ - --rm \ - --network ${project_name}_default \ - vltgroup/s3cmd:latest \ - s3cmd \ - --access_key=$MINIO_ACCESS_KEY \ - --secret_key=$MINIO_SECRET_KEY \ - --region=us-east-1 \ - --no-ssl \ - --host=minio:9000 \ - --host-bucket=minio:9000 \ - mb s3://frappe + --rm \ + --network ${project_name}_default \ + vltgroup/s3cmd:latest \ + s3cmd \ + --access_key=$MINIO_ACCESS_KEY \ + --secret_key=$MINIO_SECRET_KEY \ + --region=us-east-1 \ + --no-ssl \ + --host=minio:9000 \ + --host-bucket=minio:9000 \ + mb s3://frappe print_group Push backup docker run \ - --rm \ - -e BUCKET_NAME=frappe \ - -e REGION=us-east-1 \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge push-backup + --rm \ + -e BUCKET_NAME=frappe \ + -e REGION=us-east-1 \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge push-backup print_group Prune and restart services docker_compose_with_args stop @@ -187,17 +187,17 @@ check_health print_group Restore backup from S3 docker run \ - --rm \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=frappe \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge restore-backup + --rm \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=frappe \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge restore-backup check_health ping_site @@ -206,78 +206,78 @@ SITE_NAME=$PG_SITE_NAME ping_site EDGE_SITE_NAME=edge.localhost print_group "Create new site (edge)" docker run \ - --rm \ - -e SITE_NAME=$EDGE_SITE_NAME \ - -e INSTALL_APPS=frappe \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge new + --rm \ + -e SITE_NAME=$EDGE_SITE_NAME \ + -e INSTALL_APPS=frappe \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge new check_health SITE_NAME=$EDGE_SITE_NAME ping_site print_group Migrate edge site docker run \ - --rm \ - -e MAINTENANCE_MODE=1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \ - --network ${project_name}_default \ - frappe/frappe-worker:edge migrate + --rm \ + -e MAINTENANCE_MODE=1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + -v ${project_name}_assets-vol:/home/frappe/frappe-bench/sites/assets \ + --network ${project_name}_default \ + frappe/frappe-worker:edge migrate check_migration_complete print_group "Restore backup S3 (overwrite)" docker run \ - --rm \ - -e MYSQL_ROOT_PASSWORD=admin \ - -e BUCKET_NAME=frappe \ - -e BUCKET_DIR=local \ - -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ - -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ - -e ENDPOINT_URL=http://minio:9000 \ - -e REGION=us-east-1 \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge restore-backup + --rm \ + -e MYSQL_ROOT_PASSWORD=admin \ + -e BUCKET_NAME=frappe \ + -e BUCKET_DIR=local \ + -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \ + -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \ + -e ENDPOINT_URL=http://minio:9000 \ + -e REGION=us-east-1 \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge restore-backup check_migration_complete ping_site print_group "Check console for $SITE_NAME" docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge console $SITE_NAME + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge console $SITE_NAME print_group "Check console for $PG_SITE_NAME" docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge console $PG_SITE_NAME + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge console $PG_SITE_NAME print_group "Check drop site for $SITE_NAME (MariaDB)" docker run \ - --rm \ - -e SITE_NAME=$SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge drop + --rm \ + -e SITE_NAME=$SITE_NAME \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge drop print_group "Check drop site for $PG_SITE_NAME (Postgres)" docker run \ - --rm \ - -e SITE_NAME=$PG_SITE_NAME \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:edge drop + --rm \ + -e SITE_NAME=$PG_SITE_NAME \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:edge drop print_group Check bench --help docker run \ - --rm \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - --user frappe \ - frappe/frappe-worker:edge bench --help + --rm \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + --user frappe \ + frappe/frappe-worker:edge bench --help diff --git a/tests/test-erpnext.sh b/tests/test-erpnext.sh index e2eb6902..c83151a9 100755 --- a/tests/test-erpnext.sh +++ b/tests/test-erpnext.sh @@ -16,29 +16,29 @@ cat .env print_group Start services docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/erpnext-publish.yml \ - up -d + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/erpnext-publish.yml \ + up -d print_group Fix permissions docker run \ - --rm \ - --user root \ - -v ${project_name}_sites-vol:/sites \ - -v ${project_name}_assets-vol:/assets \ - -v ${project_name}_logs-vol:/logs \ - frappe/erpnext-worker:test chown -R 1000:1000 /logs /sites /assets + --rm \ + --user root \ + -v ${project_name}_sites-vol:/sites \ + -v ${project_name}_assets-vol:/assets \ + -v ${project_name}_logs-vol:/logs \ + frappe/erpnext-worker:test chown -R 1000:1000 /logs /sites /assets print_group Create site docker run \ - --rm \ - -e "SITE_NAME=$SITE_NAME" \ - -e "INSTALL_APPS=erpnext" \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/erpnext-worker:test new + --rm \ + -e "SITE_NAME=$SITE_NAME" \ + -e "INSTALL_APPS=erpnext" \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/erpnext-worker:test new ping_site rm .env diff --git a/tests/test-frappe.sh b/tests/test-frappe.sh index 035aa2c9..12131839 100755 --- a/tests/test-frappe.sh +++ b/tests/test-frappe.sh @@ -8,13 +8,13 @@ project_name="test_frappe" SITE_NAME="test_frappe.localhost" docker_compose_with_args() { - # shellcheck disable=SC2068 - docker-compose \ - -p $project_name \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-frappe.yml \ - -f installation/frappe-publish.yml \ - $@ + # shellcheck disable=SC2068 + docker-compose \ + -p $project_name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-frappe.yml \ + -f installation/frappe-publish.yml \ + $@ } echo ::group::Setup env @@ -29,11 +29,11 @@ docker_compose_with_args up -d print_group Create site docker run \ - --rm \ - -e "SITE_NAME=$SITE_NAME" \ - -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ - --network ${project_name}_default \ - frappe/frappe-worker:test new + --rm \ + -e "SITE_NAME=$SITE_NAME" \ + -v ${project_name}_sites-vol:/home/frappe/frappe-bench/sites \ + --network ${project_name}_default \ + frappe/frappe-worker:test new ping_site