From 242708df0c395357054955126f231d836e2d94f5 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 18:01:23 +0530 Subject: [PATCH] refactor: reuse common scripts and files move common scripts to build/common --- .travis.yml | 10 +- .../commands/background.py | 0 .../commands/backup.py | 0 .../commands/check_connection.py | 0 .../commands/console.py | 0 .../commands/doctor.py | 0 .../commands/migrate.py | 0 .../{frappe-worker => common}/commands/new.py | 0 .../commands/worker.py | 0 .../common_site_config.json.template | 0 .../nginx-default.conf.template | 0 .../worker}/docker-entrypoint.sh | 0 build/erpnext-assets/.dockerignore | 2 - build/erpnext-assets/Dockerfile | 11 +- build/erpnext-assets/sites/apps.txt | 2 - build/erpnext-assets/v11.Dockerfile | 10 +- build/erpnext-assets/v12.Dockerfile | 10 +- build/erpnext-worker/Dockerfile | 6 +- build/erpnext-worker/commands/new.py | 68 -------- build/erpnext-worker/docker-entrypoint.sh | 158 ------------------ build/erpnext-worker/v11.Dockerfile | 6 +- build/erpnext-worker/v12.Dockerfile | 6 +- build/frappe-assets/.dockerignore | 2 - build/frappe-assets/Dockerfile | 7 +- .../frappe-assets/nginx-default.conf.template | 90 ---------- build/frappe-assets/sites/apps.txt | 1 - build/frappe-assets/v11.Dockerfile | 7 +- build/frappe-assets/v12.Dockerfile | 7 +- build/frappe-socketio/Dockerfile | 6 +- build/frappe-socketio/v11.Dockerfile | 6 +- build/frappe-socketio/v12.Dockerfile | 6 +- build/frappe-worker/Dockerfile | 6 +- build/frappe-worker/commands/background.py | 7 - build/frappe-worker/commands/backup.py | 29 ---- .../commands/check_connection.py | 67 -------- build/frappe-worker/commands/doctor.py | 4 - build/frappe-worker/commands/migrate.py | 47 ------ build/frappe-worker/commands/worker.py | 7 - .../common_site_config.json.template | 7 - build/frappe-worker/v11.Dockerfile | 6 +- build/frappe-worker/v12.Dockerfile | 6 +- 41 files changed, 62 insertions(+), 545 deletions(-) rename build/{erpnext-worker => common}/commands/background.py (100%) rename build/{erpnext-worker => common}/commands/backup.py (100%) rename build/{erpnext-worker => common}/commands/check_connection.py (100%) rename build/{frappe-worker => common}/commands/console.py (100%) rename build/{erpnext-worker => common}/commands/doctor.py (100%) rename build/{erpnext-worker => common}/commands/migrate.py (100%) rename build/{frappe-worker => common}/commands/new.py (100%) rename build/{erpnext-worker => common}/commands/worker.py (100%) rename build/{erpnext-worker => common}/common_site_config.json.template (100%) rename build/{erpnext-assets => common}/nginx-default.conf.template (100%) rename build/{frappe-worker => common/worker}/docker-entrypoint.sh (100%) delete mode 100644 build/erpnext-assets/.dockerignore delete mode 100644 build/erpnext-assets/sites/apps.txt delete mode 100644 build/erpnext-worker/commands/new.py delete mode 100755 build/erpnext-worker/docker-entrypoint.sh delete mode 100644 build/frappe-assets/.dockerignore delete mode 100644 build/frappe-assets/nginx-default.conf.template delete mode 100644 build/frappe-assets/sites/apps.txt delete mode 100644 build/frappe-worker/commands/background.py delete mode 100644 build/frappe-worker/commands/backup.py delete mode 100644 build/frappe-worker/commands/check_connection.py delete mode 100644 build/frappe-worker/commands/doctor.py delete mode 100644 build/frappe-worker/commands/migrate.py delete mode 100644 build/frappe-worker/commands/worker.py delete mode 100755 build/frappe-worker/common_site_config.json.template diff --git a/.travis.yml b/.travis.yml index ae4bb89f..2797ab28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,26 +34,26 @@ matrix: - ./dbench setup docker stop - name: "Build Frappe python environment" script: - - docker build -t frappe-worker build/frappe-worker + - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge - docker push frappe/frappe-worker:edge - name: "Build Frappe nginx + static assets" script: - - docker build -t frappe-assets build/frappe-assets + - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge - docker push frappe/frappe-assets:edge - name: "Build ERPNext python environment" script: - - docker build -t erpnext-worker build/erpnext-worker + - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge - docker push frappe/erpnext-worker:edge - name: "Build ERPNext nginx + static assets" script: - - docker build -t erpnext-assets build/erpnext-assets + - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - name: "Build Frappe socketio service" script: - - docker build -t frappe-socketio build/frappe-socketio + - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge - docker push frappe/frappe-socketio:edge diff --git a/build/erpnext-worker/commands/background.py b/build/common/commands/background.py similarity index 100% rename from build/erpnext-worker/commands/background.py rename to build/common/commands/background.py diff --git a/build/erpnext-worker/commands/backup.py b/build/common/commands/backup.py similarity index 100% rename from build/erpnext-worker/commands/backup.py rename to build/common/commands/backup.py diff --git a/build/erpnext-worker/commands/check_connection.py b/build/common/commands/check_connection.py similarity index 100% rename from build/erpnext-worker/commands/check_connection.py rename to build/common/commands/check_connection.py diff --git a/build/frappe-worker/commands/console.py b/build/common/commands/console.py similarity index 100% rename from build/frappe-worker/commands/console.py rename to build/common/commands/console.py diff --git a/build/erpnext-worker/commands/doctor.py b/build/common/commands/doctor.py similarity index 100% rename from build/erpnext-worker/commands/doctor.py rename to build/common/commands/doctor.py diff --git a/build/erpnext-worker/commands/migrate.py b/build/common/commands/migrate.py similarity index 100% rename from build/erpnext-worker/commands/migrate.py rename to build/common/commands/migrate.py diff --git a/build/frappe-worker/commands/new.py b/build/common/commands/new.py similarity index 100% rename from build/frappe-worker/commands/new.py rename to build/common/commands/new.py diff --git a/build/erpnext-worker/commands/worker.py b/build/common/commands/worker.py similarity index 100% rename from build/erpnext-worker/commands/worker.py rename to build/common/commands/worker.py diff --git a/build/erpnext-worker/common_site_config.json.template b/build/common/common_site_config.json.template similarity index 100% rename from build/erpnext-worker/common_site_config.json.template rename to build/common/common_site_config.json.template diff --git a/build/erpnext-assets/nginx-default.conf.template b/build/common/nginx-default.conf.template similarity index 100% rename from build/erpnext-assets/nginx-default.conf.template rename to build/common/nginx-default.conf.template diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh similarity index 100% rename from build/frappe-worker/docker-entrypoint.sh rename to build/common/worker/docker-entrypoint.sh diff --git a/build/erpnext-assets/.dockerignore b/build/erpnext-assets/.dockerignore deleted file mode 100644 index c58728b5..00000000 --- a/build/erpnext-assets/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -/docker/ diff --git a/build/erpnext-assets/Dockerfile b/build/erpnext-assets/Dockerfile index 90bd0a4d..d316db93 100644 --- a/build/erpnext-assets/Dockerfile +++ b/build/erpnext-assets/Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,18 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/erpnext-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-assets/sites/apps.txt b/build/erpnext-assets/sites/apps.txt deleted file mode 100644 index d700de5f..00000000 --- a/build/erpnext-assets/sites/apps.txt +++ /dev/null @@ -1,2 +0,0 @@ -frappe -erpnext diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile index 8d4907d9..36e85627 100644 --- a/build/erpnext-assets/v11.Dockerfile +++ b/build/erpnext-assets/v11.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:10-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,19 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/erpnext-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile index b000a199..1840c860 100644 --- a/build/erpnext-assets/v12.Dockerfile +++ b/build/erpnext-assets/v12.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,19 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/erpnext-assets/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/common/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index 59175254..6d145d8c 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -31,11 +31,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/erpnext-worker/commands/new.py b/build/erpnext-worker/commands/new.py deleted file mode 100644 index b6589c79..00000000 --- a/build/erpnext-worker/commands/new.py +++ /dev/null @@ -1,68 +0,0 @@ -import os, frappe, json - -from frappe.commands.site import _new_site - -site_name = os.environ.get("SITE_NAME", 'site1.localhost') -mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') -mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') -install_erpnext = os.environ.get("INSTALL_ERPNEXT", None) -force = True if os.environ.get("FORCE", None) else False - -frappe.init(site_name, new_site=True) - -_new_site( - None, - site_name, - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password, - admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), - verbose=True, - install_apps=['erpnext'] if install_erpnext else [], - source_sql=None, - force=force, - reinstall=False, -) - -config = None -with open('common_site_config.json') as config_file: - config = json.load(config_file) - -site_config = None -with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: - site_config = json.load(site_config_file) - -# update User's host to '%' required to connect from any container -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) - -# Set db password -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') -) -os.system(command) - -# Grant permission to database -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) - -exit(0) diff --git a/build/erpnext-worker/docker-entrypoint.sh b/build/erpnext-worker/docker-entrypoint.sh deleted file mode 100755 index b6fc0d35..00000000 --- a/build/erpnext-worker/docker-entrypoint.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -function configureEnv() { - if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then - - if [[ -z "$MARIADB_HOST" ]]; then - echo "MARIADB_HOST is not set" - exit 1 - fi - - if [[ -z "$REDIS_CACHE" ]]; then - echo "REDIS_CACHE is not set" - exit 1 - fi - - if [[ -z "$REDIS_QUEUE" ]]; then - echo "REDIS_QUEUE is not set" - exit 1 - fi - - if [[ -z "$REDIS_SOCKETIO" ]]; then - echo "REDIS_SOCKETIO is not set" - exit 1 - fi - - if [[ -z "$SOCKETIO_PORT" ]]; then - echo "SOCKETIO_PORT is not set" - exit 1 - fi - - envsubst "${MARIADB_HOST} - ${REDIS_CACHE} - ${REDIS_QUEUE} - ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}" < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json - fi -} - -function checkConnection() { - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/check_connection.py" -} - -function checkConfigExists() { - COUNTER=0 - while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do - sleep 1 - (( COUNTER=COUNTER+1 )) - echo "config file not created, retry $COUNTER" - done - - if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" - exit 1 - fi -} - -if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt -fi - -if [ "$1" = 'start' ]; then - configureEnv - checkConnection - - chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json - - if [[ -z "$WORKERS" ]]; then - export WORKERS=2 - fi - - if [[ -z "$FRAPPE_PORT" ]]; then - export FRAPPE_PORT=8000 - fi - - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers $WORKERS \ - --worker-class=gthread \ - --log-file=- \ - -t 120 frappe.app:application --preload" - else - . /home/frappe/frappe-bench/env/bin/activate - gunicorn -b 0.0.0.0:$FRAPPE_PORT \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers $WORKERS \ - --worker-class=gthread \ - --log-file=- \ - -t 120 frappe.app:application --preload - fi - -elif [ "$1" = 'worker' ]; then - checkConfigExists - checkConnection - # default WORKER_TYPE=default - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/worker.py" - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/worker.py - fi - -elif [ "$1" = 'schedule' ]; then - checkConfigExists - checkConnection - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/background.py" - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/background.py - fi - -elif [ "$1" = 'new' ]; then - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/new.py" - exit - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/new.py - fi - -elif [ "$1" = 'migrate' ]; then - - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/migrate.py" - exit - -elif [ "$1" = 'doctor' ]; then - - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/doctor.py" - exit - -elif [ "$1" = 'backup' ]; then - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/backup.py" - exit - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/backup.py - fi - -else - - exec su frappe -c "$@" - -fi diff --git a/build/erpnext-worker/v11.Dockerfile b/build/erpnext-worker/v11.Dockerfile index c56a5e96..81a59b63 100644 --- a/build/erpnext-worker/v11.Dockerfile +++ b/build/erpnext-worker/v11.Dockerfile @@ -28,11 +28,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/erpnext-worker/v12.Dockerfile b/build/erpnext-worker/v12.Dockerfile index d892c81d..3945142a 100644 --- a/build/erpnext-worker/v12.Dockerfile +++ b/build/erpnext-worker/v12.Dockerfile @@ -31,11 +31,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-assets/.dockerignore b/build/frappe-assets/.dockerignore deleted file mode 100644 index c58728b5..00000000 --- a/build/frappe-assets/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -/docker/ diff --git a/build/frappe-assets/Dockerfile b/build/frappe-assets/Dockerfile index ef0e7dcc..762e74d1 100644 --- a/build/frappe-assets/Dockerfile +++ b/build/frappe-assets/Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-assets/nginx-default.conf.template b/build/frappe-assets/nginx-default.conf.template deleted file mode 100644 index c2e0e0ba..00000000 --- a/build/frappe-assets/nginx-default.conf.template +++ /dev/null @@ -1,90 +0,0 @@ -upstream frappe-server { - server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; -} - -upstream socketio-server { - server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; -} - -server { - listen 80; - server_name $http_host; - root /var/www/html; - - location /assets { - try_files $uri =404; - } - - location ~ ^/protected/(.*) { - internal; - try_files /sites/$http_host/$1 =404; - } - - location /socket.io { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Origin $scheme://$http_host; - proxy_set_header Host $host; - - proxy_pass http://socketio-server; - } - - location / { - try_files /sites/$http_host/public/$uri @webserver; - } - - location @webserver { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Host $host; - proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout 120; - proxy_redirect off; - - proxy_pass http://frappe-server; - } - - # error pages - error_page 502 /502.html; - location /502.html { - root /var/www/templates; - internal; - } - - # optimizations - sendfile on; - keepalive_timeout 15; - client_max_body_size 50m; - client_body_buffer_size 16K; - client_header_buffer_size 1k; - - # enable gzip compresion - # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge - gzip on; - gzip_http_version 1.1; - gzip_comp_level 5; - gzip_min_length 256; - gzip_proxied any; - gzip_vary on; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/font-woff - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component; - # text/html is always compressed by HttpGzipModule -} diff --git a/build/frappe-assets/sites/apps.txt b/build/frappe-assets/sites/apps.txt deleted file mode 100644 index f47ffaf5..00000000 --- a/build/frappe-assets/sites/apps.txt +++ /dev/null @@ -1 +0,0 @@ -frappe diff --git a/build/frappe-assets/v11.Dockerfile b/build/frappe-assets/v11.Dockerfile index ef98aded..225243d2 100644 --- a/build/frappe-assets/v11.Dockerfile +++ b/build/frappe-assets/v11.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:10-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-assets/v12.Dockerfile b/build/frappe-assets/v12.Dockerfile index 2e16296b..cbbac49b 100644 --- a/build/frappe-assets/v12.Dockerfile +++ b/build/frappe-assets/v12.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 4987af4e..a74b0d3e 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-socketio/v11.Dockerfile b/build/frappe-socketio/v11.Dockerfile index 78986e0c..fcaac8ac 100644 --- a/build/frappe-socketio/v11.Dockerfile +++ b/build/frappe-socketio/v11.Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-socketio/v12.Dockerfile b/build/frappe-socketio/v12.Dockerfile index 022000d7..adf536bd 100644 --- a/build/frappe-socketio/v12.Dockerfile +++ b/build/frappe-socketio/v12.Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 39013537..41310b0d 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -29,11 +29,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/commands/background.py b/build/frappe-worker/commands/background.py deleted file mode 100644 index 7065efd9..00000000 --- a/build/frappe-worker/commands/background.py +++ /dev/null @@ -1,7 +0,0 @@ -import frappe -from frappe.utils.scheduler import start_scheduler - -print("Starting background scheduler . . .") -start_scheduler() - -exit(0) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py deleted file mode 100644 index e6fbcbb6..00000000 --- a/build/frappe-worker/commands/backup.py +++ /dev/null @@ -1,29 +0,0 @@ -import os, frappe, compileall, re -from frappe.utils.backups import scheduled_backup -from frappe.utils import now -from frappe.utils import get_sites - -def backup(sites, with_files=False): - for site in sites: - frappe.init(site) - frappe.connect() - odb = scheduled_backup( - ignore_files=not with_files, - backup_path_db=None, - backup_path_files=None, - backup_path_private_files=None, - force=True - ) - print("database backup taken -", odb.backup_path_db, "- on", now()) - if with_files: - print("files backup taken -", odb.backup_path_files, "- on", now()) - print("private files backup taken -", odb.backup_path_private_files, "- on", now()) - frappe.destroy() - -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -with_files=True if os.environ.get("WITH_FILES") else False - -backup(sites, with_files) - -exit(0) diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py deleted file mode 100644 index abe17199..00000000 --- a/build/frappe-worker/commands/check_connection.py +++ /dev/null @@ -1,67 +0,0 @@ -import socket, os, json, time -from six.moves.urllib.parse import urlparse - -def is_open(ip, port, timeout=30): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout) - try: - s.connect((ip, int(port))) - s.shutdown(socket.SHUT_RDWR) - return True - except: - return False - finally: - s.close() - -def check_host(ip, port, retry=10, delay=3): - ipup = False - for i in range(retry): - print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) - if is_open(ip, port): - ipup = True - break - else: - time.sleep(delay) - return ipup - -# Check connection to servers -config = None -try: - with open('common_site_config.json') as config_file: - config = json.load(config_file) -except FileNotFoundError: - raise FileNotFoundError("common_site_config.json missing") -except: - raise ValueError("common_site_config.json is not valid") - -# Check mariadb -check_mariadb = False -check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) -if not check_mariadb: - raise ConnectionError("Connection to mariadb timed out") - -# Check redis queue -check_redis_queue = False -redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc -redis_queue, redis_queue_port = redis_queue_url.split(":") -check_redis_queue = check_host(redis_queue, redis_queue_port) -if not check_redis_queue: - raise ConnectionError("Connection to redis queue timed out") - -# Check redis cache -check_redis_cache = False -redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc -redis_cache, redis_cache_port = redis_cache_url.split(":") -check_redis_cache = check_host(redis_cache, redis_cache_port) -if not check_redis_cache: - raise ConnectionError("Connection to redis cache timed out") - -# Check redis socketio -check_redis_socketio = False -redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc -redis_socketio, redis_socketio_port = redis_socketio_url.split(":") -check_redis_socketio = check_host(redis_socketio, redis_socketio_port) -if not check_redis_socketio: - raise ConnectionError("Connection to redis socketio timed out") - -print('Connections OK') diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py deleted file mode 100644 index 54f508fb..00000000 --- a/build/frappe-worker/commands/doctor.py +++ /dev/null @@ -1,4 +0,0 @@ -import frappe -from frappe.utils.doctor import doctor - -doctor() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py deleted file mode 100644 index be382bea..00000000 --- a/build/frappe-worker/commands/migrate.py +++ /dev/null @@ -1,47 +0,0 @@ -import os, frappe, compileall, re, json - -from frappe.migrate import migrate -from frappe.utils import get_sites - -def get_config(): - config = None - with open('common_site_config.json') as config_file: - config = json.load(config_file) - return config - -def save_config(config): - with open('common_site_config.json', 'w') as f: - return json.dump(config, f, indent=1, sort_keys=True) - -def set_maintenance_mode(enable=True): - conf = get_config() - - if enable: - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - save_config(conf) - - if not enable: - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - save_config(conf) - - -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False - -if maintenance_mode: - set_maintenance_mode(True) - -for site in sites: - print('Migrating', site) - frappe.init(site=site) - frappe.connect() - try: - migrate() - finally: - frappe.destroy() - -if maintenance_mode: - set_maintenance_mode(False) - -exit(0) diff --git a/build/frappe-worker/commands/worker.py b/build/frappe-worker/commands/worker.py deleted file mode 100644 index 6ec0bcbf..00000000 --- a/build/frappe-worker/commands/worker.py +++ /dev/null @@ -1,7 +0,0 @@ -import os, frappe -from frappe.utils.background_jobs import start_worker - -queue = os.environ.get("WORKER_TYPE", "default") -start_worker(queue, False) - -exit(0) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template deleted file mode 100755 index 27593289..00000000 --- a/build/frappe-worker/common_site_config.json.template +++ /dev/null @@ -1,7 +0,0 @@ -{ - "db_host": "${MARIADB_HOST}", - "redis_cache": "redis://${REDIS_CACHE}", - "redis_queue": "redis://${REDIS_QUEUE}", - "redis_socketio": "redis://${REDIS_SOCKETIO}", - "socketio_port": ${SOCKETIO_PORT} -} diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile index 934a316b..fa2f01c5 100644 --- a/build/frappe-worker/v11.Dockerfile +++ b/build/frappe-worker/v11.Dockerfile @@ -26,11 +26,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile index 86e63041..1eb6d60a 100644 --- a/build/frappe-worker/v12.Dockerfile +++ b/build/frappe-worker/v12.Dockerfile @@ -29,11 +29,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites