From aeadd23214cf4e859cda7d8d00a1860e2bc20438 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 10 Feb 2020 13:30:54 +0530 Subject: [PATCH 01/29] feat: docker production images --- .travis.yml | 61 ++++--- build/erpnext-assets/.dockerignore | 2 + build/erpnext-assets/Dockerfile | 39 +++++ build/erpnext-assets/docker-entrypoint.sh | 35 ++++ .../nginx-default.conf.template | 90 ++++++++++ build/erpnext-assets/sites/apps.txt | 2 + build/erpnext-assets/v11.Dockerfile | 39 +++++ build/erpnext-assets/v12.Dockerfile | 39 +++++ build/erpnext-python/Dockerfile | 44 +++++ build/erpnext-python/commands/background.py | 7 + build/erpnext-python/commands/backup.py | 29 ++++ .../commands/check_connection.py | 67 ++++++++ build/erpnext-python/commands/doctor.py | 4 + build/erpnext-python/commands/migrate.py | 47 ++++++ build/erpnext-python/commands/new.py | 68 ++++++++ build/erpnext-python/commands/update.py | 55 ++++++ build/erpnext-python/commands/worker.py | 7 + .../common_site_config.json.template | 7 + build/erpnext-python/docker-entrypoint.sh | 158 ++++++++++++++++++ build/erpnext-python/v11.Dockerfile | 41 +++++ build/erpnext-python/v12.Dockerfile | 44 +++++ build/frappe-socketio/Dockerfile | 34 ++++ build/frappe-socketio/docker-entrypoint.sh | 29 ++++ build/frappe-socketio/health.js | 19 +++ build/frappe-socketio/package.json | 17 ++ build/frappe-socketio/v11.Dockerfile | 34 ++++ build/frappe-socketio/v12.Dockerfile | 34 ++++ 27 files changed, 1031 insertions(+), 21 deletions(-) create mode 100644 build/erpnext-assets/.dockerignore create mode 100644 build/erpnext-assets/Dockerfile create mode 100755 build/erpnext-assets/docker-entrypoint.sh create mode 100644 build/erpnext-assets/nginx-default.conf.template create mode 100644 build/erpnext-assets/sites/apps.txt create mode 100644 build/erpnext-assets/v11.Dockerfile create mode 100644 build/erpnext-assets/v12.Dockerfile create mode 100644 build/erpnext-python/Dockerfile create mode 100644 build/erpnext-python/commands/background.py create mode 100644 build/erpnext-python/commands/backup.py create mode 100644 build/erpnext-python/commands/check_connection.py create mode 100644 build/erpnext-python/commands/doctor.py create mode 100644 build/erpnext-python/commands/migrate.py create mode 100644 build/erpnext-python/commands/new.py create mode 100644 build/erpnext-python/commands/update.py create mode 100644 build/erpnext-python/commands/worker.py create mode 100755 build/erpnext-python/common_site_config.json.template create mode 100755 build/erpnext-python/docker-entrypoint.sh create mode 100644 build/erpnext-python/v11.Dockerfile create mode 100644 build/erpnext-python/v12.Dockerfile create mode 100644 build/frappe-socketio/Dockerfile create mode 100755 build/frappe-socketio/docker-entrypoint.sh create mode 100644 build/frappe-socketio/health.js create mode 100644 build/frappe-socketio/package.json create mode 100644 build/frappe-socketio/v11.Dockerfile create mode 100644 build/frappe-socketio/v12.Dockerfile diff --git a/.travis.yml b/.travis.yml index de8802ae..bb56485f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,49 @@ +sudo: required + dist: xenial -env: - - DOCKER_COMPOSE_VERSION: 1.23.1 - - services: - docker before_install: - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - if [ $BUILD == "development" ];then + sudo apt-get update; + sudo apt-get -y install docker-compose; + fi install: - - chmod ugo+x ./dbench - - chmod ugo+x ./test.sh - - ./dbench setup docker - - ./dbench init frappe-bench - - ./dbench new-site site1.local - - ./dbench setup hosts + - if [ $BUILD == "development" ];then + chmod ugo+x ./dbench; + chmod ugo+x ./test.sh; + ./dbench setup docker; + ./dbench init frappe-bench; + ./dbench new-site site1.local; + ./dbench setup hosts; + fi -script: - - ./test.sh - - ./dbench setup docker stop +after_success: + - docker --version + +matrix: + include: + - name: "Test frappe / erpnext development" + env: BUILD=development + script: + - ./test.sh + - ./dbench setup docker stop + - name: "Build frappe / erpnext python environment" + script: + - docker build -t erpnext-python build/erpnext-python + - docker tag erpnext-python frappe/erpnext-python:edge + - docker push frappe/erpnext-python:edge + - name: "Build frappe / erpnext nginx + static assets" + script: + - docker build -t erpnext-assets build/erpnext-assets + - docker tag erpnext-assets frappe/erpnext-assets:edge + - docker push frappe/erpnext-assets:edge + - name: "Build frappe socketio service" + script: + - docker build -t frappe-socketio build/frappe-socketio + - docker tag frappe-socketio frappe/frappe-socketio:edge + - docker push frappe/frappe-socketio:edge diff --git a/build/erpnext-assets/.dockerignore b/build/erpnext-assets/.dockerignore new file mode 100644 index 00000000..c58728b5 --- /dev/null +++ b/build/erpnext-assets/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +/docker/ diff --git a/build/erpnext-assets/Dockerfile b/build/erpnext-assets/Dockerfile new file mode 100644 index 00000000..90bd0a4d --- /dev/null +++ b/build/erpnext-assets/Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe \ + && git clone --depth 1 https://github.com/frappe/erpnext + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-assets/docker-entrypoint.sh b/build/erpnext-assets/docker-entrypoint.sh new file mode 100755 index 00000000..80fbf003 --- /dev/null +++ b/build/erpnext-assets/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +## Thanks +# https://serverfault.com/a/919212 +## + +set -e + +rsync -a --delete /var/www/html/assets/js /assets +rsync -a --delete /var/www/html/assets/css /assets +rsync -a --delete /var/www/html/assets/frappe /assets +rsync -a --delete /var/www/html/assets/erpnext /assets + +chmod -R 755 /assets + +if [[ -z "$ERPNEXT_PY" ]]; then + export ERPNEXT_PY=0.0.0.0 +fi + +if [[ -z "$ERPNEXT_PY_PORT" ]]; then + export ERPNEXT_PY_PORT=8000 +fi + +if [[ -z "$FRAPPE_SOCKETIO" ]]; then + export FRAPPE_SOCKETIO=0.0.0.0 +fi + +if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then + export FRAPPE_SOCKETIO_PORT=9000 +fi + +envsubst '${API_HOST} ${API_PORT} ${ERPNEXT_PY} ${ERPNEXT_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ + < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec "$@" diff --git a/build/erpnext-assets/nginx-default.conf.template b/build/erpnext-assets/nginx-default.conf.template new file mode 100644 index 00000000..17578b6a --- /dev/null +++ b/build/erpnext-assets/nginx-default.conf.template @@ -0,0 +1,90 @@ +upstream erpnext-server { + server ${ERPNEXT_PY}:${ERPNEXT_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://erpnext-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/erpnext-assets/sites/apps.txt b/build/erpnext-assets/sites/apps.txt new file mode 100644 index 00000000..d700de5f --- /dev/null +++ b/build/erpnext-assets/sites/apps.txt @@ -0,0 +1,2 @@ +frappe +erpnext diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile new file mode 100644 index 00000000..d2024670 --- /dev/null +++ b/build/erpnext-assets/v11.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:10-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-11 \ + && git clone --depth 1 https://github.com/frappe/erpnext --branch version-11 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile new file mode 100644 index 00000000..1dd2e5d6 --- /dev/null +++ b/build/erpnext-assets/v12.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-12 \ + && git clone --depth 1 https://github.com/frappe/erpnext --branch version-12 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/erpnext-python/Dockerfile b/build/erpnext-python/Dockerfile new file mode 100644 index 00000000..59175254 --- /dev/null +++ b/build/erpnext-python/Dockerfile @@ -0,0 +1,44 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/erpnext-python/commands/background.py b/build/erpnext-python/commands/background.py new file mode 100644 index 00000000..7065efd9 --- /dev/null +++ b/build/erpnext-python/commands/background.py @@ -0,0 +1,7 @@ +import frappe +from frappe.utils.scheduler import start_scheduler + +print("Starting background scheduler . . .") +start_scheduler() + +exit(0) diff --git a/build/erpnext-python/commands/backup.py b/build/erpnext-python/commands/backup.py new file mode 100644 index 00000000..e6fbcbb6 --- /dev/null +++ b/build/erpnext-python/commands/backup.py @@ -0,0 +1,29 @@ +import os, frappe, compileall, re +from frappe.utils.backups import scheduled_backup +from frappe.utils import now +from frappe.utils import get_sites + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +with_files=True if os.environ.get("WITH_FILES") else False + +backup(sites, with_files) + +exit(0) diff --git a/build/erpnext-python/commands/check_connection.py b/build/erpnext-python/commands/check_connection.py new file mode 100644 index 00000000..abe17199 --- /dev/null +++ b/build/erpnext-python/commands/check_connection.py @@ -0,0 +1,67 @@ +import socket, os, json, time +from six.moves.urllib.parse import urlparse + +def is_open(ip, port, timeout=30): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def check_host(ip, port, retry=10, delay=3): + ipup = False + for i in range(retry): + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + if is_open(ip, port): + ipup = True + break + else: + time.sleep(delay) + return ipup + +# Check connection to servers +config = None +try: + with open('common_site_config.json') as config_file: + config = json.load(config_file) +except FileNotFoundError: + raise FileNotFoundError("common_site_config.json missing") +except: + raise ValueError("common_site_config.json is not valid") + +# Check mariadb +check_mariadb = False +check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) +if not check_mariadb: + raise ConnectionError("Connection to mariadb timed out") + +# Check redis queue +check_redis_queue = False +redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc +redis_queue, redis_queue_port = redis_queue_url.split(":") +check_redis_queue = check_host(redis_queue, redis_queue_port) +if not check_redis_queue: + raise ConnectionError("Connection to redis queue timed out") + +# Check redis cache +check_redis_cache = False +redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc +redis_cache, redis_cache_port = redis_cache_url.split(":") +check_redis_cache = check_host(redis_cache, redis_cache_port) +if not check_redis_cache: + raise ConnectionError("Connection to redis cache timed out") + +# Check redis socketio +check_redis_socketio = False +redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc +redis_socketio, redis_socketio_port = redis_socketio_url.split(":") +check_redis_socketio = check_host(redis_socketio, redis_socketio_port) +if not check_redis_socketio: + raise ConnectionError("Connection to redis socketio timed out") + +print('Connections OK') diff --git a/build/erpnext-python/commands/doctor.py b/build/erpnext-python/commands/doctor.py new file mode 100644 index 00000000..54f508fb --- /dev/null +++ b/build/erpnext-python/commands/doctor.py @@ -0,0 +1,4 @@ +import frappe +from frappe.utils.doctor import doctor + +doctor() diff --git a/build/erpnext-python/commands/migrate.py b/build/erpnext-python/commands/migrate.py new file mode 100644 index 00000000..be382bea --- /dev/null +++ b/build/erpnext-python/commands/migrate.py @@ -0,0 +1,47 @@ +import os, frappe, compileall, re, json + +from frappe.migrate import migrate +from frappe.utils import get_sites + +def get_config(): + config = None + with open('common_site_config.json') as config_file: + config = json.load(config_file) + return config + +def save_config(config): + with open('common_site_config.json', 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + +def set_maintenance_mode(enable=True): + conf = get_config() + + if enable: + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + save_config(conf) + + if not enable: + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + save_config(conf) + + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + +if maintenance_mode: + set_maintenance_mode(True) + +for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() + +if maintenance_mode: + set_maintenance_mode(False) + +exit(0) diff --git a/build/erpnext-python/commands/new.py b/build/erpnext-python/commands/new.py new file mode 100644 index 00000000..b6589c79 --- /dev/null +++ b/build/erpnext-python/commands/new.py @@ -0,0 +1,68 @@ +import os, frappe, json + +from frappe.commands.site import _new_site + +site_name = os.environ.get("SITE_NAME", 'site1.localhost') +mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') +mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') +install_erpnext = os.environ.get("INSTALL_ERPNEXT", None) +force = True if os.environ.get("FORCE", None) else False + +frappe.init(site_name, new_site=True) + +_new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=['erpnext'] if install_erpnext else [], + source_sql=None, + force=force, + reinstall=False, +) + +config = None +with open('common_site_config.json') as config_file: + config = json.load(config_file) + +site_config = None +with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + +# update User's host to '%' required to connect from any container +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +# Set db password +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') +) +os.system(command) + +# Grant permission to database +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +exit(0) diff --git a/build/erpnext-python/commands/update.py b/build/erpnext-python/commands/update.py new file mode 100644 index 00000000..fa47fc44 --- /dev/null +++ b/build/erpnext-python/commands/update.py @@ -0,0 +1,55 @@ +def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): + conf = get_config(bench_path=bench_path) + version_upgrade = is_version_upgrade(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if not no_backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + import bench.utils, bench.app + print('Reloading bench...') + if sys.version_info >= (3, 4): + import importlib + importlib.reload(bench.utils) + importlib.reload(bench.app) + else: + reload(bench.utils) + reload(bench.app) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + if build: + build_assets(bench_path=bench_path) + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_"*80) + print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") + print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + print() \ No newline at end of file diff --git a/build/erpnext-python/commands/worker.py b/build/erpnext-python/commands/worker.py new file mode 100644 index 00000000..6ec0bcbf --- /dev/null +++ b/build/erpnext-python/commands/worker.py @@ -0,0 +1,7 @@ +import os, frappe +from frappe.utils.background_jobs import start_worker + +queue = os.environ.get("WORKER_TYPE", "default") +start_worker(queue, False) + +exit(0) diff --git a/build/erpnext-python/common_site_config.json.template b/build/erpnext-python/common_site_config.json.template new file mode 100755 index 00000000..27593289 --- /dev/null +++ b/build/erpnext-python/common_site_config.json.template @@ -0,0 +1,7 @@ +{ + "db_host": "${MARIADB_HOST}", + "redis_cache": "redis://${REDIS_CACHE}", + "redis_queue": "redis://${REDIS_QUEUE}", + "redis_socketio": "redis://${REDIS_SOCKETIO}", + "socketio_port": ${SOCKETIO_PORT} +} diff --git a/build/erpnext-python/docker-entrypoint.sh b/build/erpnext-python/docker-entrypoint.sh new file mode 100755 index 00000000..fef4eda9 --- /dev/null +++ b/build/erpnext-python/docker-entrypoint.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +function configureEnv() { + if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then + + if [[ -z "$MARIADB_HOST" ]]; then + echo "MARIADB_HOST is not set" + exit 1 + fi + + if [[ -z "$REDIS_CACHE" ]]; then + echo "REDIS_CACHE is not set" + exit 1 + fi + + if [[ -z "$REDIS_QUEUE" ]]; then + echo "REDIS_QUEUE is not set" + exit 1 + fi + + if [[ -z "$REDIS_SOCKETIO" ]]; then + echo "REDIS_SOCKETIO is not set" + exit 1 + fi + + if [[ -z "$SOCKETIO_PORT" ]]; then + echo "SOCKETIO_PORT is not set" + exit 1 + fi + + envsubst '${MARIADB_HOST} + ${REDIS_CACHE} + ${REDIS_QUEUE} + ${REDIS_SOCKETIO} + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +function checkConnection() { + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/check_connection.py" +} + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then + ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt +fi + +if [ "$1" = 'start' ]; then + configureEnv + checkConnection + + chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json + + if [[ -z "$WORKERS" ]]; then + export WORKERS=2 + fi + + if [[ -z "$FRAPPE_PORT" ]]; then + export FRAPPE_PORT=8000 + fi + + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload" + else + . /home/frappe/frappe-bench/env/bin/activate + gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload + fi + +elif [ "$1" = 'worker' ]; then + checkConfigExists + checkConnection + # default WORKER_TYPE=default + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/worker.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/worker.py + fi + +elif [ "$1" = 'schedule' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/background.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/background.py + fi + +elif [ "$1" = 'new' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/new.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/new.py + fi + +elif [ "$1" = 'migrate' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/migrate.py" + exit + +elif [ "$1" = 'doctor' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/doctor.py" + exit + +elif [ "$1" = 'backup' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/backup.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/backup.py + fi + +else + + exec su frappe -c "$@" + +fi diff --git a/build/erpnext-python/v11.Dockerfile b/build/erpnext-python/v11.Dockerfile new file mode 100644 index 00000000..c56a5e96 --- /dev/null +++ b/build/erpnext-python/v11.Dockerfile @@ -0,0 +1,41 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext --branch version-11 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/erpnext-python/v12.Dockerfile b/build/erpnext-python/v12.Dockerfile new file mode 100644 index 00000000..d892c81d --- /dev/null +++ b/build/erpnext-python/v12.Dockerfile @@ -0,0 +1,44 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ + && git clone --depth 1 -o upstream https://github.com/frappe/erpnext --branch version-12 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile new file mode 100644 index 00000000..4987af4e --- /dev/null +++ b/build/frappe-socketio/Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/develop/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/develop/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/docker-entrypoint.sh b/build/frappe-socketio/docker-entrypoint.sh new file mode 100755 index 00000000..600a744c --- /dev/null +++ b/build/frappe-socketio/docker-entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [ "$1" = 'start' ]; then + checkConfigExists + su frappe -c "node /home/frappe/frappe-bench/apps/frappe/socketio.js" + +elif [ "$1" = 'doctor' ]; then + + su frappe -c "node /home/frappe/frappe-bench/apps/frappe/health.js" + +else + + exec su frappe -c "$@" + +fi diff --git a/build/frappe-socketio/health.js b/build/frappe-socketio/health.js new file mode 100644 index 00000000..b85981bd --- /dev/null +++ b/build/frappe-socketio/health.js @@ -0,0 +1,19 @@ +var net = require('net'); +var { get_conf } = require('./node_utils'); + +var conf = get_conf(); +var server = ['localhost', conf.socketio_port || 9000]; + +var sock = new net.Socket(); +sock.setTimeout(2500); +sock.on('connect', function() { + console.info(server[0]+':'+server[1]+' is up.'); + sock.destroy(); + process.exit(); +}).on('error', function(e) { + console.error(server[0]+':'+server[1]+' is down: ' + e.message); + process.exit(1); +}).on('timeout', function(e) { + console.error(server[0]+':'+server[1]+' is down: timeout'); + process.exit(1); +}).connect(server[1], server[0]); diff --git a/build/frappe-socketio/package.json b/build/frappe-socketio/package.json new file mode 100644 index 00000000..4f8572d5 --- /dev/null +++ b/build/frappe-socketio/package.json @@ -0,0 +1,17 @@ +{ + "name": "frappe-socketio", + "version": "1.0.0", + "description": "Frappe SocketIO Server", + "main": "socketio.js", + "scripts": { + "start": "node socketio.js" + }, + "author": "Revant Nandgaonkar", + "license": "MIT", + "dependencies": { + "express": "^4.17.1", + "redis": "^2.8.0", + "socket.io": "^2.3.0", + "superagent": "^5.1.0" + } +} diff --git a/build/frappe-socketio/v11.Dockerfile b/build/frappe-socketio/v11.Dockerfile new file mode 100644 index 00000000..78986e0c --- /dev/null +++ b/build/frappe-socketio/v11.Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-11/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-11/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-socketio/v12.Dockerfile b/build/frappe-socketio/v12.Dockerfile new file mode 100644 index 00000000..022000d7 --- /dev/null +++ b/build/frappe-socketio/v12.Dockerfile @@ -0,0 +1,34 @@ +FROM node:slim + +# Install needed packages +RUN apt-get update && apt-get install -y curl && apt-get clean + +RUN useradd -ms /bin/bash frappe + +# Make bench directories +RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe + +COPY package.json /home/frappe/frappe-bench/apps/frappe + + +# get socketio +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-12/socketio.js \ + --output /home/frappe/frappe-bench/apps/frappe/socketio.js \ + && curl https://raw.githubusercontent.com/frappe/frappe/version-12/node_utils.js \ + --output /home/frappe/frappe-bench/apps/frappe/node_utils.js + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && npm install --only=production + +COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +RUN chown -R frappe:frappe /home/frappe + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] From e1e1c6b6f96f27135128f0cafb45bac805f1af4b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 17 Feb 2020 20:35:50 +0530 Subject: [PATCH 02/29] refactor: rename images erpnext-assets - ERPNext nginx config + static assets erpnext-worker - ERPNext python environment frappe-assets - Frappe nginx config + static assets frappe-worker - Frappe python environment frappe-socketio - Frappe socketio process common for apps --- .travis.yml | 24 ++- .../Dockerfile | 0 .../commands/background.py | 0 .../commands/backup.py | 0 .../commands/check_connection.py | 0 .../commands/doctor.py | 0 .../commands/migrate.py | 0 .../commands/new.py | 0 .../commands/update.py | 0 .../commands/worker.py | 0 .../common_site_config.json.template | 0 .../docker-entrypoint.sh | 0 .../v11.Dockerfile | 0 .../v12.Dockerfile | 0 build/frappe-assets/.dockerignore | 2 + build/frappe-assets/Dockerfile | 36 ++++ build/frappe-assets/docker-entrypoint.sh | 35 ++++ .../frappe-assets/nginx-default.conf.template | 90 ++++++++++ build/frappe-assets/sites/apps.txt | 1 + build/frappe-assets/v11.Dockerfile | 36 ++++ build/frappe-assets/v12.Dockerfile | 36 ++++ build/frappe-worker/Dockerfile | 42 +++++ build/frappe-worker/commands/background.py | 7 + build/frappe-worker/commands/backup.py | 29 ++++ .../commands/check_connection.py | 67 ++++++++ build/frappe-worker/commands/doctor.py | 4 + build/frappe-worker/commands/migrate.py | 47 ++++++ build/frappe-worker/commands/new.py | 67 ++++++++ build/frappe-worker/commands/update.py | 55 ++++++ build/frappe-worker/commands/worker.py | 7 + .../common_site_config.json.template | 7 + build/frappe-worker/docker-entrypoint.sh | 158 ++++++++++++++++++ build/frappe-worker/v11.Dockerfile | 39 +++++ build/frappe-worker/v12.Dockerfile | 42 +++++ 34 files changed, 824 insertions(+), 7 deletions(-) rename build/{erpnext-python => erpnext-worker}/Dockerfile (100%) rename build/{erpnext-python => erpnext-worker}/commands/background.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/backup.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/check_connection.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/doctor.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/migrate.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/new.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/update.py (100%) rename build/{erpnext-python => erpnext-worker}/commands/worker.py (100%) rename build/{erpnext-python => erpnext-worker}/common_site_config.json.template (100%) rename build/{erpnext-python => erpnext-worker}/docker-entrypoint.sh (100%) rename build/{erpnext-python => erpnext-worker}/v11.Dockerfile (100%) rename build/{erpnext-python => erpnext-worker}/v12.Dockerfile (100%) create mode 100644 build/frappe-assets/.dockerignore create mode 100644 build/frappe-assets/Dockerfile create mode 100755 build/frappe-assets/docker-entrypoint.sh create mode 100644 build/frappe-assets/nginx-default.conf.template create mode 100644 build/frappe-assets/sites/apps.txt create mode 100644 build/frappe-assets/v11.Dockerfile create mode 100644 build/frappe-assets/v12.Dockerfile create mode 100644 build/frappe-worker/Dockerfile create mode 100644 build/frappe-worker/commands/background.py create mode 100644 build/frappe-worker/commands/backup.py create mode 100644 build/frappe-worker/commands/check_connection.py create mode 100644 build/frappe-worker/commands/doctor.py create mode 100644 build/frappe-worker/commands/migrate.py create mode 100644 build/frappe-worker/commands/new.py create mode 100644 build/frappe-worker/commands/update.py create mode 100644 build/frappe-worker/commands/worker.py create mode 100755 build/frappe-worker/common_site_config.json.template create mode 100755 build/frappe-worker/docker-entrypoint.sh create mode 100644 build/frappe-worker/v11.Dockerfile create mode 100644 build/frappe-worker/v12.Dockerfile diff --git a/.travis.yml b/.travis.yml index bb56485f..ae4bb89f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,22 +27,32 @@ after_success: matrix: include: - - name: "Test frappe / erpnext development" + - name: "Test Frappe / ERPNext development" env: BUILD=development script: - ./test.sh - ./dbench setup docker stop - - name: "Build frappe / erpnext python environment" + - name: "Build Frappe python environment" script: - - docker build -t erpnext-python build/erpnext-python - - docker tag erpnext-python frappe/erpnext-python:edge - - docker push frappe/erpnext-python:edge - - name: "Build frappe / erpnext nginx + static assets" + - docker build -t frappe-worker build/frappe-worker + - docker tag frappe-worker frappe/frappe-worker:edge + - docker push frappe/frappe-worker:edge + - name: "Build Frappe nginx + static assets" + script: + - docker build -t frappe-assets build/frappe-assets + - docker tag frappe-assets frappe/frappe-assets:edge + - docker push frappe/frappe-assets:edge + - name: "Build ERPNext python environment" + script: + - docker build -t erpnext-worker build/erpnext-worker + - docker tag erpnext-worker frappe/erpnext-worker:edge + - docker push frappe/erpnext-worker:edge + - name: "Build ERPNext nginx + static assets" script: - docker build -t erpnext-assets build/erpnext-assets - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - - name: "Build frappe socketio service" + - name: "Build Frappe socketio service" script: - docker build -t frappe-socketio build/frappe-socketio - docker tag frappe-socketio frappe/frappe-socketio:edge diff --git a/build/erpnext-python/Dockerfile b/build/erpnext-worker/Dockerfile similarity index 100% rename from build/erpnext-python/Dockerfile rename to build/erpnext-worker/Dockerfile diff --git a/build/erpnext-python/commands/background.py b/build/erpnext-worker/commands/background.py similarity index 100% rename from build/erpnext-python/commands/background.py rename to build/erpnext-worker/commands/background.py diff --git a/build/erpnext-python/commands/backup.py b/build/erpnext-worker/commands/backup.py similarity index 100% rename from build/erpnext-python/commands/backup.py rename to build/erpnext-worker/commands/backup.py diff --git a/build/erpnext-python/commands/check_connection.py b/build/erpnext-worker/commands/check_connection.py similarity index 100% rename from build/erpnext-python/commands/check_connection.py rename to build/erpnext-worker/commands/check_connection.py diff --git a/build/erpnext-python/commands/doctor.py b/build/erpnext-worker/commands/doctor.py similarity index 100% rename from build/erpnext-python/commands/doctor.py rename to build/erpnext-worker/commands/doctor.py diff --git a/build/erpnext-python/commands/migrate.py b/build/erpnext-worker/commands/migrate.py similarity index 100% rename from build/erpnext-python/commands/migrate.py rename to build/erpnext-worker/commands/migrate.py diff --git a/build/erpnext-python/commands/new.py b/build/erpnext-worker/commands/new.py similarity index 100% rename from build/erpnext-python/commands/new.py rename to build/erpnext-worker/commands/new.py diff --git a/build/erpnext-python/commands/update.py b/build/erpnext-worker/commands/update.py similarity index 100% rename from build/erpnext-python/commands/update.py rename to build/erpnext-worker/commands/update.py diff --git a/build/erpnext-python/commands/worker.py b/build/erpnext-worker/commands/worker.py similarity index 100% rename from build/erpnext-python/commands/worker.py rename to build/erpnext-worker/commands/worker.py diff --git a/build/erpnext-python/common_site_config.json.template b/build/erpnext-worker/common_site_config.json.template similarity index 100% rename from build/erpnext-python/common_site_config.json.template rename to build/erpnext-worker/common_site_config.json.template diff --git a/build/erpnext-python/docker-entrypoint.sh b/build/erpnext-worker/docker-entrypoint.sh similarity index 100% rename from build/erpnext-python/docker-entrypoint.sh rename to build/erpnext-worker/docker-entrypoint.sh diff --git a/build/erpnext-python/v11.Dockerfile b/build/erpnext-worker/v11.Dockerfile similarity index 100% rename from build/erpnext-python/v11.Dockerfile rename to build/erpnext-worker/v11.Dockerfile diff --git a/build/erpnext-python/v12.Dockerfile b/build/erpnext-worker/v12.Dockerfile similarity index 100% rename from build/erpnext-python/v12.Dockerfile rename to build/erpnext-worker/v12.Dockerfile diff --git a/build/frappe-assets/.dockerignore b/build/frappe-assets/.dockerignore new file mode 100644 index 00000000..c58728b5 --- /dev/null +++ b/build/frappe-assets/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +/docker/ diff --git a/build/frappe-assets/Dockerfile b/build/frappe-assets/Dockerfile new file mode 100644 index 00000000..ef0e7dcc --- /dev/null +++ b/build/frappe-assets/Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-assets/docker-entrypoint.sh b/build/frappe-assets/docker-entrypoint.sh new file mode 100755 index 00000000..3cae6744 --- /dev/null +++ b/build/frappe-assets/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +## Thanks +# https://serverfault.com/a/919212 +## + +set -e + +rsync -a --delete /var/www/html/assets/js /assets +rsync -a --delete /var/www/html/assets/css /assets +rsync -a --delete /var/www/html/assets/frappe /assets +rsync -a --delete /var/www/html/assets/erpnext /assets + +chmod -R 755 /assets + +if [[ -z "$FRAPPE_PY" ]]; then + export FRAPPE_PY=0.0.0.0 +fi + +if [[ -z "$FRAPPE_PY_PORT" ]]; then + export FRAPPE_PY_PORT=8000 +fi + +if [[ -z "$FRAPPE_SOCKETIO" ]]; then + export FRAPPE_SOCKETIO=0.0.0.0 +fi + +if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then + export FRAPPE_SOCKETIO_PORT=9000 +fi + +envsubst '${API_HOST} ${API_PORT} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ + < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec "$@" diff --git a/build/frappe-assets/nginx-default.conf.template b/build/frappe-assets/nginx-default.conf.template new file mode 100644 index 00000000..c2e0e0ba --- /dev/null +++ b/build/frappe-assets/nginx-default.conf.template @@ -0,0 +1,90 @@ +upstream frappe-server { + server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://frappe-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/frappe-assets/sites/apps.txt b/build/frappe-assets/sites/apps.txt new file mode 100644 index 00000000..f47ffaf5 --- /dev/null +++ b/build/frappe-assets/sites/apps.txt @@ -0,0 +1 @@ +frappe diff --git a/build/frappe-assets/v11.Dockerfile b/build/frappe-assets/v11.Dockerfile new file mode 100644 index 00000000..ef98aded --- /dev/null +++ b/build/frappe-assets/v11.Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:10-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-11 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-assets/v12.Dockerfile b/build/frappe-assets/v12.Dockerfile new file mode 100644 index 00000000..2e16296b --- /dev/null +++ b/build/frappe-assets/v12.Dockerfile @@ -0,0 +1,36 @@ +FROM bitnami/node:12-prod + +WORKDIR /home/frappe/frappe-bench +COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt + +RUN install_packages git + +RUN mkdir -p apps sites/assets \ + && cd apps \ + && git clone --depth 1 https://github.com/frappe/frappe --branch version-12 + +RUN cd /home/frappe/frappe-bench/apps/frappe \ + && yarn \ + && yarn run production \ + && rm -fr node_modules \ + && yarn install --production=true + +RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ + && mkdir -p /var/www/error_pages \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + +RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ + && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ + +FROM nginx:latest +COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ +COPY --from=0 /var/www/error_pages /var/www/ +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / + +RUN apt-get update && apt-get install -y rsync && apt-get clean + +VOLUME [ "/assets" ] + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile new file mode 100644 index 00000000..39013537 --- /dev/null +++ b/build/frappe-worker/Dockerfile @@ -0,0 +1,42 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-worker/commands/background.py b/build/frappe-worker/commands/background.py new file mode 100644 index 00000000..7065efd9 --- /dev/null +++ b/build/frappe-worker/commands/background.py @@ -0,0 +1,7 @@ +import frappe +from frappe.utils.scheduler import start_scheduler + +print("Starting background scheduler . . .") +start_scheduler() + +exit(0) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py new file mode 100644 index 00000000..e6fbcbb6 --- /dev/null +++ b/build/frappe-worker/commands/backup.py @@ -0,0 +1,29 @@ +import os, frappe, compileall, re +from frappe.utils.backups import scheduled_backup +from frappe.utils import now +from frappe.utils import get_sites + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +with_files=True if os.environ.get("WITH_FILES") else False + +backup(sites, with_files) + +exit(0) diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py new file mode 100644 index 00000000..abe17199 --- /dev/null +++ b/build/frappe-worker/commands/check_connection.py @@ -0,0 +1,67 @@ +import socket, os, json, time +from six.moves.urllib.parse import urlparse + +def is_open(ip, port, timeout=30): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def check_host(ip, port, retry=10, delay=3): + ipup = False + for i in range(retry): + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) + if is_open(ip, port): + ipup = True + break + else: + time.sleep(delay) + return ipup + +# Check connection to servers +config = None +try: + with open('common_site_config.json') as config_file: + config = json.load(config_file) +except FileNotFoundError: + raise FileNotFoundError("common_site_config.json missing") +except: + raise ValueError("common_site_config.json is not valid") + +# Check mariadb +check_mariadb = False +check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) +if not check_mariadb: + raise ConnectionError("Connection to mariadb timed out") + +# Check redis queue +check_redis_queue = False +redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc +redis_queue, redis_queue_port = redis_queue_url.split(":") +check_redis_queue = check_host(redis_queue, redis_queue_port) +if not check_redis_queue: + raise ConnectionError("Connection to redis queue timed out") + +# Check redis cache +check_redis_cache = False +redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc +redis_cache, redis_cache_port = redis_cache_url.split(":") +check_redis_cache = check_host(redis_cache, redis_cache_port) +if not check_redis_cache: + raise ConnectionError("Connection to redis cache timed out") + +# Check redis socketio +check_redis_socketio = False +redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc +redis_socketio, redis_socketio_port = redis_socketio_url.split(":") +check_redis_socketio = check_host(redis_socketio, redis_socketio_port) +if not check_redis_socketio: + raise ConnectionError("Connection to redis socketio timed out") + +print('Connections OK') diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py new file mode 100644 index 00000000..54f508fb --- /dev/null +++ b/build/frappe-worker/commands/doctor.py @@ -0,0 +1,4 @@ +import frappe +from frappe.utils.doctor import doctor + +doctor() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py new file mode 100644 index 00000000..be382bea --- /dev/null +++ b/build/frappe-worker/commands/migrate.py @@ -0,0 +1,47 @@ +import os, frappe, compileall, re, json + +from frappe.migrate import migrate +from frappe.utils import get_sites + +def get_config(): + config = None + with open('common_site_config.json') as config_file: + config = json.load(config_file) + return config + +def save_config(config): + with open('common_site_config.json', 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + +def set_maintenance_mode(enable=True): + conf = get_config() + + if enable: + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + save_config(conf) + + if not enable: + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + save_config(conf) + + +installed_sites = ":".join(get_sites()) +sites = os.environ.get("SITES", installed_sites).split(":") +maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + +if maintenance_mode: + set_maintenance_mode(True) + +for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() + +if maintenance_mode: + set_maintenance_mode(False) + +exit(0) diff --git a/build/frappe-worker/commands/new.py b/build/frappe-worker/commands/new.py new file mode 100644 index 00000000..10d6a8e4 --- /dev/null +++ b/build/frappe-worker/commands/new.py @@ -0,0 +1,67 @@ +import os, frappe, json + +from frappe.commands.site import _new_site + +site_name = os.environ.get("SITE_NAME", 'site1.localhost') +mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') +mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') +force = True if os.environ.get("FORCE", None) else False + +frappe.init(site_name, new_site=True) + +_new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=[], + source_sql=None, + force=force, + reinstall=False, +) + +config = None +with open('common_site_config.json') as config_file: + config = json.load(config_file) + +site_config = None +with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + +# update User's host to '%' required to connect from any container +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +# Set db password +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') +) +os.system(command) + +# Grant permission to database +command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password +) +command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') +) +os.system(command) + +exit(0) diff --git a/build/frappe-worker/commands/update.py b/build/frappe-worker/commands/update.py new file mode 100644 index 00000000..fa47fc44 --- /dev/null +++ b/build/frappe-worker/commands/update.py @@ -0,0 +1,55 @@ +def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, + restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): + conf = get_config(bench_path=bench_path) + version_upgrade = is_version_upgrade(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + + before_update(bench_path=bench_path, requirements=requirements) + + conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) + update_config(conf, bench_path=bench_path) + + if not no_backup: + print('Backing up sites...') + backup_all_sites(bench_path=bench_path) + + if pull: + pull_all_apps(bench_path=bench_path, reset=reset) + + if requirements: + update_requirements(bench_path=bench_path) + update_node_packages(bench_path=bench_path) + + if version_upgrade[0] or (not version_upgrade[0] and force): + pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + import bench.utils, bench.app + print('Reloading bench...') + if sys.version_info >= (3, 4): + import importlib + importlib.reload(bench.utils) + importlib.reload(bench.app) + else: + reload(bench.utils) + reload(bench.app) + + if patch: + print('Patching sites...') + patch_sites(bench_path=bench_path) + if build: + build_assets(bench_path=bench_path) + if version_upgrade[0] or (not version_upgrade[0] and force): + post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) + if restart_supervisor or conf.get('restart_supervisor_on_update'): + restart_supervisor_processes(bench_path=bench_path) + if restart_systemd or conf.get('restart_systemd_on_update'): + restart_systemd_processes(bench_path=bench_path) + + conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) + update_config(conf, bench_path=bench_path) + + print("_"*80) + print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") + print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") + print() \ No newline at end of file diff --git a/build/frappe-worker/commands/worker.py b/build/frappe-worker/commands/worker.py new file mode 100644 index 00000000..6ec0bcbf --- /dev/null +++ b/build/frappe-worker/commands/worker.py @@ -0,0 +1,7 @@ +import os, frappe +from frappe.utils.background_jobs import start_worker + +queue = os.environ.get("WORKER_TYPE", "default") +start_worker(queue, False) + +exit(0) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template new file mode 100755 index 00000000..27593289 --- /dev/null +++ b/build/frappe-worker/common_site_config.json.template @@ -0,0 +1,7 @@ +{ + "db_host": "${MARIADB_HOST}", + "redis_cache": "redis://${REDIS_CACHE}", + "redis_queue": "redis://${REDIS_QUEUE}", + "redis_socketio": "redis://${REDIS_SOCKETIO}", + "socketio_port": ${SOCKETIO_PORT} +} diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh new file mode 100755 index 00000000..fef4eda9 --- /dev/null +++ b/build/frappe-worker/docker-entrypoint.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +function configureEnv() { + if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then + + if [[ -z "$MARIADB_HOST" ]]; then + echo "MARIADB_HOST is not set" + exit 1 + fi + + if [[ -z "$REDIS_CACHE" ]]; then + echo "REDIS_CACHE is not set" + exit 1 + fi + + if [[ -z "$REDIS_QUEUE" ]]; then + echo "REDIS_QUEUE is not set" + exit 1 + fi + + if [[ -z "$REDIS_SOCKETIO" ]]; then + echo "REDIS_SOCKETIO is not set" + exit 1 + fi + + if [[ -z "$SOCKETIO_PORT" ]]; then + echo "SOCKETIO_PORT is not set" + exit 1 + fi + + envsubst '${MARIADB_HOST} + ${REDIS_CACHE} + ${REDIS_QUEUE} + ${REDIS_SOCKETIO} + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +function checkConnection() { + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/check_connection.py" +} + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + let COUNTER=COUNTER+1 + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then + ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt +fi + +if [ "$1" = 'start' ]; then + configureEnv + checkConnection + + chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json + + if [[ -z "$WORKERS" ]]; then + export WORKERS=2 + fi + + if [[ -z "$FRAPPE_PORT" ]]; then + export FRAPPE_PORT=8000 + fi + + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload" + else + . /home/frappe/frappe-bench/env/bin/activate + gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload + fi + +elif [ "$1" = 'worker' ]; then + checkConfigExists + checkConnection + # default WORKER_TYPE=default + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/worker.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/worker.py + fi + +elif [ "$1" = 'schedule' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/background.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/background.py + fi + +elif [ "$1" = 'new' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/new.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/new.py + fi + +elif [ "$1" = 'migrate' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/migrate.py" + exit + +elif [ "$1" = 'doctor' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/doctor.py" + exit + +elif [ "$1" = 'backup' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/backup.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/backup.py + fi + +else + + exec su frappe -c "$@" + +fi diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile new file mode 100644 index 00000000..934a316b --- /dev/null +++ b/build/frappe-worker/v11.Dockerfile @@ -0,0 +1,39 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile new file mode 100644 index 00000000..86e63041 --- /dev/null +++ b/build/frappe-worker/v12.Dockerfile @@ -0,0 +1,42 @@ +FROM bitnami/python:latest-prod + +RUN useradd -ms /bin/bash frappe +WORKDIR /home/frappe/frappe-bench +RUN install_packages \ + git \ + wkhtmltopdf \ + mariadb-client \ + gettext-base \ + wget \ + # for PDF + libssl-dev \ + fonts-cantarell \ + xfonts-75dpi \ + xfonts-base \ + # For psycopg2 + libpq-dev \ + build-essential + +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb + +RUN mkdir -p apps logs commands + +RUN virtualenv env \ + && . env/bin/activate \ + && cd apps \ + && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ + && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe + +COPY ./commands/* /home/frappe/frappe-bench/commands/ +COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template + +# Setup docker-entrypoint +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +WORKDIR /home/frappe/frappe-bench/sites + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["start"] From 6677885e78d48ab7e5c598f4b438f262a7ecea04 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 19 Feb 2020 19:31:00 +0530 Subject: [PATCH 03/29] chore: reorder commands for clarity Signed-off-by: Chinmay D. Pai --- build/erpnext-assets/v11.Dockerfile | 6 +++--- build/erpnext-assets/v12.Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile index d2024670..8d4907d9 100644 --- a/build/erpnext-assets/v11.Dockerfile +++ b/build/erpnext-assets/v11.Dockerfile @@ -18,12 +18,12 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ - && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile index 1dd2e5d6..b000a199 100644 --- a/build/erpnext-assets/v12.Dockerfile +++ b/build/erpnext-assets/v12.Dockerfile @@ -18,12 +18,12 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ - && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ From 0910278efdab255609d93b197c4d35eadfe3f752 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 12:08:51 +0530 Subject: [PATCH 04/29] feat: add support to start bench console Signed-off-by: Chinmay D. Pai --- build/frappe-worker/commands/console.py | 25 ++++++++++++++++++++++++ build/frappe-worker/docker-entrypoint.sh | 17 ++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 build/frappe-worker/commands/console.py diff --git a/build/frappe-worker/commands/console.py b/build/frappe-worker/commands/console.py new file mode 100644 index 00000000..20b3fd34 --- /dev/null +++ b/build/frappe-worker/commands/console.py @@ -0,0 +1,25 @@ +import sys +import frappe +import IPython + +from frappe.utils import get_sites + + +def console(site): + "Start ipython console for a site" + if site not in get_sites(): + print("Site {0} does not exist on the current bench".format(site)) + return + + frappe.init(site=site) + frappe.connect() + frappe.local.lang = frappe.db.get_default("lang") + all_apps = frappe.get_installed_apps() + for app in all_apps: + locals()[app] = __import__(app) + print("Apps in this namespace:\n{}".format(", ".join(all_apps))) + IPython.embed(display_banner="", header="") + + +site = sys.argv[-1] +console(site) diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh index fef4eda9..d5b65abf 100755 --- a/build/frappe-worker/docker-entrypoint.sh +++ b/build/frappe-worker/docker-entrypoint.sh @@ -151,6 +151,23 @@ elif [ "$1" = 'backup' ]; then python /home/frappe/frappe-bench/commands/backup.py fi +elif [ "$1" = 'console' ]; then + + if [[ -z "$2" ]]; then + echo "Need to specify a sitename with the command:" + echo "console " + exit 1 + fi + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/console.py $2" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/console.py "$2" + fi + else exec su frappe -c "$@" From 3b5f829248b1198a5498c5ad38a2835c160fd3fd Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 12:09:51 +0530 Subject: [PATCH 05/29] chore: remove unused update.py Signed-off-by: Chinmay D. Pai --- build/erpnext-worker/commands/update.py | 55 ------------------------- build/frappe-worker/commands/update.py | 55 ------------------------- 2 files changed, 110 deletions(-) delete mode 100644 build/erpnext-worker/commands/update.py delete mode 100644 build/frappe-worker/commands/update.py diff --git a/build/erpnext-worker/commands/update.py b/build/erpnext-worker/commands/update.py deleted file mode 100644 index fa47fc44..00000000 --- a/build/erpnext-worker/commands/update.py +++ /dev/null @@ -1,55 +0,0 @@ -def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): - conf = get_config(bench_path=bench_path) - version_upgrade = is_version_upgrade(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - - before_update(bench_path=bench_path, requirements=requirements) - - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - update_config(conf, bench_path=bench_path) - - if not no_backup: - print('Backing up sites...') - backup_all_sites(bench_path=bench_path) - - if pull: - pull_all_apps(bench_path=bench_path, reset=reset) - - if requirements: - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - import bench.utils, bench.app - print('Reloading bench...') - if sys.version_info >= (3, 4): - import importlib - importlib.reload(bench.utils) - importlib.reload(bench.app) - else: - reload(bench.utils) - reload(bench.app) - - if patch: - print('Patching sites...') - patch_sites(bench_path=bench_path) - if build: - build_assets(bench_path=bench_path) - if version_upgrade[0] or (not version_upgrade[0] and force): - post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - update_config(conf, bench_path=bench_path) - - print("_"*80) - print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") - print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") - print() \ No newline at end of file diff --git a/build/frappe-worker/commands/update.py b/build/frappe-worker/commands/update.py deleted file mode 100644 index fa47fc44..00000000 --- a/build/frappe-worker/commands/update.py +++ /dev/null @@ -1,55 +0,0 @@ -def update(pull=False, patch=False, build=False, update_bench=False, auto=False, restart_supervisor=False, - restart_systemd=False, requirements=False, no_backup=False, bench_path='.', force=False, reset=False): - conf = get_config(bench_path=bench_path) - version_upgrade = is_version_upgrade(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - validate_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - - before_update(bench_path=bench_path, requirements=requirements) - - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - update_config(conf, bench_path=bench_path) - - if not no_backup: - print('Backing up sites...') - backup_all_sites(bench_path=bench_path) - - if pull: - pull_all_apps(bench_path=bench_path, reset=reset) - - if requirements: - update_requirements(bench_path=bench_path) - update_node_packages(bench_path=bench_path) - - if version_upgrade[0] or (not version_upgrade[0] and force): - pre_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - import bench.utils, bench.app - print('Reloading bench...') - if sys.version_info >= (3, 4): - import importlib - importlib.reload(bench.utils) - importlib.reload(bench.app) - else: - reload(bench.utils) - reload(bench.app) - - if patch: - print('Patching sites...') - patch_sites(bench_path=bench_path) - if build: - build_assets(bench_path=bench_path) - if version_upgrade[0] or (not version_upgrade[0] and force): - post_upgrade(version_upgrade[1], version_upgrade[2], bench_path=bench_path) - if restart_supervisor or conf.get('restart_supervisor_on_update'): - restart_supervisor_processes(bench_path=bench_path) - if restart_systemd or conf.get('restart_systemd_on_update'): - restart_systemd_processes(bench_path=bench_path) - - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - update_config(conf, bench_path=bench_path) - - print("_"*80) - print("Bench: Deployment tool for Frappe and ERPNext (https://erpnext.org).") - print("Open source depends on your contributions, so please contribute bug reports, patches, fixes or cash and be a part of the community") - print() \ No newline at end of file From 347b94dee23b8d5866e25b3ab103ed85218e9e95 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 20 Feb 2020 12:33:52 +0530 Subject: [PATCH 06/29] chore: fixes to shell scripts * envsubst: expression may fail to expand in single quotes * use (()) instead of let * use find instead of ls to better hand alphanumeric folders Signed-off-by: Chinmay D. Pai --- build/erpnext-assets/docker-entrypoint.sh | 2 +- build/erpnext-worker/docker-entrypoint.sh | 8 ++++---- build/frappe-assets/docker-entrypoint.sh | 2 +- build/frappe-worker/docker-entrypoint.sh | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/build/erpnext-assets/docker-entrypoint.sh b/build/erpnext-assets/docker-entrypoint.sh index 80fbf003..90bdf4b2 100755 --- a/build/erpnext-assets/docker-entrypoint.sh +++ b/build/erpnext-assets/docker-entrypoint.sh @@ -29,7 +29,7 @@ if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then export FRAPPE_SOCKETIO_PORT=9000 fi -envsubst '${API_HOST} ${API_PORT} ${ERPNEXT_PY} ${ERPNEXT_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ +envsubst "${API_HOST} ${API_PORT} ${ERPNEXT_PY} ${ERPNEXT_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}" \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" diff --git a/build/erpnext-worker/docker-entrypoint.sh b/build/erpnext-worker/docker-entrypoint.sh index fef4eda9..b6fc0d35 100755 --- a/build/erpnext-worker/docker-entrypoint.sh +++ b/build/erpnext-worker/docker-entrypoint.sh @@ -28,11 +28,11 @@ function configureEnv() { exit 1 fi - envsubst '${MARIADB_HOST} + envsubst "${MARIADB_HOST} ${REDIS_CACHE} ${REDIS_QUEUE} ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + ${SOCKETIO_PORT}" < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json fi } @@ -45,7 +45,7 @@ function checkConfigExists() { COUNTER=0 while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do sleep 1 - let COUNTER=COUNTER+1 + (( COUNTER=COUNTER+1 )) echo "config file not created, retry $COUNTER" done @@ -56,7 +56,7 @@ function checkConfigExists() { } if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt + find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt fi if [ "$1" = 'start' ]; then diff --git a/build/frappe-assets/docker-entrypoint.sh b/build/frappe-assets/docker-entrypoint.sh index 3cae6744..4b791d10 100755 --- a/build/frappe-assets/docker-entrypoint.sh +++ b/build/frappe-assets/docker-entrypoint.sh @@ -29,7 +29,7 @@ if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then export FRAPPE_SOCKETIO_PORT=9000 fi -envsubst '${API_HOST} ${API_PORT} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}' \ +envsubst "${API_HOST} ${API_PORT} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}" \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh index d5b65abf..da1934b6 100755 --- a/build/frappe-worker/docker-entrypoint.sh +++ b/build/frappe-worker/docker-entrypoint.sh @@ -28,11 +28,11 @@ function configureEnv() { exit 1 fi - envsubst '${MARIADB_HOST} + envsubst "${MARIADB_HOST} ${REDIS_CACHE} ${REDIS_QUEUE} ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + ${SOCKETIO_PORT}" < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json fi } @@ -45,7 +45,7 @@ function checkConfigExists() { COUNTER=0 while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do sleep 1 - let COUNTER=COUNTER+1 + (( COUNTER=COUNTER+1 )) echo "config file not created, retry $COUNTER" done @@ -56,7 +56,7 @@ function checkConfigExists() { } if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - ls -1 /home/frappe/frappe-bench/apps | sort -r > /home/frappe/frappe-bench/sites/apps.txt + find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt fi if [ "$1" = 'start' ]; then From 242708df0c395357054955126f231d836e2d94f5 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 18:01:23 +0530 Subject: [PATCH 07/29] refactor: reuse common scripts and files move common scripts to build/common --- .travis.yml | 10 +- .../commands/background.py | 0 .../commands/backup.py | 0 .../commands/check_connection.py | 0 .../commands/console.py | 0 .../commands/doctor.py | 0 .../commands/migrate.py | 0 .../{frappe-worker => common}/commands/new.py | 0 .../commands/worker.py | 0 .../common_site_config.json.template | 0 .../nginx-default.conf.template | 0 .../worker}/docker-entrypoint.sh | 0 build/erpnext-assets/.dockerignore | 2 - build/erpnext-assets/Dockerfile | 11 +- build/erpnext-assets/sites/apps.txt | 2 - build/erpnext-assets/v11.Dockerfile | 10 +- build/erpnext-assets/v12.Dockerfile | 10 +- build/erpnext-worker/Dockerfile | 6 +- build/erpnext-worker/commands/new.py | 68 -------- build/erpnext-worker/docker-entrypoint.sh | 158 ------------------ build/erpnext-worker/v11.Dockerfile | 6 +- build/erpnext-worker/v12.Dockerfile | 6 +- build/frappe-assets/.dockerignore | 2 - build/frappe-assets/Dockerfile | 7 +- .../frappe-assets/nginx-default.conf.template | 90 ---------- build/frappe-assets/sites/apps.txt | 1 - build/frappe-assets/v11.Dockerfile | 7 +- build/frappe-assets/v12.Dockerfile | 7 +- build/frappe-socketio/Dockerfile | 6 +- build/frappe-socketio/v11.Dockerfile | 6 +- build/frappe-socketio/v12.Dockerfile | 6 +- build/frappe-worker/Dockerfile | 6 +- build/frappe-worker/commands/background.py | 7 - build/frappe-worker/commands/backup.py | 29 ---- .../commands/check_connection.py | 67 -------- build/frappe-worker/commands/doctor.py | 4 - build/frappe-worker/commands/migrate.py | 47 ------ build/frappe-worker/commands/worker.py | 7 - .../common_site_config.json.template | 7 - build/frappe-worker/v11.Dockerfile | 6 +- build/frappe-worker/v12.Dockerfile | 6 +- 41 files changed, 62 insertions(+), 545 deletions(-) rename build/{erpnext-worker => common}/commands/background.py (100%) rename build/{erpnext-worker => common}/commands/backup.py (100%) rename build/{erpnext-worker => common}/commands/check_connection.py (100%) rename build/{frappe-worker => common}/commands/console.py (100%) rename build/{erpnext-worker => common}/commands/doctor.py (100%) rename build/{erpnext-worker => common}/commands/migrate.py (100%) rename build/{frappe-worker => common}/commands/new.py (100%) rename build/{erpnext-worker => common}/commands/worker.py (100%) rename build/{erpnext-worker => common}/common_site_config.json.template (100%) rename build/{erpnext-assets => common}/nginx-default.conf.template (100%) rename build/{frappe-worker => common/worker}/docker-entrypoint.sh (100%) delete mode 100644 build/erpnext-assets/.dockerignore delete mode 100644 build/erpnext-assets/sites/apps.txt delete mode 100644 build/erpnext-worker/commands/new.py delete mode 100755 build/erpnext-worker/docker-entrypoint.sh delete mode 100644 build/frappe-assets/.dockerignore delete mode 100644 build/frappe-assets/nginx-default.conf.template delete mode 100644 build/frappe-assets/sites/apps.txt delete mode 100644 build/frappe-worker/commands/background.py delete mode 100644 build/frappe-worker/commands/backup.py delete mode 100644 build/frappe-worker/commands/check_connection.py delete mode 100644 build/frappe-worker/commands/doctor.py delete mode 100644 build/frappe-worker/commands/migrate.py delete mode 100644 build/frappe-worker/commands/worker.py delete mode 100755 build/frappe-worker/common_site_config.json.template diff --git a/.travis.yml b/.travis.yml index ae4bb89f..2797ab28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,26 +34,26 @@ matrix: - ./dbench setup docker stop - name: "Build Frappe python environment" script: - - docker build -t frappe-worker build/frappe-worker + - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge - docker push frappe/frappe-worker:edge - name: "Build Frappe nginx + static assets" script: - - docker build -t frappe-assets build/frappe-assets + - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge - docker push frappe/frappe-assets:edge - name: "Build ERPNext python environment" script: - - docker build -t erpnext-worker build/erpnext-worker + - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge - docker push frappe/erpnext-worker:edge - name: "Build ERPNext nginx + static assets" script: - - docker build -t erpnext-assets build/erpnext-assets + - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - name: "Build Frappe socketio service" script: - - docker build -t frappe-socketio build/frappe-socketio + - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge - docker push frappe/frappe-socketio:edge diff --git a/build/erpnext-worker/commands/background.py b/build/common/commands/background.py similarity index 100% rename from build/erpnext-worker/commands/background.py rename to build/common/commands/background.py diff --git a/build/erpnext-worker/commands/backup.py b/build/common/commands/backup.py similarity index 100% rename from build/erpnext-worker/commands/backup.py rename to build/common/commands/backup.py diff --git a/build/erpnext-worker/commands/check_connection.py b/build/common/commands/check_connection.py similarity index 100% rename from build/erpnext-worker/commands/check_connection.py rename to build/common/commands/check_connection.py diff --git a/build/frappe-worker/commands/console.py b/build/common/commands/console.py similarity index 100% rename from build/frappe-worker/commands/console.py rename to build/common/commands/console.py diff --git a/build/erpnext-worker/commands/doctor.py b/build/common/commands/doctor.py similarity index 100% rename from build/erpnext-worker/commands/doctor.py rename to build/common/commands/doctor.py diff --git a/build/erpnext-worker/commands/migrate.py b/build/common/commands/migrate.py similarity index 100% rename from build/erpnext-worker/commands/migrate.py rename to build/common/commands/migrate.py diff --git a/build/frappe-worker/commands/new.py b/build/common/commands/new.py similarity index 100% rename from build/frappe-worker/commands/new.py rename to build/common/commands/new.py diff --git a/build/erpnext-worker/commands/worker.py b/build/common/commands/worker.py similarity index 100% rename from build/erpnext-worker/commands/worker.py rename to build/common/commands/worker.py diff --git a/build/erpnext-worker/common_site_config.json.template b/build/common/common_site_config.json.template similarity index 100% rename from build/erpnext-worker/common_site_config.json.template rename to build/common/common_site_config.json.template diff --git a/build/erpnext-assets/nginx-default.conf.template b/build/common/nginx-default.conf.template similarity index 100% rename from build/erpnext-assets/nginx-default.conf.template rename to build/common/nginx-default.conf.template diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh similarity index 100% rename from build/frappe-worker/docker-entrypoint.sh rename to build/common/worker/docker-entrypoint.sh diff --git a/build/erpnext-assets/.dockerignore b/build/erpnext-assets/.dockerignore deleted file mode 100644 index c58728b5..00000000 --- a/build/erpnext-assets/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -/docker/ diff --git a/build/erpnext-assets/Dockerfile b/build/erpnext-assets/Dockerfile index 90bd0a4d..d316db93 100644 --- a/build/erpnext-assets/Dockerfile +++ b/build/erpnext-assets/Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,18 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/erpnext-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-assets/sites/apps.txt b/build/erpnext-assets/sites/apps.txt deleted file mode 100644 index d700de5f..00000000 --- a/build/erpnext-assets/sites/apps.txt +++ /dev/null @@ -1,2 +0,0 @@ -frappe -erpnext diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile index 8d4907d9..36e85627 100644 --- a/build/erpnext-assets/v11.Dockerfile +++ b/build/erpnext-assets/v11.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:10-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,19 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/erpnext-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile index b000a199..1840c860 100644 --- a/build/erpnext-assets/v12.Dockerfile +++ b/build/erpnext-assets/v12.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -18,18 +19,19 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ && mkdir -p /var/www/error_pages \ - && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext + && cp -r /tmp/bench/bench/config/templates/502.html /var/www/error_pages RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ + && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ && cp -r /tmp/bench/bench/config/templates /var/www/error_pages FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/erpnext-assets/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/common/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index 59175254..6d145d8c 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -31,11 +31,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/erpnext-worker/commands/new.py b/build/erpnext-worker/commands/new.py deleted file mode 100644 index b6589c79..00000000 --- a/build/erpnext-worker/commands/new.py +++ /dev/null @@ -1,68 +0,0 @@ -import os, frappe, json - -from frappe.commands.site import _new_site - -site_name = os.environ.get("SITE_NAME", 'site1.localhost') -mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') -mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') -install_erpnext = os.environ.get("INSTALL_ERPNEXT", None) -force = True if os.environ.get("FORCE", None) else False - -frappe.init(site_name, new_site=True) - -_new_site( - None, - site_name, - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password, - admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), - verbose=True, - install_apps=['erpnext'] if install_erpnext else [], - source_sql=None, - force=force, - reinstall=False, -) - -config = None -with open('common_site_config.json') as config_file: - config = json.load(config_file) - -site_config = None -with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: - site_config = json.load(site_config_file) - -# update User's host to '%' required to connect from any container -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) - -# Set db password -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') -) -os.system(command) - -# Grant permission to database -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) - -exit(0) diff --git a/build/erpnext-worker/docker-entrypoint.sh b/build/erpnext-worker/docker-entrypoint.sh deleted file mode 100755 index b6fc0d35..00000000 --- a/build/erpnext-worker/docker-entrypoint.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -function configureEnv() { - if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then - - if [[ -z "$MARIADB_HOST" ]]; then - echo "MARIADB_HOST is not set" - exit 1 - fi - - if [[ -z "$REDIS_CACHE" ]]; then - echo "REDIS_CACHE is not set" - exit 1 - fi - - if [[ -z "$REDIS_QUEUE" ]]; then - echo "REDIS_QUEUE is not set" - exit 1 - fi - - if [[ -z "$REDIS_SOCKETIO" ]]; then - echo "REDIS_SOCKETIO is not set" - exit 1 - fi - - if [[ -z "$SOCKETIO_PORT" ]]; then - echo "SOCKETIO_PORT is not set" - exit 1 - fi - - envsubst "${MARIADB_HOST} - ${REDIS_CACHE} - ${REDIS_QUEUE} - ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}" < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json - fi -} - -function checkConnection() { - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/check_connection.py" -} - -function checkConfigExists() { - COUNTER=0 - while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do - sleep 1 - (( COUNTER=COUNTER+1 )) - echo "config file not created, retry $COUNTER" - done - - if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" - exit 1 - fi -} - -if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then - find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt -fi - -if [ "$1" = 'start' ]; then - configureEnv - checkConnection - - chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json - - if [[ -z "$WORKERS" ]]; then - export WORKERS=2 - fi - - if [[ -z "$FRAPPE_PORT" ]]; then - export FRAPPE_PORT=8000 - fi - - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers $WORKERS \ - --worker-class=gthread \ - --log-file=- \ - -t 120 frappe.app:application --preload" - else - . /home/frappe/frappe-bench/env/bin/activate - gunicorn -b 0.0.0.0:$FRAPPE_PORT \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers $WORKERS \ - --worker-class=gthread \ - --log-file=- \ - -t 120 frappe.app:application --preload - fi - -elif [ "$1" = 'worker' ]; then - checkConfigExists - checkConnection - # default WORKER_TYPE=default - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/worker.py" - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/worker.py - fi - -elif [ "$1" = 'schedule' ]; then - checkConfigExists - checkConnection - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/background.py" - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/background.py - fi - -elif [ "$1" = 'new' ]; then - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/new.py" - exit - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/new.py - fi - -elif [ "$1" = 'migrate' ]; then - - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/migrate.py" - exit - -elif [ "$1" = 'doctor' ]; then - - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/doctor.py" - exit - -elif [ "$1" = 'backup' ]; then - - if [[ -z "$RUN_AS_ROOT" ]]; then - su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ - && python /home/frappe/frappe-bench/commands/backup.py" - exit - else - . /home/frappe/frappe-bench/env/bin/activate - python /home/frappe/frappe-bench/commands/backup.py - fi - -else - - exec su frappe -c "$@" - -fi diff --git a/build/erpnext-worker/v11.Dockerfile b/build/erpnext-worker/v11.Dockerfile index c56a5e96..81a59b63 100644 --- a/build/erpnext-worker/v11.Dockerfile +++ b/build/erpnext-worker/v11.Dockerfile @@ -28,11 +28,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/erpnext-worker/v12.Dockerfile b/build/erpnext-worker/v12.Dockerfile index d892c81d..3945142a 100644 --- a/build/erpnext-worker/v12.Dockerfile +++ b/build/erpnext-worker/v12.Dockerfile @@ -31,11 +31,11 @@ RUN virtualenv env \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/erpnext -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-assets/.dockerignore b/build/frappe-assets/.dockerignore deleted file mode 100644 index c58728b5..00000000 --- a/build/frappe-assets/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -/docker/ diff --git a/build/frappe-assets/Dockerfile b/build/frappe-assets/Dockerfile index ef0e7dcc..762e74d1 100644 --- a/build/frappe-assets/Dockerfile +++ b/build/frappe-assets/Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-assets/nginx-default.conf.template b/build/frappe-assets/nginx-default.conf.template deleted file mode 100644 index c2e0e0ba..00000000 --- a/build/frappe-assets/nginx-default.conf.template +++ /dev/null @@ -1,90 +0,0 @@ -upstream frappe-server { - server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; -} - -upstream socketio-server { - server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; -} - -server { - listen 80; - server_name $http_host; - root /var/www/html; - - location /assets { - try_files $uri =404; - } - - location ~ ^/protected/(.*) { - internal; - try_files /sites/$http_host/$1 =404; - } - - location /socket.io { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Origin $scheme://$http_host; - proxy_set_header Host $host; - - proxy_pass http://socketio-server; - } - - location / { - try_files /sites/$http_host/public/$uri @webserver; - } - - location @webserver { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Host $host; - proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout 120; - proxy_redirect off; - - proxy_pass http://frappe-server; - } - - # error pages - error_page 502 /502.html; - location /502.html { - root /var/www/templates; - internal; - } - - # optimizations - sendfile on; - keepalive_timeout 15; - client_max_body_size 50m; - client_body_buffer_size 16K; - client_header_buffer_size 1k; - - # enable gzip compresion - # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge - gzip on; - gzip_http_version 1.1; - gzip_comp_level 5; - gzip_min_length 256; - gzip_proxied any; - gzip_vary on; - gzip_types - application/atom+xml - application/javascript - application/json - application/rss+xml - application/vnd.ms-fontobject - application/x-font-ttf - application/font-woff - application/x-web-app-manifest+json - application/xhtml+xml - application/xml - font/opentype - image/svg+xml - image/x-icon - text/css - text/plain - text/x-component; - # text/html is always compressed by HttpGzipModule -} diff --git a/build/frappe-assets/sites/apps.txt b/build/frappe-assets/sites/apps.txt deleted file mode 100644 index f47ffaf5..00000000 --- a/build/frappe-assets/sites/apps.txt +++ /dev/null @@ -1 +0,0 @@ -frappe diff --git a/build/frappe-assets/v11.Dockerfile b/build/frappe-assets/v11.Dockerfile index ef98aded..225243d2 100644 --- a/build/frappe-assets/v11.Dockerfile +++ b/build/frappe-assets/v11.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:10-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-assets/v12.Dockerfile b/build/frappe-assets/v12.Dockerfile index 2e16296b..cbbac49b 100644 --- a/build/frappe-assets/v12.Dockerfile +++ b/build/frappe-assets/v12.Dockerfile @@ -1,7 +1,8 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench -COPY sites/apps.txt /home/frappe/frappe-bench/sites/apps.txt +RUN mkdir -p /home/frappe/frappe-bench/sites \ + && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt RUN install_packages git @@ -25,8 +26,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY docker-entrypoint.sh / +COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY build/frappe-assets/docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 4987af4e..a74b0d3e 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-socketio/v11.Dockerfile b/build/frappe-socketio/v11.Dockerfile index 78986e0c..fcaac8ac 100644 --- a/build/frappe-socketio/v11.Dockerfile +++ b/build/frappe-socketio/v11.Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-socketio/v12.Dockerfile b/build/frappe-socketio/v12.Dockerfile index 022000d7..adf536bd 100644 --- a/build/frappe-socketio/v12.Dockerfile +++ b/build/frappe-socketio/v12.Dockerfile @@ -8,7 +8,7 @@ RUN useradd -ms /bin/bash frappe # Make bench directories RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frappe -COPY package.json /home/frappe/frappe-bench/apps/frappe +COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe # get socketio @@ -21,11 +21,11 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm install --only=production -COPY health.js /home/frappe/frappe-bench/apps/frappe/health.js +COPY build/frappe-socketio/health.js /home/frappe/frappe-bench/apps/frappe/health.js RUN chown -R frappe:frappe /home/frappe # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 39013537..41310b0d 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -29,11 +29,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/commands/background.py b/build/frappe-worker/commands/background.py deleted file mode 100644 index 7065efd9..00000000 --- a/build/frappe-worker/commands/background.py +++ /dev/null @@ -1,7 +0,0 @@ -import frappe -from frappe.utils.scheduler import start_scheduler - -print("Starting background scheduler . . .") -start_scheduler() - -exit(0) diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py deleted file mode 100644 index e6fbcbb6..00000000 --- a/build/frappe-worker/commands/backup.py +++ /dev/null @@ -1,29 +0,0 @@ -import os, frappe, compileall, re -from frappe.utils.backups import scheduled_backup -from frappe.utils import now -from frappe.utils import get_sites - -def backup(sites, with_files=False): - for site in sites: - frappe.init(site) - frappe.connect() - odb = scheduled_backup( - ignore_files=not with_files, - backup_path_db=None, - backup_path_files=None, - backup_path_private_files=None, - force=True - ) - print("database backup taken -", odb.backup_path_db, "- on", now()) - if with_files: - print("files backup taken -", odb.backup_path_files, "- on", now()) - print("private files backup taken -", odb.backup_path_private_files, "- on", now()) - frappe.destroy() - -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -with_files=True if os.environ.get("WITH_FILES") else False - -backup(sites, with_files) - -exit(0) diff --git a/build/frappe-worker/commands/check_connection.py b/build/frappe-worker/commands/check_connection.py deleted file mode 100644 index abe17199..00000000 --- a/build/frappe-worker/commands/check_connection.py +++ /dev/null @@ -1,67 +0,0 @@ -import socket, os, json, time -from six.moves.urllib.parse import urlparse - -def is_open(ip, port, timeout=30): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout) - try: - s.connect((ip, int(port))) - s.shutdown(socket.SHUT_RDWR) - return True - except: - return False - finally: - s.close() - -def check_host(ip, port, retry=10, delay=3): - ipup = False - for i in range(retry): - print("Attempt {i} to connect to {ip}:{port}".format(ip=ip,port=port,i=i+1)) - if is_open(ip, port): - ipup = True - break - else: - time.sleep(delay) - return ipup - -# Check connection to servers -config = None -try: - with open('common_site_config.json') as config_file: - config = json.load(config_file) -except FileNotFoundError: - raise FileNotFoundError("common_site_config.json missing") -except: - raise ValueError("common_site_config.json is not valid") - -# Check mariadb -check_mariadb = False -check_mariadb = check_host(config.get('db_host', 'mariadb'), 3306) -if not check_mariadb: - raise ConnectionError("Connection to mariadb timed out") - -# Check redis queue -check_redis_queue = False -redis_queue_url = urlparse(config.get("redis_queue","redis://redis:6379")).netloc -redis_queue, redis_queue_port = redis_queue_url.split(":") -check_redis_queue = check_host(redis_queue, redis_queue_port) -if not check_redis_queue: - raise ConnectionError("Connection to redis queue timed out") - -# Check redis cache -check_redis_cache = False -redis_cache_url = urlparse(config.get("redis_cache","redis://redis:6379")).netloc -redis_cache, redis_cache_port = redis_cache_url.split(":") -check_redis_cache = check_host(redis_cache, redis_cache_port) -if not check_redis_cache: - raise ConnectionError("Connection to redis cache timed out") - -# Check redis socketio -check_redis_socketio = False -redis_socketio_url = urlparse(config.get("redis_socketio","redis://redis:6379")).netloc -redis_socketio, redis_socketio_port = redis_socketio_url.split(":") -check_redis_socketio = check_host(redis_socketio, redis_socketio_port) -if not check_redis_socketio: - raise ConnectionError("Connection to redis socketio timed out") - -print('Connections OK') diff --git a/build/frappe-worker/commands/doctor.py b/build/frappe-worker/commands/doctor.py deleted file mode 100644 index 54f508fb..00000000 --- a/build/frappe-worker/commands/doctor.py +++ /dev/null @@ -1,4 +0,0 @@ -import frappe -from frappe.utils.doctor import doctor - -doctor() diff --git a/build/frappe-worker/commands/migrate.py b/build/frappe-worker/commands/migrate.py deleted file mode 100644 index be382bea..00000000 --- a/build/frappe-worker/commands/migrate.py +++ /dev/null @@ -1,47 +0,0 @@ -import os, frappe, compileall, re, json - -from frappe.migrate import migrate -from frappe.utils import get_sites - -def get_config(): - config = None - with open('common_site_config.json') as config_file: - config = json.load(config_file) - return config - -def save_config(config): - with open('common_site_config.json', 'w') as f: - return json.dump(config, f, indent=1, sort_keys=True) - -def set_maintenance_mode(enable=True): - conf = get_config() - - if enable: - conf.update({ "maintenance_mode": 1, "pause_scheduler": 1 }) - save_config(conf) - - if not enable: - conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) - save_config(conf) - - -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False - -if maintenance_mode: - set_maintenance_mode(True) - -for site in sites: - print('Migrating', site) - frappe.init(site=site) - frappe.connect() - try: - migrate() - finally: - frappe.destroy() - -if maintenance_mode: - set_maintenance_mode(False) - -exit(0) diff --git a/build/frappe-worker/commands/worker.py b/build/frappe-worker/commands/worker.py deleted file mode 100644 index 6ec0bcbf..00000000 --- a/build/frappe-worker/commands/worker.py +++ /dev/null @@ -1,7 +0,0 @@ -import os, frappe -from frappe.utils.background_jobs import start_worker - -queue = os.environ.get("WORKER_TYPE", "default") -start_worker(queue, False) - -exit(0) diff --git a/build/frappe-worker/common_site_config.json.template b/build/frappe-worker/common_site_config.json.template deleted file mode 100755 index 27593289..00000000 --- a/build/frappe-worker/common_site_config.json.template +++ /dev/null @@ -1,7 +0,0 @@ -{ - "db_host": "${MARIADB_HOST}", - "redis_cache": "redis://${REDIS_CACHE}", - "redis_queue": "redis://${REDIS_QUEUE}", - "redis_socketio": "redis://${REDIS_SOCKETIO}", - "socketio_port": ${SOCKETIO_PORT} -} diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile index 934a316b..fa2f01c5 100644 --- a/build/frappe-worker/v11.Dockerfile +++ b/build/frappe-worker/v11.Dockerfile @@ -26,11 +26,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-11 \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile index 86e63041..1eb6d60a 100644 --- a/build/frappe-worker/v12.Dockerfile +++ b/build/frappe-worker/v12.Dockerfile @@ -29,11 +29,11 @@ RUN virtualenv env \ && git clone --depth 1 -o upstream https://github.com/frappe/frappe --branch version-12 \ && pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/frappe -COPY ./commands/* /home/frappe/frappe-bench/commands/ -COPY ./common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY build/common/commands/* /home/frappe/frappe-bench/commands/ +COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template # Setup docker-entrypoint -COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites From 9b6a641f4d273490eca2abfaa28b0c5d764a9d3d Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 21:48:51 +0530 Subject: [PATCH 08/29] fix: config generation fix envsubst single quote SHELL-FORMAT env var changed from ERPNEXT_PY to FRAPPE_PY --- build/common/nginx-default.conf.template | 6 +++--- build/common/worker/docker-entrypoint.sh | 4 ++-- build/erpnext-assets/docker-entrypoint.sh | 15 ++++++++++----- build/frappe-assets/docker-entrypoint.sh | 8 ++++++-- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/build/common/nginx-default.conf.template b/build/common/nginx-default.conf.template index 17578b6a..c2e0e0ba 100644 --- a/build/common/nginx-default.conf.template +++ b/build/common/nginx-default.conf.template @@ -1,5 +1,5 @@ -upstream erpnext-server { - server ${ERPNEXT_PY}:${ERPNEXT_PY_PORT} fail_timeout=0; +upstream frappe-server { + server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; } upstream socketio-server { @@ -44,7 +44,7 @@ server { proxy_read_timeout 120; proxy_redirect off; - proxy_pass http://erpnext-server; + proxy_pass http://frappe-server; } # error pages diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index da1934b6..a191b159 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -28,11 +28,11 @@ function configureEnv() { exit 1 fi - envsubst "${MARIADB_HOST} + envsubst '${MARIADB_HOST} ${REDIS_CACHE} ${REDIS_QUEUE} ${REDIS_SOCKETIO} - ${SOCKETIO_PORT}" < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json fi } diff --git a/build/erpnext-assets/docker-entrypoint.sh b/build/erpnext-assets/docker-entrypoint.sh index 90bdf4b2..17b712e3 100755 --- a/build/erpnext-assets/docker-entrypoint.sh +++ b/build/erpnext-assets/docker-entrypoint.sh @@ -13,12 +13,12 @@ rsync -a --delete /var/www/html/assets/erpnext /assets chmod -R 755 /assets -if [[ -z "$ERPNEXT_PY" ]]; then - export ERPNEXT_PY=0.0.0.0 +if [[ -z "$FRAPPE_PY" ]]; then + export FRAPPE_PY=0.0.0.0 fi -if [[ -z "$ERPNEXT_PY_PORT" ]]; then - export ERPNEXT_PY_PORT=8000 +if [[ -z "$FRAPPE_PY_PORT" ]]; then + export FRAPPE_PY_PORT=8000 fi if [[ -z "$FRAPPE_SOCKETIO" ]]; then @@ -29,7 +29,12 @@ if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then export FRAPPE_SOCKETIO_PORT=9000 fi -envsubst "${API_HOST} ${API_PORT} ${ERPNEXT_PY} ${ERPNEXT_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}" \ +envsubst '${API_HOST} + ${API_PORT} + ${FRAPPE_PY} + ${FRAPPE_PY_PORT} + ${FRAPPE_SOCKETIO} + ${FRAPPE_SOCKETIO_PORT}' \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" diff --git a/build/frappe-assets/docker-entrypoint.sh b/build/frappe-assets/docker-entrypoint.sh index 4b791d10..f1caec04 100755 --- a/build/frappe-assets/docker-entrypoint.sh +++ b/build/frappe-assets/docker-entrypoint.sh @@ -9,7 +9,6 @@ set -e rsync -a --delete /var/www/html/assets/js /assets rsync -a --delete /var/www/html/assets/css /assets rsync -a --delete /var/www/html/assets/frappe /assets -rsync -a --delete /var/www/html/assets/erpnext /assets chmod -R 755 /assets @@ -29,7 +28,12 @@ if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then export FRAPPE_SOCKETIO_PORT=9000 fi -envsubst "${API_HOST} ${API_PORT} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} ${FRAPPE_SOCKETIO_PORT}" \ +envsubst '${API_HOST} + ${API_PORT} + ${FRAPPE_PY} + ${FRAPPE_PY_PORT} + ${FRAPPE_SOCKETIO} + ${FRAPPE_SOCKETIO_PORT}' \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" From da53bff5df6756f5863f643e27b8f04dd547a4da Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 22:35:49 +0530 Subject: [PATCH 09/29] feat: docker-compose services common services redis and mariadb frappe services gunicorn, schedule, workers, socketio erpnext services gunicorm, schedule, workers, socketio --- .gitignore | 6 ++ installation/docker-compose-common.yml | 35 +++++++ installation/docker-compose-erpnext.yml | 111 +++++++++++++++++++++++ installation/docker-compose-frappe.yml | 111 +++++++++++++++++++++++ installation/docker-compose-networks.yml | 6 ++ installation/env-example | 4 + installation/frappe-mariadb.cnf | 7 ++ 7 files changed, 280 insertions(+) create mode 100644 installation/docker-compose-common.yml create mode 100644 installation/docker-compose-erpnext.yml create mode 100644 installation/docker-compose-frappe.yml create mode 100644 installation/docker-compose-networks.yml create mode 100644 installation/env-example create mode 100644 installation/frappe-mariadb.cnf diff --git a/.gitignore b/.gitignore index 4d67604d..502c3673 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ *.code-workspace + +# Environment Variables +.env + +# mounted volume +sites diff --git a/installation/docker-compose-common.yml b/installation/docker-compose-common.yml new file mode 100644 index 00000000..ede3abe1 --- /dev/null +++ b/installation/docker-compose-common.yml @@ -0,0 +1,35 @@ +version: '3' + +services: + redis-cache: + image: redis:latest + restart: on-failure + volumes: + - redis-cache-vol:/data + + redis-queue: + image: redis:latest + restart: on-failure + volumes: + - redis-queue-vol:/data + + redis-socketio: + image: redis:latest + restart: on-failure + volumes: + - redis-socketio-vol:/data + + mariadb: + image: mariadb:10.3 + restart: on-failure + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + volumes: + - ./frappe-mariadb.cnf:/etc/mysql/conf.d/frappe.cnf + - mariadb-vol:/var/lib/mysql + +volumes: + mariadb-vol: + redis-cache-vol: + redis-queue-vol: + redis-socketio-vol: diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml new file mode 100644 index 00000000..a62b0f33 --- /dev/null +++ b/installation/docker-compose-erpnext.yml @@ -0,0 +1,111 @@ +version: '3' + +services: + erpnext-assets: + image: frappe/erpnext-assets:${VERSION} + restart: on-failure + environment: + - FRAPPE_PY=erpnext-python + - FRAPPE_PY_PORT=8000 + - FRAPPE_SOCKETIO=frappe-socketio + - FRAPPE_SOCKETIO_PORT=9000 + - LETSENCRYPT_HOST=${SITES} + - VIRTUAL_HOST=${SITES} + - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} + depends_on: + - erpnext-python + - frappe-socketio + - frappe-worker-default + - frappe-worker-long + - frappe-worker-short + links: + - erpnext-python + - frappe-socketio + - frappe-worker-default + - frappe-worker-long + - frappe-worker-short + volumes: + - ./sites:/var/www/html/sites:rw + - assets-vol:/assets:rw + + erpnext-python: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + environment: + - MARIADB_HOST=mariadb + - REDIS_CACHE=redis-cache:6379 + - REDIS_QUEUE=redis-queue:6379 + - REDIS_SOCKETIO=redis-socketio:6379 + - SOCKETIO_PORT=9000 + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + - assets-vol:/home/frappe/frappe-bench/sites/assets:rw + + frappe-socketio: + image: frappe/frappe-socketio:${VERSION} + restart: on-failure + depends_on: + - redis-socketio + links: + - redis-socketio + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-default: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-short: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=short + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-long: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=long + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-schedule: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: schedule + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + +volumes: + assets-vol: diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml new file mode 100644 index 00000000..a41f3770 --- /dev/null +++ b/installation/docker-compose-frappe.yml @@ -0,0 +1,111 @@ +version: '3' + +services: + frappe-assets: + image: frappe/frappe-assets:${VERSION} + restart: on-failure + environment: + - FRAPPE_PY=frappe-python + - FRAPPE_PY_PORT=8000 + - FRAPPE_SOCKETIO=frappe-socketio + - FRAPPE_SOCKETIO_PORT=9000 + - LETSENCRYPT_HOST=${SITES} + - VIRTUAL_HOST=${SITES} + - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} + depends_on: + - frappe-python + - frappe-socketio + - frappe-worker-default + - frappe-worker-long + - frappe-worker-short + links: + - frappe-python + - frappe-socketio + - frappe-worker-default + - frappe-worker-long + - frappe-worker-short + volumes: + - ./sites:/var/www/html/sites:rw + - assets-vol:/assets:rw + + frappe-python: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + environment: + - MARIADB_HOST=mariadb + - REDIS_CACHE=redis-cache:6379 + - REDIS_QUEUE=redis-queue:6379 + - REDIS_SOCKETIO=redis-socketio:6379 + - SOCKETIO_PORT=9000 + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + - assets-vol:/home/frappe/frappe-bench/sites/assets:rw + + frappe-socketio: + image: frappe/frappe-socketio:${VERSION} + restart: on-failure + depends_on: + - redis-socketio + links: + - redis-socketio + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-default: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-short: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=short + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-worker-long: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: worker + environment: + - WORKER_TYPE=long + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + + frappe-schedule: + image: frappe/frappe-worker:${VERSION} + restart: on-failure + command: schedule + depends_on: + - redis-queue + - redis-cache + links: + - redis-queue + - redis-cache + volumes: + - ./sites:/home/frappe/frappe-bench/sites:rw + +volumes: + assets-vol: diff --git a/installation/docker-compose-networks.yml b/installation/docker-compose-networks.yml new file mode 100644 index 00000000..377f01d2 --- /dev/null +++ b/installation/docker-compose-networks.yml @@ -0,0 +1,6 @@ +version: '3' + +networks: + default: + external: + name: webproxy diff --git a/installation/env-example b/installation/env-example new file mode 100644 index 00000000..66d9dedf --- /dev/null +++ b/installation/env-example @@ -0,0 +1,4 @@ +VERSION=edge +MYSQL_ROOT_PASSWORD=admin +SITES=your.domain.com +LETSENCRYPT_EMAIL=your.email@your.domain.com diff --git a/installation/frappe-mariadb.cnf b/installation/frappe-mariadb.cnf new file mode 100644 index 00000000..83b2c703 --- /dev/null +++ b/installation/frappe-mariadb.cnf @@ -0,0 +1,7 @@ +[mysqld] +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 From ab4cb7936c2691f7c0acda1dc0d8f48b4f93079b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 23:07:18 +0530 Subject: [PATCH 10/29] ci: build images for edge, v12 and v11 --- .travis.yml | 113 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2797ab28..938235c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ services: before_install: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - sudo apt-get update && sudo apt-get -y install git - if [ $BUILD == "development" ];then - sudo apt-get update; sudo apt-get -y install docker-compose; fi @@ -32,28 +32,129 @@ matrix: script: - ./test.sh - ./dbench setup docker stop - - name: "Build Frappe python environment" + - name: "Build Frappe python environment (edge)" script: - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge - docker push frappe/frappe-worker:edge - - name: "Build Frappe nginx + static assets" + - name: "Build Frappe nginx + static assets (edge)" script: - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge - docker push frappe/frappe-assets:edge - - name: "Build ERPNext python environment" + - name: "Build ERPNext python environment (edge)" script: - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge - docker push frappe/erpnext-worker:edge - - name: "Build ERPNext nginx + static assets" + - name: "Build ERPNext nginx + static assets (edge)" script: - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - - name: "Build Frappe socketio service" + - name: "Build Frappe socketio service (edge)" script: - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge - docker push frappe/frappe-socketio:edge + - name: "Build Frappe python environment (v12)" + script: + - git clone https://github.com/frappe/frappe --branch version-12 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker tag frappe-worker:$VERSION frappe/frappe-worker:v12 + - docker tag frappe-worker:$VERSION frappe/frappe-worker:$VERSION + - docker push frappe/frappe-worker:$VERSION + - name: "Build Frappe nginx + static assets (v12)" + script: + - git clone https://github.com/frappe/frappe --branch version-12 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker tag frappe-assets:$VERSION frappe/frappe-assets:v12 + - docker push frappe/frappe-assets:$VERSION + - name: "Build ERPNext python environment (v12)" + script: + - git clone https://github.com/frappe/erpnext --branch version-12 + - cd erpnext + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker tag erpnext-worker:$VERSION frappe/erpnext-worker:v12 + - docker push frappe/erpnext-worker:$VERSION + - name: "Build ERPNext nginx + static assets (v12)" + script: + - git clone https://github.com/frappe/erpnext --branch version-12 + - cd erpnext + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker tag erpnext-assets:$VERSION frappe/erpnext-assets:v12 + - docker push frappe/erpnext-assets:$VERSION + - name: "Build Frappe socketio service (v12)" + script: + - git clone https://github.com/frappe/frappe --branch version-12 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker tag frappe-socketio:$VERSION frappe/frappe-socketio:v12 + - docker push frappe/frappe-socketio:$VERSION + - name: "Build Frappe python environment (v11)" + script: + - git clone https://github.com/frappe/frappe --branch version-11 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker tag frappe-worker:$VERSION frappe/frappe-worker:v11 + - docker push frappe/frappe-worker:$VERSION + - name: "Build Frappe nginx + static assets (v11)" + script: + - git clone https://github.com/frappe/frappe --branch version-11 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker tag frappe-assets:$VERSION frappe/frappe-assets:v11 + - docker push frappe/frappe-assets:$VERSION + - name: "Build ERPNext python environment (v11)" + script: + - git clone https://github.com/frappe/erpnext --branch version-11 + - cd erpnext + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker tag erpnext-worker:$VERSION frappe/erpnext-worker:v11 + - docker push frappe/erpnext-worker:$VERSION + - name: "Build ERPNext nginx + static assets (v11)" + script: + - git clone https://github.com/frappe/erpnext --branch version-11 + - cd erpnext + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker tag erpnext-assets:$VERSION frappe/erpnext-assets:v11 + - docker push frappe/erpnext-assets:$VERSION + - name: "Build Frappe socketio service (v11)" + script: + - git clone https://github.com/frappe/frappe --branch version-11 + - cd frappe + - git fetch --tags + - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') + - cd .. + - docker build -t frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker tag frappe-socketio:$VERSION frappe/frappe-socketio:v11 + - docker push frappe/frappe-socketio:$VERSION From ee43d7d6ef89da447c1ddc47ba078bcc02581994 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 23:28:18 +0530 Subject: [PATCH 11/29] ci: fix docker image name --- .travis.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 938235c1..6cebe644 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,8 +65,8 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . - - docker tag frappe-worker:$VERSION frappe/frappe-worker:v12 - - docker tag frappe-worker:$VERSION frappe/frappe-worker:$VERSION + - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v12 + - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:$VERSION - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v12)" script: @@ -75,8 +75,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . - - docker tag frappe-assets:$VERSION frappe/frappe-assets:v12 + - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v12 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v12)" script: @@ -85,8 +85,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . - - docker tag erpnext-worker:$VERSION frappe/erpnext-worker:v12 + - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v12 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v12)" script: @@ -95,8 +95,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . - - docker tag erpnext-assets:$VERSION frappe/erpnext-assets:v12 + - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v12 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v12)" script: @@ -105,8 +105,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . - - docker tag frappe-socketio:$VERSION frappe/frappe-socketio:v12 + - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v12 - docker push frappe/frappe-socketio:$VERSION - name: "Build Frappe python environment (v11)" script: @@ -115,8 +115,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . - - docker tag frappe-worker:$VERSION frappe/frappe-worker:v11 + - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v11 - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v11)" script: @@ -125,8 +125,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . - - docker tag frappe-assets:$VERSION frappe/frappe-assets:v11 + - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v11 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v11)" script: @@ -135,8 +135,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . - - docker tag erpnext-worker:$VERSION frappe/erpnext-worker:v11 + - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v11 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v11)" script: @@ -145,8 +145,8 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . - - docker tag erpnext-assets:$VERSION frappe/erpnext-assets:v11 + - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v11 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v11)" script: @@ -155,6 +155,6 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . - - docker tag frappe-socketio:$VERSION frappe/frappe-socketio:v11 + - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v11 - docker push frappe/frappe-socketio:$VERSION From c5d31ef16497286d65995bae26f058c2e5e3761c Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 20 Feb 2020 23:44:25 +0530 Subject: [PATCH 12/29] ci: fix frappe-worker docker image name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6cebe644..6e172ee7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v12 - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:$VERSION - docker push frappe/frappe-worker:$VERSION From 5d1dd3c0c47fe43c9b3f83c5687b8ffee6d31128 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 Feb 2020 13:09:10 +0530 Subject: [PATCH 13/29] ci: branch based builds --- .travis.yml | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e172ee7..60cdbb03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,17 +9,17 @@ before_install: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - sudo apt-get update && sudo apt-get -y install git - if [ $BUILD == "development" ];then - sudo apt-get -y install docker-compose; + sudo apt-get -y install docker-compose; fi install: - if [ $BUILD == "development" ];then - chmod ugo+x ./dbench; - chmod ugo+x ./test.sh; - ./dbench setup docker; - ./dbench init frappe-bench; - ./dbench new-site site1.local; - ./dbench setup hosts; + chmod ugo+x ./dbench; + chmod ugo+x ./test.sh; + ./dbench setup docker; + ./dbench init frappe-bench; + ./dbench new-site site1.local; + ./dbench setup hosts; fi after_success: @@ -33,128 +33,143 @@ matrix: - ./test.sh - ./dbench setup docker stop - name: "Build Frappe python environment (edge)" + if: branch = develop script: - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge - docker push frappe/frappe-worker:edge - name: "Build Frappe nginx + static assets (edge)" + if: branch = develop script: - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge - docker push frappe/frappe-assets:edge - name: "Build ERPNext python environment (edge)" + if: branch = develop script: - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge - docker push frappe/erpnext-worker:edge - name: "Build ERPNext nginx + static assets (edge)" + if: branch = develop script: - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - name: "Build Frappe socketio service (edge)" + if: branch = develop script: - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge - docker push frappe/frappe-socketio:edge - name: "Build Frappe python environment (v12)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/v12.Dockerfile . - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v12 - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:$VERSION - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v12)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/v12.Dockerfile . - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v12 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v12)" + if: branch = master script: - git clone https://github.com/frappe/erpnext --branch version-12 - cd erpnext - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/v12.Dockerfile . - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v12 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v12)" + if: branch = master script: - git clone https://github.com/frappe/erpnext --branch version-12 - cd erpnext - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/v12.Dockerfile . - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v12 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v12)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/v12.Dockerfile . - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v12 - docker push frappe/frappe-socketio:$VERSION - name: "Build Frappe python environment (v11)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/Dockerfile . + - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/v11.Dockerfile . - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v11 - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v11)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/Dockerfile . + - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/v11.Dockerfile . - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v11 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v11)" + if: branch = master script: - git clone https://github.com/frappe/erpnext --branch version-11 - cd erpnext - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/Dockerfile . + - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/v11.Dockerfile . - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v11 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v11)" + if: branch = master script: - git clone https://github.com/frappe/erpnext --branch version-11 - cd erpnext - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/Dockerfile . + - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/v11.Dockerfile . - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v11 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v11)" + if: branch = master script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe - git fetch --tags - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/Dockerfile . + - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/v11.Dockerfile . - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v11 - docker push frappe/frappe-socketio:$VERSION From 1cc7c7050f2f5e0f9212275922579966086d7126 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 Feb 2020 16:55:13 +0530 Subject: [PATCH 14/29] ci: build images not to run on PR --- .travis.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60cdbb03..09c14fdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,37 +33,37 @@ matrix: - ./test.sh - ./dbench setup docker stop - name: "Build Frappe python environment (edge)" - if: branch = develop + if: branch = develop AND type != pull_request script: - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge - docker push frappe/frappe-worker:edge - name: "Build Frappe nginx + static assets (edge)" - if: branch = develop + if: branch = develop AND type != pull_request script: - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge - docker push frappe/frappe-assets:edge - name: "Build ERPNext python environment (edge)" - if: branch = develop + if: branch = develop AND type != pull_request script: - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge - docker push frappe/erpnext-worker:edge - name: "Build ERPNext nginx + static assets (edge)" - if: branch = develop + if: branch = develop AND type != pull_request script: - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge - docker push frappe/erpnext-assets:edge - name: "Build Frappe socketio service (edge)" - if: branch = develop + if: branch = develop AND type != pull_request script: - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge - docker push frappe/frappe-socketio:edge - name: "Build Frappe python environment (v12)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe @@ -75,7 +75,7 @@ matrix: - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:$VERSION - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v12)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe @@ -86,7 +86,7 @@ matrix: - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v12 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v12)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/erpnext --branch version-12 - cd erpnext @@ -97,7 +97,7 @@ matrix: - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v12 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v12)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/erpnext --branch version-12 - cd erpnext @@ -108,7 +108,7 @@ matrix: - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v12 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v12)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-12 - cd frappe @@ -119,7 +119,7 @@ matrix: - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v12 - docker push frappe/frappe-socketio:$VERSION - name: "Build Frappe python environment (v11)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe @@ -130,7 +130,7 @@ matrix: - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v11 - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v11)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe @@ -141,7 +141,7 @@ matrix: - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v11 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v11)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/erpnext --branch version-11 - cd erpnext @@ -152,7 +152,7 @@ matrix: - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v11 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v11)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/erpnext --branch version-11 - cd erpnext @@ -163,7 +163,7 @@ matrix: - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v11 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v11)" - if: branch = master + if: branch = master AND type != pull_request script: - git clone https://github.com/frappe/frappe --branch version-11 - cd frappe From 5bd812ccddc8ed2d99c1dd6d9b7cfc0cd6d3e10e Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 Feb 2020 17:35:26 +0530 Subject: [PATCH 15/29] ci: docker login only before building images --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09c14fdc..fe893af2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,12 @@ services: - docker before_install: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - sudo apt-get update && sudo apt-get -y install git + - if [ $BUILD != "development"]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + sudo apt-get update && sudo apt-get -y install git + fi - if [ $BUILD == "development" ];then - sudo apt-get -y install docker-compose; + sudo apt-get update && sudo apt-get -y install docker-compose; fi install: From e16f41b94720a64b265bdb4dbab2a6b6c4e7614a Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 Feb 2020 17:44:40 +0530 Subject: [PATCH 16/29] ci: fix travis bash conditions --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe893af2..989bfdf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,16 +6,16 @@ services: - docker before_install: - - if [ $BUILD != "development"]; then + - if [[ "$BUILD" -ne "development" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin sudo apt-get update && sudo apt-get -y install git fi - - if [ $BUILD == "development" ];then + - if [[ $BUILD == "development" ]];then sudo apt-get update && sudo apt-get -y install docker-compose; fi install: - - if [ $BUILD == "development" ];then + - if [[ $BUILD == "development" ]];then chmod ugo+x ./dbench; chmod ugo+x ./test.sh; ./dbench setup docker; From 1a6aa403ee7fc0ba107b13d7da7f651c30c59058 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 21 Feb 2020 17:48:20 +0530 Subject: [PATCH 17/29] ci: fix travis bash script --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 989bfdf0..4539fafc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ services: before_install: - if [[ "$BUILD" -ne "development" ]]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - sudo apt-get update && sudo apt-get -y install git + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; + sudo apt-get update && sudo apt-get -y install git; fi - if [[ $BUILD == "development" ]];then sudo apt-get update && sudo apt-get -y install docker-compose; From f48e1ce4a6a51059bedb7ff024dc5c3ac3e21e8a Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 24 Feb 2020 12:57:20 +0530 Subject: [PATCH 18/29] fix: docker-compose-erpnext --- installation/docker-compose-erpnext.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index a62b0f33..62e3d545 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -29,7 +29,7 @@ services: - assets-vol:/assets:rw erpnext-python: - image: frappe/frappe-worker:${VERSION} + image: frappe/erpnext-worker:${VERSION} restart: on-failure environment: - MARIADB_HOST=mariadb @@ -52,7 +52,7 @@ services: - ./sites:/home/frappe/frappe-bench/sites:rw frappe-worker-default: - image: frappe/frappe-worker:${VERSION} + image: frappe/erpnext-worker:${VERSION} restart: on-failure command: worker depends_on: @@ -65,7 +65,7 @@ services: - ./sites:/home/frappe/frappe-bench/sites:rw frappe-worker-short: - image: frappe/frappe-worker:${VERSION} + image: frappe/erpnext-worker:${VERSION} restart: on-failure command: worker environment: @@ -80,7 +80,7 @@ services: - ./sites:/home/frappe/frappe-bench/sites:rw frappe-worker-long: - image: frappe/frappe-worker:${VERSION} + image: frappe/erpnext-worker:${VERSION} restart: on-failure command: worker environment: @@ -95,7 +95,7 @@ services: - ./sites:/home/frappe/frappe-bench/sites:rw frappe-schedule: - image: frappe/frappe-worker:${VERSION} + image: frappe/erpnext-worker:${VERSION} restart: on-failure command: schedule depends_on: From e32d3b4e51f497fd18b0b18ab80397046d48161a Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 1 Mar 2020 15:38:05 +0530 Subject: [PATCH 19/29] fix: ERPNext assets image build --- build/erpnext-assets/Dockerfile | 2 +- build/erpnext-assets/v11.Dockerfile | 7 +++---- build/erpnext-assets/v12.Dockerfile | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/build/erpnext-assets/Dockerfile b/build/erpnext-assets/Dockerfile index d316db93..7fcb2b20 100644 --- a/build/erpnext-assets/Dockerfile +++ b/build/erpnext-assets/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git +RUN install_packages git python2 RUN mkdir -p apps sites/assets \ && cd apps \ diff --git a/build/erpnext-assets/v11.Dockerfile b/build/erpnext-assets/v11.Dockerfile index 36e85627..6db3b415 100644 --- a/build/erpnext-assets/v11.Dockerfile +++ b/build/erpnext-assets/v11.Dockerfile @@ -2,9 +2,9 @@ FROM bitnami/node:10-prod WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ - && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt + && echo "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git +RUN install_packages git python2 RUN mkdir -p apps sites/assets \ && cd apps \ @@ -23,9 +23,8 @@ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ - && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ diff --git a/build/erpnext-assets/v12.Dockerfile b/build/erpnext-assets/v12.Dockerfile index 1840c860..85dac5ac 100644 --- a/build/erpnext-assets/v12.Dockerfile +++ b/build/erpnext-assets/v12.Dockerfile @@ -2,9 +2,9 @@ FROM bitnami/node:12-prod WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ - && echo -e "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt + && echo "frappe\nerpnext" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git +RUN install_packages git python2 RUN mkdir -p apps sites/assets \ && cd apps \ @@ -23,9 +23,8 @@ RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/frappe-bench/sites/assets/frappe \ && cp -R /home/frappe/frappe-bench/apps/frappe/node_modules /home/frappe/frappe-bench/sites/assets/frappe/ \ - && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext \ && mkdir -p /home/frappe/frappe-bench/sites/assets/erpnext \ - && cp -r /tmp/bench/bench/config/templates /var/www/error_pages + && cp -R /home/frappe/frappe-bench/apps/erpnext/erpnext/public/* /home/frappe/frappe-bench/sites/assets/erpnext FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ From 500b8925cb69d9f423aaff8b952507a641f6ce7e Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 1 Mar 2020 15:39:26 +0530 Subject: [PATCH 20/29] chore: README for installation --- installation/README.md | 153 ++++++++++++++++++++++++ installation/docker-compose-erpnext.yml | 2 +- installation/docker-compose-frappe.yml | 2 +- installation/env-example | 1 + 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 installation/README.md diff --git a/installation/README.md b/installation/README.md new file mode 100644 index 00000000..ae38630d --- /dev/null +++ b/installation/README.md @@ -0,0 +1,153 @@ +# Production deployment using docker + +### Setup Letsencrypt Nginx Proxy Companion + +Read more: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion + +```sh +cd $HOME +git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git +cd docker-compose-letsencrypt-nginx-proxy-companion +cp .env.sample .env +./start.sh +``` + +### Clone frappe_docker repository + +```sh +cd $HOME +git clone https://github.com/frappe/frappe_docker.git +cd frappe_docker/installation +cp env-example .env + +# make directory for sites +mkdir sites +``` + +### Setup Environment Variables + +3 Environment variables are set to pass secret and variable data. + +If `env-example` is copied to `.env` following values are set. + +- `VERSION=edge` set version tag or latest for major version e.g. v12.3.0, v12 +- `MYSQL_ROOT_PASSWORD=admin`, set mariadb root password (bootstraps a mariadb container with this root password). If managed database mariadb is used NO need to set the password here +- `MARIADB_HOST=mariadb` set hostname to `mariadb` in case of docker container for mariadb is used. In case managed db is used set the hostname/IP/domain name here +- `SITES=site1.domain.com,site2.domain.com` these are list of sites that are part of the deployment "bench". Each site is separated by (,) comma +- `LETSENCRYPT_EMAIL=your.email@your.domain.com` email for letsencrypt expiry notification + +### Start frappe-bench services + +DNS needs to be configured for following to work + +```sh +docker-compose \ + --project-name frappe-bench-00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-conpose-erpnext.yml \ + --project-directory . up -d +``` + +Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed + +### Create new sites + +Note: Wait for mariadb to start. If new site creation fails re-try again after mariadb container is up and running + +```sh +# Create ERPNext site +docker exec \ + -e "SITE_NAME=site1.domain.com" \ + -e "DB_ROOT_USER=root" \ + -e "DB_ROOT_PASSWORD=admin" \ + -e "ADMIN_PASSWORD=admin" \ + -e "INSTALL_ERPNEXT=1" \ + frappe-bench-00_erpnext-python_1 new +``` + +Environment Variables: + +- `SITE_NAME`, name of the new site to create +- `DB_ROOT_USER`, MariaDB Root user. The user that can create databases +- `DB_ROOT_PASSWORD`, In case of mariadb docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of managed database use appropriate password +- `ADMIN_PASSWORD` set the administrator password for new site +- `INSTALL_ERPNEXT=1` available only in erpnext-worker and erpnext containers. Installs ERPNext on this new site +- `FORCE=1` is optional variable which force installs the same site. + +### Backup sites + +Environment Variables + +- `SITES` is list of sites separated by (:) colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up +- `WITH_FILES` if set to 1, it will back up user uploaded files for the sites + +```sh +docker exec \ + -e "SITES=site1.domain.com:site2.domain.com" \ + -e "WITH_FILES=1" \ + frappe-bench-00_erpnext-python_1 backup +``` + +Backup will be available in `sites` mounted volume + +### Update and migrate site + +```sh +# Change to repo root +cd $HOME/frappe_docker + +# Pull new images +docker-compose \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + pull + +# Restart containers +docker-compose \ + --project-name frappe-bench-00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + --project-directory . restart + +docker exec \ + -e "MAINTENANCE_MODE=1" \ + frappe-bench-00_erpnext-python_1 migrate +``` + +### Troubleshoot + +1. Clearing redis cache: + +``` +# change to repo root +cd $HOME/frappe_docker + +# Stop all bench containers +docker-compose \ + --project-name frappe-bench-00 \ + -f docker/docker-compose-common.yml \ + -f docker/docker-compose-erpnext.yml \ + --project-directory . stop + +# Remove redis containers +docker-compose \ + --project-name frappe-bench-00 \ + -f docker/docker-compose-common.yml \ + -f docker/docker-compose-erpnext.yml \ + --project-directory . rm redis-cache redis-queue redis-socketio + +# Clean redis volumes +docker volume rm \ + frappe-bench-00_redis-cache-vol \ + frappe-bench-00_redis-queue-vol \ + frappe-bench-00_redis-socketio-vol + +# Restart project +docker-compose \ + --project-name frappe-bench-00 \ + -f docker/docker-compose-common.yml \ + -f docker/docker-compose-erpnext.yml \ + --project-directory . up -d +``` + +Note: Environment variables from `.env` file located at current working directory will be used. diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index 62e3d545..4d4713fb 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -32,7 +32,7 @@ services: image: frappe/erpnext-worker:${VERSION} restart: on-failure environment: - - MARIADB_HOST=mariadb + - MARIADB_HOST=${MARIADB_HOST} - REDIS_CACHE=redis-cache:6379 - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml index a41f3770..f0af5ba3 100644 --- a/installation/docker-compose-frappe.yml +++ b/installation/docker-compose-frappe.yml @@ -32,7 +32,7 @@ services: image: frappe/frappe-worker:${VERSION} restart: on-failure environment: - - MARIADB_HOST=mariadb + - MARIADB_HOST=${MARIADB_HOST} - REDIS_CACHE=redis-cache:6379 - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 diff --git a/installation/env-example b/installation/env-example index 66d9dedf..0d12cca6 100644 --- a/installation/env-example +++ b/installation/env-example @@ -1,4 +1,5 @@ VERSION=edge +MARIADB_HOST=mariadb MYSQL_ROOT_PASSWORD=admin SITES=your.domain.com LETSENCRYPT_EMAIL=your.email@your.domain.com From 33b5071aaab51bbf06a5fd68d5f109d9af9d9cd7 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 1 Mar 2020 16:45:34 +0530 Subject: [PATCH 21/29] fix: Frappe/ERPNext worker image new site command --- build/common/commands/new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 10d6a8e4..ebf029b9 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -6,7 +6,7 @@ site_name = os.environ.get("SITE_NAME", 'site1.localhost') mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') force = True if os.environ.get("FORCE", None) else False - +install_apps = ['erpnext'] if os.environ.get("INSTALL_ERPNEXT", None) else False frappe.init(site_name, new_site=True) _new_site( @@ -16,7 +16,7 @@ _new_site( mariadb_root_password=mariadb_root_password, admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), verbose=True, - install_apps=[], + install_apps=install_apps, source_sql=None, force=force, reinstall=False, From 8d9d583a8f1616bb64a8581063e0b3d1e162cbcc Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 1 Mar 2020 16:45:54 +0530 Subject: [PATCH 22/29] fix: production installation README --- installation/README.md | 65 +++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/installation/README.md b/installation/README.md index ae38630d..2cde41fa 100644 --- a/installation/README.md +++ b/installation/README.md @@ -17,11 +17,11 @@ cp .env.sample .env ```sh cd $HOME git clone https://github.com/frappe/frappe_docker.git -cd frappe_docker/installation -cp env-example .env +cd frappe_docker +cp installation/env-example .env # make directory for sites -mkdir sites +mkdir installation/sites ``` ### Setup Environment Variables @@ -42,10 +42,11 @@ DNS needs to be configured for following to work ```sh docker-compose \ - --project-name frappe-bench-00 \ + --project-name frappebench00 \ -f installation/docker-compose-common.yml \ - -f installation/docker-conpose-erpnext.yml \ - --project-directory . up -d + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d ``` Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed @@ -56,13 +57,13 @@ Note: Wait for mariadb to start. If new site creation fails re-try again after m ```sh # Create ERPNext site -docker exec \ +docker exec -it \ -e "SITE_NAME=site1.domain.com" \ -e "DB_ROOT_USER=root" \ -e "DB_ROOT_PASSWORD=admin" \ -e "ADMIN_PASSWORD=admin" \ -e "INSTALL_ERPNEXT=1" \ - frappe-bench-00_erpnext-python_1 new + frappebench00_erpnext-python_1 docker-entrypoint.sh new ``` Environment Variables: @@ -82,10 +83,10 @@ Environment Variables - `WITH_FILES` if set to 1, it will back up user uploaded files for the sites ```sh -docker exec \ +docker exec -it \ -e "SITES=site1.domain.com:site2.domain.com" \ -e "WITH_FILES=1" \ - frappe-bench-00_erpnext-python_1 backup + frappebench00_erpnext-python_1 docker-entrypoint.sh backup ``` Backup will be available in `sites` mounted volume @@ -104,19 +105,20 @@ docker-compose \ # Restart containers docker-compose \ - --project-name frappe-bench-00 \ + --project-name frappebench00 \ -f installation/docker-compose-common.yml \ -f installation/docker-compose-erpnext.yml \ - --project-directory . restart + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d -docker exec \ +docker exec -it \ -e "MAINTENANCE_MODE=1" \ - frappe-bench-00_erpnext-python_1 migrate + frappebench00_erpnext-python_1 docker-entrypoint.sh migrate ``` ### Troubleshoot -1. Clearing redis cache: +Clearing redis cache: ``` # change to repo root @@ -124,30 +126,33 @@ cd $HOME/frappe_docker # Stop all bench containers docker-compose \ - --project-name frappe-bench-00 \ - -f docker/docker-compose-common.yml \ - -f docker/docker-compose-erpnext.yml \ - --project-directory . stop + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation stop # Remove redis containers docker-compose \ - --project-name frappe-bench-00 \ - -f docker/docker-compose-common.yml \ - -f docker/docker-compose-erpnext.yml \ - --project-directory . rm redis-cache redis-queue redis-socketio + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation rm redis-cache redis-queue redis-socketio # Clean redis volumes docker volume rm \ - frappe-bench-00_redis-cache-vol \ - frappe-bench-00_redis-queue-vol \ - frappe-bench-00_redis-socketio-vol + frappebench00_redis-cache-vol \ + frappebench00_redis-queue-vol \ + frappebench00_redis-socketio-vol # Restart project docker-compose \ - --project-name frappe-bench-00 \ - -f docker/docker-compose-common.yml \ - -f docker/docker-compose-erpnext.yml \ - --project-directory . up -d + --project-name frappebench00 \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d ``` Note: Environment variables from `.env` file located at current working directory will be used. From 2f59ebbae02a5c33d14bab09e9f6459e7c44611d Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 1 Mar 2020 17:17:01 +0530 Subject: [PATCH 23/29] fix: installation README for migration --- installation/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/installation/README.md b/installation/README.md index 2cde41fa..fdcb3d97 100644 --- a/installation/README.md +++ b/installation/README.md @@ -2,6 +2,8 @@ ### Setup Letsencrypt Nginx Proxy Companion +DNS needs to be configured for following to work + Read more: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion ```sh @@ -26,7 +28,7 @@ mkdir installation/sites ### Setup Environment Variables -3 Environment variables are set to pass secret and variable data. +Environment variables are set to pass secret and variable data. If `env-example` is copied to `.env` following values are set. @@ -38,8 +40,6 @@ If `env-example` is copied to `.env` following values are set. ### Start frappe-bench services -DNS needs to be configured for following to work - ```sh docker-compose \ --project-name frappebench00 \ @@ -97,6 +97,9 @@ Backup will be available in `sites` mounted volume # Change to repo root cd $HOME/frappe_docker +# Update environment variable VERSION +nano .env + # Pull new images docker-compose \ -f installation/docker-compose-common.yml \ From 4364330fe9ff28efd47c89ebd8e6d1705a0a9eba Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 2 Mar 2020 05:07:40 +0530 Subject: [PATCH 24/29] fix: improve installation README --- installation/README.md | 76 ++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/installation/README.md b/installation/README.md index fdcb3d97..831bd00a 100644 --- a/installation/README.md +++ b/installation/README.md @@ -2,7 +2,7 @@ ### Setup Letsencrypt Nginx Proxy Companion -DNS needs to be configured for following to work +DNS needs to be configured for following to work. Read more: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion @@ -32,11 +32,11 @@ Environment variables are set to pass secret and variable data. If `env-example` is copied to `.env` following values are set. -- `VERSION=edge` set version tag or latest for major version e.g. v12.3.0, v12 -- `MYSQL_ROOT_PASSWORD=admin`, set mariadb root password (bootstraps a mariadb container with this root password). If managed database mariadb is used NO need to set the password here -- `MARIADB_HOST=mariadb` set hostname to `mariadb` in case of docker container for mariadb is used. In case managed db is used set the hostname/IP/domain name here -- `SITES=site1.domain.com,site2.domain.com` these are list of sites that are part of the deployment "bench". Each site is separated by (,) comma -- `LETSENCRYPT_EMAIL=your.email@your.domain.com` email for letsencrypt expiry notification +- `VERSION=edge` set version tag or latest for major version e.g. v12.3.0, v12. +- `MYSQL_ROOT_PASSWORD=admin`, set mariadb root password (bootstraps a mariadb container with this root password). If managed database mariadb is used NO need to set the password here. +- `MARIADB_HOST=mariadb` set hostname to `mariadb` in case of docker container for mariadb is used. In case managed db is used set the hostname/IP/domain name here. +- `SITES=site1.domain.com,site2.domain.com` these are list of sites that are part of the deployment "bench". Each site is separated by (,) comma. +- `LETSENCRYPT_EMAIL=your.email@your.domain.com` email for letsencrypt expiry notification. ### Start frappe-bench services @@ -49,38 +49,41 @@ docker-compose \ --project-directory installation up -d ``` -Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed +Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed. ### Create new sites -Note: Wait for mariadb to start. If new site creation fails re-try again after mariadb container is up and running +Note: + +- Wait for mariadb to start. If new site creation fails re-try again after mariadb container is up and running. +- Use `.env` file or environment variables instead of passing secrets as command arguments. ```sh # Create ERPNext site docker exec -it \ - -e "SITE_NAME=site1.domain.com" \ - -e "DB_ROOT_USER=root" \ - -e "DB_ROOT_PASSWORD=admin" \ - -e "ADMIN_PASSWORD=admin" \ + -e "SITE_NAME=$SITE_NAME" \ + -e "DB_ROOT_USER=$DB_ROOT_USER" \ + -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ + -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ -e "INSTALL_ERPNEXT=1" \ frappebench00_erpnext-python_1 docker-entrypoint.sh new ``` -Environment Variables: +Environment Variables needed: -- `SITE_NAME`, name of the new site to create -- `DB_ROOT_USER`, MariaDB Root user. The user that can create databases -- `DB_ROOT_PASSWORD`, In case of mariadb docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of managed database use appropriate password -- `ADMIN_PASSWORD` set the administrator password for new site -- `INSTALL_ERPNEXT=1` available only in erpnext-worker and erpnext containers. Installs ERPNext on this new site +- `SITE_NAME`, name of the new site to create. +- `DB_ROOT_USER`, MariaDB Root user. The user that can create databases. +- `MYSQL_ROOT_PASSWORD`, In case of mariadb docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of managed database use appropriate password. +- `ADMIN_PASSWORD` set the administrator password for new site. +- `INSTALL_ERPNEXT=1` available only in erpnext-worker and erpnext containers. Installs ERPNext on this new site. - `FORCE=1` is optional variable which force installs the same site. ### Backup sites Environment Variables -- `SITES` is list of sites separated by (:) colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up -- `WITH_FILES` if set to 1, it will back up user uploaded files for the sites +- `SITES` is list of sites separated by (:) colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. +- `WITH_FILES` if set to 1, it will back up user uploaded files for the sites. ```sh docker exec -it \ @@ -89,7 +92,7 @@ docker exec -it \ frappebench00_erpnext-python_1 docker-entrypoint.sh backup ``` -Backup will be available in `sites` mounted volume +Backup will be available in `sites` mounted volume. ### Update and migrate site @@ -121,7 +124,9 @@ docker exec -it \ ### Troubleshoot -Clearing redis cache: +1. Remove containers and volumes clear redis cache: + +This can be used when images are upgraded and if migration fails. ``` # change to repo root @@ -158,4 +163,31 @@ docker-compose \ --project-directory installation up -d ``` +2. Clear redis cache by exec command: + +In case of following error during container restarts, + +``` +frappe-worker-short_1 | Traceback (most recent call last): +frappe-worker-short_1 | File "/home/frappe/frappe-bench/commands/worker.py", line 5, in +frappe-worker-short_1 | start_worker(queue, False) +frappe-worker-short_1 | File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker +frappe-worker-short_1 | Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) +frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work +frappe-worker-short_1 | self.register_birth() +frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth +frappe-worker-short_1 | raise ValueError(msg.format(self.name)) +frappe-worker-short_1 | ValueError: There exists an active worker named '8dfe5c234085.10.short' already +``` + +Use commands : + +```sh +# Clear the cache which is causing problem. + +docker exec -it frappebench00_redis-cache_1 redis-cli FLUSHALL +docker exec -it frappebench00_redis-queue_1 redis-cli FLUSHALL +docker exec -it frappebench00_redis-socketio_1 redis-cli FLUSHALL +``` + Note: Environment variables from `.env` file located at current working directory will be used. From c2c09400da0494582d7cb46ac9f51a87a97c2cc9 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 2 Mar 2020 05:08:49 +0530 Subject: [PATCH 25/29] fix: use commonly known variable for MYSQL_ROOT_PASSWORD --- build/common/commands/new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index ebf029b9..b87641db 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -4,7 +4,7 @@ from frappe.commands.site import _new_site site_name = os.environ.get("SITE_NAME", 'site1.localhost') mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') -mariadb_root_password = os.environ.get("DB_ROOT_PASSWORD", 'admin') +mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') force = True if os.environ.get("FORCE", None) else False install_apps = ['erpnext'] if os.environ.get("INSTALL_ERPNEXT", None) else False frappe.init(site_name, new_site=True) From 0df0d6d11fa226e9c00c61352f890ded84491c2c Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 2 Mar 2020 05:09:57 +0530 Subject: [PATCH 26/29] fix: use SOCKETIO_PORT instead of FRAPPE_SOCKETIO_PORT --- build/common/nginx-default.conf.template | 2 +- build/erpnext-assets/docker-entrypoint.sh | 6 +++--- build/frappe-assets/docker-entrypoint.sh | 6 +++--- installation/docker-compose-erpnext.yml | 2 +- installation/docker-compose-frappe.yml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build/common/nginx-default.conf.template b/build/common/nginx-default.conf.template index c2e0e0ba..b9fc161a 100644 --- a/build/common/nginx-default.conf.template +++ b/build/common/nginx-default.conf.template @@ -3,7 +3,7 @@ upstream frappe-server { } upstream socketio-server { - server ${FRAPPE_SOCKETIO}:${FRAPPE_SOCKETIO_PORT} fail_timeout=0; + server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0; } server { diff --git a/build/erpnext-assets/docker-entrypoint.sh b/build/erpnext-assets/docker-entrypoint.sh index 17b712e3..4919fb34 100755 --- a/build/erpnext-assets/docker-entrypoint.sh +++ b/build/erpnext-assets/docker-entrypoint.sh @@ -25,8 +25,8 @@ if [[ -z "$FRAPPE_SOCKETIO" ]]; then export FRAPPE_SOCKETIO=0.0.0.0 fi -if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then - export FRAPPE_SOCKETIO_PORT=9000 +if [[ -z "$SOCKETIO_PORT" ]]; then + export SOCKETIO_PORT=9000 fi envsubst '${API_HOST} @@ -34,7 +34,7 @@ envsubst '${API_HOST} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} - ${FRAPPE_SOCKETIO_PORT}' \ + ${SOCKETIO_PORT}' \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" diff --git a/build/frappe-assets/docker-entrypoint.sh b/build/frappe-assets/docker-entrypoint.sh index f1caec04..f68e7fd3 100755 --- a/build/frappe-assets/docker-entrypoint.sh +++ b/build/frappe-assets/docker-entrypoint.sh @@ -24,8 +24,8 @@ if [[ -z "$FRAPPE_SOCKETIO" ]]; then export FRAPPE_SOCKETIO=0.0.0.0 fi -if [[ -z "$FRAPPE_SOCKETIO_PORT" ]]; then - export FRAPPE_SOCKETIO_PORT=9000 +if [[ -z "$SOCKETIO_PORT" ]]; then + export SOCKETIO_PORT=9000 fi envsubst '${API_HOST} @@ -33,7 +33,7 @@ envsubst '${API_HOST} ${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} - ${FRAPPE_SOCKETIO_PORT}' \ + ${SOCKETIO_PORT}' \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index 4d4713fb..93b7c6a1 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -8,7 +8,7 @@ services: - FRAPPE_PY=erpnext-python - FRAPPE_PY_PORT=8000 - FRAPPE_SOCKETIO=frappe-socketio - - FRAPPE_SOCKETIO_PORT=9000 + - SOCKETIO_PORT=9000 - LETSENCRYPT_HOST=${SITES} - VIRTUAL_HOST=${SITES} - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml index f0af5ba3..869fb73c 100644 --- a/installation/docker-compose-frappe.yml +++ b/installation/docker-compose-frappe.yml @@ -8,7 +8,7 @@ services: - FRAPPE_PY=frappe-python - FRAPPE_PY_PORT=8000 - FRAPPE_SOCKETIO=frappe-socketio - - FRAPPE_SOCKETIO_PORT=9000 + - SOCKETIO_PORT=9000 - LETSENCRYPT_HOST=${SITES} - VIRTUAL_HOST=${SITES} - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} From b2f1bdcc273332b27500790135428313a60d031d Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Mon, 2 Mar 2020 10:59:25 +0530 Subject: [PATCH 27/29] chore: clean up and update readme Signed-off-by: Chinmay D. Pai --- README.md | 247 +++++++++++++++++++++++++++++++++-------- installation/README.md | 193 -------------------------------- 2 files changed, 203 insertions(+), 237 deletions(-) delete mode 100644 installation/README.md diff --git a/README.md b/README.md index b51b0885..611cde45 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,232 @@ -# Frappe on Docker +### Getting Started -[![Build Status](https://travis-ci.com/frappe/frappe_docker.svg)](https://travis-ci.com/frappe/frappe_docker) +The templates in this repository will help deploy Frappe/ERPNext docker in a production environment. -This is a repo designed to aide setting up frappe/ERPNext on docker. +This docker installation takes care of the following: -## Getting Started +* Setting up the desired version of Frappe/ERPNext. +* Setting up all the system requirements: eg. MariaDB, Node, Redis. +* [OPTIONAL] Configuring networking for remote access and setting up LetsEncrypt -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. +### Installation Process -Unfortunately, this container is not currently suited for a production environment (but we're working towards that goal!). +#### Setting up Pre-requisites -### Build the container and initialize the bench +This repository requires Docker and Git to be setup on the instance to be used. -**Note:** These instructions assume you have both [Docker](https://docs.docker.com/engine/installation) and [Docker Compose](https://docs.docker.com/compose/install/) installed on your system. +#### Setup Letsencrypt Nginx Proxy Companion -1. Clone this repo and change your working directory to it: +This is an optional first step. This step is only required if you want to have SSL setup on your docker instance. - ```bash - git clone https://github.com/frappe/frappe_docker.git - cd frappe_docker/ - ``` +This step also assumes that the DNS is preconfigured since it automatically handles setup and renewal of SSL certificates. -2. Build and start the container, and initialize the bench: +For more details, see: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion - ```bash - ./dbench setup docker - ./dbench init - ``` +To setup the proxy companion, run the following steps: - **Note:** This will take a while, as docker will now build the container. +```sh +cd $HOME +git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git +cd docker-compose-letsencrypt-nginx-proxy-companion +cp .env.sample .env +./start.sh +``` -3. Add a new site and start Frappe: +#### Setting up Frappe/ERPNext Docker - ```bash - ./dbench new-site site1.local - ./dbench setup hosts - ./dbench start - ``` +Clone this repository somewhere in your system: -4. Use Frappe: - Open your browser to `localhost:8000/login`. Then log in using the username `Administrator` and the password `admin`. +```sh +git clone https://github.com/frappe/frappe_docker.git +cd frappe_docker +``` -### Basic Usage of `./dbench` +Copy the example docker environment file to `.env`: -**IMPORTANT: Always make sure that your current directory is the root directory of the repo (i.e. `frappe_docker/`)** +``` +cp installation/env-example installation/.env +``` -- `./dbench`: Launches you into an interactive shell in the container as the user `frappe`. +Make a directory for sites: -- `./dbench setup docker [ stop | down ]`: Starts and builds the docker containers using `docker-compose up -d`. - - `stop`: Stops the containers with `docker-compose stop`. - - `down`: Deletes the containers and the corresponding volumes with `docker-compose down`. +```sh +mkdir installation/sites +``` -- `./dbench setup hosts`: Adds all sites to the containers hosts file. - **Note:** Run this after you've added a new site to avoid errors. +#### Setup Environment Variables -- `./dbench -c frappe | root `: Runs a command in the container, as the selected user. +Docker allows passing an environment file to aide in setting up containers, which is used by this repository to pass secret and variable data. -- `./dbench -h`: Shows this help message. +To get started, copy the existing `env-example` file to `.env` inside the `installation` directory. By default, the file will contain the following variables: -- `./dbench `: Runs a command in bench (i.e. Running `./dbench new-site site1.local`, will run `bench new-site site1.local` in the container). +- `VERSION=edge` + - In this case, `edge` corresponds to `develop`. To setup any other version, you may use the branch name or version specific tags. (eg. version-12, v11.1.15, v11) +- `MYSQL_ROOT_PASSWORD=admin` + - Bootstraps a MariaDB container with this value set as the root password. If a managed MariaDB instance is to be used, there is no need to set the password here. +- `MARIADB_HOST=mariadb` + - Sets the hostname to `mariadb`. This is required if the database is managed with the containerized MariaDB instance. + - In case of a separately managed database setup, set the value to the database's hostname/IP/domain. +- `SITES=site1.domain.com,site2.domain.com` + - List of sites that are part of the deployment "bench". Each site is separated by a comma(,). + - If LetsEncrypt is being setup, make sure that the DNS for all the site domains are pointing to the current instance. +- `LETSENCRYPT_EMAIL=your.email@your.domain.com` + - Email for LetsEncrypt expiry notification. This is only required if you are setting up the nginx proxy companion. -## For More Info +#### Start Frappe/ERPNext Services -For more info on building this docker container refer to this [Wiki](https://github.com/frappe/frappe_docker/wiki/Hitchhiker's-guide-to-building-this-frappe_docker-image) +To start the Frappe/ERPNext services, run the following command: -## Contributing +```sh +docker-compose \ + --project-name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d +``` -Feel free to contribute to this project and make it better. +Make sure to replace `` with any desired name you wish to set for the project. -## License +Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed. -This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details +#### Setup New Sites + +Note: + +- Wait for the mariadb service to start before trying to create a new site. + - If new site creation fails, retry after the mariadb container is up and running. + - If you're using a managed database instance, make sure that the database is running before setting up a new site. +- Use `.env` file or environment variables instead of passing secrets as command arguments. + +```sh +# Create ERPNext site +docker exec -it \ + -e "SITE_NAME=$SITE_NAME" \ + -e "DB_ROOT_USER=$DB_ROOT_USER" \ + -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ + -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ + -e "INSTALL_ERPNEXT=1" \ + _erpnext-python_1 docker-entrypoint.sh new +``` + +Environment Variables needed: + +- `SITE_NAME`: name of the new site to create. +- `DB_ROOT_USER`: MariaDB Root user. The user that can create databases. +- `MYSQL_ROOT_PASSWORD`: In case of mariadb docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of managed database use appropriate password. +- `ADMIN_PASSWORD`: set the administrator password for new site. +- `INSTALL_ERPNEXT=1`: available only in erpnext-worker and erpnext containers. Installs ERPNext on this new site. +- `FORCE=1`: is optional variable which force installs the same site. + +#### Backup Sites + +Environment Variables + +- `SITES` is list of sites separated by (:) colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. +- `WITH_FILES` if set to 1, it will backup user uploaded files for the sites. + +```sh +docker exec -it \ + -e "SITES=site1.domain.com:site2.domain.com" \ + -e "WITH_FILES=1" \ + _erpnext-python_1 docker-entrypoint.sh backup +``` + +Backup will be available in the `sites` mounted volume. + +#### Updating and Migrating Sites + +Switch to the root of the `frappe_docker` directory before running the following commands: + +```sh +# Update environment variable VERSION +nano .env + +# Pull new images +docker-compose \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + pull + +# Restart containers +docker-compose \ + --project-name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d + +docker exec -it \ + -e "MAINTENANCE_MODE=1" \ + _erpnext-python_1 docker-entrypoint.sh migrate +``` + +### Troubleshoot + +1. Remove containers and volumes, and clear redis cache: + +This can be used when existing images are upgraded and migration fails. + +``` +# change to repo root +cd $HOME/frappe_docker + +# Stop all bench containers +docker-compose \ + --project-name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation stop + +# Remove redis containers +docker-compose \ + --project-name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation rm redis-cache redis-queue redis-socketio + +# Clean redis volumes +docker volume rm \ + _redis-cache-vol \ + _redis-queue-vol \ + _redis-socketio-vol + +# Restart project +docker-compose \ + --project-name \ + -f installation/docker-compose-common.yml \ + -f installation/docker-compose-erpnext.yml \ + -f installation/docker-compose-networks.yml \ + --project-directory installation up -d +``` + +2. Clear redis cache using `docker exec` command: + +In case of following error during container restarts: + +``` +frappe-worker-short_1 | Traceback (most recent call last): +frappe-worker-short_1 | File "/home/frappe/frappe-bench/commands/worker.py", line 5, in +frappe-worker-short_1 | start_worker(queue, False) +frappe-worker-short_1 | File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker +frappe-worker-short_1 | Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) +frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work +frappe-worker-short_1 | self.register_birth() +frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth +frappe-worker-short_1 | raise ValueError(msg.format(self.name)) +frappe-worker-short_1 | ValueError: There exists an active worker named '8dfe5c234085.10.short' already +``` + +Use commands : + +```sh +# Clear the cache which is causing problem. + +docker exec -it _redis-cache_1 redis-cli FLUSHALL +docker exec -it _redis-queue_1 redis-cli FLUSHALL +docker exec -it _redis-socketio_1 redis-cli FLUSHALL +``` + +Note: Environment variables from `.env` file located at current working directory will be used. diff --git a/installation/README.md b/installation/README.md deleted file mode 100644 index 831bd00a..00000000 --- a/installation/README.md +++ /dev/null @@ -1,193 +0,0 @@ -# Production deployment using docker - -### Setup Letsencrypt Nginx Proxy Companion - -DNS needs to be configured for following to work. - -Read more: https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion - -```sh -cd $HOME -git clone https://github.com/evertramos/docker-compose-letsencrypt-nginx-proxy-companion.git -cd docker-compose-letsencrypt-nginx-proxy-companion -cp .env.sample .env -./start.sh -``` - -### Clone frappe_docker repository - -```sh -cd $HOME -git clone https://github.com/frappe/frappe_docker.git -cd frappe_docker -cp installation/env-example .env - -# make directory for sites -mkdir installation/sites -``` - -### Setup Environment Variables - -Environment variables are set to pass secret and variable data. - -If `env-example` is copied to `.env` following values are set. - -- `VERSION=edge` set version tag or latest for major version e.g. v12.3.0, v12. -- `MYSQL_ROOT_PASSWORD=admin`, set mariadb root password (bootstraps a mariadb container with this root password). If managed database mariadb is used NO need to set the password here. -- `MARIADB_HOST=mariadb` set hostname to `mariadb` in case of docker container for mariadb is used. In case managed db is used set the hostname/IP/domain name here. -- `SITES=site1.domain.com,site2.domain.com` these are list of sites that are part of the deployment "bench". Each site is separated by (,) comma. -- `LETSENCRYPT_EMAIL=your.email@your.domain.com` email for letsencrypt expiry notification. - -### Start frappe-bench services - -```sh -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation up -d -``` - -Note: use `docker-compose-frappe.yml` in case you need bench with just frappe installed. - -### Create new sites - -Note: - -- Wait for mariadb to start. If new site creation fails re-try again after mariadb container is up and running. -- Use `.env` file or environment variables instead of passing secrets as command arguments. - -```sh -# Create ERPNext site -docker exec -it \ - -e "SITE_NAME=$SITE_NAME" \ - -e "DB_ROOT_USER=$DB_ROOT_USER" \ - -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \ - -e "ADMIN_PASSWORD=$ADMIN_PASSWORD" \ - -e "INSTALL_ERPNEXT=1" \ - frappebench00_erpnext-python_1 docker-entrypoint.sh new -``` - -Environment Variables needed: - -- `SITE_NAME`, name of the new site to create. -- `DB_ROOT_USER`, MariaDB Root user. The user that can create databases. -- `MYSQL_ROOT_PASSWORD`, In case of mariadb docker container use the one set in `MYSQL_ROOT_PASSWORD` in previous steps. In case of managed database use appropriate password. -- `ADMIN_PASSWORD` set the administrator password for new site. -- `INSTALL_ERPNEXT=1` available only in erpnext-worker and erpnext containers. Installs ERPNext on this new site. -- `FORCE=1` is optional variable which force installs the same site. - -### Backup sites - -Environment Variables - -- `SITES` is list of sites separated by (:) colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. -- `WITH_FILES` if set to 1, it will back up user uploaded files for the sites. - -```sh -docker exec -it \ - -e "SITES=site1.domain.com:site2.domain.com" \ - -e "WITH_FILES=1" \ - frappebench00_erpnext-python_1 docker-entrypoint.sh backup -``` - -Backup will be available in `sites` mounted volume. - -### Update and migrate site - -```sh -# Change to repo root -cd $HOME/frappe_docker - -# Update environment variable VERSION -nano .env - -# Pull new images -docker-compose \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - pull - -# Restart containers -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation up -d - -docker exec -it \ - -e "MAINTENANCE_MODE=1" \ - frappebench00_erpnext-python_1 docker-entrypoint.sh migrate -``` - -### Troubleshoot - -1. Remove containers and volumes clear redis cache: - -This can be used when images are upgraded and if migration fails. - -``` -# change to repo root -cd $HOME/frappe_docker - -# Stop all bench containers -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation stop - -# Remove redis containers -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation rm redis-cache redis-queue redis-socketio - -# Clean redis volumes -docker volume rm \ - frappebench00_redis-cache-vol \ - frappebench00_redis-queue-vol \ - frappebench00_redis-socketio-vol - -# Restart project -docker-compose \ - --project-name frappebench00 \ - -f installation/docker-compose-common.yml \ - -f installation/docker-compose-erpnext.yml \ - -f installation/docker-compose-networks.yml \ - --project-directory installation up -d -``` - -2. Clear redis cache by exec command: - -In case of following error during container restarts, - -``` -frappe-worker-short_1 | Traceback (most recent call last): -frappe-worker-short_1 | File "/home/frappe/frappe-bench/commands/worker.py", line 5, in -frappe-worker-short_1 | start_worker(queue, False) -frappe-worker-short_1 | File "/home/frappe/frappe-bench/apps/frappe/frappe/utils/background_jobs.py", line 147, in start_worker -frappe-worker-short_1 | Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) -frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 474, in work -frappe-worker-short_1 | self.register_birth() -frappe-worker-short_1 | File "/home/frappe/frappe-bench/env/lib/python3.7/site-packages/rq/worker.py", line 261, in register_birth -frappe-worker-short_1 | raise ValueError(msg.format(self.name)) -frappe-worker-short_1 | ValueError: There exists an active worker named '8dfe5c234085.10.short' already -``` - -Use commands : - -```sh -# Clear the cache which is causing problem. - -docker exec -it frappebench00_redis-cache_1 redis-cli FLUSHALL -docker exec -it frappebench00_redis-queue_1 redis-cli FLUSHALL -docker exec -it frappebench00_redis-socketio_1 redis-cli FLUSHALL -``` - -Note: Environment variables from `.env` file located at current working directory will be used. From 9adaee274f7fb50c62dd8ea6a56686428adb4368 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 2 Mar 2020 14:58:16 +0530 Subject: [PATCH 28/29] ci: tag git branches as docker images --- .travis.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4539fafc..f3724041 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,30 +39,35 @@ matrix: script: - docker build -t frappe-worker -f build/frappe-worker/Dockerfile . - docker tag frappe-worker frappe/frappe-worker:edge + - docker tag frappe-worker frappe/frappe-worker:develop - docker push frappe/frappe-worker:edge - name: "Build Frappe nginx + static assets (edge)" if: branch = develop AND type != pull_request script: - docker build -t frappe-assets -f build/frappe-assets/Dockerfile . - docker tag frappe-assets frappe/frappe-assets:edge + - docker tag frappe-assets frappe/frappe-assets:develop - docker push frappe/frappe-assets:edge - name: "Build ERPNext python environment (edge)" if: branch = develop AND type != pull_request script: - docker build -t erpnext-worker -f build/erpnext-worker/Dockerfile . - docker tag erpnext-worker frappe/erpnext-worker:edge + - docker tag erpnext-worker frappe/erpnext-worker:develop - docker push frappe/erpnext-worker:edge - name: "Build ERPNext nginx + static assets (edge)" if: branch = develop AND type != pull_request script: - docker build -t erpnext-assets -f build/erpnext-assets/Dockerfile . - docker tag erpnext-assets frappe/erpnext-assets:edge + - docker tag erpnext-assets frappe/erpnext-assets:develop - docker push frappe/erpnext-assets:edge - name: "Build Frappe socketio service (edge)" if: branch = develop AND type != pull_request script: - docker build -t frappe-socketio -f build/frappe-socketio/Dockerfile . - docker tag frappe-socketio frappe/frappe-socketio:edge + - docker tag frappe-socketio frappe/frappe-socketio:develop - docker push frappe/frappe-socketio:edge - name: "Build Frappe python environment (v12)" if: branch = master AND type != pull_request @@ -73,6 +78,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/v12.Dockerfile . + - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:version-12 - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v12 - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:$VERSION - docker push frappe/frappe-worker:$VERSION @@ -85,6 +91,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/v12.Dockerfile . + - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:version-12 - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v12 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v12)" @@ -96,6 +103,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/v12.Dockerfile . + - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:version-12 - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v12 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v12)" @@ -107,6 +115,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/v12.Dockerfile . + - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:version-12 - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v12 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v12)" @@ -118,6 +127,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v12*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/v12.Dockerfile . + - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:version-12 - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v12 - docker push frappe/frappe-socketio:$VERSION - name: "Build Frappe python environment (v11)" @@ -129,6 +139,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-worker:$VERSION -f build/frappe-worker/v11.Dockerfile . + - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:version-11 - docker tag frappe/frappe-worker:$VERSION frappe/frappe-worker:v11 - docker push frappe/frappe-worker:$VERSION - name: "Build Frappe nginx + static assets (v11)" @@ -140,6 +151,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-assets:$VERSION -f build/frappe-assets/v11.Dockerfile . + - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:version-11 - docker tag frappe/frappe-assets:$VERSION frappe/frappe-assets:v11 - docker push frappe/frappe-assets:$VERSION - name: "Build ERPNext python environment (v11)" @@ -151,6 +163,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/erpnext-worker:$VERSION -f build/erpnext-worker/v11.Dockerfile . + - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:version-11 - docker tag frappe/erpnext-worker:$VERSION frappe/erpnext-worker:v11 - docker push frappe/erpnext-worker:$VERSION - name: "Build ERPNext nginx + static assets (v11)" @@ -162,6 +175,7 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/erpnext-assets:$VERSION -f build/erpnext-assets/v11.Dockerfile . + - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:version-11 - docker tag frappe/erpnext-assets:$VERSION frappe/erpnext-assets:v11 - docker push frappe/erpnext-assets:$VERSION - name: "Build Frappe socketio service (v11)" @@ -173,5 +187,6 @@ matrix: - export VERSION=$(git tag --list --sort=-version:refname "v11*" | sed -n 1p | sed -e 's#.*@\(\)#\1#') - cd .. - docker build -t frappe/frappe-socketio:$VERSION -f build/frappe-socketio/v11.Dockerfile . + - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:version-11 - docker tag frappe/frappe-socketio:$VERSION frappe/frappe-socketio:v11 - docker push frappe/frappe-socketio:$VERSION From 07bcaec7239d435839fde7d7833099033439708f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 2 Mar 2020 16:01:35 +0530 Subject: [PATCH 29/29] fix: allow user process to create logs allow user process to create logs in worker images --- build/common/worker/docker-entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index a191b159..bf8a90c2 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -59,6 +59,9 @@ if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt fi +# Allow user process to create files in logs directory +chown -R frappe:frappe /home/frappe/frappe-bench/logs + if [ "$1" = 'start' ]; then configureEnv checkConnection