Simplify Dockerfiles and custom app guide (#714)

* Add assets builder image

* Use assets builder in custom_app tutorial

* Use erpnext in custom app tutorial

* Add info about base images (frappe or erpnext)

* Add assets-builder image to frappe group so it is built in CI

* Update backend image:
- Fix mounted caching
- Uncomplicate ERPNext build
- Fix root-frappe permissions

* Add build-assets script for simpler frontend build

* Add install-app script for backend build

* Rename build-assets to install-app for frontend build

* Update custom app builds according to new main dockerfiles

* Cache pip packages in custom app example backend dockerfile

* Update custom app guide

* Fix typo in backend dockerfile

* Add info about install-app scripts in readme

custom_app/README.md

@@ -18,15 +18,17 @@ To get started, install Docker and [Buildx](https://github.com/docker/buildx#ins
 Before the next step—to build images—replace "custom_app" with your app's name in `docker-bake.hcl`. After that, let's try to build:
 
 ```bash
-FRAPPE_VERSION=<Frappe version you need> docker buildx bake
+FRAPPE_VERSION=... ERPNEXT_VERSION=... docker buildx bake
 ```
 
+> 💡 We assume that majority of our users use ERPNext, that's why images in this tutorial are based on ERPNext images. If don't want ERPNext, change base image in Dockerfiles and remove ERPNEXT_VERSION from bake file.
+
 If something goes wrong feel free to leave an issue.
 
 To test if site works, setup `.env` file (check [example](<(https://github.com/frappe/frappe_docker/blob/main/example.env)>)) and run:
 
 ```bash
-docker-compose up -d
+docker-compose -f compose.yaml -f overrides/compose.noproxy.yaml -f overrides/compose.mariadb.yaml -f overrides/compose.redis.yaml -f custom_app/compose.override.yaml up -d
 docker-compose exec backend \
   bench new-site 127.0.0.1 \
     --mariadb-root-password 123 \
@@ -36,3 +38,7 @@ docker-compose restart backend
 ```
 
 Cool! You just containerized your app!
+
+## Installing multiple apps
+
+Both backend and frontend builds contain `install-app` script that places app where it should be. Each call to script installs given app. Usage: `install-app [APP_NAME] [BRANCH?] [GIT_URL?]`.

custom_app/backend.Dockerfile

@@ -1,8 +1,20 @@
-ARG FRAPPE_VERSION
+# syntax=docker/dockerfile:1.3
-FROM frappe/frappe-worker:${FRAPPE_VERSION}
+
+ARG ERPNEXT_VERSION
+FROM frappe/erpnext-worker:${ERPNEXT_VERSION}
+
+USER root
 
 ARG APP_NAME
-COPY --chown=frappe . ../apps/${APP_NAME}
+COPY . ../apps/${APP_NAME}
 
-RUN echo "frappe\n${APP_NAME}" >/home/frappe/frappe-bench/sites/apps.txt \
+RUN --mount=type=cache,target=/root/.cache/pip \
-    && ../env/bin/pip install --no-cache-dir -e ../apps/${APP_NAME}
+    install-app ${APP_NAME}
+
+# or with git:
+# ARG APP_NAME
+# ARG BRANCH
+# ARG GIT_URL
+# RUN install-assets ${APP_NAME} ${BRANCH} ${GIT_URL}
+
+USER frappe

custom_app/docker-bake.hcl

@@ -1,6 +1,7 @@
 APP_NAME="custom_app"
 
 variable "FRAPPE_VERSION" {}
+variable "ERPNEXT_VERSION" {}
 
 group "default" {
     targets = ["backend", "frontend"]
@@ -10,7 +11,7 @@ target "backend" {
     dockerfile = "backend.Dockerfile"
     tags = ["custom_app/worker:latest"]
     args = {
-      "FRAPPE_VERSION" = FRAPPE_VERSION
+      "ERPNEXT_VERSION" = ERPNEXT_VERSION
       "APP_NAME" = APP_NAME
     }
 }
@@ -20,6 +21,7 @@ target "frontend" {
     tags = ["custom_app/nginx:latest"]
     args = {
       "FRAPPE_VERSION" = FRAPPE_VERSION
+      "ERPNEXT_VERSION" = ERPNEXT_VERSION
       "APP_NAME" = APP_NAME
     }
 }

custom_app/frontend.Dockerfile

@@ -1,52 +1,19 @@
 ARG FRAPPE_VERSION
-FROM node:14-bullseye-slim as prod_node_modules
+ARG ERPNEXT_VERSION
-
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    git \
-    build-essential \
-    python \
-    ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /root/frappe-bench
-RUN mkdir -p sites/assets
-
-ARG FRAPPE_VERSION
-RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe
-
-RUN yarn --cwd apps/frappe
 
+FROM frappe/assets-builder:${FRAPPE_VERSION} as assets
 
 ARG APP_NAME
 COPY . apps/${APP_NAME}
+RUN install-app ${APP_NAME}
 
-# Install production node modules
+# or with git:
-RUN yarn --cwd apps/${APP_NAME} --prod
+# ARG APP_NAME
+# ARG BRANCH
+# ARG GIT_URL
+# RUN install-app ${APP_NAME} ${BRANCH} ${GIT_URL}
 
 
+FROM frappe/erpnext-nginx:${ERPNEXT_VERSION}
 
-FROM prod_node_modules as assets
+COPY --from=assets /out /usr/share/nginx/html
-
-ARG APP_NAME
-
-# Install development node modules
-RUN yarn --cwd apps/${APP_NAME}
-
-# Build assets
-RUN echo "frappe\n${APP_NAME}" >sites/apps.txt \
-    && yarn --cwd apps/frappe production --app ${APP_NAME} \
-    && rm sites/apps.txt
-
-
-
-FROM frappe/frappe-nginx:${FRAPPE_VERSION}
-
-ARG APP_NAME
-
-# Copy all not built assets
-COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public /usr/share/nginx/html/assets/${APP_NAME}
-# Copy production node modules
-COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/node_modules /usr/share/nginx/html/assets/${APP_NAME}/node_modules
-# Copy built assets
-COPY --from=assets /root/frappe-bench/sites /usr/share/nginx/html

docker-bake.hcl

@@ -30,7 +30,7 @@ target "bench-test" {
 # Base for all other targets
 
 group "frappe" {
-    targets = ["frappe-worker", "frappe-nginx", "frappe-socketio"]
+    targets = ["frappe-worker", "frappe-nginx", "frappe-socketio", "assets-builder"]
 }
 
 group "erpnext" {
@@ -81,6 +81,13 @@ target "frappe-nginx" {
     tags =  tag("frappe-nginx", "${FRAPPE_VERSION}")
 }
 
+target "assets-builder" {
+  inherits = ["default-args"]
+  context = "images/nginx"
+  target = "assets_builder"
+  tags = tag("assets-builder", "${FRAPPE_VERSION}")
+}
+
 target "erpnext-nginx" {
     inherits = ["default-args"]
     context = "images/nginx"

images/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-bullseye-slim as base
+FROM node:14-bullseye-slim as assets_builder
 
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
@@ -8,81 +8,55 @@ RUN apt-get update \
     ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root/frappe-bench
+WORKDIR /frappe-bench
-RUN mkdir -p sites/assets
+
+RUN mkdir -p sites/assets /out/assets \
+    && echo frappe >sites/apps.txt
 
 ARG FRAPPE_VERSION
 RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe
 
-
-
-FROM base as frappe_prod_node_modules
-
-# Install production node modules
-RUN yarn --cwd apps/frappe --prod
-
-
-
-FROM frappe_prod_node_modules as frappe_assets
-
 # Install development node modules
-RUN yarn --cwd apps/frappe
+RUN yarn --cwd apps/frappe \
-
-# Build assets they're stored in frappe-bench/sites/assets
-RUN echo "frappe" >sites/apps.txt \
-    && yarn --cwd apps/frappe run production \
-    && rm sites/apps.txt \
     # TODO: Currently `yarn run production` doesn't create .build on develop branch: https://github.com/frappe/frappe/issues/15396
-    && if [ ! -f sites/.build ]; then touch sites/.build; fi
+    && if [ ! -f sites/.build ]; then touch sites/.build; fi \
+    && cp sites/.build /out
+
+COPY install-app.sh /usr/local/bin/install-app
 
 
+FROM assets_builder as frappe_assets
 
-FROM base as erpnext_prod_node_modules
+RUN install-app frappe
+
+
+FROM assets_builder as erpnext_assets
 
 ARG ERPNEXT_VERSION
-RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext
+RUN install-app erpnext ${ERPNEXT_VERSION} https://github.com/frappe/erpnext
-
-RUN yarn --cwd apps/erpnext --prod
 
 
+FROM alpine/git as bench
 
-FROM erpnext_prod_node_modules as erpnext_assets
+# Error pages
-
+RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \
-RUN yarn --cwd apps/erpnext
+    && mkdir /out \
-
+    && mv /tmp/bench/bench/config/templates/502.html /out/
-COPY --from=frappe_assets /root/frappe-bench/apps/frappe/node_modules /root/frappe-bench/apps/frappe/node_modules
-COPY --from=frappe_assets /root/frappe-bench/apps/frappe/package.json /root/frappe-bench/apps/frappe/yarn.lock /root/frappe-bench/apps/frappe/
-
-RUN echo "frappe\nerpnext" >sites/apps.txt \
-    && yarn --cwd apps/frappe run production --app erpnext \
-    && rm sites/apps.txt
-
-
-
-FROM base as error_pages
-
-RUN git clone --depth 1 https://github.com/frappe/bench /root/bench
-
 
 
 FROM nginxinc/nginx-unprivileged:1.21.6-alpine as frappe
 
-COPY --from=error_pages /root/bench/bench/config/templates/502.html /usr/share/nginx/html
-COPY --from=base /root/frappe-bench/apps/frappe/frappe/public /usr/share/nginx/html/assets/frappe
-COPY --from=frappe_prod_node_modules /root/frappe-bench/apps/frappe/node_modules /usr/share/nginx/html/assets/frappe/node_modules
-COPY --from=frappe_assets /root/frappe-bench/sites /usr/share/nginx/html
-
 # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/20-envsubst-on-templates.sh
 COPY nginx-template.conf /etc/nginx/templates/default.conf.template
 # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/docker-entrypoint.sh
 COPY entrypoint.sh /docker-entrypoint.d/frappe-entrypoint.sh
 
-USER 1000
+COPY --from=bench /out /usr/share/nginx/html/
+COPY --from=frappe_assets /out /usr/share/nginx/html
 
+USER 1000
 
 
 FROM frappe as erpnext
 
-COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/erpnext/public /usr/share/nginx/html/assets/erpnext
+COPY --from=erpnext_assets /out /usr/share/nginx/html
-COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/node_modules /usr/share/nginx/html/assets/erpnext/node_modules
-COPY --from=erpnext_assets /root/frappe-bench/sites /usr/share/nginx/html

images/nginx/install-app.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+set -x
+
+APP=$1 BRANCH=$2 GIT_URL=$3
+
+cd /frappe-bench
+
+if test "$BRANCH" && test "$GIT_URL"; then
+  # Clone in case not copied manually
+  git clone --depth 1 -b "$BRANCH" "$GIT_URL" "apps/$APP"
+fi
+
+# Add all not built assets
+cp -r "apps/$APP/$APP/public" "/out/assets/$APP"
+
+# Add production node modules
+yarn --cwd "apps/$APP" --prod
+cp -r "apps/$APP/node_modules" "/out/assets/$APP/node_modules"
+
+# Add built assets
+yarn --cwd "apps/$APP"
+echo "$APP" >>sites/apps.txt
+yarn --cwd apps/frappe run production --app "$APP"
+cp -r sites/assets /out
+
+# Cleanup
+rm -rf "apps/$APP"
+rm -rf sites/assets

images/worker/Dockerfile

@@ -14,12 +14,10 @@ USER frappe
 RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites
 WORKDIR /home/frappe/frappe-bench
 
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
+USER root
-    pip install -U pip wheel \
+RUN pip install -U pip wheel \
     && python -m venv env \
     && env/bin/pip install -U pip wheel
-USER root
-
 
 
 FROM base as build_deps
@@ -37,44 +35,26 @@ RUN apt-get update \
     # Make is required to build wheels of ERPNext deps in develop branch for linux/arm64
     make \
     && rm -rf /var/lib/apt/lists/*
-USER frappe
 
+COPY install-app.sh /usr/local/bin/install-app
 
 
 FROM build_deps as frappe_builder
 
 ARG FRAPPE_VERSION
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache/pip \
-    git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \
+    install-app frappe ${FRAPPE_VERSION} https://github.com/frappe/frappe \
-    && env/bin/pip install -e apps/frappe \
     && env/bin/pip install -U gevent \
-    && rm -r apps/frappe/.git \
     # Link Frappe's node_modules/ to make Website Theme work
     && mkdir -p /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
     && ln -s /home/frappe/frappe-bench/sites/assets/frappe/node_modules /home/frappe/frappe-bench/apps/frappe/node_modules
 
 
-
-# We split ERPNext wheels build in separate stage to achieve concurrency with Frappe build
-FROM build_deps as erpnext_wheels
-
-ARG ERPNEXT_VERSION
-RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \
-    && rm -r apps/erpnext/.git
-
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
-    pip wheel --wheel-dir /home/frappe/erpnext-wheels -r apps/erpnext/requirements.txt
-
-
-
 FROM frappe_builder as erpnext_builder
 
-COPY --from=erpnext_wheels --chown=frappe /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
+ARG ERPNEXT_VERSION
-RUN --mount=type=bind,target=/home/frappe/erpnext-wheels,source=/home/frappe/erpnext-wheels,from=erpnext_wheels \
+RUN --mount=type=cache,target=/root/.cache/pip \
-    --mount=type=cache,target=/home/frappe/.cache/pip \
+    install-app erpnext ${ERPNEXT_VERSION} https://github.com/frappe/erpnext
-    --mount=type=cache,target=/home/frappe/.cache/pip,source=/home/frappe/.cache/pip,from=erpnext_wheels \
-    env/bin/pip install --find-links=/home/frappe/erpnext-wheels -e apps/erpnext
-
 
 
 FROM base as configured_base
@@ -107,8 +87,6 @@ RUN apt-get update \
     nodejs \
     && rm -rf /var/lib/apt/lists/*
 
-USER frappe
-
 COPY pretend-bench.sh /usr/local/bin/bench
 COPY push_backup.py /usr/local/bin/push-backup
 COPY configure.py patched_bench_helper.py /usr/local/bin/
@@ -121,14 +99,19 @@ CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frapp
 
 FROM configured_base as frappe
 
-RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt
 COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
 COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
+COPY --from=frappe_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/
+
+USER frappe
 
 
+# Split frappe and erpnext to reduce image size (because of frappe-bench/env/ directory)
 FROM configured_base as erpnext
 
-RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt
 COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
 COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
 COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
+COPY --from=erpnext_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/
+
+USER frappe

images/worker/install-app.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set -x
+
+APP=$1 BRANCH=$2 GIT_URL=$3
+
+cd /home/frappe/frappe-bench
+
+if test "$BRANCH" && test "$GIT_URL"; then
+  # Clone in case not copied manually
+  git clone --depth 1 -b "$BRANCH" "$GIT_URL" "apps/$APP"
+  rm -r "apps/$APP/.git"
+fi
+
+env/bin/pip install -e "apps/$APP"
+
+echo "$APP" >>sites/apps.txt