From 8be0ad130ad412a43793d23e1368a672822a0e17 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Wed, 1 Mar 2023 17:53:28 +0100 Subject: [PATCH] Add a development setup --- README.md | 20 +++++++ dev.env | 25 +++++++++ docker-compose.dev.yml | 83 +++++++++++++++++++++++++++++ indico-dev/worker/.transifexrc | 5 ++ indico-dev/worker/Dockerfile | 61 +++++++++++++++++++++ indico-dev/worker/bootstrap_user.py | 25 +++++++++ indico-dev/worker/logging.yaml | 16 ++++++ indico-dev/worker/run_celery.sh | 18 +++++++ indico-dev/worker/run_indico.sh | 40 ++++++++++++++ indico-dev/worker/uwsgi.ini | 37 +++++++++++++ indico.dev.conf | 28 ++++++++++ 11 files changed, 358 insertions(+) create mode 100644 dev.env create mode 100644 docker-compose.dev.yml create mode 100644 indico-dev/worker/.transifexrc create mode 100644 indico-dev/worker/Dockerfile create mode 100644 indico-dev/worker/bootstrap_user.py create mode 100644 indico-dev/worker/logging.yaml create mode 100644 indico-dev/worker/run_celery.sh create mode 100644 indico-dev/worker/run_indico.sh create mode 100644 indico-dev/worker/uwsgi.ini create mode 100644 indico.dev.conf diff --git a/README.md b/README.md index e9d05d9..6b06e9b 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,26 @@ $ docker run getindico/indico /opt/indico/run_celery.sh In the above, we omit the network setup to make e.g. the DB accessible from the containers. +## Development setup + +The development setup builds an image from the latest master. Essentially, it puts this [tutorial](https://docs.getindico.io/en/stable/installation/development/) into a Dockerfile. The main purpose of this setup is to allow anyone to relatively quickly and painlessly try out the latest translations from Transifex without having to manually setup an Indico instance. + +### Quickstart + +To start the containers, run: +```sh +$ docker compose --env-file dev.env --file docker-compose.dev.yml up +``` +Indico should now be accessible at [localhost:8080](localhost:8080). + +In its simplest configuration, you just need to provide your Transifex API token in the [dev.env](dev.env) file (`TRANSIFEX_API_TOKEN=1/xxx`). The container will pull and compile the latest translations from Transifex and make them available in Indico. + +The development setup creates a default admin user with username `admin` and password `indiko42`. + +Since emails also get translated, you can use maildump to check those as well. Maildump is available at [localhost:60000](localhost:60000). + +Check the [production setup](#production-like-setup) to see more configuration options. + ## OpenShift ```sh diff --git a/dev.env b/dev.env new file mode 100644 index 0000000..55403a6 --- /dev/null +++ b/dev.env @@ -0,0 +1,25 @@ +## DB config +## =========================================== +PGHOST=indico-postgres +PGUSER=indico +PGPASSWORD=indicopass +PGDATABASE=indico + +## Celery +## =========================================== +C_FORCE_ROOT=true + +## Nginx +## =========================================== +NGINX_PORT=8080 +# Should match the 'BASE_URL' in indico.conf +NGINX_SERVER_NAME=localhost:8080 + +## Indico +## =========================================== +# Path to the indico.conf file +INDICO_CONFIG=indico.dev.conf + +## Transifex +## =========================================== +TRANSIFEX_API_TOKEN=1/xxx diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..a6a89d9 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,83 @@ +version: "3" +services: + # The main Indico container which runs flask + # The same image is also used to run celery + indico-web: &indico-web + build: indico-dev/worker + command: /opt/indico/run_indico.sh + depends_on: + - indico-redis + - indico-celery + environment: + - PGHOST=${PGHOST} + - PGUSER=${PGUSER} + - PGPASSWORD=${PGPASSWORD} + - PGDATABASE=${PGDATABASE} + - C_FORCE_ROOT=${C_FORCE_ROOT} + - TRANSIFEX_API_TOKEN=${TRANSIFEX_API_TOKEN} + networks: + - backend + - frontend + ports: + # Indico is accessible either via nginx (localhost:8080 by default), or + # directly via localhost:9090. In that case, static assets are served by flask + - "9090:59999" + # Maildump is accessible at localhost:60000 + - "60000:60000" + volumes: + - 'archive:/opt/indico/archive' # file storage + - 'customization:/opt/indico/custom' + - 'static-files:/opt/indico/static' + - './data/indico/log:/opt/indico/log' # logs + - type: bind + source: ${INDICO_CONFIG} + target: /opt/indico/etc/indico.conf + read_only: true + tmpfs: + - /opt/indico/tmp + # Indico celery + indico-celery: + <<: *indico-web + command: /opt/indico/run_celery.sh + depends_on: + - indico-redis + networks: + - backend + ports: [] + # Redis + indico-redis: + image: redis + networks: + - backend + volumes: + - './data/redis:/data' + # Postgres + indico-postgres: + image: centos/postgresql-13-centos7 + environment: + - POSTGRESQL_USER=${PGUSER} + - POSTGRESQL_PASSWORD=${PGPASSWORD} + - POSTGRESQL_DATABASE=${PGDATABASE} + - POSTGRESQL_ADMIN_PASSWORD=${PGPASSWORD} + networks: + - backend + # Nginx proxy + # Indico can be accessed via localhost:8080 + indico-nginx: + build: nginx + environment: + - NGINX_SERVER_NAME=${NGINX_SERVER_NAME} + networks: + - frontend + ports: + - "${NGINX_PORT:-8080}:8080" + volumes: + - 'static-files:/opt/indico/static:ro' + - './data/nginx/log:/var/log/nginx' # logs +volumes: + archive: + static-files: + customization: +networks: + backend: {} + frontend: {} diff --git a/indico-dev/worker/.transifexrc b/indico-dev/worker/.transifexrc new file mode 100644 index 0000000..bea6cb2 --- /dev/null +++ b/indico-dev/worker/.transifexrc @@ -0,0 +1,5 @@ +[https://www.transifex.com] +api_hostname = https://api.transifex.com +hostname = https://www.transifex.com +username = api +rest_hostname = https://rest.api.transifex.com diff --git a/indico-dev/worker/Dockerfile b/indico-dev/worker/Dockerfile new file mode 100644 index 0000000..fc97a96 --- /dev/null +++ b/indico-dev/worker/Dockerfile @@ -0,0 +1,61 @@ +FROM python:3.10-alpine + +ENV INDICO_VIRTUALENV="/opt/indico/.venv" INDICO_CONFIG="/opt/indico/etc/indico.conf" + +ARG pip="${INDICO_VIRTUALENV}/bin/pip" + +USER root + +# Install dependencies +RUN set -ex && apk update && \ + apk add --no-cache libpq-dev postgresql-client vim less bash curl gettext git nodejs npm \ + gcc g++ make musl-dev linux-headers mupdf-dev freetype-dev + # gcc, g++, make, musl-dev, linux-headers, mupdf-dev and freetype-dev are needed to build some python and/or node libraries + +# Install maildump +RUN pip install --no-cache-dir pipx && \ + pipx install maildump && \ + pip uninstall pipx -y + +# Create a virtual environment +RUN python -m venv ${INDICO_VIRTUALENV} +RUN ${pip} install --no-cache-dir --upgrade pip uwsgi + +# Install the Transifex client +RUN mkdir -p /opt/tx +RUN cd /opt/tx && curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash + +WORKDIR /opt/indico + +# Clone indico +RUN git clone https://github.com/indico/indico.git src --depth 1 + +# Clone indico-plugins +RUN git clone https://github.com/indico/indico-plugins.git indico-plugins --depth 1 + +# Install requirements +RUN cd /opt/indico/src && ${pip} install --no-cache-dir -e '.[dev]' +RUN for dir in /opt/indico/indico-plugins/*/; do (cd "${dir}" && ${pip} install --no-cache-dir -e '.[dev]'); done + +# Run webpack for indico and plugins +RUN cd /opt/indico/src && npm ci && \ + ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py indico --dev && \ + ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py all-plugins --dev /opt/indico/indico-plugins && \ + npm cache clean --force + +RUN ["/bin/bash", "-c", "mkdir -p --mode=775 /opt/indico/{etc,tmp,log,cache,archive}"] + +RUN ${INDICO_VIRTUALENV}/bin/indico setup create-symlinks /opt/indico +RUN ${INDICO_VIRTUALENV}/bin/indico setup create-logging-config /opt/indico/etc + +COPY .transifexrc /opt/indico/etc/ + +WORKDIR /opt/indico + +# uwsgi & maildump ports +EXPOSE 59999 60000 + +COPY uwsgi.ini /etc/uwsgi.ini + +COPY run_indico.sh run_celery.sh bootstrap_user.py /opt/indico/ +RUN chmod 755 /opt/indico/*.sh diff --git a/indico-dev/worker/bootstrap_user.py b/indico-dev/worker/bootstrap_user.py new file mode 100644 index 0000000..44e4d44 --- /dev/null +++ b/indico-dev/worker/bootstrap_user.py @@ -0,0 +1,25 @@ +from indico.web.flask.app import make_app + +# Create an admin user so we can skip the manual /bootstrap setup +with make_app().app_context(): + from indico.core.config import config + from indico.core.db import db + from indico.modules.auth import Identity + from indico.modules.users import User + + user = User() + user.first_name = "John" + user.last_name = "Doe" + user.affiliation = "CERN" + user.email = "john.doe@example.com" + user.is_admin = True + + identity = Identity(provider='indico', identifier="admin", password="indiko42") + user.identities.add(identity) + + db.session.add(user) + db.session.flush() + + user.settings.set('timezone', config.DEFAULT_TIMEZONE) + user.settings.set('lang', config.DEFAULT_LOCALE) + db.session.commit() diff --git a/indico-dev/worker/logging.yaml b/indico-dev/worker/logging.yaml new file mode 100644 index 0000000..04733b3 --- /dev/null +++ b/indico-dev/worker/logging.yaml @@ -0,0 +1,16 @@ +version: 1 + +root: + level: INFO + handlers: [stderr] + +loggers: {} + +handlers: + stderr: + class: logging.StreamHandler + formatter: default + +formatters: + default: + format: '%(asctime)s %(levelname)-7s %(request_id)s %(name)-25s %(message)s' diff --git a/indico-dev/worker/run_celery.sh b/indico-dev/worker/run_celery.sh new file mode 100644 index 0000000..c44dab1 --- /dev/null +++ b/indico-dev/worker/run_celery.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +. /opt/indico/.venv/bin/activate + +check_db_ready() { + psql -c 'SELECT COUNT(*) FROM events.events' +} + +# Wait until the DB becomes ready +check_db_ready +until [ $? -eq 0 ]; do + echo "Waiting for DB to be ready..." + sleep 10 + check_db_ready +done + +echo 'Starting Celery...' +indico celery worker -B diff --git a/indico-dev/worker/run_indico.sh b/indico-dev/worker/run_indico.sh new file mode 100644 index 0000000..e88b303 --- /dev/null +++ b/indico-dev/worker/run_indico.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +. /opt/indico/.venv/bin/activate + +connect_to_db() { + psql -lqt | cut -d \| -f 1 | grep -qw $PGDATABASE +} + +# Wait until the DB becomes available +connect_to_db +until [ $? -eq 0 ]; do + echo "Waiting for DB to become available..." + sleep 1 + connect_to_db +done + +# Check whether the DB is already setup +psql -c 'SELECT COUNT(*) FROM events.events' + +if [ $? -eq 1 ]; then + echo 'Preparing DB...' + echo 'CREATE EXTENSION unaccent;' | psql + echo 'CREATE EXTENSION pg_trgm;' | psql + indico db prepare + echo 'Bootstrapping admin user...' + python /opt/indico/bootstrap_user.py +fi + +echo 'Pulling translations...' +cd /opt/indico/src && /opt/tx/tx --token=$TRANSIFEX_API_TOKEN --root-config=/opt/indico/etc/.transifexrc pull --all -f + +echo 'Compiling translations...' +indico i18n compile-catalog +indico i18n compile-catalog-react + +echo 'Starting maildump...' +/root/.local/bin/maildump -n --http-ip 0.0.0.0 --http-port 60000 --db /tmp/maildump.sqlite --smtp-ip 127.0.0.1 --smtp-port 25 & + +echo 'Starting Indico...' +uwsgi /etc/uwsgi.ini diff --git a/indico-dev/worker/uwsgi.ini b/indico-dev/worker/uwsgi.ini new file mode 100644 index 0000000..e369a9f --- /dev/null +++ b/indico-dev/worker/uwsgi.ini @@ -0,0 +1,37 @@ +[uwsgi] +;uid = indico +;gid = nginx +;umask = 027 +;pidfile = /run/uwsgi/uwsgi.pid +;stats = /opt/indico/web/uwsgi-stats.sock + +processes = 4 +enable-threads = true +http-socket = 0.0.0.0:59999 +protocol = http + +master = true +auto-procname = true +procname-prefix-spaced = indico +disable-logging = true + +;plugin = python +single-interpreter = true + +touch-reload = /opt/indico/indico.wsgi +wsgi-file = /opt/indico/indico.wsgi +virtualenv = /opt/indico/.venv + +ignore-sigpipe = true +ignore-write-errors = true +disable-write-exception = true + +vacuum = true +memory-report = true +max-requests = 2500 +harakiri = 900 +harakiri-verbose = true +reload-on-rss = 2048 +evil-reload-on-rss = 8192 + +offload-threads = 4 diff --git a/indico.dev.conf b/indico.dev.conf new file mode 100644 index 0000000..972c79c --- /dev/null +++ b/indico.dev.conf @@ -0,0 +1,28 @@ +# General settings +SQLALCHEMY_DATABASE_URI = 'postgresql://indico:indicopass@indico-postgres:5432/indico' +SECRET_KEY = 'super_secret_random_key' +BASE_URL = 'http://localhost:8080' + +DEFAULT_TIMEZONE = 'Europe/Zurich' +DEFAULT_LOCALE = 'en_GB' + +REDIS_CACHE_URL = 'redis://indico-redis:6379/0' +CELERY_BROKER = 'redis://indico-redis:6379/1' + +ENABLE_ROOMBOOKING = True + +LOG_DIR = '/opt/indico/log' +TEMP_DIR = '/opt/indico/tmp' +CACHE_DIR = '/opt/indico/cache' +CUSTOMIZATION_DIR = '/opt/indico/custom' + +STORAGE_BACKENDS = {'default': 'fs:/opt/indico/archive'} +ATTACHMENT_STORAGE = 'default' + +PLUGINS = {'previewer_code', 'vc_zoom', 'payment_manual'} + +# Development settings +DB_LOG = True +DEBUG = True +SMTP_USE_CELERY = False +LOCAL_MODERATION = True