Add a development setup

.github/workflows/ci.yml

@@ -2,11 +2,11 @@ name: CI
 
 on:
   push:
-    branches: [master]
+    branches: [ci-testing]
 
 jobs:
-  test:
-    name: "Test docker compose"
+  test-prod:
+    name: "[Prod] Test docker compose"
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
@@ -24,4 +24,17 @@ jobs:
           docker compose -f docker-compose.local.yml build
 
       - name: Test if Indico works
-        run: ./test.sh
+        run: ./test.sh prod
+  test-dev:
+    name: "[Dev] Test docker compose"
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build containers
+        run: |
+          cd indico-dev
+          docker compose build
+
+      - name: Test if Indico works
+        run: ./test.sh dev

.gitignore

@@ -1,5 +1,6 @@
 data/*
 indico-prod/data/*
+indico-dev/data/*
 
 # IDE files
 .idea/

README.md

@@ -50,6 +50,26 @@ $ docker run getindico/indico /opt/indico/run_celery.sh
 
 In the above, we omit the network setup to make e.g. the DB accessible from the containers.
 
+## Development setup
+
+The development setup builds an image from the latest master. Essentially, it puts this [tutorial](https://docs.getindico.io/en/stable/installation/development/) into a Dockerfile. The main purpose of this setup is to allow anyone to relatively quickly and painlessly try out the latest translations from Transifex without having to manually setup an Indico instance.
+
+### Quickstart
+
+To start the containers, run:
+```sh
+$ cd indico-dev && docker compose up
+```
+Indico should now be accessible at [localhost:8080](localhost:8080).
+
+The Indico container by default pulls the latest translations from a public mirror which does not need authentication. The mirror updates its translations once every hour. If you want to pull directly from Transifex, you can provide your Transifex API token in the [dev.env](dev.env) file (`TRANSIFEX_API_TOKEN=1/xxx`).
+
+The development setup creates a default admin user with username `admin` and password `indiko42`.
+
+Since emails also get translated, you can use maildump to check those as well. Maildump is available at [localhost:60000](localhost:60000).
+
+Check the [production setup](#production-like-setup) to see more configuration options. 
+
 ## OpenShift
 
 ```sh

indico-dev/.env

@@ -0,0 +1,26 @@
+## DB config
+## ===========================================
+PGHOST=indico-postgres
+PGUSER=indico
+PGPASSWORD=indicopass
+PGDATABASE=indico
+
+## Celery
+## ===========================================
+C_FORCE_ROOT=true
+
+## Nginx
+## ===========================================
+NGINX_PORT=8080
+# Should match the 'BASE_URL' in indico.conf
+NGINX_SERVER_NAME=localhost:8080
+
+## Indico
+## ===========================================
+# Path to the indico.conf file
+INDICO_CONFIG=indico.conf
+
+## Transifex
+## ===========================================
+# If not specified, the translations are pulled from a public mirror
+TRANSIFEX_API_TOKEN=

indico-dev/docker-compose.yml

@@ -0,0 +1,83 @@
+version: "3"
+services:
+  # The main Indico container which runs flask
+  # The same image is also used to run celery
+  indico-web: &indico-web
+    build: worker
+    command: /opt/indico/run_indico.sh
+    depends_on:
+      - indico-redis
+      - indico-celery
+    environment:
+      - PGHOST=${PGHOST}
+      - PGUSER=${PGUSER}
+      - PGPASSWORD=${PGPASSWORD}
+      - PGDATABASE=${PGDATABASE}
+      - C_FORCE_ROOT=${C_FORCE_ROOT}
+      - TRANSIFEX_API_TOKEN=${TRANSIFEX_API_TOKEN}
+    networks:
+      - backend
+      - frontend
+    ports:
+      # Indico is accessible either via nginx (localhost:8080 by default), or
+      # directly via localhost:9090. In that case, static assets are served by flask
+      - "9090:59999"
+      # Maildump is accessible at localhost:60000
+      - "60000:60000"
+    volumes:
+      - 'archive:/opt/indico/archive' # file storage
+      - 'customization:/opt/indico/custom'
+      - 'static-files:/opt/indico/static'
+      - './data/indico/log:/opt/indico/log' # logs
+      - type: bind
+        source: ${INDICO_CONFIG}
+        target: /opt/indico/etc/indico.conf
+        read_only: true
+    tmpfs:
+      - /opt/indico/tmp
+  # Indico celery
+  indico-celery:
+    <<: *indico-web
+    command: /opt/indico/run_celery.sh
+    depends_on:
+      - indico-redis
+    networks:
+      - backend
+    ports: []
+  # Redis
+  indico-redis:
+    image: redis
+    networks:
+      - backend
+    volumes:
+      - './data/redis:/data'
+  # Postgres
+  indico-postgres:
+    image: centos/postgresql-13-centos7
+    environment:
+      - POSTGRESQL_USER=${PGUSER}
+      - POSTGRESQL_PASSWORD=${PGPASSWORD}
+      - POSTGRESQL_DATABASE=${PGDATABASE}
+      - POSTGRESQL_ADMIN_PASSWORD=${PGPASSWORD}
+    networks:
+      - backend
+  # Nginx proxy
+  # Indico can be accessed via localhost:8080
+  indico-nginx:
+    build: nginx
+    environment:
+      - NGINX_SERVER_NAME=${NGINX_SERVER_NAME}
+    networks:
+      - frontend
+    ports:
+      - "${NGINX_PORT:-8080}:8080"
+    volumes:
+      - 'static-files:/opt/indico/static:ro'
+      - './data/nginx/log:/var/log/nginx' # logs
+volumes:
+  archive:
+  static-files:
+  customization:
+networks:
+  backend: {}
+  frontend: {}

indico-dev/indico.conf

@@ -0,0 +1,28 @@
+# General settings
+SQLALCHEMY_DATABASE_URI = 'postgresql://indico:indicopass@indico-postgres:5432/indico'
+SECRET_KEY = 'super_secret_random_key'
+BASE_URL = 'http://localhost:8080'
+
+DEFAULT_TIMEZONE = 'Europe/Zurich'
+DEFAULT_LOCALE   = 'en_GB'
+
+REDIS_CACHE_URL = 'redis://indico-redis:6379/0'
+CELERY_BROKER   = 'redis://indico-redis:6379/1'
+
+ENABLE_ROOMBOOKING = True
+
+LOG_DIR           = '/opt/indico/log'
+TEMP_DIR          = '/opt/indico/tmp'
+CACHE_DIR         = '/opt/indico/cache'
+CUSTOMIZATION_DIR = '/opt/indico/custom'
+
+STORAGE_BACKENDS   = {'default': 'fs:/opt/indico/archive'}
+ATTACHMENT_STORAGE = 'default'
+
+PLUGINS = {'previewer_code', 'vc_zoom', 'payment_manual'}
+
+# Development settings
+DB_LOG           = True
+DEBUG            = True
+SMTP_USE_CELERY  = False
+LOCAL_MODERATION = True

indico-dev/nginx/Dockerfile

@@ -0,0 +1,20 @@
+FROM nginxinc/nginx-unprivileged:stable-alpine
+
+USER root
+
+RUN set -ex && \
+        apk add --update py-pip unzip && \
+        rm -rf /var/cache/apk/*
+
+RUN rm /etc/nginx/conf.d/default.conf
+
+EXPOSE 8080
+
+# OpenShift runs containers using an arbitrarily assigned user ID for security reasons
+# This user is always in the root group so it is needed to grant privileges to group 0.
+RUN chgrp -R 0 /var/* /etc/nginx && chmod -R g+rwX /var/* /etc/nginx
+
+COPY run_nginx.sh indico.conf.template /
+RUN chmod +x /run_nginx.sh
+
+ENTRYPOINT ["/run_nginx.sh"]

indico-dev/nginx/indico.conf.template

@@ -0,0 +1,32 @@
+server {
+  # localhost:8080 is the main entrypoint of this docker-compose setup
+  listen 8080;
+  listen [::]:8080;
+  server_name ${NGINX_SERVER_NAME};
+
+  access_log /var/log/nginx/access.log combined;
+  access_log /dev/stdout combined;
+  error_log  /var/log/nginx/error.log info;
+  error_log  stderr info;
+
+  root       /var/empty;
+
+  sendfile on;
+
+  # Serve static files
+  location ~ ^/(images|fonts)(.*)/(.+?)(__v[0-9a-f]+)?\.([^.]+)$ {
+    alias /opt/indico/static/$1$2/$3.$5;
+  }
+
+  location ~ ^/(css|dist|images|fonts)/(.*)$ {
+    alias /opt/indico/static/$1/$2;
+  }
+
+  location / {
+    # indico-web is the container running Indico
+    proxy_pass http://indico-web:59999;
+    proxy_set_header Host $server_name;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+}

indico-dev/nginx/run_nginx.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+envsubst '\$NGINX_SERVER_NAME' < /indico.conf.template > /etc/nginx/conf.d/indico.conf
+nginx -g "daemon off;"

indico-dev/worker/.transifexrc

@@ -0,0 +1,5 @@
+[https://www.transifex.com]
+api_hostname  = https://api.transifex.com
+hostname      = https://www.transifex.com
+username      = api
+rest_hostname = https://rest.api.transifex.com

indico-dev/worker/Dockerfile

@@ -0,0 +1,61 @@
+FROM python:3.10-alpine
+
+ENV INDICO_VIRTUALENV="/opt/indico/.venv" INDICO_CONFIG="/opt/indico/etc/indico.conf"
+
+ARG pip="${INDICO_VIRTUALENV}/bin/pip"
+
+USER root
+
+# Install dependencies
+RUN set -ex && apk update && \
+    apk add --no-cache libpq-dev postgresql-client vim less bash curl gettext git nodejs npm zip wget \
+                       gcc g++ make musl-dev linux-headers mupdf-dev freetype-dev
+    # gcc, g++, make, musl-dev, linux-headers, mupdf-dev and freetype-dev are needed to build some python and/or node libraries
+
+# Install maildump
+RUN pip install --no-cache-dir pipx && \
+    pipx install maildump && \
+    pip uninstall pipx -y
+
+# Create a virtual environment
+RUN python -m venv ${INDICO_VIRTUALENV}
+RUN ${pip} install --no-cache-dir --upgrade pip uwsgi
+
+# Install the Transifex client
+RUN mkdir -p /opt/tx
+RUN cd /opt/tx && curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
+
+WORKDIR /opt/indico
+
+# Clone indico
+RUN git clone https://github.com/indico/indico.git src --depth 1
+
+# Clone indico-plugins
+RUN git clone https://github.com/indico/indico-plugins.git indico-plugins --depth 1
+
+# Install requirements
+RUN cd /opt/indico/src && ${pip} install --no-cache-dir -e '.[dev]'
+RUN for dir in /opt/indico/indico-plugins/*/; do (cd "${dir}" && ${pip} install --no-cache-dir -e '.[dev]'); done
+
+# Run webpack for indico and plugins
+RUN cd /opt/indico/src && npm ci && \
+    ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py indico --dev && \
+    ${INDICO_VIRTUALENV}/bin/python ./bin/maintenance/build-assets.py all-plugins --dev /opt/indico/indico-plugins && \
+    npm cache clean --force
+
+RUN ["/bin/bash", "-c", "mkdir -p --mode=775 /opt/indico/{etc,tmp,log,cache,archive}"]
+
+RUN ${INDICO_VIRTUALENV}/bin/indico setup create-symlinks /opt/indico
+RUN ${INDICO_VIRTUALENV}/bin/indico setup create-logging-config /opt/indico/etc
+
+COPY .transifexrc /opt/indico/etc/
+
+WORKDIR /opt/indico
+
+# uwsgi & maildump ports
+EXPOSE 59999 60000
+
+COPY uwsgi.ini /etc/uwsgi.ini
+
+COPY run_indico.sh run_celery.sh pull_translations.sh run_initial_setup.py /opt/indico/
+RUN chmod 755 /opt/indico/*.sh

indico-dev/worker/pull_translations.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [[ -z $TRANSIFEX_API_TOKEN ]]; then
+    echo "Transifex API token not provided"
+    echo "Pulling translations from a mirror..."
+    cd /opt/indico/src
+    wget --tries=5 https://test-indico-transifex-mirror.app.cern.ch/translations.zip
+    if [ $? -ne 0 ]; then
+        echo "Failed to pull from mirror"
+        exit 1
+    fi
+    unzip translations.zip
+    # Delete existing translations
+    rm -R indico/translations/*/
+    # Move the current translations
+    mv translations/* indico/translations/
+    rm -r translations
+else
+    echo "Transifex API token provided"
+    echo "Pulling translations from Transifex..."
+    cd /opt/indico/src && /opt/tx/tx --token=$TRANSIFEX_API_TOKEN --root-config=/opt/indico/etc/.transifexrc pull --all -f
+fi

indico-dev/worker/run_celery.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. /opt/indico/.venv/bin/activate
+
+check_db_ready() {
+    psql -c 'SELECT COUNT(*) FROM events.events'
+}
+
+# Wait until the DB becomes ready
+check_db_ready
+until [ $? -eq 0 ]; do
+    echo "Waiting for DB to be ready..."
+    sleep 10
+    check_db_ready
+done
+
+echo 'Starting Celery...'
+indico celery worker -B