Loading templates from external repository (#18)

* remove books from mail templates

* add en.docs.sensebox.de link to hackair mail

* update luftdaten url

* change books to docs link

* use go modules

* Update package name

* update to go 1.15

* Update Docker base image to go 1.15

* Load templates from external repository (#14)

* pull images from github

* update dockerfile and install git

* just pull main branch

* add github actions stuff stub

* restructure ci

* remove v

* seems there is no GOPATH

* debug

* GOPATH?!

* Add /bin to PATH

* debug

* more ci?

* Remove Gopkg files

* use templates from remote git repo

* install git in builder container image

* install ca-certificates correctly

* fix clone

* set global templates after slurping

* expand testing

* use main branch of templates

* update go mod templates

* Cleanup (#16)

* remove honeybadger

* add env variables for templater

* add prefixes

* print logs in actions

* oops removed to much stuff

Co-authored-by: Matthias Pfeil <matthias.pfeil@wwu.de>

* move js testing into test directory (#15)

* document stuff (#17)

* document templater variables

* add optional flags

Co-authored-by: Gerald Pape <ubergesundheit@users.noreply.github.com>

* Fix logging from templating

* Forgot this in e98b956

* Add error for template length

* Update mailer/templates/templates.go

Co-authored-by: Matthias Pfeil <matthias.pfeil@wwu.de>

Co-authored-by: umut0 <umut08152@gmail.com>
Co-authored-by: Mario <mario.pesch@uni-muenster.de>
Co-authored-by: umut <u.tas@reedu.de>
Co-authored-by: Matthias Pfeil <matthias.pfeil@wwu.de>
Co-authored-by: Gerald Pape <ubergesundheit@users.noreply.github.com>

.github/workflows/test.yaml

@@ -0,0 +1,80 @@
+name: Build and test mailer
+
+on:
+  push:
+    branches:
+    - master
+    - development
+    - cleanup
+  pull_request:
+    branches:
+    - master
+    - development
+    - ci-stuff
+
+jobs:
+  test-mailer:
+    name: Test sensebox-mailer
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+
+    - uses: webiny/action-post-run@1.0.0
+      id: docker-logs
+      with:
+        command: docker logs mailer
+
+    # no need to setup go or docker, ubuntu-20.04 has
+    # reasonable versions installed ok for our use case
+    - run: |
+        set -e
+        export GOPATH=${HOME}/go
+
+        GO111MODULE=on go get github.com/mailhog/MailHog@v1.0.1
+        GO111MODULE=on go get github.com/square/certstrap@7cda9d4
+
+        export PATH=${PATH}:${GOPATH}/bin
+
+        docker build -t ci-mailer .
+
+        ./genCerts.sh
+
+        MailHog &
+        MAILHOG_PID=$!
+
+        export SENSEBOX_MAILER_SERVER_CERT=$(cat out/mailer_server.crt)
+        export SENSEBOX_MAILER_SERVER_KEY=$(cat out/mailer_server.key)
+        export SENSEBOX_MAILER_CA_CERT=$(cat out/openSenseMapCA.crt)
+        export SENSEBOX_MAILER_SMTP_SERVER=localhost
+        export SENSEBOX_MAILER_SMTP_PORT=1025
+        export SENSEBOX_MAILER_SMTP_USER=smtpuser
+        export SENSEBOX_MAILER_SMTP_PASSWORD=smtppassword
+        export SENSEBOX_MAILER_FROM_DOMAIN=sensebox.de
+        export SENSEBOX_MAILER_FROM_NAME_PREFIX=senseBox
+        export SENSEBOX_MAILER_TEMPLATES_FETCH_INTERVAL=5s
+
+        docker run \
+          --detach \
+          --name=mailer \
+          --network=host \
+          -e SENSEBOX_MAILER_SERVER_CERT \
+          -e SENSEBOX_MAILER_SERVER_KEY \
+          -e SENSEBOX_MAILER_CA_CERT \
+          -e SENSEBOX_MAILER_SMTP_SERVER \
+          -e SENSEBOX_MAILER_SMTP_PORT \
+          -e SENSEBOX_MAILER_SMTP_USER \
+          -e SENSEBOX_MAILER_SMTP_PASSWORD \
+          -e SENSEBOX_MAILER_FROM_DOMAIN \
+          -e SENSEBOX_MAILER_FROM_NAME_PREFIX \
+          -e SENSEBOX_MAILER_TEMPLATES_FETCH_INTERVAL \
+          ci-mailer
+
+        echo "Sleeping 15 seconds to test git pull"
+
+        sleep 15
+
+        node test/
+
+        docker stop mailer
+        docker rm mailer
+        kill $MAILHOG_PID

Dockerfile

@@ -1,23 +1,20 @@
-FROM golang:1.10 as builder
+FROM golang:1.15-alpine as builder
 
 ENV IMPORTPATH=github.com/sensebox/sensebox-mailer
 
 WORKDIR /go/src/${IMPORTPATH}
 
 COPY . ./
 
-# Compile static assets
-RUN go get github.com/rakyll/statik && \
-  statik -src=/go/src/${IMPORTPATH}/templates
-
-RUN export branch=$(git rev-parse --abbrev-ref HEAD) && \
+RUN apk --no-cache add git && \
+  export branch=$(git rev-parse --abbrev-ref HEAD) && \
   export ts=$(TZ=UTC git log --date=local --pretty=format:"%ct" -n 1) && \
   export hash=$(TZ=UTC git log --date=local --pretty=format:"%h" -n 1) && \
   CGO_ENABLED=0 go install -a -tags netgo -ldflags "-extldflags -static -X main.branch=$branch -X main.ts=$ts -X main.hash=$hash" ${IMPORTPATH}/cmd/sensebox-mailer
 
-FROM scratch
+FROM alpine:3.12
 
-COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs
+RUN apk add --no-cache git ca-certificates
 
 COPY --from=builder /go/bin/sensebox-mailer /sensebox-mailer
 

README.md

@@ -4,20 +4,16 @@ This project is the mailer used by the [openSenseMap-API](https://github.com/sen
 
 ## Development
 
-- Clone the project into your GOPATH.
-- Install certstrap (`go get -u github.com/square/certstrap`)
-- Install statik (`go get -u github.com/rakyll/statik`)
+This project is written in Go. Useful tools for development
+are certstrap (`go get -u github.com/square/certstrap`) and mailhog (`go get github.com/mailhog/MailHog`).
 
 ### Compilation
 
     go build -o sensebox-mailer cmd/sensebox-mailer/*.go
 
 ### Adding new mail templates
 
-1. Create new html files in `templates`. Ideally named `yourNewTemplate_en.html` or `yourNewTemplate_de.html`. Please always create both files for german (`_de.html`) and english (`_en.html`).
-1. Edit `templates/templates.json` and add your template to the array.
-1. Run `statik -src=templates -f`
-1. Compile the project `go build -o sensebox-mailer cmd/sensebox-mailer/*.go`
+Templates are loaded from repository [https://github.com/sensebox/sensebox-mailer-templates](https://github.com/sensebox/sensebox-mailer-templates).
 
 ### Running
 
@@ -29,13 +25,7 @@ A good mailserver for development and testing is [mailhog](https://github.com/ma
 
 A good starting point for a bash script for development is `mailhog.sh`.
 
-Running this script:
-
-- starts `mailhog` docker container
-- runs statik and compiles all templates
-- compiles the sensebox-mailer go binary
-- exports all necessary env variables
-- runs the sensebox-mailer go binary
+Running this script starts a `mailhog` docker container.
 
 Before running this script its important to run [`genCerts.sh`](#1-generate-server-and-client-certificates).
 
@@ -107,4 +97,7 @@ You should configure the following variables:
 | `SMTP_USER` | the smtp server user |  |
 | `SMTP_PASSWORD` | the smtp server password |  |
 | `FROM_DOMAIN` | the domain you are sending from |  |
-| `HONEYBADGER_APIKEY` | api key for honeybadger error reporting | y |
+| `TEMPLATES_REPOSITORY` | git url of the repository which contains the templates | y |
+| `TEMPLATES_BRANCH` | use this branch of the repository for templates | y |
+| `TEMPLATES_FS_PATH` | path where to clone repository | y |
+| `TEMPLATES_FETCH_INTERVAL` | time interval how often to pull repository. Specify in [go duration](https://golang.org/pkg/time/#ParseDuration) format.  | y |

cmd/sensebox-mailer/config.go

@@ -4,22 +4,14 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-
-	"github.com/honeybadger-io/honeybadger-go"
+	"time"
 )
 
 const envPrefix = "SENSEBOX_MAILER_"
 
-func initConfigFromEnv() (caCert, serverCert, serverKey []byte, smtpServer, smtpUser, smtpPassword, fromDomain string, smtpPort int, errors []error) {
+func initConfigFromEnv() (caCert, serverCert, serverKey []byte, smtpServer, smtpUser, smtpPassword, fromDomain string, smtpPort int, repository string, branch string, fsPath string, fetchInterval time.Duration, errors []error) {
 	errors = make([]error, 0)
 
-	// try to configure honeybadger integration..
-	honeybadgerAPIKey, _ := getStringFromEnv("HONEYBADGER_APIKEY")
-	if honeybadgerAPIKey != "" {
-		honeybadger.Configure(honeybadger.Configuration{APIKey: honeybadgerAPIKey})
-		fmt.Println("startup, enabled honeybadger integration")
-	}
-
 	caCert, caCertBytesErr := getBytesFromEnv("CA_CERT")
 	if caCertBytesErr != nil {
 		errors = append(errors, caCertBytesErr)
@@ -60,6 +52,30 @@ func initConfigFromEnv() (caCert, serverCert, serverKey []byte, smtpServer, smtp
 		errors = append(errors, fromDomainErr)
 	}
 
+	repository, repositoryErr := getStringFromEnvWithDefault("TEMPLATES_REPOSITORY","https://github.com/sensebox/sensebox-mailer-templates.git")
+	if repositoryErr != nil {
+		errors = append(errors, repositoryErr)
+	}
+
+	branch, branchErr := getStringFromEnvWithDefault("TEMPLATES_BRANCH","main")
+	if branchErr != nil {
+		errors = append(errors, branchErr)
+	}
+
+	fsPath, fsPathErr := getStringFromEnvWithDefault("TEMPLATES_FS_PATH","./mailer-templates")
+	if fsPathErr != nil {
+		errors = append(errors, fsPathErr)
+	}
+
+	fetchIntervalStr, fetchIntervalStrErr := getStringFromEnvWithDefault("TEMPLATES_FETCH_INTERVAL", "5m")
+	if fetchIntervalStrErr != nil {
+		errors = append(errors, fetchIntervalStrErr)
+	}
+	fetchInterval, fetchIntervalErr := time.ParseDuration(fetchIntervalStr)
+	if fetchIntervalErr != nil {
+		errors = append(errors, fetchIntervalErr)
+	}
+
 	if len(errors) != 0 {
 		return
 	}
@@ -74,6 +90,14 @@ func getStringFromEnv(key string) (string, error) {
 	return str, nil
 }
 
+func getStringFromEnvWithDefault(key string, defaultValue string) (string, error) {
+	str := os.Getenv(envPrefix + key)
+	if len(str) == 0 {
+		return defaultValue, nil
+	}
+	return str, nil
+}
+
 func getBytesFromEnv(key string) ([]byte, error) {
 	str, err := getStringFromEnv(key)
 	if err != nil {

cmd/sensebox-mailer/sensebox-mailer.go

@@ -6,8 +6,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/honeybadger-io/honeybadger-go"
 	"github.com/sensebox/sensebox-mailer/mailer"
+	"github.com/sensebox/sensebox-mailer/mailer/templates"
 	// should be "github.com/jordan-wright/email"
 	// but we wait until https://github.com/jordan-wright/email/pull/61 is merged
 )
@@ -29,10 +29,8 @@ func logStartup() {
 }
 
 func main() {
-	defer honeybadger.Monitor()
-
 	logStartup()
-	caCert, serverCert, serverKey, smtpServer, smtpUser, smtpPassword, fromDomain, smtpPort, errors := initConfigFromEnv()
+	caCert, serverCert, serverKey, smtpServer, smtpUser, smtpPassword, fromDomain, smtpPort, repository, branch, fsPath, fetchInterval, errors := initConfigFromEnv()
 	if len(errors) != 0 {
 		for _, err := range errors {
 			fmt.Println(err.Error())
@@ -51,7 +49,13 @@ func main() {
 		FromDomain:   fromDomain,
 	}
 
-	err := mailer.Start()
+	err := templates.NewTemplater(repository, branch, fsPath, fetchInterval)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	err = mailer.Start()
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

docker-compose.yml

@@ -29,3 +29,7 @@ services:
       SENSEBOX_MAILER_SMTP_USER: <your smtp server user>
       SENSEBOX_MAILER_SMTP_PASSWORD: <your smtp server password>
       SENSEBOX_MAILER_FROM_DOMAIN: <the domain you want your mails to originate from>
+      SENSEBOX_MAILER_TEMPLATES_REPOSITORY: <templates repository git url>
+      SENSEBOX_MAILER_TEMPLATES_BRANCH: <branch name>
+      SENSEBOX_MAILER_TEMPLATES_FS_PATH: <templates path within the branch>
+      SENSEBOX_MAILER_TEMPLATES_FETCH_INTERVAL: <interval to fetch templates in ms >

docker.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+function cleanup() {
+  echo 'cleanup!'
+  sudo docker stop mailer
+  sudo docker rm mailer
+}
+trap cleanup EXIT
+
+sudo docker run \
+  --name=mailer \
+  --network=host \
+  -e "SENSEBOX_MAILER_SERVER_CERT=$(cat out/mailer_server.crt)" \
+  -e "SENSEBOX_MAILER_SERVER_KEY=$(cat out/mailer_server.key)" \
+  -e "SENSEBOX_MAILER_CA_CERT=$(cat out/openSenseMapCA.crt)" \
+  -e "SENSEBOX_MAILER_SMTP_SERVER=localhost" \
+  -e "SENSEBOX_MAILER_SMTP_PORT=1025" \
+  -e "SENSEBOX_MAILER_SMTP_USER=smtpuser" \
+  -e "SENSEBOX_MAILER_SMTP_PASSWORD=smtppassword" \
+  -e "SENSEBOX_MAILER_FROM_DOMAIN=sensebox.de" \
+  -e "SENSEBOX_MAILER_FROM_NAME_PREFIX=senseBox" \
+  mailer

genCerts.sh

@@ -10,8 +10,8 @@ echo "Generate root CA"
 certstrap init --passphrase "" --expires "10 years" --common-name "${CA_NAME}"
 
 echo "Create certificate requests for the server and client"
-certstrap request-cert --passphrase "" --key "out/${CA_NAME}.key" --key-bits "4096" --common-name "${SERVICE}_server" --domain "${SERVICE},localhost"
-certstrap request-cert --passphrase "" --key "out/${CA_NAME}.key" --key-bits "4096" --common-name "${SERVICE}_client" --domain "${SERVICE},localhost"
+certstrap request-cert --passphrase "" --key-bits "4096" --common-name "${SERVICE}_server" --domain "${SERVICE},localhost"
+certstrap request-cert --passphrase "" --key-bits "4096" --common-name "${SERVICE}_client" --domain "${SERVICE},localhost"
 
 echo "Sign the certificate requests"
 certstrap sign --passphrase "" --expires "10 years" --CA "$CA_NAME" "${SERVICE}_server"

go.mod

@@ -0,0 +1,8 @@
+module github.com/sensebox/sensebox-mailer
+
+go 1.15
+
+require (
+	github.com/lovego/email v0.0.0-20171113032818-f4db5d7e012a
+	github.com/sensebox/sensebox-mailer-templates v0.0.0-20210117123008-cba07d01969c
+)