DPM 200 logging ASIM formatter

README.md

@@ -5,7 +5,7 @@
 ## Dependencies
 
 Python 3.9.7
-Django 2+  
+Django 4.2.11+  
 Postgres 10  
 
 ## Prerequisites

config/local.py

@@ -0,0 +1,40 @@
+from config.settings import *
+
+# For local reset Django logging to "plain" Python logging (read non ASIM formatted)
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'root': {
+        'level': 'INFO',
+        'handlers': ['console'],
+    },
+    'formatters': {
+        'verbose': {
+            'format': '[%(levelname)s] [%(name)s] %(message)s',
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        'werkzeug': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'django': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'django.server': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+    },
+}

config/settings.py

@@ -1,10 +1,12 @@
 import os
+import sys
 
 import dj_database_url
 import environ
 
 import sentry_sdk
 from celery.schedules import crontab
+from django_log_formatter_asim import ASIMFormatter
 from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 
@@ -174,6 +176,43 @@
 DEFAULT_AWS_ACCESS_KEY_ID = env('DEFAULT_AWS_ACCESS_KEY_ID')
 DEFAULT_AWS_SECRET_ACCESS_KEY = env('DEFAULT_AWS_SECRET_ACCESS_KEY')
 
+# Logging
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(asctime)s [%(levelname)s] [%(name)s] %(message)s'
+        },
+        'asim_formatter': {
+            '()': ASIMFormatter,
+        },
+    },
+    'handlers': {
+        'asim': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'asim_formatter',
+            'stream': sys.stdout,
+        },
+    },
+    'root': {
+        'level': 'INFO',
+        'handlers': ['asim'],
+    },
+    'loggers': {
+        'django': {
+            'level': 'INFO',
+            'handlers': ['asim'],
+            'propagate': False,
+        },
+        'django.db.backends': {
+            'level': 'ERROR',
+            'handlers': ['asim'],
+            'propagate': False,
+        },
+    },
+}
+
 # Redis
 
 if 'redis' in VCAP_SERVICES:
@@ -263,6 +302,6 @@
 ELASTIC_APM = {
   'SERVICE_NAME': 'dnb-service',
   'SECRET_TOKEN': env('ELASTIC_APM_SECRET_TOKEN'),
-  'SERVER_URL' : env('ELASTIC_APM_URL'),
+  'SERVER_URL': env('ELASTIC_APM_URL'),
   'ENVIRONMENT': ELASTIC_APM_ENVIRONMENT,
 }

requirements-dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --output-file=requirements-dev.txt requirements-dev.in
@@ -50,6 +50,10 @@ certifi==2023.7.22
     #   elastic-apm
     #   requests
     #   sentry-sdk
+cfgv==3.4.0
+    # via
+    #   -r requirements.txt
+    #   pre-commit
 charset-normalizer==3.1.0
     # via
     #   -r requirements.txt
@@ -83,13 +87,18 @@ cron-descriptor==1.4.3
     # via
     #   -r requirements.txt
     #   django-celery-beat
+distlib==0.3.8
+    # via
+    #   -r requirements.txt
+    #   virtualenv
 dj-database-url==0.5.0
     # via -r requirements.txt
 django==4.2.11
     # via
     #   -r requirements.txt
     #   django-celery-beat
     #   django-extensions
+    #   django-log-formatter-asim
     #   django-staff-sso-client
     #   djangorestframework
     #   drf-spectacular
@@ -101,6 +110,8 @@ django-environ==0.4.5
     # via -r requirements.txt
 django-extensions==3.2.3
     # via -r requirements-dev.in
+django-log-formatter-asim==0.0.4
+    # via -r requirements.txt
 django-prometheus==2.3.1
     # via -r requirements.txt
 django-staff-sso-client==4.2.2
@@ -132,6 +143,10 @@ factory-boy==2.12.0
     # via -r requirements-dev.in
 faker==2.0.4
     # via factory-boy
+filelock==3.13.4
+    # via
+    #   -r requirements.txt
+    #   virtualenv
 flake8==3.7.7
     # via
     #   -r requirements-dev.in
@@ -152,6 +167,10 @@ freezegun==0.3.12
     # via -r requirements-dev.in
 gunicorn==22.0.0
     # via -r requirements.txt
+identify==2.5.35
+    # via
+    #   -r requirements.txt
+    #   pre-commit
 idna==3.7
     # via
     #   -r requirements.txt
@@ -183,6 +202,10 @@ mccabe==0.6.1
     # via flake8
 more-itertools==7.0.0
     # via pytest
+nodeenv==1.8.0
+    # via
+    #   -r requirements.txt
+    #   pre-commit
 notifications-python-client==6.2.1
     # via -r requirements.txt
 oauthlib==3.2.2
@@ -196,8 +219,16 @@ packaging==24.0
     #   pytest
 pathtools==0.1.2
     # via watchdog
+platformdirs==4.2.0
+    # via
+    #   -r requirements.txt
+    #   virtualenv
 pluggy==0.13.1
     # via pytest
+pre-commit==3.7.0
+    # via
+    #   -r requirements.txt
+    #   django-log-formatter-asim
 prometheus-client==0.11.0
     # via
     #   -r requirements.txt
@@ -258,6 +289,7 @@ pyyaml==6.0.1
     # via
     #   -r requirements.txt
     #   drf-spectacular
+    #   pre-commit
     #   watchdog
 redis==4.5.4
     # via -r requirements.txt
@@ -308,6 +340,10 @@ sqlparse==0.5.0
     #   django
 text-unidecode==1.3
     # via faker
+typing-extensions==4.11.0
+    # via
+    #   -r requirements.txt
+    #   asgiref
 tzdata==2024.1
     # via
     #   -r requirements.txt
@@ -329,6 +365,10 @@ vine==5.0.0
     #   amqp
     #   celery
     #   kombu
+virtualenv==20.25.2
+    # via
+    #   -r requirements.txt
+    #   pre-commit
 watchdog[watchmedo]==0.10.2
     # via -r requirements-dev.in
 wcwidth==0.2.5

requirements.in

@@ -17,6 +17,7 @@ django-prometheus==2.3.1
 smart-open
 notifications-python-client==6.2.1
 elastic-apm
+django-log-formatter-asim==0.0.4
 
 whitenoise
 gunicorn

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
@@ -35,6 +35,8 @@ certifi==2023.7.22
     #   elastic-apm
     #   requests
     #   sentry-sdk
+cfgv==3.4.0
+    # via pre-commit
 charset-normalizer==3.1.0
     # via requests
 click==8.0.3
@@ -51,12 +53,15 @@ click-repl==0.2.0
     # via celery
 cron-descriptor==1.4.3
     # via django-celery-beat
+distlib==0.3.8
+    # via virtualenv
 dj-database-url==0.5.0
     # via -r requirements.in
 django==4.2.11
     # via
     #   -r requirements.in
     #   django-celery-beat
+    #   django-log-formatter-asim
     #   django-staff-sso-client
     #   djangorestframework
     #   drf-spectacular
@@ -66,6 +71,8 @@ django-celery-results==2.4.0
     # via -r requirements.in
 django-environ==0.4.5
     # via -r requirements.in
+django-log-formatter-asim==0.0.4
+    # via -r requirements.in
 django-prometheus==2.3.1
     # via -r requirements.in
 django-staff-sso-client==4.2.2
@@ -82,8 +89,12 @@ drf-spectacular==0.27.1
     # via -r requirements.in
 elastic-apm==6.2.0
     # via -r requirements.in
+filelock==3.13.4
+    # via virtualenv
 gunicorn==22.0.0
     # via -r requirements.in
+identify==2.5.35
+    # via pre-commit
 idna==3.7
     # via requests
 inflection==0.5.1
@@ -98,12 +109,18 @@ jsonschema-specifications==2023.12.1
     # via jsonschema
 kombu==5.2.3
     # via celery
+nodeenv==1.8.0
+    # via pre-commit
 notifications-python-client==6.2.1
     # via -r requirements.in
 oauthlib==3.2.2
     # via requests-oauthlib
 packaging==24.0
     # via gunicorn
+platformdirs==4.2.0
+    # via virtualenv
+pre-commit==3.7.0
+    # via django-log-formatter-asim
 prometheus-client==0.11.0
     # via django-prometheus
 prompt-toolkit==3.0.18
@@ -127,6 +144,7 @@ pyyaml==6.0.1
     # via
     #   -r requirements.in
     #   drf-spectacular
+    #   pre-commit
 redis==4.5.4
     # via -r requirements.in
 referencing==0.34.0
@@ -158,6 +176,8 @@ sqlparse==0.5.0
     # via
     #   -r requirements.in
     #   django
+typing-extensions==4.11.0
+    # via asgiref
 tzdata==2024.1
     # via django-celery-beat
 uritemplate==3.0.1
@@ -174,6 +194,8 @@ vine==5.0.0
     #   amqp
     #   celery
     #   kombu
+virtualenv==20.25.2
+    # via pre-commit
 wcwidth==0.2.5
     # via prompt-toolkit
 whitenoise==5.2.0

sample_env

@@ -24,3 +24,5 @@ DEV_AUTH_EMAIL=datahub.dev@digital.trade.gov.uk
 DEV_AUTH_TOKEN=cc373a2a49ce7143817a9d036fa2a0be92da0d6a
 # Replace GOVUK_NOTIFICATIONS_API_KEY with a real key to use notify functionality
 GOVUK_NOTIFICATIONS_API_KEY=ainaidahNgaeteghei3yooshaiyeeShi8heSie3Ba9AGhoos8eicie5lei2nahue9DaiBait5Ba4ajeiMee6Photh4alegh4Eez8Quopaith5B
+# Extend settings with local development configuration
+DJANGO_SETTINGS_MODULE=config.local