Merge pull request #27 from chadell/release-v1.1.0

Release v1.1.0

.github/workflows/ci.yml

@@ -104,7 +104,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.6", "3.7", "3.8", "3.9"]
-        nautobot-version: ["1.0.3"]
+        nautobot-version: ["1.0.3", "1.1.4"]
     runs-on: "ubuntu-20.04"
     env:
       INVOKE_NAUTOBOT_SSOT_PYTHON_VER: "${{ matrix.python-version }}"

development/Dockerfile

@@ -3,22 +3,33 @@ ARG PYTHON_VER
 ARG NAUTOBOT_VER
 FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER}
 
+ENV prometheus_multiproc_dir=/prom_cache
+
+ARG NAUTOBOT_ROOT=/opt/nautobot
+
+ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT}
+
+WORKDIR $NAUTOBOT_ROOT
+
+# Configure poetry
+RUN poetry config virtualenvs.create false \
+    && poetry config installer.parallel false
+
+# -------------------------------------------------------------------------------------
+# Install Nautobot Plugin
+# -------------------------------------------------------------------------------------
+
 WORKDIR /source
 
 # Copy in only pyproject.toml/poetry.lock to help with caching this layer if no updates to dependencies
 COPY poetry.lock pyproject.toml /source/
+
 # --no-root declares not to install the project package since we're wanting to take advantage of caching dependency installation
 # and the project is copied in and installed after this step
 RUN poetry install --no-interaction --no-ansi --no-root
 
-# Add worker plugin(s) if present
-COPY packages/placeholder packages/*.whl /tmp/packages/
-RUN for wheel in /tmp/packages/*.whl; do [ -f "$wheel" ] || continue; pip install "$wheel"; done
-
 # Copy in the rest of the source code and install local Nautobot plugin
-WORKDIR /source
 COPY . /source
 RUN poetry install --no-interaction --no-ansi
 
-WORKDIR /source
 COPY development/nautobot_config.py /opt/nautobot/nautobot_config.py

development/creds.example.env

@@ -1,10 +1,9 @@
 POSTGRES_PASSWORD=notverysecurepwd
-REDIS_PASSWORD=notverysecurepwd
+NAUTOBOT_REDIS_PASSWORD=notverysecurepwd
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 NAUTOBOT_CREATE_SUPERUSER=true
 NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567
 NAUTOBOT_SUPERUSER_PASSWORD=admin
 # POSTGRES_HOST=localhost
 # REDIS_HOST=localhost
 # NAUTOBOT_ROOT=./development
-

development/dev.env

@@ -9,9 +9,10 @@ NAUTOBOT_ROOT=/opt/nautobot
 POSTGRES_DB=nautobot
 POSTGRES_HOST=postgres
 POSTGRES_USER=nautbot
-REDIS_HOST=redis
-REDIS_PORT=6379
+NAUTOBOT_REDIS_HOST=redis
+NAUTOBOT_REDIS_PORT=6379
 # REDIS_SSL=True
 # Uncomment REDIS_SSL if using SSL
 SUPERUSER_EMAIL=admin@example.com
 SUPERUSER_NAME=admin
+NAUTOBOT_LOG_LEVEL=DEBUG

development/docker-compose.base.yml

@@ -25,8 +25,23 @@ services:
     <<: *nautobot-base
   worker:
     entrypoint: "nautobot-server rqworker"
+    healthcheck:
+      interval: "5s"
+      timeout: "5s"
+      start_period: "5s"
+      retries: 3
+      test: ["CMD", "nautobot-server", "health_check"]
     depends_on:
       - "nautobot"
+    <<: *nautobot-base
+  celery_worker:
     healthcheck:
       disable: true
+    depends_on:
+      - "nautobot"
+      - "redis"
+    entrypoint:
+      - "sh"
+      - "-c"  # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env
+      - "nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL"  # $$ because of docker-compose
     <<: *nautobot-base

development/docker-compose.dev.yml

@@ -1,16 +1,17 @@
-# We can't remove volumes in a compose override, for the test configuration using the final containers
-# we don't want the volumes so this is the default override file to add the volumes in the dev case
-# any override will need to include these volumes to use them.
-# see:  https://github.com/docker/compose/issues/3729
 ---
+x-nautobot-base: &nautobot-base
+  volumes:
+    - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
+    - "../:/source"
+
 version: "3.4"
 services:
   nautobot:
+    healthcheck:
+      disable: true
     command: "nautobot-server runserver 0.0.0.0:8080"
-    volumes:
-      - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
-      - "../:/source"
+    <<: *nautobot-base
+  celery_worker:
+    <<: *nautobot-base
   worker:
-    volumes:
-      - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
-      - "../:/source"
+    <<: *nautobot-base

development/docker-compose.requirements.yml

@@ -15,7 +15,7 @@ services:
     command:
       - "sh"
       - "-c"  # this is to evaluate the $REDIS_PASSWORD from the env
-      - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD"
+      - "redis-server --appendonly yes --requirepass $$NAUTOBOT_REDIS_PASSWORD"
     env_file:
       - "dev.env"
       - "creds.env"

development/nautobot_config.py

@@ -1,15 +1,12 @@
-#########################
-#                       #
-#   Required settings   #
-#                       #
-#########################
+"""Nautobot development configuration file."""
 
 import os
 import sys
 
-from distutils.util import strtobool
 from django.core.exceptions import ImproperlyConfigured
-from nautobot.core import settings
+from nautobot.core.settings import *  # noqa: F403
+from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
+
 
 # Enforce required configuration parameters
 for key in [
@@ -18,29 +15,12 @@
     "POSTGRES_USER",
     "POSTGRES_HOST",
     "POSTGRES_PASSWORD",
-    "REDIS_HOST",
-    "REDIS_PASSWORD",
     "SECRET_KEY",
 ]:
     if not os.environ.get(key):
         raise ImproperlyConfigured(f"Required environment variable {key} is missing.")
 
 
-def is_truthy(arg):
-    """Convert "truthy" strings into Booleans.
-
-    Examples:
-        >>> is_truthy('yes')
-        True
-    Args:
-        arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
-        f, false, off and 0. Raises ValueError if val is anything else.
-    """
-    if isinstance(arg, bool):
-        return arg
-    return bool(strtobool(arg))
-
-
 TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
 
 # This is a list of valid fully-qualified domain names (FQDNs) for the Nautobot server. Nautobot will not permit write
@@ -63,47 +43,29 @@ def is_truthy(arg):
     }
 }
 
-# Redis variables
-REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
-REDIS_PORT = os.getenv("REDIS_PORT", 6379)
-REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
 
-# Check for Redis SSL
-REDIS_SCHEME = "redis"
-REDIS_SSL = is_truthy(os.environ.get("REDIS_SSL", False))
-if REDIS_SSL:
-    REDIS_SCHEME = "rediss"
+# Nautobot uses Cacheops for database query caching. These are the following defaults.
+# For detailed configuration see: https://github.com/Suor/django-cacheops#setup
 
-# The django-redis cache is used to establish concurrent locks using Redis. The
-# django-rq settings will use the same instance/database by default.
-#
-# This "default" server is now used by RQ_QUEUES.
-# >> See: nautobot.core.settings.RQ_QUEUES
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": f"{REDIS_SCHEME}://{REDIS_HOST}:{REDIS_PORT}/0",
+        "LOCATION": parse_redis_connection(redis_database=0),
         "TIMEOUT": 300,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "PASSWORD": REDIS_PASSWORD,
         },
     }
 }
+CACHEOPS_REDIS = parse_redis_connection(redis_database=1)
 
-# RQ_QUEUES is not set here because it just uses the default that gets imported
-# up top via `from nautobot.core.settings import *`.
-
-# REDIS CACHEOPS
-CACHEOPS_REDIS = f"{REDIS_SCHEME}://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/1"
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
 # symbols. Nautobot will not run without this defined. For more information, see
 # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY
 SECRET_KEY = os.environ["SECRET_KEY"]
 
-
 #########################
 #                       #
 #   Optional settings   #
@@ -357,9 +319,9 @@ def is_truthy(arg):
 EXTRA_INSTALLED_APPS = os.environ["EXTRA_INSTALLED_APPS"].split(",") if os.environ.get("EXTRA_INSTALLED_APPS") else []
 
 # Django Debug Toolbar
-DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING}
-
-if "debug_toolbar" not in EXTRA_INSTALLED_APPS:
-    EXTRA_INSTALLED_APPS.append("debug_toolbar")
-if "debug_toolbar.middleware.DebugToolbarMiddleware" not in settings.MIDDLEWARE:
-    settings.MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")
+if DEBUG:
+    DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING}
+    if "debug_toolbar" not in INSTALLED_APPS:  # noqa: F405
+        INSTALLED_APPS.append("debug_toolbar")  # noqa: F405
+    if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE:  # noqa: F405
+        MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")  # noqa: F405

docs/changelog.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v1.1.0 - 2022-02-03
+
+### Added
+
+- [#11](https://github.com/nautobot/nautobot-plugin-ssot/issues/11) - Add option to profile memory usage during job execution.
+- [#14](https://github.com/nautobot/nautobot-plugin-ssot/pull/14), [#22](https://github.com/nautobot/nautobot-plugin-ssot/pull/22) - Added Prefix sync to example jobs, added Celery worker to dev environment.
+- [#15](https://github.com/nautobot/nautobot-plugin-ssot/pull/15) - Added `load_source_adapter`, `load_target_adapter`, `calculate_diff`, and `execute_sync` API methods that Job implementations can override instead of overriding the generalized `sync_data` method.
+- [#21](https://github.com/nautobot/nautobot-plugin-ssot/pull/21) - Add performance stats in Sync job detail view.
+
+### Fixed
+
+- [#13](https://github.com/nautobot/nautobot-plugin-ssot/issues/13) - Handle pagination of Nautobot REST API in example jobs.
+- [#18](https://github.com/nautobot/nautobot-plugin-ssot/pull/18) - Don't skip diff and sync if either of the source or target adapters initially contains no data.
+
 ## v1.0.1 - 2021-10-18
 
 ### Changed

docs/developing_jobs.md

@@ -8,35 +8,55 @@ In brief, the following general steps can be followed:
 
 1. Define one or more `DiffSyncModel` data model class(es) representing the common data record(s) to be synchronized between the two systems.
 
-    * Define your models based on the data you want to sync. For example, if you are syncing specific attributes (e.g. tags) of a device, but not the devices themselves, your parent model
-    should be the tags. This approach prevents unnecessary `create`, `update`, and `delete` calls for models that are not sync'd.
-    * For each model class, implement the `create`, `update`, and `delete` DiffSyncModel APIs for writing data to the Data Target system.
+   - Define your models based on the data you want to sync. For example, if you are syncing specific attributes (e.g. tags) of a device, but not the devices themselves, your parent model
+     should be the tags. This approach prevents unnecessary `create`, `update`, and `delete` calls for models that are not sync'd.
+   - For each model class, implement the `create`, `update`, and `delete` DiffSyncModel APIs for writing data to the Data Target system.
 
 2. Define a `DiffSync` adapter class for loading initial data from Nautobot and constructing instances of each `DiffSyncModel` class to represent that data.
 3. Define a `DiffSync` adapter class for loading initial data from the Data Source or Data Target system and constructing instances of the `DiffSyncModel` classes to represent that data.
-4. Develop a Job class, derived from either the `DataSource` or `DataTarget` classes provided by this plugin, and implement its `sync_data` API function to make use of the DiffSync adapters and data models defined previously. Typically this will look something like:
 
-        def sync_data(self):
-            """Sync data from Data Source to Nautobot."""
-            self.log_info(message="Loading current data from Data Source...")
-            diffsync1 = DataSourceDiffSync(job=self, sync=self.sync)
-            diffsync1.load()
+4. Develop a Job class, derived from either the `DataSource` or `DataTarget` classes provided by this plugin, and implement the adapters to populate the `self.source_adapter` and `self.target_adapter` that are used by the built-in implementation of `sync_data`. This `sync_data` method is an opinionated way of running the process including some performance data, more in [next section](#analyze-job-performance), but you could overwrite it completely or any of the key hooks that it calls:
 
-            self.log_info(message="Loading current data from Nautobot...")
-            diffsync2 = NautobotDiffSync(job=self, sync=self.sync)
-            diffsync2.load()
+   - `self.load_source_adapter`: This is mandatory to be implemented. As an example:
 
-            diffsync_flags = DiffSyncFlags.CONTINUE_ON_FAILURE
+     ```python
+     def load_source_adapter(self):
+         """Method to instantiate and load the SOURCE adapter into `self.source_adapter`."""
+         self.source_adapter = NautobotRemote(url=self.kwargs["source_url"], token=self.kwargs["source_token"], job=self)
+         self.source_adapter.load()
+     ```
 
-            self.log_info(message="Calculating diffs...")
-            diff = diffsync1.diff_to(diffsync_1, flags=diffsync_flags)
-            self.sync.diff = diff.dict()
-            self.sync.save()
+   - `self.load_target_adapter`: This is mandatory to be implemented. As an example:
 
-            if not self.kwargs["dry_run"]:
-                self.log_info(message="Syncing from Data Source to Nautobot...")
-                diffsync1.sync_to(diffsync2, flags=diffsync_flags)
-                self.log_info(message="Sync complete")
+     ```python
+     def load_target_adapter(self):
+        """Method to instantiate and load the TARGET adapter into `self.target_adapter`."""
+        self.target_adapter = NautobotLocal(job=self)
+        self.target_adapter.load()
+     ```
+
+   - `self.calculate_diff`: This method is implemented by default, using the output from load_adapter methods.
+
+   - `self.execute_sync`: This method is implemented by default, using the output from load_adapter methods. Only executed if it's not a `dry-run` execution.
 
 5. Optionally, on your Job class, also implement the `lookup_object`, `data_mappings`, and/or `config_information` APIs (to provide more information to the end user about the details of this Job), as well as the various metadata properties on your Job's `Meta` inner class. Refer to the example Jobs provided in this plugin for examples and further details.
 6. Install your Job via any of the supported Nautobot methods (installation into the `JOBS_ROOT` directory, inclusion in a Git repository, or packaging as part of a plugin) and it should automatically become available!
+
+## Analyze Job performance
+
+The built-in implementation of `sync_data` is composed by 4 steps:
+
+- Loading data from source adapter
+- Loading data from target adapter
+- Calculating diff
+- Executing synchronization (if not `dry-run`)
+
+For each one of these 4 steps we can capture data for performance analysis:
+
+- Time spent: available in the "Data Sync" detail view under Duration section
+- Memory used at the end of the step execution: available in the "Data Sync" detail view under Memory Usage Stats section
+- Peak Memory used during the step execution: available in the "Data Sync" detail view under Memory Usage Stats section
+
+> Memory performance stats are optional, and you must enable them per Job execution with the related checkbox.
+
+This data could give you some insights about where most of the time is spent and how efficient in memory your process is (if there is a big difference between the peak and the final numbers is a hint of something not going well). Understanding it, you could focus on the step that needs more attention.

docs/requirements.txt

@@ -1 +1 @@
-mkdocs==1.1.2
+mkdocs==1.2.3