Baselined from internal Repository

last_commit:e443600c3b2154bdf4749b4eb027d35f30256ddf

docker-compose.yml

@@ -2,13 +2,15 @@ version: "3.5"
 
 services:
   gve_devnet_meraki_mx_security_baseline:
-    image: ghcr.io/gve-sw/gve_devnet_meraki_mx_security_baseline:latest
-#    build: . (local docker file build)
+#    image: ghcr.io/gve-sw/gve_devnet_meraki_mx_security_baseline:latest
+    build: .
     container_name: gve_devnet_meraki_mx_security_baseline
     environment:
       - MERAKI_API_KEY=${MERAKI_API_KEY}
     ports:
       - "5000:5000"
     volumes:
-      - ./flask_app:/app
+      - ./flask_app/logs:/app/logs
+      - ./flask_app/mx_configs:/app/mx_configs
+      - ./flask_app/sqlite.db:/app/sqlite.db
     restart: "always"

flask_app/app.py

@@ -79,55 +79,93 @@
 logger.addHandler(file_handler)
 logger.addHandler(stream_handler)
 
-## One time actions ##
+org_to_id = {}
+network_to_id = {}
 
-# Build drop down menus for organization and network selection, mapping or orgs/networks to id
-organizations = dashboard.organizations.getOrganizations()
-sorted_organizations = sorted(organizations, key=lambda x: x['name'])
 
-# Connection to DB (one-time)
-conn_one_time = db.create_connection(app.config['DATABASE'])
+# Methods
+@cache.memoize(timeout=300)  # Cache the result for 5 minutes
+def dropdown():
+    """
+    Return Drop Down Content (wrapped in method to support new networks and organizations) - cached
+    :return: A list of orgs and the corresponding networks
+    """
+    dropdown_content = []
 
-org_to_id = {}
-network_to_id = {}
-DROPDOWN_CONTENT = []
-for organization in sorted_organizations:
-    # Add Org to Base Templates DB Table (if not present already)
-    db.add_template(conn_one_time, 'base', organization['id'], None)
-    org_data = {'orgaid': organization['id'], 'organame': organization['name']}
+    # Build drop down menus for organization and network selection, mapping or orgs/networks to id
+    organizations = dashboard.organizations.getOrganizations()
+    sorted_organizations = sorted(organizations, key=lambda x: x['name'])
 
-    try:
-        networks = dashboard.organizations.getOrganizationNetworks(organization['id'], total_pages='all')
+    # Connection to DB (one-time)
+    conn = db.create_connection(app.config['DATABASE'])
 
-        network_data = []
-        network_ids = []
-        for network in networks:
-            # Filter for networks with appliance (MX) only
-            if 'appliance' in network['productTypes']:
-                # Add Network to Exception Template DB Table (if not present already)
-                db.add_template(conn_one_time, 'exception', network['id'], None)
+    # Get the table of base templates and exception templates
+    base_templates = db.query_all_base_templates(conn)
+    base_templates = {item[0]: item[1] for item in base_templates}
 
-                network_data.append({'networkid': network['id'], 'networkname': network['name']})
-                network_ids.append(network['id'])
+    exception_templates = db.query_all_exception_templates(conn)
+    exception_templates = {item[0]: item[1] for item in exception_templates}
 
-                # Add new entries to data structures
-                network_to_id[network['name']] = network['id']
+    # Track all the current orgs and networks (used to remove stale data in DB)
+    current_org_ids = []
+    current_network_ids = []
+    for organization in sorted_organizations:
+        # Add Org to Base Templates DB Table (if not present already)
+        db.add_template(conn, 'base', organization['id'], None)
+        org_data = {'orgaid': organization['id'], 'organame': organization['name']}
+        current_org_ids.append(organization['id'])
 
-        # Associate networks with their org
-        org_data['networks'] = network_data
+        try:
+            networks = dashboard.organizations.getOrganizationNetworks(organization['id'], total_pages='all')
 
-        # Add new entries to data structures
-        DROPDOWN_CONTENT.append(org_data)
-        org_to_id[organization['name']] = {'id': organization['id'], 'network_ids': network_ids}
+            network_data = []
+            network_ids = []
+            for network in networks:
+                # Filter for networks with appliance (MX) only
+                if 'appliance' in network['productTypes']:
+                    # Add Network to Exception Template DB Table (if not present already)
+                    db.add_template(conn, 'exception', network['id'], None)
 
-    except Exception as e:
-        logger.error(f"Error retrieving networks for organization ID {organization['id']}: {e}")
+                    network_data.append({'networkid': network['id'], 'networkname': network['name']})
+                    network_ids.append(network['id'])
+                    current_network_ids.append(network['id'])
 
-# Close DB connection (one-time)
-db.close_connection(conn_one_time)
+                    # Add new entries to data structures
+                    network_to_id[network['name']] = network['id']
+
+            # Associate networks with their org
+            org_data['networks'] = network_data
+
+            # Add new entries to data structures
+            dropdown_content.append(org_data)
+            org_to_id[organization['name']] = {'id': organization['id'], 'network_ids': network_ids}
+
+        except Exception as e:
+            logger.error(f"Error retrieving networks for organization ID {organization['id']}: {e}")
+
+    # Clean up any orgs or networks still in db but not in the current list
+    for org_id in base_templates:
+        if org_id not in current_org_ids:
+            db.delete_template(conn, 'base', org_id)
+
+            # Remove from other dicts as well
+            if org_id in org_to_id:
+                del org_to_id[org_id]
+
+    for net_id in exception_templates:
+        if net_id not in current_network_ids:
+            db.delete_template(conn, 'exception', net_id)
+
+            # Remove from other dicts as well
+            if net_id in network_to_id:
+                del network_to_id[net_id]
+
+    # Close DB connection (one-time)
+    db.close_connection(conn)
+
+    return dropdown_content
 
 
-# Methods
 def getSystemTimeAndLocation():
     """
     Return location and time of accessing device (used on all webpage footers)
@@ -195,7 +233,7 @@ def thread_wrapper(current_config, progress_inc, baseline_filename, exception_fi
             upload_errors[error['network']] = [error['error']]
 
 
-@cache.memoize(timeout=120)  # Cache the result for 2 minutes
+@cache.memoize(timeout=60)  # Cache the result for 1 minute
 def get_mx_config_information(selected_organization, selected_network):
     """
     Get the current MX Security configs for the selected Network (wrapped in a separate method to support caching results)
@@ -231,6 +269,8 @@ def index():
     """
     logger.info(f"Main Index {request.method} Request:")
 
+    dropdown_content = dropdown()
+
     # Get DB connection
     conn = get_conn()
 
@@ -246,7 +286,7 @@ def index():
 
     # Build a display list for each orgs networks (show network name, base template, exception template)
     network_displays = []
-    for org in DROPDOWN_CONTENT:
+    for org in dropdown_content:
         org_networks = []
         for network in org['networks']:
             network_display = {'id': network['networkid'], 'org_name': org['organame'],
@@ -271,6 +311,8 @@ def download_baseline():
     """
     logger.info(f'Download Baseline {request.method} Request:')
 
+    dropdown_content = dropdown()
+
     # If success is present (during redirect after successfully updating SSID), extract URL param
     if request.args.get('success'):
         success = request.args.get('success')
@@ -306,7 +348,7 @@ def download_baseline():
             return redirect(url_for('download_baseline', success=True))
 
     # Render page
-    return render_template('download_baseline.html', hiddenLinks=False, dropdown_content=DROPDOWN_CONTENT,
+    return render_template('download_baseline.html', hiddenLinks=False, dropdown_content=dropdown_content,
                            selected_elements=selected_elements, current_config=current_config, success=success,
                            timeAndLocation=getSystemTimeAndLocation(), tracked_settings=config.tracked_settings)
 
@@ -318,6 +360,9 @@ def assign_baseline():
     """
     logger.info(f'Assign Baseline Template {request.method} Request:')
 
+    # Unused, but triggers adding any new network or org to the dictionary
+    dropdown_content = dropdown()
+
     # Get DB connection
     conn = get_conn()
 
@@ -399,6 +444,8 @@ def assign_exception():
     """
     logger.info(f'Assign Exception {request.method} Request:')
 
+    dropdown_content = dropdown()
+
     # Get DB connection
     conn = get_conn()
 
@@ -425,7 +472,7 @@ def assign_exception():
 
     # Build a display list for each orgs networks (show network name, base template, exception template)
     network_displays = []
-    for org in DROPDOWN_CONTENT:
+    for org in dropdown_content:
         org_networks = []
         for network in org['networks']:
             network_display = {'id': network['networkid'], 'org_name': org['organame'],
@@ -484,6 +531,8 @@ def deploy_templates():
     global progress, upload_errors
     logger.info(f'Deploy Templates {request.method} Request:')
 
+    dropdown_content = dropdown()
+
     # Get DB connection
     conn = get_conn()
 
@@ -510,7 +559,7 @@ def deploy_templates():
 
     # Build a display list for each orgs networks (show network name, base template, exception template)
     network_displays = []
-    for org in DROPDOWN_CONTENT:
+    for org in dropdown_content:
         for network in org['networks']:
             network_display = {'id': network['networkid'], 'org_name': org['organame'],
                                'net_name': network['networkname'],
@@ -573,6 +622,10 @@ def deploy_templates():
                 current_config = MerakiMXConfig(org_to_id[template_selection['orgName']]['id'],
                                                 network_to_id[template_selection['netName']], logger)
 
+                # If name is still none, there was a failure
+                if current_config.net_name is None:
+                    continue
+
                 # Spawn a background thread
                 thread = threading.Thread(target=thread_wrapper,
                                           args=(current_config, progress_inc, baseline_filename, exception_filename,))

flask_app/db.py

@@ -17,10 +17,15 @@
 __copyright__ = "Copyright (c) 2023 Cisco and/or its affiliates."
 __license__ = "Cisco Sample Code License, Version 1.1"
 
+import os
 import sqlite3
 from pprint import pprint
 from sqlite3 import Error
 
+# Absolute Paths
+script_dir = os.path.dirname(os.path.abspath(__file__))
+db_path = os.path.join(script_dir, 'sqlite.db')
+
 
 def create_connection(db_file):
     """
@@ -175,7 +180,7 @@ def update_template(conn, template_type, meraki_id, file_name):
 
 def remove_template(conn, template_type, meraki_id):
     """
-    Remove template from baseline or exception table
+    "Soft Delete" template from baseline or exception table - Set to Null
     :param conn: DB connection object
     :param template_type: The type of template (controls table selection): base, exception
     :param meraki_id: ID used to search for a template (org id for baseline table, network id for exception table)
@@ -198,6 +203,31 @@ def remove_template(conn, template_type, meraki_id):
     conn.commit()
 
 
+def delete_template(conn, template_type, meraki_id):
+    """
+    Hard Delete  template from baseline or exception table
+    :param conn: DB connection object
+    :param template_type: The type of template (controls table selection): base, exception
+    :param meraki_id: ID used to search for a template (org id for baseline table, network id for exception table)
+    """
+    c = conn.cursor()
+
+    if template_type == "base":
+        # Base template case
+        table = "base_templates"
+
+        delete_statement = f"DELETE FROM {table} WHERE org_id = ?"
+        c.execute(delete_statement, (meraki_id,))
+    elif template_type == "exception":
+        # Exception template case
+        table = "exception_templates"
+
+        delete_statement = f"DELETE FROM {table} WHERE net_id = ?"
+        c.execute(delete_statement, (meraki_id,))
+
+    conn.commit()
+
+
 def close_connection(conn):
     """
     Close DB Connection
@@ -209,7 +239,7 @@ def close_connection(conn):
 # If running this python file, create connection to database, create tables, and print out the results of queries of
 # every table
 if __name__ == "__main__":
-    conn = create_connection("sqlite.db")
+    conn = create_connection(db_path)
     create_tables(conn)
     pprint(query_all_base_templates(conn))
     pprint(query_all_exception_templates(conn))

flask_app/mx_config.py

@@ -110,8 +110,11 @@ def get_network_name(self):
         """
         Get network name (useful for webpage table displays)
         """
-        network = dashboard.networks.getNetwork(self.net_id)
-        self.net_name = network['name']
+        try:
+            network = dashboard.networks.getNetwork(self.net_id)
+            self.net_name = network['name']
+        except Exception as e:
+            self.upload_errors.append({'network': self.net_name, 'error': str(e)})
 
     def get_l3_out_rules(self):
         """