diff --git a/IMAGES/download_templates.png b/IMAGES/download_templates.png index 02384b2..ed0061e 100644 Binary files a/IMAGES/download_templates.png and b/IMAGES/download_templates.png differ diff --git a/docker-compose.yml b/docker-compose.yml index bbbbab4..bf72cca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,15 @@ version: "3.5" services: gve_devnet_meraki_mx_security_baseline: - image: ghcr.io/gve-sw/gve_devnet_meraki_mx_security_baseline:latest -# build: . (local docker file build) +# image: ghcr.io/gve-sw/gve_devnet_meraki_mx_security_baseline:latest + build: . container_name: gve_devnet_meraki_mx_security_baseline environment: - MERAKI_API_KEY=${MERAKI_API_KEY} ports: - "5000:5000" volumes: - - ./flask_app:/app + - ./flask_app/logs:/app/logs + - ./flask_app/mx_configs:/app/mx_configs + - ./flask_app/sqlite.db:/app/sqlite.db restart: "always" diff --git a/flask_app/app.py b/flask_app/app.py index 4c75de1..e18dfae 100644 --- a/flask_app/app.py +++ b/flask_app/app.py @@ -79,55 +79,93 @@ logger.addHandler(file_handler) logger.addHandler(stream_handler) -## One time actions ## +org_to_id = {} +network_to_id = {} -# Build drop down menus for organization and network selection, mapping or orgs/networks to id -organizations = dashboard.organizations.getOrganizations() -sorted_organizations = sorted(organizations, key=lambda x: x['name']) -# Connection to DB (one-time) -conn_one_time = db.create_connection(app.config['DATABASE']) +# Methods +@cache.memoize(timeout=300) # Cache the result for 5 minutes +def dropdown(): + """ + Return Drop Down Content (wrapped in method to support new networks and organizations) - cached + :return: A list of orgs and the corresponding networks + """ + dropdown_content = [] -org_to_id = {} -network_to_id = {} -DROPDOWN_CONTENT = [] -for organization in sorted_organizations: - # Add Org to Base Templates DB Table (if not present already) - db.add_template(conn_one_time, 'base', organization['id'], None) - org_data = {'orgaid': organization['id'], 'organame': organization['name']} + # Build drop down menus for organization and network selection, mapping or orgs/networks to id + organizations = dashboard.organizations.getOrganizations() + sorted_organizations = sorted(organizations, key=lambda x: x['name']) - try: - networks = dashboard.organizations.getOrganizationNetworks(organization['id'], total_pages='all') + # Connection to DB (one-time) + conn = db.create_connection(app.config['DATABASE']) - network_data = [] - network_ids = [] - for network in networks: - # Filter for networks with appliance (MX) only - if 'appliance' in network['productTypes']: - # Add Network to Exception Template DB Table (if not present already) - db.add_template(conn_one_time, 'exception', network['id'], None) + # Get the table of base templates and exception templates + base_templates = db.query_all_base_templates(conn) + base_templates = {item[0]: item[1] for item in base_templates} - network_data.append({'networkid': network['id'], 'networkname': network['name']}) - network_ids.append(network['id']) + exception_templates = db.query_all_exception_templates(conn) + exception_templates = {item[0]: item[1] for item in exception_templates} - # Add new entries to data structures - network_to_id[network['name']] = network['id'] + # Track all the current orgs and networks (used to remove stale data in DB) + current_org_ids = [] + current_network_ids = [] + for organization in sorted_organizations: + # Add Org to Base Templates DB Table (if not present already) + db.add_template(conn, 'base', organization['id'], None) + org_data = {'orgaid': organization['id'], 'organame': organization['name']} + current_org_ids.append(organization['id']) - # Associate networks with their org - org_data['networks'] = network_data + try: + networks = dashboard.organizations.getOrganizationNetworks(organization['id'], total_pages='all') - # Add new entries to data structures - DROPDOWN_CONTENT.append(org_data) - org_to_id[organization['name']] = {'id': organization['id'], 'network_ids': network_ids} + network_data = [] + network_ids = [] + for network in networks: + # Filter for networks with appliance (MX) only + if 'appliance' in network['productTypes']: + # Add Network to Exception Template DB Table (if not present already) + db.add_template(conn, 'exception', network['id'], None) - except Exception as e: - logger.error(f"Error retrieving networks for organization ID {organization['id']}: {e}") + network_data.append({'networkid': network['id'], 'networkname': network['name']}) + network_ids.append(network['id']) + current_network_ids.append(network['id']) -# Close DB connection (one-time) -db.close_connection(conn_one_time) + # Add new entries to data structures + network_to_id[network['name']] = network['id'] + + # Associate networks with their org + org_data['networks'] = network_data + + # Add new entries to data structures + dropdown_content.append(org_data) + org_to_id[organization['name']] = {'id': organization['id'], 'network_ids': network_ids} + + except Exception as e: + logger.error(f"Error retrieving networks for organization ID {organization['id']}: {e}") + + # Clean up any orgs or networks still in db but not in the current list + for org_id in base_templates: + if org_id not in current_org_ids: + db.delete_template(conn, 'base', org_id) + + # Remove from other dicts as well + if org_id in org_to_id: + del org_to_id[org_id] + + for net_id in exception_templates: + if net_id not in current_network_ids: + db.delete_template(conn, 'exception', net_id) + + # Remove from other dicts as well + if net_id in network_to_id: + del network_to_id[net_id] + + # Close DB connection (one-time) + db.close_connection(conn) + + return dropdown_content -# Methods def getSystemTimeAndLocation(): """ Return location and time of accessing device (used on all webpage footers) @@ -195,7 +233,7 @@ def thread_wrapper(current_config, progress_inc, baseline_filename, exception_fi upload_errors[error['network']] = [error['error']] -@cache.memoize(timeout=120) # Cache the result for 2 minutes +@cache.memoize(timeout=60) # Cache the result for 1 minute def get_mx_config_information(selected_organization, selected_network): """ Get the current MX Security configs for the selected Network (wrapped in a separate method to support caching results) @@ -231,6 +269,8 @@ def index(): """ logger.info(f"Main Index {request.method} Request:") + dropdown_content = dropdown() + # Get DB connection conn = get_conn() @@ -246,7 +286,7 @@ def index(): # Build a display list for each orgs networks (show network name, base template, exception template) network_displays = [] - for org in DROPDOWN_CONTENT: + for org in dropdown_content: org_networks = [] for network in org['networks']: network_display = {'id': network['networkid'], 'org_name': org['organame'], @@ -271,6 +311,8 @@ def download_baseline(): """ logger.info(f'Download Baseline {request.method} Request:') + dropdown_content = dropdown() + # If success is present (during redirect after successfully updating SSID), extract URL param if request.args.get('success'): success = request.args.get('success') @@ -306,7 +348,7 @@ def download_baseline(): return redirect(url_for('download_baseline', success=True)) # Render page - return render_template('download_baseline.html', hiddenLinks=False, dropdown_content=DROPDOWN_CONTENT, + return render_template('download_baseline.html', hiddenLinks=False, dropdown_content=dropdown_content, selected_elements=selected_elements, current_config=current_config, success=success, timeAndLocation=getSystemTimeAndLocation(), tracked_settings=config.tracked_settings) @@ -318,6 +360,9 @@ def assign_baseline(): """ logger.info(f'Assign Baseline Template {request.method} Request:') + # Unused, but triggers adding any new network or org to the dictionary + dropdown_content = dropdown() + # Get DB connection conn = get_conn() @@ -399,6 +444,8 @@ def assign_exception(): """ logger.info(f'Assign Exception {request.method} Request:') + dropdown_content = dropdown() + # Get DB connection conn = get_conn() @@ -425,7 +472,7 @@ def assign_exception(): # Build a display list for each orgs networks (show network name, base template, exception template) network_displays = [] - for org in DROPDOWN_CONTENT: + for org in dropdown_content: org_networks = [] for network in org['networks']: network_display = {'id': network['networkid'], 'org_name': org['organame'], @@ -484,6 +531,8 @@ def deploy_templates(): global progress, upload_errors logger.info(f'Deploy Templates {request.method} Request:') + dropdown_content = dropdown() + # Get DB connection conn = get_conn() @@ -510,7 +559,7 @@ def deploy_templates(): # Build a display list for each orgs networks (show network name, base template, exception template) network_displays = [] - for org in DROPDOWN_CONTENT: + for org in dropdown_content: for network in org['networks']: network_display = {'id': network['networkid'], 'org_name': org['organame'], 'net_name': network['networkname'], @@ -573,6 +622,10 @@ def deploy_templates(): current_config = MerakiMXConfig(org_to_id[template_selection['orgName']]['id'], network_to_id[template_selection['netName']], logger) + # If name is still none, there was a failure + if current_config.net_name is None: + continue + # Spawn a background thread thread = threading.Thread(target=thread_wrapper, args=(current_config, progress_inc, baseline_filename, exception_filename,)) diff --git a/flask_app/db.py b/flask_app/db.py index 51310b3..b4f607b 100644 --- a/flask_app/db.py +++ b/flask_app/db.py @@ -17,10 +17,15 @@ __copyright__ = "Copyright (c) 2023 Cisco and/or its affiliates." __license__ = "Cisco Sample Code License, Version 1.1" +import os import sqlite3 from pprint import pprint from sqlite3 import Error +# Absolute Paths +script_dir = os.path.dirname(os.path.abspath(__file__)) +db_path = os.path.join(script_dir, 'sqlite.db') + def create_connection(db_file): """ @@ -175,7 +180,7 @@ def update_template(conn, template_type, meraki_id, file_name): def remove_template(conn, template_type, meraki_id): """ - Remove template from baseline or exception table + "Soft Delete" template from baseline or exception table - Set to Null :param conn: DB connection object :param template_type: The type of template (controls table selection): base, exception :param meraki_id: ID used to search for a template (org id for baseline table, network id for exception table) @@ -198,6 +203,31 @@ def remove_template(conn, template_type, meraki_id): conn.commit() +def delete_template(conn, template_type, meraki_id): + """ + Hard Delete template from baseline or exception table + :param conn: DB connection object + :param template_type: The type of template (controls table selection): base, exception + :param meraki_id: ID used to search for a template (org id for baseline table, network id for exception table) + """ + c = conn.cursor() + + if template_type == "base": + # Base template case + table = "base_templates" + + delete_statement = f"DELETE FROM {table} WHERE org_id = ?" + c.execute(delete_statement, (meraki_id,)) + elif template_type == "exception": + # Exception template case + table = "exception_templates" + + delete_statement = f"DELETE FROM {table} WHERE net_id = ?" + c.execute(delete_statement, (meraki_id,)) + + conn.commit() + + def close_connection(conn): """ Close DB Connection @@ -209,7 +239,7 @@ def close_connection(conn): # If running this python file, create connection to database, create tables, and print out the results of queries of # every table if __name__ == "__main__": - conn = create_connection("sqlite.db") + conn = create_connection(db_path) create_tables(conn) pprint(query_all_base_templates(conn)) pprint(query_all_exception_templates(conn)) diff --git a/flask_app/mx_config.py b/flask_app/mx_config.py index b0b33c7..6e15213 100644 --- a/flask_app/mx_config.py +++ b/flask_app/mx_config.py @@ -110,8 +110,11 @@ def get_network_name(self): """ Get network name (useful for webpage table displays) """ - network = dashboard.networks.getNetwork(self.net_id) - self.net_name = network['name'] + try: + network = dashboard.networks.getNetwork(self.net_id) + self.net_name = network['name'] + except Exception as e: + self.upload_errors.append({'network': self.net_name, 'error': str(e)}) def get_l3_out_rules(self): """