New repo, added base_url into config

.gitignore

@@ -0,0 +1,7 @@
+test
+logs
+src/__pycache__
+config.json
+build
+dist
+*.spec

README.md

@@ -1,2 +1,14 @@
 # Qualys-MITRE-ATTACK-Grabber
- Pulls data from Qualys' API and is grouped by the MITRE ATT&CK matrix. Creates a csv file that allows for easy data visualization via a pivot table. Created per Larry Lotspeich's, Threat and Vulnerability Management Manager at Entergy, request. 
+ Pulls data from Qualys' API and is grouped by the MITRE ATT&CK matrix. Creates a csv file that allows for easy data visualization via a pivot table.<br />
+ Created per Larry Lotspeich's, Threat and Vulnerability Management Manager at Entergy, request.
+
+# Setup
+1. Download the latest release package or just download the source code! <br />
+**NOTE:** If running via the .exe provided be sure to place this .exe in it's own folder as the program populates and reads from it's own directories <br />
+    **NOTE:** If running via the source code, be sure to download all packages used (for quick install of all packages download the `requirements.txt` in the `src` folder) 
+2. After program finishes a csv should populate in the directory `/logs/CSVs` <br />
+   The CSV naming format is `tactic_technique_counts_YYYY-MM-DDThh-mm-ss.csv`
+3. Follow the steps below to create the pivot table <br />
+   `Open the CSV in excel -> Select all the data (header included) -> (Insert tab -> PivotTable) -> Select where you want the table -> Check mark all the PivotTable Fields`
+
+Pretty straight forward thank you :D

src/ExportToEXCEL.py

@@ -0,0 +1,17 @@
+import csv
+
+def saveCSVData(input, fileName):
+    data = []
+    for entry in input:
+        tactic = entry['tactic']
+        techniques = entry['techniques']
+        if not techniques:
+            data.append({'Tactic': tactic, 'Technique': 'No Instances', 'Count': 0})
+        for technique in techniques:
+            data.append({'Tactic': tactic, 'Technique': technique['name'], 'Count': int(technique['count'])})
+
+    header = data[0].keys()
+    with open(fileName, 'w', newline='') as file:
+        writer = csv.DictWriter(file, fieldnames=header)
+        writer.writeheader()
+        writer.writerows(data)

src/MitreMatrixUpdate.py

@@ -0,0 +1,23 @@
+import requests
+from mitreattack.stix20 import MitreAttackData
+from os import remove, rmdir, mkdir
+
+def updateJson():
+    response = requests.get(url="https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json")
+    mkdir("temp")
+    f = open('temp/enterprise-attack.json', 'w')
+    f.write(response.text)
+    f.close()
+
+def getTactics():
+    updateJson()
+    mitre_attack_data = MitreAttackData("temp/enterprise-attack.json")
+    output = []
+    tactics = mitre_attack_data.get_tactics(remove_revoked_deprecated=True)
+    for tactic in tactics:
+        for reference in tactic["external_references"]:
+            if reference["source_name"] == "mitre-attack":
+                output.append({"id":reference["external_id"],"name":tactic["name"]})
+    remove("temp/enterprise-attack.json")
+    rmdir("temp")
+    return output

src/QualysAPIRequest.py

@@ -0,0 +1,44 @@
+import requests
+import json
+from requests.auth import HTTPBasicAuth
+from time import sleep
+
+def getTechniqueCounts(tacticID):
+    with open("config/config.json") as config_file:
+        config = json.load(config_file)
+        username = config["username"]
+        password = config["password"]
+        BASE_URL = config["base_url"]
+
+    if username == "API_USERNAME" or password == "API_PASSWORD" or BASE_URL == "https://BASE_URL":
+        raise RuntimeError("Please update the default config/config.json file!")
+    url = f'{BASE_URL}/portal-front/rest/assetview/1.0/assetvuln/pivotList?limit=200&offset=0&query=&groupByPivot=VM&havingQuery=(vulnerabilities.typeDetected:Confirmed and (vulnerabilities.severity:3 or vulnerabilities.severity:4 or vulnerabilities.severity:5) and vulnerabilities.nonRunningKernel:FALSE ) and (vulnerabilities.typeDetected: [Confirmed, Potential] and vulnerabilities.found: TRUE and vulnerabilities.disabled: FALSE and vulnerabilities.ignored: FALSE)&groupBy=vulnerabilities.vulnerability.mitre.attack.tactic.id:{tacticID}&invalidateCache=true'
+    output = []
+
+    # Sometimes qualys will give us a landing page saying "Please wait"
+    # When this occurs just retry 3 times and if it still doesn't work then return
+    retry_count = 3
+    delay = 30
+
+    for attempt in range(retry_count):
+        response = requests.get(url, auth=HTTPBasicAuth(username, password))
+        if response.status_code == 200:
+            if("</html>" in response.text):
+                print(f"Rate limited, retrying...")
+                sleep(delay)
+                delay *= 2
+                continue
+
+            data = response.json()
+            if data["total"] == 0:
+                return []
+
+            for technique in data["data"]:
+                output.append({"name":technique["vulnerabilities.vulnerability.mitre.attack.technique.name"],"count":technique["count"]})
+
+            return output
+        else:
+            print(f"Error: {response.status_code} - {response.text}")
+            return []
+    print("Retry limit reached")
+    return []

src/main.py

@@ -0,0 +1,37 @@
+from ExportToEXCEL import saveCSVData
+from MitreMatrixUpdate import getTactics
+from QualysAPIRequest import getTechniqueCounts
+import json
+from datetime import datetime
+from os import path, makedirs
+
+if not path.exists("logs"):
+    makedirs("logs")
+    if not path.exists("logs/CSVs"):
+        makedirs("logs/CSVs")
+
+if not path.exists("config"):
+    makedirs("config")
+    f = open("config/config.json", "w")
+    f.write("{\"username\":\"API_USERNAME\", \"password\":\"API_PASSWORD\", \"base_url\":\"https://BASE_URL\"}")
+    f.close()
+    raise RuntimeError("Please update the default config/config.json file!")
+
+#I don't like that I've had to hardcode this but it helps prevent rate limiting
+mitreTacticsInVMDR = ["Initial Access", "Execution", "Persistence", "Privilege Escalation", "Defense Evasion", "Credential Access", "Lateral Movement", "Collection", "Impact"]
+data = []
+for tactic in getTactics():
+    if tactic["name"] in mitreTacticsInVMDR:
+        data.append({"tactic":tactic["name"], "techniques":getTechniqueCounts(tactic["id"])})
+        print(data[-1])
+
+now = datetime.now()
+formatted_now = now.strftime("%Y-%m-%dT%H-%M-%S")
+
+f = open("logs/data_log.txt", "a")
+f.write(f"\n\nTIMEDATE: {formatted_now}\n")
+f.write(json.dumps(data))
+f.close()
+
+filename = f"logs/CSVs/tactic_technique_counts_{formatted_now}.csv"
+saveCSVData(data, filename)