HFE_GA_v19.4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "436402bd",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "import os\n",
    "os.environ['OPENBLAS_NUM_THREADS'] = '30'\n",
    "\n",
    "#!jupyter nbconvert --to script geHFE.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72f68280",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#!/usr/bin/env python\n",
    "# coding: utf-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd4f892",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import sys\n",
    "if sys.version_info[0] < 3: \n",
    "    from StringIO import StringIO\n",
    "else:\n",
    "    from io import StringIO\n",
    "import pandas as pd\n",
    "\n",
    "import itertools\n",
    "import multiprocessing\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import time\n",
    "import warnings\n",
    "from datetime import date, datetime\n",
    "from multiprocessing import Pool, cpu_count\n",
    "from operator import attrgetter, itemgetter\n",
    "from pathlib import Path\n",
    "from sys import getsizeof\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "import graphviz\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pydot\n",
    "import pydotplus\n",
    "import pylab\n",
    "import seaborn as sns\n",
    "import sklearn\n",
    "#from ann_visualizer.visualize import ann_viz\n",
    "from deap import algorithms, base, creator, tools\n",
    "from genetic_selection import GeneticSelectionCV\n",
    "# from keras.layers import Dense\n",
    "# from keras.models import Sequential\n",
    "#from keras.utils.vis_utils import plot_model\n",
    "from matplotlib import pyplot\n",
    "from numpy import absolute, asarray, loadtxt, mean, std\n",
    "from numpy.linalg import norm\n",
    "from pandas import DataFrame\n",
    "from scipy import stats\n",
    "from scoop import futures\n",
    "from sklearn import (datasets, feature_selection, linear_model, metrics, svm,\n",
    "                     tree)\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.linear_model import (ElasticNet, LinearRegression,\n",
    "                                  LogisticRegression, Perceptron)\n",
    "from sklearn.metrics import (accuracy_score, classification_report,\n",
    "                             confusion_matrix, f1_score, matthews_corrcoef,\n",
    "                             mean_squared_error, plot_confusion_matrix,\n",
    "                             precision_recall_curve, r2_score, roc_auc_score,\n",
    "                             roc_curve)\n",
    "from sklearn.model_selection import (GridSearchCV, KFold, RepeatedKFold,\n",
    "                                     RepeatedStratifiedKFold,\n",
    "                                     cross_val_predict, cross_val_score,\n",
    "                                     train_test_split)\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from tqdm import tqdm\n",
    "from xgboost import XGBClassifier, XGBRegressor\n",
    "from sklearn.compose import ColumnTransformer\n",
    "import numpy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.preprocessing import StandardScaler    \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import logging\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from sklearn.impute import KNNImputer\n",
    "import scipy\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "import traceback\n",
    "from fuzzysearch import find_near_matches#\n",
    "import json\n",
    "import re\n",
    "import gc\n",
    "import pathlib\n",
    "import sys\n",
    "import logging\n",
    "from pathlib import Path\n",
    "import itertools as it\n",
    "import datetime\n",
    "import pathlib\n",
    "from fuzzysearch import find_near_matches\n",
    "from scipy.stats import skewnorm\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d73044a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchmetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88621157",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "random.seed()\n",
    "\n",
    "\n",
    "debug = True\n",
    "model_train_time_warning_threshold = 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e275cb74",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "log_small = np.logspace(-1, -5, 3)\n",
    "print(log_small)\n",
    "bool_param = [True, False]\n",
    "print(bool_param)\n",
    "log_large = np.logspace(0, 2, 3).astype(int)\n",
    "print(log_large)\n",
    "log_large_long = np.floor(np.logspace(2, 2.8, 6)).astype(int)\n",
    "print(log_large_long)\n",
    "log_med_long = np.floor(np.logspace(0, 1.5, 5)).astype(int)\n",
    "print(log_med_long)\n",
    "log_med = np.floor(np.logspace(0, 1.5, 3)).astype(int)\n",
    "print(log_med)\n",
    "nstep = 3\n",
    "log_zero_one = np.logspace(0.0, 1.0, nstep) / 10\n",
    "print(log_zero_one)\n",
    "lin_zero_one = np.linspace(0.01, 1.0, nstep) / 10\n",
    "print(lin_zero_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6ab443c",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def normalize(weights):\n",
    "    # calculate l1 vector norm\n",
    "    result = norm(weights, 1)\n",
    "    # check for a vector of all zeros\n",
    "    if result == 0.0:\n",
    "        return weights\n",
    "    # return normalized vector (unit norm)\n",
    "    return weights / result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52d1702c",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "def get_free_gpu():\n",
    "    gpu_stats = subprocess.check_output([\"nvidia-smi\", \"--format=csv\", \"--query-gpu=memory.used,memory.free\"])\n",
    "    gpu_df = pd.read_csv(StringIO(gpu_stats.decode('utf-8')),\n",
    "                         names=['memory.used', 'memory.free'],\n",
    "                         skiprows=1)\n",
    "    #print('GPU usage:\\n{}'.format(gpu_df))\n",
    "    gpu_df['memory.free'] = gpu_df['memory.free'].map(lambda x: x.rstrip(' [MiB]'))\n",
    "    idx = gpu_df['memory.free'].astype(int).idxmax()\n",
    "    print('Returning GPU{} with {} free MiB'.format(idx, gpu_df.iloc[idx]['memory.free']))\n",
    "    return int(idx)\n",
    "\n",
    "\n",
    "def ratioFunction(num1, num2):\n",
    "    #num1 = input('Enter the first number: ')\n",
    "    if (num1 == 0 or num2 == 0):\n",
    "        #print(\"zero found\")\n",
    "        return np.nan\n",
    "    num1 = float(num1)  # Now we are good\n",
    "    #num2 = input('Enter the second number: ')\n",
    "    num2 = float(num2)  # Good, good\n",
    "    ratio12 = float(num1/num2)\n",
    "    #print('The ratio of', str(num1), 'and', str(num2),'is', ratio12 + '.')\n",
    "    return ratio12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34c3640f",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def getNfeaturesANOVAF(n):\n",
    "    res = []\n",
    "    for colName in X_train.columns:\n",
    "        if colName != \"intercept\":\n",
    "            res.append(\n",
    "                (\n",
    "                    colName,\n",
    "                    sklearn.feature_selection.f_classif(\n",
    "                        np.array(X_train[colName]).reshape(-1, 1), y_train\n",
    "                    )[0],\n",
    "                )\n",
    "            )\n",
    "    sortedList = sorted(res, key=lambda x: x[1])\n",
    "    sortedList.reverse()\n",
    "    nFeatures = sortedList[:n]\n",
    "    finalColNames = []\n",
    "    for elem in nFeatures:\n",
    "        finalColNames.append(elem[0])\n",
    "    return finalColNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00ddd3b1",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def getRandomForestFeatureColumns(X, y, n):\n",
    "    try:\n",
    "        X = X.drop(\"index\", axis=1)\n",
    "    except:\n",
    "        pass\n",
    "    independentVariables = X\n",
    "    X_train = X\n",
    "    y_train = y\n",
    "    forest = RandomForestClassifier(random_state=1)\n",
    "    forest.fit(X_train, y_train)\n",
    "    importances = forest.feature_importances_\n",
    "    namedFeatures = []\n",
    "    for i in range(0, len(X_train.columns)):\n",
    "        namedFeatures.append((list(X_train.columns)[i], importances[i]))\n",
    "    sortedNamedFeatures = sorted(namedFeatures, key=itemgetter(1))\n",
    "    sortedNamedFeatures.reverse()\n",
    "    nFeatures = sortedNamedFeatures[:n]\n",
    "    finalColNames = []\n",
    "    for elem in nFeatures:\n",
    "        finalColNames.append(elem[0])\n",
    "    return finalColNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "319c568c",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def getXGBoostFeatureColumns(X, y, n):\n",
    "    try:\n",
    "        X = X.drop(\"index\", axis=1)\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        model = XGBClassifier()\n",
    "        model.fit(X, y)\n",
    "        importances = model.feature_importances_\n",
    "        namedFeatures = []\n",
    "        for i in range(0, len(X_train.columns)):\n",
    "            namedFeatures.append((list(X_train.columns)[i], importances[i]))\n",
    "        sortedNamedFeatures = sorted(namedFeatures, key=itemgetter(1))\n",
    "        sortedNamedFeatures.reverse()\n",
    "        nFeatures = sortedNamedFeatures[:n]\n",
    "        finalColNames = []\n",
    "        for elem in nFeatures:\n",
    "            finalColNames.append(elem[0])\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(\"Failed to get xgboost feature columns\")\n",
    "\n",
    "    return finalColNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22e13cbb",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def getExtraTreesFeatureColumns(X, y, n):\n",
    "    independentVariables = X\n",
    "\n",
    "    X_train = X\n",
    "    y_train = y\n",
    "    try:\n",
    "        X = X.drop(\"index\", axis=1)\n",
    "    except:\n",
    "        X = X\n",
    "    forest = ExtraTreesClassifier(random_state=1)\n",
    "    forest.fit(X_train, y_train)\n",
    "    importances = forest.feature_importances_\n",
    "    namedFeatures = []\n",
    "    for i in range(0, len(X_train.columns)):\n",
    "        namedFeatures.append((list(X_train.columns)[i], importances[i]))\n",
    "    sortedNamedFeatures = sorted(namedFeatures, key=itemgetter(1))\n",
    "    sortedNamedFeatures.reverse()\n",
    "\n",
    "    #     xFeatureColNames = []\n",
    "    #     for i in range(0, n):\n",
    "    #         xFeatureColNames.append(sortedNamedFeatures[i][0])\n",
    "    nFeatures = sortedNamedFeatures[:n]\n",
    "    finalColNames = []\n",
    "    for elem in nFeatures:\n",
    "        finalColNames.append(elem[0])\n",
    "    return finalColNames\n",
    "\n",
    "\n",
    "# # Base learner and ensemble generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e673883",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_featured_selected_training_data(method='anova'):\n",
    "    #global X_train, X_test, y_train, y_test\n",
    "        # Select n features------------------------------------------------------------------------\n",
    "    f = feature_parameter_vector\n",
    "    nFeatures = random.choice(f)\n",
    "    if(method=='anova'):\n",
    "        xFeatureColumnNames = getNfeaturesANOVAF(\n",
    "            nFeatures)\n",
    "    \n",
    "    elif(method=='randomforest'):\n",
    "        xFeatureColumnNames = getRandomForestFeatureColumns(\n",
    "        X_train, y_train, nFeatures\n",
    "    )\n",
    "    elif(method=='extratrees'):\n",
    "        xFeatureColumnNames = getExtraTreesFeatureColumns(\n",
    "        X_train, y_train, nFeatures\n",
    "    )\n",
    "    elif(method=='xgb'):\n",
    "        xFeatureColumnNames = getXGBoostFeatureColumns(\n",
    "        X_train, y_train, nFeatures\n",
    "    )\n",
    "        \n",
    "        \n",
    "    X_train_fs = X_train[xFeatureColumnNames].copy()\n",
    "    X_test_fs = X_test[xFeatureColumnNames].copy()\n",
    "    \n",
    "    return X_train_fs, X_test_fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc9f1fca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def debug_base_learner(model, mccscore, X_train, auc_score, model_train_time):\n",
    "    print(str(model).split(\"(\")[0], round(mccscore, 5), len(X_train.columns), auc_score, model_train_time)\n",
    "    if(model_train_time> model_train_time_warning_threshold):\n",
    "        print(\"Warning long train time, \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9066b9f3",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def perceptronModelGen():\n",
    "    start = time.time()\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "\n",
    "    # Initialise parameter space----------------------------------------------------------------\n",
    "    maxIterList = [5, 7, 10, 12, 15, 20, 25, 50, 75]\n",
    "    sc = StandardScaler()\n",
    "    sc.fit(X_train)\n",
    "    # Apply the scaler to the X training data\n",
    "    X_train_std = sc.transform(X_train)\n",
    "\n",
    "    # Apply the SAME scaler to the X test data\n",
    "    X_test_std = sc.transform(X_test)\n",
    "\n",
    "    # Create a perceptron object with the parameters: n iterations (epochs) over the data, and a learning rate of 0.1\n",
    "    model = Perceptron(max_iter=random.choice(\n",
    "        maxIterList), eta0=0.1, random_state=0)\n",
    "\n",
    "    # Train the perceptron--------------------------------------------------------------------\n",
    "    model.fit(X_train_std, y_train)\n",
    "    y_train_hat = model.predict(X_train)# predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97b2f160",
   "metadata": {},
   "outputs": [],
   "source": [
    "#perceptronModelGen()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e5e9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def store_model(mccscore, model, feature_list, model_train_time, auc_score, y_pred, model_type='sklearn'): #**kwargs ):\n",
    "    #print(kwargs)\n",
    "    #torch=kwargs['torch'], \n",
    "    #xgb=kwargs['xgb']\n",
    "    \n",
    "    \n",
    "    with open(model_store_path, 'r') as f:\n",
    "        model_store_data = json.load(f)\n",
    "    \n",
    "    idx = len(model_store_data['models']) + 1\n",
    "    \n",
    "    time_stamp =  time.time_ns() \n",
    "    \n",
    "    \n",
    "    if(model_type=='sklearn'):\n",
    "        model = str(model)\n",
    "    \n",
    "    elif(model_type=='torch'):\n",
    "        y_pred = y_pred.astype(float)\n",
    "        torch.save(model, f=f\"{global_param_str + additional_naming}/\"+\"/torch/\"+str(time_stamp))\n",
    "        model=time_stamp\n",
    "        \n",
    "    elif(model_type=='xgb'):    \n",
    "        pickle.dump(model, open(f\"{global_param_str + additional_naming}/\"+\"/xgb/\"+str(time_stamp), \"wb\"))\n",
    "        model=time_stamp\n",
    "        y_pred = y_pred.astype(float)\n",
    "        \n",
    "    #print(type(model))\n",
    "    scale = global_param_dict.get('scale')\n",
    "    if(scale):\n",
    "        y_pred = y_pred.astype(float)\n",
    "    \n",
    "    model_store_entry = {\n",
    "        'index':idx,\n",
    "        'mcc_score':mccscore,\n",
    "        'model': model, \n",
    "        'feature_list':feature_list,\n",
    "        'model_train_time':model_train_time,\n",
    "        'auc_score':auc_score,\n",
    "        'y_pred': list(y_pred),\n",
    "        'model_type': model_type,\n",
    "\n",
    "    }\n",
    "    #print(model_store_entry)\n",
    "    \n",
    "    model_store_data['models'].update({idx: model_store_entry})\n",
    "    \n",
    "    jsonString = json.dumps(model_store_data)\n",
    "    jsonFile = open(model_store_path, \"w\", encoding='utf-8')\n",
    "    jsonFile.write(jsonString)\n",
    "    jsonFile.close()\n",
    "    \n",
    "    torch.cuda.empty_cache() #exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f7fbc2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stored_model():\n",
    "    \n",
    "    with open(model_store_path, 'r', encoding='utf-8') as f:\n",
    "        model_store_data = json.load(f)\n",
    "    \n",
    "    model_key_list = list(model_store_data['models'].keys())\n",
    "    \n",
    "    try:\n",
    "        model_key = str(random.choice(model_key_list))\n",
    "\n",
    "        print(f\"Returning stored model at index {model_key}/{len(model_key_list)}\")\n",
    "        \n",
    "         \n",
    "        \n",
    "        if(model_store_data['models'].get(model_key)['model_type'] == 'sklearn'):\n",
    "            model = eval(model_store_data['models'].get(model_key)['model'])\n",
    "            \n",
    "        elif(model_store_data['models'].get(model_key)['model_type'] == 'torch'):\n",
    "            time_stamp = model_store_data['models'].get(model_key)['model']\n",
    "            model = torch.load(f=f\"{global_param_str + additional_naming}/\"+\"/torch/\"+str(time_stamp))\n",
    "            \n",
    "        elif(model_store_data['models'].get(model_key)['model_type'] == 'xgb'):\n",
    "            time_stamp = model_store_data['models'].get(model_key)['model']\n",
    "            model = pickle.load(open(f\"{global_param_str + additional_naming}/\"+\"/xgb/\"+str(time_stamp), \"rb\"))\n",
    "            \n",
    "        \n",
    "\n",
    "        return (model_store_data['models'].get(model_key)['mcc_score'],\n",
    "                model,\n",
    "               model_store_data['models'].get(model_key)['feature_list'],\n",
    "               model_store_data['models'].get(model_key)['model_train_time'],\n",
    "               model_store_data['models'].get(model_key)['auc_score'],\n",
    "                np.array(model_store_data['models'].get(model_key)['y_pred'])\n",
    "               )\n",
    "    except Exception as e:\n",
    "        print(\"Failed inside getting stored model, returning random new model\")\n",
    "        index = random.randint(0, len(modelFuncList)-1)\n",
    "\n",
    "\n",
    "        return modelFuncList[index]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48dfd519",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def perceptronModelGen_dummy():\n",
    "    start = time.time()\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "\n",
    "    # Initialise parameter space----------------------------------------------------------------\n",
    "    maxIterList = [5, 7, 10, 12, 15, 20, 25, 50, 75]\n",
    "    sc = StandardScaler()\n",
    "    sc.fit(X_train)\n",
    "    # Apply the scaler to the X training data\n",
    "    X_train_std = sc.transform(X_train)\n",
    "\n",
    "    # Apply the SAME scaler to the X test data\n",
    "    X_test_std = sc.transform(X_test)\n",
    "\n",
    "    # Create a perceptron object with the parameters: n iterations (epochs) over the data, and a learning rate of 0.1\n",
    "    model = Perceptron(max_iter=random.choice(\n",
    "        maxIterList), eta0=0.1, random_state=0)\n",
    "\n",
    "    # Train the perceptron--------------------------------------------------------------------\n",
    "    model.fit(X_train_std, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_train_hat = model.predict(X_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    \n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            #print(sample_parameter_space)\n",
    "#     if(store_base_learners):\n",
    "#         store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred) #Don't store dummy\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaaadab4",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def extraTreesModelGen():\n",
    "    start = time.time()\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    seed = 7\n",
    "    num_trees = 30\n",
    "    max_features_n = random.choice(list(np.arange(0.05, 0.2, 0.001)))\n",
    "    min_samples_leaf_n = random.choice([2, 5, 8, 10, 15, 20, 25, 40, 50])\n",
    "    n_estimators_n = random.choice(\n",
    "        [\n",
    "            10,\n",
    "            20,\n",
    "            30,\n",
    "            40,\n",
    "            50,\n",
    "            60,\n",
    "            70,\n",
    "            80,\n",
    "            90,\n",
    "            100,\n",
    "            125,\n",
    "            150,\n",
    "            175,\n",
    "            200,\n",
    "            225,\n",
    "            250,\n",
    "            275,\n",
    "            300,\n",
    "            325,\n",
    "            350,\n",
    "            375,\n",
    "            400,\n",
    "            425,\n",
    "            450,\n",
    "            475,\n",
    "            500,\n",
    "            550,\n",
    "            600,\n",
    "            650,\n",
    "            700,\n",
    "        ]\n",
    "    )\n",
    "    max_depth_n = random.choice([2, 4, 8, 10, None])\n",
    "    class_weight_n = random.choice([\"balanced\"])\n",
    "\n",
    "    # warm_start = True\n",
    "    model = ExtraTreesClassifier(\n",
    "        n_estimators=n_estimators_n,\n",
    "        max_features=max_features_n,\n",
    "        min_samples_leaf=min_samples_leaf_n,\n",
    "        max_depth=max_depth_n,\n",
    "        class_weight=class_weight_n,\n",
    "    )\n",
    "\n",
    "    # Fit model-------------------------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "    y_train_hat = model.predict(X_train)\n",
    "    score = model.score(X_test, y_test)\n",
    "    # print(score)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    \n",
    "    \n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "316f071f",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def logModelGen():\n",
    "    start = time.time()\n",
    "    \n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    solver_n = random.choice([\"sag\"])\n",
    "    class_weight_n = random.choice([\"balanced\"])\n",
    "    max_iter_n = random.choice([5, 7, 10, 12, 15, 20, 25, 50, 75])\n",
    "    C_n = random.choice([1e-5, 1e-4, 1e-3, 1e-2, 1e1,\n",
    "                        1, 1e1, 1e2, 1e3, 1e4, 1e5])\n",
    "    model = LogisticRegression(\n",
    "        solver=solver_n, class_weight=class_weight_n, max_iter=max_iter_n, C=C_n\n",
    "    )\n",
    "\n",
    "    # Fit model---------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "    y_train_hat = model.predict(X_train)\n",
    "    score = model.score(X_test, y_test)\n",
    "    # print(score)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            #print(sample_parameter_space)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce237a1d",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def randomForestModelGen():\n",
    "    start = time.time()\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='randomforest')\n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    num_trees = 30\n",
    "    max_features_n = random.choice(list(np.arange(0.05, 0.2, 0.001)))\n",
    "    min_samples_leaf_n = random.choice([2, 5, 8, 10, 15, 20, 25, 40, 50])\n",
    "    n_estimators_n = random.choice(\n",
    "        [\n",
    "            10,\n",
    "            20,\n",
    "            30,\n",
    "            40,\n",
    "            50,\n",
    "            60,\n",
    "            70,\n",
    "            80,\n",
    "            90,\n",
    "            100,\n",
    "            125,\n",
    "            150,\n",
    "            175,\n",
    "            200,\n",
    "            225,\n",
    "            250,\n",
    "            275,\n",
    "            300,\n",
    "            325,\n",
    "            350,\n",
    "            375,\n",
    "            400,\n",
    "            425,\n",
    "            450,\n",
    "            475,\n",
    "            500,\n",
    "            550,\n",
    "            600,\n",
    "            650,\n",
    "            700,\n",
    "        ]\n",
    "    )\n",
    "    max_depth_n = random.choice([2, 4, 8, 10, None])\n",
    "    class_weight_n = random.choice([\"balanced\"])\n",
    "\n",
    "    # warm_start = True\n",
    "    model = RandomForestClassifier(\n",
    "        n_estimators=n_estimators_n,\n",
    "        max_features=max_features_n,\n",
    "        min_samples_leaf=min_samples_leaf_n,\n",
    "        max_depth=max_depth_n,\n",
    "        class_weight=class_weight_n,\n",
    "    )\n",
    "\n",
    "    # Fit model-------------------------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "    y_train_hat = model.predict(X_train)\n",
    "    score = model.score(X_test, y_test)\n",
    "    # print(score)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4367db53",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def kNearestNeighborsModelGen(*kwargs):\n",
    "    \n",
    "    start = time.time()\n",
    "    #print(\"Calling knn at \", start)\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    n_neighbours_n = random.choice(\n",
    "        [2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 50, 100]\n",
    "    )\n",
    "    \n",
    "    mccscore = 0\n",
    "    auc_score = 0.5\n",
    "    model_train_time = 0\n",
    "\n",
    "    if(n_neighbours_n > len(X_train)):\n",
    "        n_neighbours_n = len(X_train)-1\n",
    "        print(\"warning kNearestNeighborsModelGen\", \"nn > sample\")\n",
    "    \n",
    "    \n",
    "    weights_n = random.choice([\"uniform\", \"distance\"])\n",
    "    try:\n",
    "        #print(\"Creating knn model\")\n",
    "         # warm_start = True\n",
    "        model = KNeighborsClassifier(n_neighbors=n_neighbours_n, weights=weights_n, n_jobs=1)\n",
    "    except Exception as e:\n",
    "        print(\"Failed to fit knn\")\n",
    "        print(e)\n",
    "   \n",
    "    try:\n",
    "        #print(\"Fitting knn\")\n",
    "        # Fit model-------------------------------------------------------------------------------------\n",
    "        #print(kwargs)\n",
    "        model.fit(X_train, y_train)\n",
    "    except Exception as e:\n",
    "        print(\"Failed to fit knn\")\n",
    "        print(e)\n",
    "\n",
    "        \n",
    "    #score = model.score(X_test, y_test)\n",
    "    # print(score)\n",
    "    #print(model.__dict__)\n",
    "    \n",
    "    \n",
    "    try:    \n",
    "        #print(\"predicting x test\")\n",
    "        #print(kwargs)\n",
    "        #display(X_test)\n",
    "        #y_train_hat = model.predict(X_train)\n",
    "        y_pred = model.predict(X_test)\n",
    "    except Exception as e:\n",
    "        print(\"Failed to predict knn\")\n",
    "        print(e)\n",
    "    \n",
    "    try:\n",
    "        #print(\"matthews scoring\")\n",
    "        #print(kwargs)\n",
    "        mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    try:\n",
    "        #print(\"Auc scoring\")\n",
    "        #print(kwargs)\n",
    "        auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9355dada",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def elasticNeuralNetworkModelGen():\n",
    "    start = time.time()\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    alpha_n = random.choice(\n",
    "        [1, 0.5, 0.1, 0.05, 0.04, 0.03, 0.02, 0.01, 0.005, 0.001, 0.0001]\n",
    "    )\n",
    "    max_iter_n = random.choice([5, 7, 10, 12, 15, 20, 25, 50, 75])\n",
    "    l1_ratio_n = random.choice(\n",
    "        [0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1])\n",
    "    loss_n = random.choice([\"log\"])\n",
    "    penalty_n = random.choice([\"elasticnet\"])\n",
    "    class_weight_n = random.choice([\"balanced\"])\n",
    "    shuffle_n = random.choice([True])\n",
    "\n",
    "    # warm_start = True\n",
    "    model = OneVsRestClassifier(\n",
    "        ElasticNet(\n",
    "            alpha=alpha_n,  # untested OneVsRestClassifier\n",
    "            max_iter=max_iter_n,\n",
    "            l1_ratio=l1_ratio_n,\n",
    "        )\n",
    "    )\n",
    "\n",
    "    # Fit model-------------------------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "    y_train_hat = model.predict(X_train)\n",
    "    score = model.score(X_test, y_test)\n",
    "    # print(score)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce0c4e45",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def svcModelGen():\n",
    "    start = time.time()\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "\n",
    "    # warm_start = True\n",
    "    model = SVC(c=1)\n",
    "\n",
    "    # penalty = penalty_n,\n",
    "    # missing loss = loss_n,\n",
    "    \n",
    "    \n",
    "    #dual coefficients or intercepts are not finite\n",
    "    try:\n",
    "        # Fit model-------------------------------------------------------------------------------------\n",
    "        model.fit(X_train, y_train)\n",
    "        y_train_hat = model.predict(X_train)\n",
    "        score = model.score(X_test, y_test)\n",
    "        # print(score)\n",
    "        y_pred = model.predict(X_test)\n",
    "        mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "        auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        if(debug):\n",
    "            debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    \n",
    "        return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(X_train.columns)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        \n",
    "        \n",
    "        #return 0 dud random prediction\n",
    "        return (0, model, list(X_train.columns), model_train_time, 0.5, (np.random.choice(a=[False, True], size=(len(y_test,)))).astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5417d338",
   "metadata": {},
   "outputs": [],
   "source": [
    "#XGBoostModelGen()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "723c443d",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def XGBoostModelGen(*kwargs):\n",
    "    \n",
    "    counter = 0\n",
    "    \n",
    "    start = time.time()\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='xgb')\n",
    "    \n",
    "    \n",
    "    # Initialise parameter space-----------------------------------------------------------------\n",
    "    gamma_n = random.choice([0.01, 0.1, 1, 3, 5, 7, 9, 10, 15])\n",
    "    reg_alpha_n = random.choice([0, 0.001, 0.005, 0.01, 0.1, 1, 3, 5])\n",
    "    reg_gamma_n = random.choice([0, 0.001, 0.005, 0.01, 0.1, 1, 3, 5])\n",
    "    learning_rate_n = random.choice(\n",
    "        [0.5, 0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]\n",
    "    )\n",
    "    subsample_n = random.choice([1, 0.98, 0.95, 0.9, 0.85, 0.8, 0.75, 0.7])\n",
    "    colsample_bytree_n = random.choice(\n",
    "        [1, 0.98, 0.95, 0.9, 0.85, 0.8, 0.75, 0.7])\n",
    "    max_depth_n = random.choice([3, 4, 5, 6, 7, 8, 9, 10, 15])\n",
    "    min_child_weight_n = random.choice(\n",
    "        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]\n",
    "    )\n",
    "    n_estimators_n = random.choice(\n",
    "        [\n",
    "            10,\n",
    "            20,\n",
    "            30,\n",
    "            40,\n",
    "            50,\n",
    "            60,\n",
    "            70,\n",
    "            80,\n",
    "            90,\n",
    "            100,\n",
    "            125,\n",
    "            150,\n",
    "            175,\n",
    "            200,\n",
    "            225,\n",
    "            250,\n",
    "            275,\n",
    "            300,\n",
    "            325,\n",
    "            350,\n",
    "            375,\n",
    "            400,\n",
    "            425,\n",
    "            450,\n",
    "            475,\n",
    "            500,\n",
    "            550,\n",
    "            600,\n",
    "            650,\n",
    "            700,\n",
    "        ]\n",
    "    )\n",
    "    #gpu_id_n = random.randint(0, 7)\n",
    "    gpu_id_n = get_free_gpu()\n",
    "    \n",
    "    #print(f\"{kwargs}defining xgb {gpu_id_n}\")\n",
    "    try:\n",
    "        # warm_start = True\n",
    "        model = XGBClassifier(\n",
    "            gamma=gamma_n,\n",
    "            reg_alpha=reg_alpha_n,\n",
    "            # reg_gamma = reg_gamma_n,   #unused?\n",
    "            learning_rate=learning_rate_n,\n",
    "            subsample=subsample_n,\n",
    "            colsample_bytree=colsample_bytree_n,\n",
    "            max_depth=max_depth_n,\n",
    "            min_child_weight=min_child_weight_n,\n",
    "            n_estimators=n_estimators_n,\n",
    "            tree_method=\"gpu_hist\",\n",
    "            gpu_id = gpu_id_n,\n",
    "            #nthread=1,\n",
    "            #silent=True,\n",
    "            verbosity = 0,\n",
    "            eval_metric = 'logloss',\n",
    "            #max_workers=10\n",
    "        )\n",
    "\n",
    "        #print(f\"{kwargs}fitting xgb \")\n",
    "        # Fit model-------------------------------------------------------------------------------------\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        y_pred = model.predict(X_test)\n",
    "        #print(f\"{kwargs}getting mccscore \")\n",
    "        mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "\n",
    "        #print(f\"{kwargs}getting auc_score \")\n",
    "        auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        if(debug):\n",
    "            print(str(model).split(\"(\")[0], round(mccscore, 5), len(X_train.columns), auc_score, model_train_time)\n",
    "            if(model_train_time> model_train_time_warning_threshold):\n",
    "                print(\"Warning long train time, \")\n",
    "                \n",
    "        #print(type(model))\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to run XGBoost on GPU {gpu_id_n}\") \n",
    "        print(e)\n",
    "        # warm_start = True\n",
    "        model = XGBClassifier(\n",
    "            gamma=gamma_n,\n",
    "            reg_alpha=reg_alpha_n,\n",
    "            # reg_gamma = reg_gamma_n,   #unused?\n",
    "            learning_rate=learning_rate_n,\n",
    "            subsample=subsample_n,\n",
    "            colsample_bytree=colsample_bytree_n,\n",
    "            max_depth=max_depth_n,\n",
    "            min_child_weight=min_child_weight_n,\n",
    "            n_estimators=n_estimators_n,\n",
    "            tree_method=\"hist\",\n",
    "            #gpu_id = gpu_id_n,\n",
    "            #nthread=1,\n",
    "            #silent=True,\n",
    "            verbosity = 0,\n",
    "            eval_metric = 'logloss',\n",
    "            #max_workers=10\n",
    "        )\n",
    "\n",
    "        #print(f\"{kwargs}fitting xgb \")\n",
    "        # Fit model-------------------------------------------------------------------------------------\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        y_pred = model.predict(X_test)\n",
    "        #print(f\"{kwargs}getting mccscore \")\n",
    "        mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "\n",
    "        #print(f\"{kwargs}getting auc_score \")\n",
    "        auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        if(debug):\n",
    "            debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "                \n",
    "        if(store_base_learners):\n",
    "            try:\n",
    "                store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred, model_type = 'xgb')\n",
    "            except Exception as e:\n",
    "                print(e, 'store base learners')\n",
    "                \n",
    "        return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "\n",
    "    if(store_base_learners):\n",
    "        try:\n",
    "            store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred, model_type = 'xgb')\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "\n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1441ae20",
   "metadata": {},
   "outputs": [],
   "source": [
    "#XGBoostModelGen()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85bd1717",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def DecisionTreeClassifier_ModelGen():\n",
    "    start = time.time()\n",
    "    \n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"ccp_alpha\": log_small,\n",
    "        \"class_weight\": [None, \"balanced\"]\n",
    "        + [{0: w} for w in [1, 2, 4, 6, 10]],  # enumerate?\n",
    "        \"criterion\": [\"gini\", \"entropy\" ],# \"log_loss\"],\n",
    "        \"max_depth\": log_large_long,\n",
    "        \"max_features\": [\"sqrt\", \"log2\"], #\"auto\", \n",
    "        'max_leaf_nodes': log_large_long,\n",
    "        \"min_impurity_decrease\": log_small,\n",
    "        'min_samples_leaf': log_med,\n",
    "        'min_samples_split': lin_zero_one,\n",
    "        \"min_weight_fraction_leaf\": log_small,\n",
    "        'random_state': [None],\n",
    "        \"splitter\": [\"best\", \"random\"],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = DecisionTreeClassifier(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    #y_train_hat = model.predict(X_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d02241f4",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def AdaBoostClassifier_ModelGen():\n",
    "    start = time.time()\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"algorithm\": [\"SAMME.R\", \"SAMME\"],\n",
    "        \"base_estimator\": [None],\n",
    "        \"learning_rate\": log_small,\n",
    "        \"n_estimators\": log_large_long,\n",
    "        \"random_state\": [None],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = AdaBoostClassifier(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e68719e2",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def GaussianNB_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    new_list = list(log_small).copy()\n",
    "    new_list.append(1e-09)\n",
    "    parameter_space = {\n",
    "        \"priors\": [\n",
    "            None,\n",
    "            [0.1, 0.9],\n",
    "            [0.9, 0.1],\n",
    "            [0.7, 0.3],\n",
    "            [0.3, 0.7],\n",
    "            [0.5, 0.5],\n",
    "            [0.6, 0.4],\n",
    "            [0.4, 0.6],\n",
    "        ],  # enumerate\n",
    "        \"var_smoothing\": new_list,\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = GaussianNB(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7529291",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def QuadraticDiscriminantAnalysis_ModelGen():\n",
    "    start = time.time()\n",
    "    \n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"priors\": [None],\n",
    "        \"reg_param\": log_small,\n",
    "        \"store_covariance\": [False],\n",
    "        \"tol\": log_small,\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = QuadraticDiscriminantAnalysis(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a909067d",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def SVC_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "\n",
    "    # Import base data set and outcome variable-----------------------------------------------\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"C\": log_small,\n",
    "        \"break_ties\": bool_param,\n",
    "        'cache_size': [200],\n",
    "        'class_weight': [None, 'balanced'] + [{0: w} for w in [1, 2, 4, 6, 10]], # enumerate class weight\n",
    "        \"coef0\": log_small,\n",
    "        \"decision_function_shape\": [\"ovr\"],  # , 'ovo'\n",
    "        \"degree\": log_med,\n",
    "        \"gamma\": [\"scale\", \"auto\"],\n",
    "        \"kernel\": [\"rbf\", \"linear\", \"poly\", \"sigmoid\"],\n",
    "        \"max_iter\": log_large_long,\n",
    "        'probability': [False],\n",
    "        'random_state': [None],\n",
    "        \"shrinking\": bool_param,\n",
    "        \"tol\": log_small,\n",
    "        \"verbose\": [False],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = SVC(**sample_parameter_space)\n",
    "    \n",
    "    try:\n",
    "        # Train the model--------------------------------------------------------------------\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        # predict\n",
    "        y_pred = model.predict(X_test)\n",
    "        mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "\n",
    "        auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        if(debug):\n",
    "            debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "        if(store_base_learners):\n",
    "            store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)        \n",
    "        \n",
    "        return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        end = time.time()\n",
    "        model_train_time = int(end-start)\n",
    "        return (0, model, list(X_train.columns), model_train_time, 0.5, (np.random.choice(a=[False, True], size=(len(y_test,))).astype(int)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8e09721",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def GradientBoostingClassifier_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "    parameter_space = {\n",
    "        \"ccp_alpha\": log_small,\n",
    "        \"criterion\": [\"friedman_mse\"],\n",
    "        \"init\": [None],\n",
    "        \"learning_rate\": log_small,\n",
    "        #\"loss\": [\"log_loss\", \"exponential\"],\n",
    "        'max_depth': log_med,\n",
    "        \"max_features\": [1.0, \"sqrt\", \"log2\"],\n",
    "        'max_leaf_nodes': log_large_long,\n",
    "        'min_impurity_decrease': log_small,\n",
    "        'min_samples_leaf': log_med,\n",
    "        'min_samples_split': lin_zero_one,\n",
    "        'min_weight_fraction_leaf': log_small,\n",
    "        \"n_estimators\": log_large_long,\n",
    "        'n_iter_no_change': log_large_long,\n",
    "        \"random_state\": [None],\n",
    "        \"subsample\": lin_zero_one,\n",
    "        \"tol\": log_small,\n",
    "        \"validation_fraction\": [0.1],\n",
    "        \"verbose\": [0],\n",
    "        \"warm_start\": [0],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = GradientBoostingClassifier(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e75526be",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def RandomForestClassifier_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"bootstrap\": bool_param,\n",
    "        \"ccp_alpha\": log_small,\n",
    "        'class_weight': [None, 'balanced'] + [{0: w} for w in [1, 2, 4, 6, 10]],\n",
    "        \"criterion\": [\"gini\", \"entropy\" ],# , \"log_loss\"],\n",
    "        \"max_depth\": log_med,\n",
    "        \"max_features\": [1.0, \"sqrt\", \"log2\"],\n",
    "        'max_leaf_nodes': log_large_long,\n",
    "        \"max_samples\": [None],\n",
    "        'min_impurity_decrease': log_small,\n",
    "        \"min_samples_leaf\": log_med,\n",
    "        \"min_samples_split\": lin_zero_one,\n",
    "        'min_weight_fraction_leaf': log_small,\n",
    "        \"n_estimators\": log_large_long,\n",
    "        \"n_jobs\": [-1],\n",
    "        \"oob_score\": [False],\n",
    "        \"random_state\": [None],\n",
    "        \"verbose\": [0],\n",
    "        \"warm_start\": [False],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = RandomForestClassifier(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    #print(plot_cf_matrix(y_test, y_pred))\n",
    "    \n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb83d3e0",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def MLPClassifier_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        \"activation\": [\"relu\"],\n",
    "        \"alpha\": log_small,\n",
    "        \"batch_size\": [\"auto\"],\n",
    "        'beta_1': log_small,\n",
    "        'beta_2': log_small,\n",
    "        'early_stopping': bool_param,\n",
    "        'epsilon': log_small,\n",
    "        \"hidden_layer_sizes\": [5, 10, 50, 100],#log_large_long, # Possible DGX hang on 630\n",
    "        \"learning_rate\": [\"constant\"], #, \"adaptive\"],\n",
    "        'learning_rate_init': log_small,\n",
    "        'max_fun': [15000],\n",
    "        'max_iter': log_large_long,\n",
    "        \"momentum\": lin_zero_one,\n",
    "        'n_iter_no_change': log_large_long,\n",
    "        'nesterovs_momentum': [True],\n",
    "        'power_t': [0.5],\n",
    "        \"random_state\": [None],\n",
    "        \"shuffle\": bool_param,\n",
    "        'solver': ['adam', 'lbfgs', 'sgd'],\n",
    "        \"tol\": log_small,\n",
    "        \"validation_fraction\": [0.1],\n",
    "        \"verbose\": [False],\n",
    "        \"warm_start\": [False],\n",
    "    }\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    # fit model with random sample of global parameter space\n",
    "    #display(sample_parameter_space)\n",
    "    model = MLPClassifier(**sample_parameter_space)\n",
    "\n",
    "    # Train the model--------------------------------------------------------------------\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # predict\n",
    "    y_pred = model.predict(X_test)\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "    if(store_base_learners):\n",
    "        store_model(mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3990f0ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy_model_gen():\n",
    "    \n",
    "    mccscore = 0.5\n",
    "    #model = fitted_perceptron\n",
    "    model_train_time = 0\n",
    "    auc_score = 0.5\n",
    "    auc_score = random.uniform(0.5, 1)\n",
    "    #y_pred = y_test_orig\n",
    "    y_pred = y_test\n",
    "    y_pred = np.random.choice(a=[False, True], size=(len(y_test,)))\n",
    "    \n",
    "    \n",
    "    return (mccscore, fitted_perceptron, dummy_columns, model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93676f04",
   "metadata": {},
   "outputs": [],
   "source": [
    "## train data\n",
    "class TrainData(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data, y_data):\n",
    "        self.X_data = X_data\n",
    "        self.y_data = y_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index], self.y_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "\n",
    "\n",
    "\n",
    "## test data    \n",
    "class TestData(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data):\n",
    "        self.X_data = X_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ed471ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#should deep model evaluation metric be mcc...?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d018d9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_acc(y_pred, y_test):\n",
    "    y_pred_tag = torch.round(torch.sigmoid(y_pred))\n",
    "\n",
    "    correct_results_sum = (y_pred_tag == y_test).sum().float()\n",
    "    acc = correct_results_sum/y_test.shape[0]\n",
    "    acc = torch.round(acc * 100)\n",
    "    \n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec8d0954",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BinaryClassification(nn.Module):\n",
    "    def __init__(self, column_length, deep_layers_1, batch_size, dropout_val):\n",
    "        super(BinaryClassification, self).__init__()\n",
    "        # Number of input features is 12.\n",
    "        self.layer_1 = nn.Linear(column_length, batch_size) \n",
    "        self.layer_2 = nn.Linear(batch_size, batch_size)\n",
    "        layers = []\n",
    "        for i in range(0, deep_layers_1):\n",
    "            layers.append(nn.Linear(batch_size, batch_size))\n",
    "\n",
    "        self.deep_layers = nn.Sequential(*layers)\n",
    "    \n",
    "        \n",
    "        self.layer_out = nn.Linear(batch_size, 1) \n",
    "        \n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(p=dropout_val)\n",
    "        self.batchnorm1 = nn.BatchNorm1d(batch_size)\n",
    "        self.batchnorm2 = nn.BatchNorm1d(batch_size)\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        x = self.relu(self.layer_1(inputs))\n",
    "        x = self.batchnorm1(x)\n",
    "        x = self.relu(self.layer_2(x))\n",
    "        x = self.relu(self.deep_layers(x))\n",
    "        x = self.batchnorm2(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.layer_out(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca4a403e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def Pytorch_binary_class_ModelGen():\n",
    "    start = time.time()\n",
    "    # Feature selection supposed to be done each iteration of cross validation\n",
    "\n",
    "    X_train, X_test = get_featured_selected_training_data(method='anova')\n",
    "    \n",
    "    if(scale==False):\n",
    "        scaler = StandardScaler()\n",
    "        X_train = scaler.fit_transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "\n",
    "    train_data = TrainData(torch.FloatTensor(X_train.to_numpy()), \n",
    "                           torch.FloatTensor(y_train.to_numpy()))    \n",
    "    test_data = TestData(torch.FloatTensor(X_test.to_numpy()))\n",
    "\n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        'column_length':[len(X_train.columns)],\n",
    "       #'epochs': [50, 200],\n",
    "    'batch_size': [int(X.shape[0]/100), int(X.shape[0]/200)],\n",
    "    #'learning_rate': lr_space,\n",
    "    #'learning_rate': [0.1, 0.001, 0.0005, 0.0001],\n",
    "    'deep_layers_1':[2, 4, 8, 16, 32],\n",
    "    'dropout_val':[0.1, 0.01, 0.001]\n",
    "    }\n",
    "    \n",
    "    additional_grid = {\n",
    "        'epochs': [10, 50, 100],\n",
    "    'learning_rate': [0.1, 0.001, 0.0005, 0.0001]\n",
    "    }\n",
    "    size_test = []\n",
    "    # Loop over al grid search combinations\n",
    "    for values in itertools.product(*additional_grid.values()):\n",
    "        point = dict(zip(additional_grid.keys(), values))\n",
    "        # merge the general settings\n",
    "        settings = { **point}\n",
    "        #print(settings)\n",
    "        size_test.append(settings)\n",
    "\n",
    "    #print(len(size_test))\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "        \n",
    "    additional_param_sample = random.choice(size_test)\n",
    "        \n",
    "    additional_param_sample = {}\n",
    "    for key in additional_grid.keys():\n",
    "        additional_param_sample[key] = random.choice(additional_grid.get(key))    \n",
    "        \n",
    "        \n",
    "    print(sample_parameter_space)\n",
    "    \n",
    "    print(additional_param_sample)\n",
    "        \n",
    "        \n",
    "    free_gpu = str(get_free_gpu())\n",
    "    \n",
    "    #os.environ[\"CUDA_VISIBLE_DEVICES\"]=free_gpu\n",
    "    \n",
    "    device = torch.device(f\"cuda:{free_gpu}\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(device)\n",
    "    \n",
    "    train_loader = DataLoader(dataset=train_data, batch_size=sample_parameter_space['batch_size'], shuffle=True)\n",
    "    test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "    \n",
    "    \n",
    "    # fit model with random sample of global parameter space\n",
    "    model = BinaryClassification(**sample_parameter_space)\n",
    "    \n",
    "    model.to(device)\n",
    "    #print(model)    \n",
    "        \n",
    "        \n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=additional_param_sample['learning_rate'])\n",
    "    model.train()\n",
    "    for e in range(1, additional_param_sample['epochs']+1):\n",
    "        epoch_loss = 0\n",
    "        epoch_acc = 0\n",
    "        for X_batch, y_batch in train_loader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            y_pred = model(X_batch)\n",
    "\n",
    "            loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "            acc = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "\n",
    "\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f}')\n",
    "        \n",
    "    para_str = (str(settings).replace(\"'\", \"_\").replace(\":\", \"_\").replace(\",\", \"_\").replace(\"{\", \"_\").replace(\"}\", \"_\").replace(\" \", \"_\")).replace(\"__\", \"_\")    \n",
    "        \n",
    "    y_pred = model(test_data.X_data.to(device))\n",
    "    \n",
    "    y_pred = torch.round(torch.sigmoid(y_pred)).cpu().detach().numpy().flatten()\n",
    "    \n",
    "    \n",
    "    \n",
    "    if(any(np.isnan(y_pred))):\n",
    "        print(\"Torch model nan, returning random y pred vector\")\n",
    "        #zero_vector = [x for x in range(0, len(y_pred))]\n",
    "        #y_pred = zero_vector\n",
    "        random_y_pred_vector = (np.random.choice(a=[False, True], size=(len(y_test,)))).astype(int)\n",
    "        y_pred = random_y_pred_vector\n",
    "    else:\n",
    "        #plot_auc(y_hat, f\"Deep ANN Torch {para_str}\")    \n",
    "        pass\n",
    "        \n",
    "\n",
    "    mccscore = matthews_corrcoef(y_test, y_pred)\n",
    "    \n",
    "    auc_score = round(metrics.roc_auc_score(y_test,y_pred), 4)\n",
    "    \n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "            \n",
    "    if(store_base_learners):\n",
    "        try:\n",
    "            store_model(mccscore, model , list(X_train.columns), model_train_time, auc_score, y_pred,  model_type = 'torch')\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    \n",
    "    torch.cuda.empty_cache() #exp\n",
    "    \n",
    "    return (mccscore, model, list(X_train.columns), model_train_time, auc_score, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc625d91",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "outputs": [],
   "source": [
    "\n",
    "#v3\n",
    "def do_work(n):\n",
    "\n",
    "    index = random.randint(0, len(modelFuncList)-1)\n",
    "\n",
    "    try:\n",
    "        if(use_stored_base_learners and random.random()>0.5):\n",
    "            \n",
    "            return get_stored_model()\n",
    "\n",
    "        else:\n",
    "            return modelFuncList[index]()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(f\"Failed to return model at index {index}, returning perceptron\")\n",
    "        return modelFuncList[1]()\n",
    "    \n",
    "    #return random.choice(modelFuncList)\n",
    "\n",
    "from multiprocessing import Process, Manager\n",
    "import warnings\n",
    "#warnings.filterwarnings(\"RuntimeWarning\")\n",
    "#warnings.filterwarnings('error')\n",
    "np.seterr(all=\"ignore\")\n",
    "\n",
    "def multi_run_wrapper(args):\n",
    "           return do_work(*args)\n",
    "\n",
    "def ensembleGenerator(nb_val=28):\n",
    "    #print(\"ensembleGenerator: \", nb_val)\n",
    "    \n",
    "    #nb_val = random.randint(2, nb_val) # dynamic individual size\n",
    "    max_Value = nb_val - 2\n",
    "    skewness = +5\n",
    "    random_val = skewnorm.rvs(a = skewness,loc=max_Value, size=10000)  #Skewnorm function\n",
    "    \n",
    "    random_val = random_val - min(random_val)      #Shift the set so the minimum value is equal to zero.\n",
    "    random_val = random_val / max(random_val)      #Standadize all the vlues between 0 and 1. \n",
    "    random_val = random_val * max_Value         #Multiply the standardized values by the maximum value.\n",
    "\n",
    "    random_val = random_val.astype(int)  +2\n",
    "\n",
    "    nb_val = np.random.choice(random_val)\n",
    "    \n",
    "    \n",
    "\n",
    "    ensemble = []\n",
    "\n",
    "    \n",
    "    dummy_list = [x for x in range(0, nb_val)]\n",
    "   \n",
    "\n",
    "\n",
    "    if(nb_val>1):\n",
    "                            \n",
    "#         \n",
    "        if __name__ == \"__main__\":\n",
    "            from eventlet import GreenPool\n",
    "            pool = GreenPool()\n",
    "            for _ in tqdm(pool.imap(do_work, dummy_list), total=len(dummy_list)):\n",
    "                ensemble.append(_)\n",
    "                pass    \n",
    "\n",
    "            if(len(ensemble)!=nb_val):\n",
    "\n",
    "                print(f\"Error generating ensemble {len(ensemble)} {nb_val} {len(dummy_list)}\")\n",
    "                raise Exception(f\"Error generating ensemble {len(ensemble)} {nb_val} {len(dummy_list)}\")\n",
    "\n",
    "\n",
    "\n",
    "        #print(\"Returning ensemble of size \", len(ensemble))\n",
    "        return ensemble\n",
    "    else:\n",
    "        print(\"Nb_val passed <1 Returning individual of size from baseLearnerGenerator\", nb_val)\n",
    "        return baseLearnerGenerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "879406c3",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def baseLearnerGenerator():\n",
    "    \n",
    "    index = random.randint(0, len(modelFuncList)-1)\n",
    "\n",
    "    return modelFuncList[index]() #store as functions, pass as result of executed function\n",
    "\n",
    "\n",
    "# Model will be fit in generation stage and pass fitted state with training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63829fa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure_binary_vector_diversity(ensemble, metric='jaccard'):\n",
    "    #beta\n",
    "    #can select any from scipy spatial distance pdist\n",
    "    #if two datasets have a Jaccard Similarity of 80% then they would have a Jaccard distance of 1 – 0.8 = 0.2 or 20%\n",
    "    #closer to zero, the more similar the vectors\n",
    "    #if all are similar the less diverse they are\n",
    "    ##assume mcc is 0.5, n = 2\n",
    "    #then (0.5 * 0.1)*2  = 0.1\n",
    "    #and  (0.5 * 0.5)*2) = 0.5\n",
    "    #needs diversity_parameter to modify strength?\n",
    "    \n",
    "    n_y_pred = len(ensemble[0]) #check level\n",
    "    \n",
    "    all_y_pred_arrays = []\n",
    "    \n",
    "    for i in range(0, n_y_pred):\n",
    "        \n",
    "        all_y_pred_arrays.append(ensemble[0][i][5])\n",
    "    \n",
    "    distance_vector = scipy.spatial.distance.pdist(all_y_pred_arrays,  \n",
    "                                       metric=metric)\n",
    "    \n",
    "\n",
    "    return np.mean(distance_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa7d9133",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def evaluate(individual):\n",
    "    # Evaluate ensemble\n",
    "    #X_test = X_test_orig.copy()\n",
    "    #y_test = y_test_orig.copy()\n",
    "\n",
    "    mccScoresList = []\n",
    "    \n",
    "    aucScoresList = []\n",
    "    \n",
    "    print(\"Evaluating individual of size: \", len(individual[0]))\n",
    "    \n",
    "    individual_data = individual[0]\n",
    "    \n",
    "    with open(file_path+\"/progress_logs_scores/\"+current_log_scores_filename, \"a\") as myfile:\n",
    "            myfile.write(f\"Evaluating individual of size: {str(len(individual[0]))}\"\n",
    "                        )\n",
    "    myfile.close()\n",
    "\n",
    "    for modelSet in individual_data:\n",
    "        #print(modelSet)\n",
    "        \n",
    "        try:\n",
    "\n",
    "            mccscore = modelSet[0]\n",
    "\n",
    "            mccScoresList.append(mccscore)\n",
    "            \n",
    "            aucscore = modelSet[4]\n",
    "            \n",
    "            aucScoresList.append(aucscore)\n",
    "\n",
    "            try:\n",
    "                mod_name = str(modelSet[1]).split(\"(\")[0]\n",
    "                #mod_name = modelSet[0][1]\n",
    "            except:\n",
    "                mod_name = modelSet[1]\n",
    "            if(debug):\n",
    "                print(\"Evaluate: \", mod_name, \"mccscore: \", mccscore, \"AUC:\", aucscore) \n",
    "            with open(file_path+\"/progress_logs_scores/\"+current_log_scores_filename, \"a\") as myfile:\n",
    "                myfile.write(f\"\\n Evaluate: mccscore:  {mccscore} AUC: {aucscore} {mod_name}\"\n",
    "                        )\n",
    "            myfile.close()\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(\"Failed, writing zero for mcc score\")\n",
    "            mccScoresList.append(0)\n",
    "    print(\n",
    "        \"Individuals mccscores list\",\n",
    "        mccScoresList)\n",
    "        \n",
    "        \n",
    "    print(\"Mean of mccscores:\",\n",
    "        (np.mean(np.array(mccScoresList))),\n",
    "          'n =',\n",
    "           len(mccScoresList)\n",
    "    )\n",
    "    #print(\"AUC scores:\", aucScoresList, \"mean AUC:\", str(round(np.mean(np.array(aucScoresList)),4)))\n",
    "    \n",
    "    print(f\"Mean MCC scores: {(round(np.mean(np.array(mccScoresList)), 4))} mean AUC: {(round(np.mean(np.array(aucScoresList)),4))} \\n\")\n",
    "    \n",
    "    with open(file_path+\"/progress_logs_scores/\"+current_log_scores_filename, \"a\") as myfile:\n",
    "                myfile.write(f\"\\n Mean MCC scores: {(round(np.mean(np.array(mccScoresList)), 4))} mean AUC: {(round(np.mean(np.array(aucScoresList)),4))} \\n\")\n",
    "    myfile.close()\n",
    "    \n",
    "     \n",
    "    #measure and incorporate diversity\n",
    "    diversity_metric = measure_binary_vector_diversity(individual)\n",
    "    \n",
    "#     nv = 10\n",
    "\n",
    "#     diversity_score = 0.4\n",
    "#     vec = [(x/nv* diversity_score * 0.1) for x in range(nv)]\n",
    "#     vec2 = [(x/nv* diversity_score * 0.5) for x in range(nv)]\n",
    "#     vec3 = [(x/nv* diversity_score * 0.9) for x in range(nv)]\n",
    "\n",
    "\n",
    "#     ypoints = np.array(vec)\n",
    "#     plt.plot(ypoints, linestyle = 'dotted', color='g')\n",
    "#     ypoints = np.array(vec2)\n",
    "#     plt.plot(ypoints, linestyle = 'dotted', color='b')\n",
    "#     ypoints = np.array(vec3)\n",
    "#     plt.plot(ypoints, linestyle = 'dotted', color='y')\n",
    "\n",
    "#     plt.show()\n",
    "    \n",
    "    \n",
    "    #return mcc for genetic algorithm evalutation\n",
    "    #return (((np.mean(np.array(mccScoresList))),)\n",
    "    final_score =  ((np.mean(np.array(mccScoresList))),)\n",
    "    diversity_parameter = 0.1\n",
    "    final_score = (final_score* diversity_metric * diversity_parameter)/2\n",
    "    \n",
    "    return final_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1998d36a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_y_pred_resolver(ensemble, valid=False):\n",
    "#     if(debug):\n",
    "#         print(f\"get_y_pred_resolver {global_param_dict.get('weighted')}\")\n",
    "    if(global_param_dict.get('weighted')== None or global_param_dict.get('weighted')=='unweighted'):\n",
    "        y_pred = get_best_y_pred_unweighted(ensemble, valid=valid)\n",
    "    elif(global_param_dict.get('weighted')=='de'):\n",
    "        \n",
    "        y_pred = get_weighted_ensemble_prediction_de_y_pred_valid(ensemble, super_ensemble_weight_finder_differential_evolution(ensemble, valid=valid), valid=valid)\n",
    "        #print(y_pred.shape)\n",
    "    elif(global_param_dict.get('weighted')=='ann'):\n",
    "        y_pred = get_y_pred_ann_torch_weighting(ensemble, valid=valid)\n",
    "    \n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dedf5edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_weighted_ensemble_auc(individual):\n",
    "    global log_store_dataframe_path\n",
    "    # Evaluate ensemble\n",
    "    #X_test = X_test_orig.copy()\n",
    "    #y_test = y_test_orig.copy()\n",
    "\n",
    "    mccScoresList = []\n",
    "    \n",
    "    aucScoresList = []\n",
    "    \n",
    "    #print(\"Evaluating individual of size: \", len(individual[0]))\n",
    "    \n",
    "    #individual_data = individual[0]\n",
    "    \n",
    "    with open(file_path+\"/progress_logs_scores/\"+current_log_scores_filename, \"a\") as myfile:\n",
    "            myfile.write(f\"Evaluating individual of size: {str(len(individual[0]))}\"\n",
    "                        )\n",
    "    myfile.close()\n",
    "    \n",
    "    y_pred = get_y_pred_resolver(individual, valid=False)\n",
    "    \n",
    "    #print(y_test.shape, y_pred.shape, )\n",
    "            #should write mcc parallel instead of auc also, need pair functions\n",
    "    auc = metrics.roc_auc_score(y_test, y_pred)\n",
    "    mcc = matthews_corrcoef(y_test, y_pred)\n",
    "    f1  = f1_score(y_test, y_pred, average='binary')\n",
    "    precision = precision_score(y_test, y_pred, average='binary')\n",
    "    recall = recall_score(y_test, y_pred, average='binary')\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    \n",
    "    \n",
    "    with open(file_path+\"/progress_logs_scores/\"+current_log_scores_filename, \"a\") as myfile:\n",
    "                myfile.write(f\"\\n ensemble MCC score: {round(mcc, 4)} ensemble AUC: {(round(auc,4))} \\n\")\n",
    "    myfile.close()\n",
    "    \n",
    "    if(debug):\n",
    "        print(f\"Ensemble MCC {mcc}, AUC {auc}, nb {len(individual[0])}\")\n",
    "    \n",
    "    #?? how does the diversity weighting and the order of operations interact with the ensemble weights already\n",
    "    #measure and incorporate diversity\n",
    "    diversity_metric = measure_binary_vector_diversity(individual)\n",
    "    \n",
    "    \n",
    "    diversity_parameter = global_param_dict.get('div_p')\n",
    "#     final_score = (auc* diversity_metric * diversity_parameter)/2\n",
    "#     final_score_mcc = (mcc* diversity_metric * diversity_parameter)/2\n",
    "    \n",
    "    #Simple diversity incorporation\n",
    "    auc_div = auc * ((-1 + (diversity_metric + diversity_parameter) )) \n",
    "    # div metric should decrease score as 1=similar, we want to penalise similarity of prediction\n",
    "    \n",
    "    mcc_div = mcc * ((-1 + (diversity_metric + diversity_parameter) ))\n",
    "    \n",
    "    #blank init with headers in main log_store_dataframe_path\n",
    "    ensemble_model_list = []\n",
    "    feature_count_list  = []\n",
    "    auc_score_list = []\n",
    "    mcc_score_list = []\n",
    "    #For each member in the ensemble\n",
    "    for i in range(0, len(individual[0])-1):\n",
    "        ensemble_model_list.append(str(individual[0][i][1])) # str or model?\n",
    "        feature_count_list.append(individual[0][i][2]) #could replace with length of list to reduce size on disk\n",
    "        auc_score_list.append(individual[0][i][4])\n",
    "        mcc_score_list.append(individual[0][i][0])\n",
    "        \n",
    "    \n",
    "    \n",
    "    orignal_feature_names  \n",
    "    \n",
    "    feature_map_vector = []\n",
    "    for col in orignal_feature_names:\n",
    "        if(col in feature_count_list):\n",
    "            feature_map_vector.append(1)\n",
    "        else:\n",
    "            feature_map_vector.append(0)\n",
    "        \n",
    "    feature_map_vector = np.array(feature_map_vector)\n",
    "    \n",
    "    #set score log dataframe in main\n",
    "    df_data = [[len(individual[0]),auc, np.mean(auc_score_list), auc_div, mcc, np.mean(mcc_score_list), mcc_div, f1, precision, recall, accuracy,\n",
    "                ensemble_model_list, feature_map_vector, auc_score_list, mcc_score_list\n",
    "               ]]\n",
    "    column_headers = ['n','auc', 'auc_mean', 'auc_div', 'mcc','mcc_mean', 'mcc_div', 'f1', 'precision', 'recall', 'accuracy',\n",
    "                'ensemble_model_list', 'feature_count_list', 'auc_score_list', 'mcc_score_list']\n",
    "    \n",
    "#     column_headers = ['nb_size', 'f_list', 'auc','mcc','f1','precision','recall','accuracy', 'nb_val', 'pop_val', 'g_val', 'g', 'weighted', 'use_stored_base_learners', 'store_base_learners',\n",
    "#        'resample', 'scale', 'n_features', 'param_space_size', 'n_unique_out',\n",
    "#        'outcome_var_n', 'div_p', 'percent_missing', 'corr', \n",
    "#                        'age', 'sex', 'bmi','ethnicity', 'bloods', 'diagnostic_order',\n",
    "#                       'drug_order', 'annotation_n', 'meta_sp_annotation_n',\n",
    "#                       'X_train_size', 'X_test_orig_size', 'X_test_size',\n",
    "#                    'run_time', 'cxpb', 'mutpb', 'indpb', 't_size']\n",
    "    \n",
    "    \n",
    "    df = pd.DataFrame(data = df_data, columns=column_headers)\n",
    "    df.to_csv(f\"{file_path}/progress_logs_scores/{log_store_dataframe_path}.csv\", mode='a', index=True, header=False)\n",
    "    \n",
    "    if(debug):\n",
    "        print(f\"\"\"Ensemble MCC {mcc}, diversity weighted MCC {mcc_div}\n",
    "        , \\n f1 {f1} precision {precision} recall {recall} accuracy {accuracy}\n",
    "        , \\n AUC {auc}, diversity weighted AUC {auc_div}\n",
    "        , \\n nb {len(individual[0])}, diversity_score: {diversity_metric}, diff: {auc_div-auc} \"\"\")\n",
    "    \n",
    "    \n",
    "    #return mcc for genetic algorithm evalutation\n",
    "    if(diversity_parameter>0):\n",
    "        return (auc_div,)\n",
    "    else:\n",
    "        return (auc,)\n",
    "    #return (auc,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8f3bac9",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def mutateEnsemble(individual):\n",
    "    # print(individual[0])\n",
    "    # print(len(individual[0]))\n",
    "    try:\n",
    "        print(f\"original individual of size {len(individual[0])-1}:\")\n",
    "        n = random.randint(0, len(individual[0])-1)\n",
    "        print(f\"Mutating individual at index {n}\")\n",
    "        try:\n",
    "            individual[0].pop(n)\n",
    "            print(f\"Successfully popped {n} from individual\")\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to pop {n} from individual of length {len(individual[0])} , popping zero\")\n",
    "            individual[0].pop(0)\n",
    "            \n",
    "            print(e)\n",
    "        \n",
    "        \n",
    "        \n",
    "        # print(n)\n",
    "        #individual[0].pop(n)\n",
    "        # print(individual)\n",
    "        individual[0].append(baseLearnerGenerator())\n",
    "        # print(individual)\n",
    "        #print(\"Mutated individual:\")\n",
    "        #print(individual)\n",
    "        return individual\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(\"Failed to mutate Ensemble\")\n",
    "        print(\"Len individual\", len(individual))\n",
    "        #print(individual[0])\n",
    "        #print(individual)\n",
    "        return individual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c17172e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weighted_ensemble_prediction_de_y_pred(best, weights):\n",
    "    \"\"\"Return the weighted prediction vector\"\"\"\n",
    "\n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    for i in range(0, len(target_ensemble)):\n",
    "        # For model i, predict it's x_test\n",
    "        feature_columns = target_ensemble[i][2]\n",
    "        y_pred = target_ensemble[i][5]\n",
    "\n",
    "\n",
    "        prediction_array.append(y_pred)\n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "    prediction_matrix = prediction_matrix.astype(float)\n",
    "    prediction_matrix_raw = prediction_matrix   \n",
    "    #auc = metrics.roc_auc_score(y_test, y_pred_best)\n",
    "    #print(\"Unweighted ensemble AUC: \", auc)\n",
    "\n",
    "    clean_prediction_matrix = prediction_matrix_raw.copy()\n",
    "    weights = normalize(weights)\n",
    "\n",
    "    weighted_prediction_matrix_array = (np.array(clean_prediction_matrix) * weights[:, None])\n",
    "    collapsed_weighted_prediction_matrix_array = weighted_prediction_matrix_array.sum(axis=0)\n",
    "\n",
    "    y_pred_weighted = round_v(collapsed_weighted_prediction_matrix_array)\n",
    "    \n",
    "    return y_pred_weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f9a823a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weighted_ensemble_prediction_de_y_pred_valid(best, weights, valid=False):\n",
    "    \"\"\"Return the weighted prediction vector\"\"\"\n",
    "\n",
    "    #global get_weighted_ensemble_prediction\n",
    "    \n",
    "    #Need to calculate weights from x_train (weights are passed in already)\n",
    "    #then call each model in the ensemble and have them predict on the x_test_orig validation \n",
    "    #Then apply the weights learned from training set\n",
    "    \n",
    "    # Get prediction matrix:\n",
    "\n",
    "    target_ensemble = best[0]\n",
    "    if(valid):\n",
    "        print(\"Evaluating weighted ensemble on validation set\")\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test = y_test_orig.copy()\n",
    "    \n",
    "        prediction_array = []\n",
    "\n",
    "        for i in range(0, len(target_ensemble)):\n",
    "            # For model i, predict it's x_test\n",
    "            feature_columns = target_ensemble[i][2]\n",
    "\n",
    "            model = target_ensemble[i][1]\n",
    "\n",
    "\n",
    "    #         is evaluate calling model text also fitting?\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                model.fit(X_train[feature_columns], y_train)\n",
    "                prediction_array.append(\n",
    "                    model.predict(x_test[feature_columns])) # Target for resolving GPU/torch model types\n",
    "\n",
    "            else:\n",
    "                print(\"get_weighted_ensemble_prediction_de_y_pred_valid, handling torch model prediction\")\n",
    "\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "    #             model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "                model.to(device)\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "\n",
    "                print(\"numpy.isnan(y_hat).any()\", numpy.isnan(y_hat).any())\n",
    "                if(numpy.isnan(y_hat).any()):\n",
    "                    print(\"Returning dummy random yhat vector for torch pred, nan found\")\n",
    "                    y_hat = np.random.choice(a=[False, True], size=(len(y_hat,)))\n",
    "\n",
    "                prediction_array.append(y_hat                                   \n",
    "                       )\n",
    "    else:\n",
    "        prediction_array = []\n",
    "\n",
    "        for i in range(0, len(target_ensemble)):\n",
    "            #print(target_ensemble[i][5].shape)\n",
    "            prediction_array.append(\n",
    "                    target_ensemble[i][5]) \n",
    "\n",
    "        \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "    prediction_matrix = prediction_matrix.astype(float)\n",
    "    prediction_matrix_raw = prediction_matrix   \n",
    "\n",
    "    \n",
    "    clean_prediction_matrix = prediction_matrix_raw.copy()\n",
    "    weights = normalize(weights)\n",
    "\n",
    "    weighted_prediction_matrix_array = (np.array(clean_prediction_matrix) * weights[:, None])\n",
    "    collapsed_weighted_prediction_matrix_array = weighted_prediction_matrix_array.sum(axis=0)\n",
    "\n",
    "    y_pred_weighted = round_v(collapsed_weighted_prediction_matrix_array)\n",
    "    \n",
    "    torch.cuda.empty_cache() # exp\n",
    "\n",
    "    return y_pred_weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24905574",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ensemble=best\n",
    "# valid=True\n",
    "# get_weighted_ensemble_prediction_de_y_pred_valid(ensemble, super_ensemble_weight_finder_differential_evolution(ensemble, valid=valid), valid=valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74065985",
   "metadata": {},
   "outputs": [],
   "source": [
    "#keep me\n",
    "round_v = np.vectorize(round) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20de8e7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_weighted_ensemble_prediction_de(weights, prediction_matrix_raw):\n",
    "        \"\"\"Function used by DE algo to search for optimal weights with scoring\"\"\"\n",
    "        #print(score_list[0:5])\n",
    "        #global prediction_matrix_raw\n",
    "        #y_test = y_test_orig.copy()\n",
    "\n",
    "        clean_prediction_matrix = prediction_matrix_raw.copy()\n",
    "        weights = normalize(weights)\n",
    "\n",
    "\n",
    "        #profiler points at this matrix definition:\n",
    "        #for each case, for the column of models in the ensemble...\n",
    "        # for i in range(0, len(prediction_array[0])):\n",
    "        #     clean_prediction_matrix[:,i] = np.matrix(np.array(clean_prediction_matrix[:,i]).transpose()[0] * weights).transpose()\n",
    "\n",
    "        weighted_prediction_matrix_array = (np.array(clean_prediction_matrix) * weights[:, None])\n",
    "        collapsed_weighted_prediction_matrix_array = weighted_prediction_matrix_array.sum(axis=0)\n",
    "\n",
    "        # can be done outside function\n",
    "\n",
    "        y_pred_best = round_v(collapsed_weighted_prediction_matrix_array)\n",
    "        \n",
    "        auc = metrics.roc_auc_score(y_test, y_pred_best) #where on earth is this y_test taken from... it is the same global one?\n",
    "        score = auc\n",
    "        #         if(score > best_score):\n",
    "        #             best_score = score\n",
    "        #             best_weights = weights\n",
    "        #             print(best_score, best_weights, \"-\" + \"\\n\")\n",
    "        #score_list.append((score, weights))\n",
    "        mcc = matthews_corrcoef(y_test, y_pred_best)\n",
    "        #score = mcc\n",
    "        #print(f\"AUC: {score}\" )\n",
    "        return 1-score\n",
    "        #print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b858d339",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cython call function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "398b2249",
   "metadata": {},
   "outputs": [],
   "source": [
    "#redundant? weights only derived from xtrain, weight vec is size of ensemble not train set\n",
    "\n",
    "#Only get weights from xtrain/ytrain, never get weights from xtest y test. Use weights on x_validation yhat to compare to ytrue_valid\n",
    "def super_ensemble_weight_finder_differential_evolution(best, valid=False):\n",
    "\n",
    "    model_train_time_warning_threshold = 5\n",
    "    # Get prediction matrix:\n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    for i in range(0, len(target_ensemble)):\n",
    "        # For model i, predict it's x_test\n",
    "        #feature_columns = target_ensemble[i][2]\n",
    "        y_pred = target_ensemble[i][5]\n",
    "        #print(y_pred.shape)\n",
    "\n",
    "\n",
    "        prediction_array.append(y_pred)\n",
    "    \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "    #print(prediction_matrix.shape)\n",
    "    prediction_matrix = prediction_matrix.astype(float)\n",
    "    prediction_matrix_raw = prediction_matrix\n",
    "    y_pred_best = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        y_pred_best.append(round(np.mean(prediction_matrix_raw[:,i])))    \n",
    "    auc = metrics.roc_auc_score(y_test, y_pred_best)\n",
    "    print(\"Unweighted ensemble AUC: \", auc)\n",
    "    \n",
    "    \n",
    "    #Set valid set after #don't set valid ever\n",
    "    #print(\"super_ensemble_weight_finder_differential_evolution\", valid)\n",
    "#     if(valid):\n",
    "#         x_test = X_test_orig.copy()\n",
    "#         y_test = y_test_orig.copy()\n",
    "\n",
    "\n",
    "    bounds = [(0, 1) for x in range(0, len(best[0]))]\n",
    "\n",
    "    start = time.time()\n",
    "    de = scipy.optimize.differential_evolution(get_weighted_ensemble_prediction_de_cython,\n",
    "                                  bounds,\n",
    "                                  args=((prediction_matrix_raw,y_test)),\n",
    "                                  strategy='best1bin',\n",
    "                                  maxiter=20,\n",
    "                                  popsize=15,\n",
    "                                  tol=0.01,\n",
    "                                  mutation=(0.5, 1),\n",
    "                                  recombination=0.7,\n",
    "                                  seed=None,\n",
    "                                  callback=None,\n",
    "                                  disp=False,\n",
    "                                  polish=True,\n",
    "                                  init='latinhypercube',\n",
    "                                  atol=0,\n",
    "                                  updating='immediate',\n",
    "                                  workers=4,\n",
    "                                  constraints=(),\n",
    "                                  x0=None,\n",
    "                                  #integrality=None,\n",
    "                                  #vectorized=False\n",
    "                                              )\n",
    "    \n",
    "    score = (1-de.fun)\n",
    "    optimal_weights = de.x\n",
    "    \n",
    "        \n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        \n",
    "        if(model_train_time> model_train_time_warning_threshold):\n",
    "            print(\"Warning long DE weights train time, \", model_train_time, model_train_time_warning_threshold)\n",
    "\n",
    "\n",
    "    print(\"best weighted score: \", score, \"difference:\", score-auc)\n",
    "    #print(\"best weights\", optimal_weights, optimal_weights.shape)\n",
    "    return optimal_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef61e861",
   "metadata": {},
   "outputs": [],
   "source": [
    "#bayesian model selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90f33003",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_y_pred_ann_torch_weighting(best, valid=False):\n",
    "    \n",
    "    \n",
    "    \n",
    "    y_test_ann = y_test.copy()\n",
    "    \n",
    "    model_train_time_warning_threshold = 15\n",
    "    start = time.time()\n",
    "\n",
    "    target_ensemble = best[0]\n",
    "    #Get prediction matrix\n",
    "    #print('Get prediction matrix')\n",
    "    if(valid):\n",
    "        print(f\"get_y_pred_ann_torch_weighting {valid}\")\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test_ann = y_test_orig.copy()\n",
    "\n",
    "        prediction_array = []\n",
    "\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            # For model i, predict it's x_test\n",
    "            feature_columns = list(target_ensemble[i][2]) #get features model was trained on\n",
    "\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                \n",
    "                model = target_ensemble[i][1]\n",
    "                \n",
    "                model.fit(X_train[feature_columns], y_train)\n",
    "                \n",
    "                prediction_array.append(\n",
    "                    model.predict(X_train[feature_columns])) #Use model to predict x train\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(X_train[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     \n",
    "                model.to(device) # Has this model been fitted??\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                                    \n",
    "                    )\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train #Store predictions from x_train into matrix\n",
    "        \n",
    "        X_prediction_matrix_raw_X_train = prediction_matrix_raw_X_train.T\n",
    "\n",
    "        #Produce test results for valid   \n",
    "        prediction_array = []\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            feature_columns = list(target_ensemble[i][2])\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                prediction_array.append(\n",
    "                    target_ensemble[i][1].predict(x_test[feature_columns])) #Generate predictions from stored models on validset\n",
    "\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     \n",
    "                model.to(device)\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                               \n",
    "                    )\n",
    "\n",
    "\n",
    "        prediction_matrix_X_test = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_test = prediction_matrix_X_test.astype(float)\n",
    "        prediction_matrix_raw_X_test = prediction_matrix_X_test\n",
    "\n",
    "        prediction_matrix_raw_X_test = prediction_matrix_raw_X_test.T #Transpose predictions into columns for each model. X >>y\n",
    "        test_data = TestData(torch.FloatTensor(prediction_matrix_raw_X_test))\n",
    "\n",
    "    elif valid==False:\n",
    "        #Make predictions on xtrain and y train, feed results into nn to learn weights to map ensemble to true. Apply nn to test ensemble preds\n",
    "        prediction_array = []\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "\n",
    "\n",
    "                prediction_array.append(\n",
    "                        target_ensemble[i][5]) #Get stored y_pred from x_test (non validation set)\n",
    "\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train\n",
    "\n",
    "        X_prediction_matrix_raw_X_train = prediction_matrix_raw_X_train.T # transpose to matrix, columns are each model yhat vector\n",
    "\n",
    "        test_data = TestData(torch.FloatTensor(X_prediction_matrix_raw_X_train)) #set test data to train set, only learn weights from training\n",
    "        #y_test = y_train.copy()\n",
    "        prediction_matrix_raw_X_test = X_prediction_matrix_raw_X_train \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    train_data = TrainData(torch.FloatTensor(X_prediction_matrix_raw_X_train), \n",
    "                           torch.FloatTensor(np.array(y_train)))    #data set to learn weights for x_train model preds to y_train labels\n",
    "\n",
    "\n",
    "\n",
    "    #print(len(prediction_array[0]))\n",
    "    #print(prediction_matrix_raw_X_test.shape)\n",
    "\n",
    "\n",
    "    y_pred_unweighted = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        y_pred_unweighted.append(round(np.mean(prediction_matrix_raw_X_test.T[:,i])))    \n",
    "\n",
    "\n",
    "\n",
    "    auc = metrics.roc_auc_score(y_test_ann, y_pred_unweighted)\n",
    "\n",
    "    mccscore_unweighted = matthews_corrcoef(y_test_ann, y_pred_unweighted)\n",
    "\n",
    "\n",
    "    y_pred_ensemble = train_ann_weight(X_prediction_matrix_raw_X_train.shape[1], int(X_prediction_matrix_raw_X_train.shape[0]), train_data, test_data)\n",
    "\n",
    "    #print(\"Ensemble ANN weighting training AUC: \", auc_score_weighted)\n",
    "\n",
    "\n",
    "    if(any(np.isnan(y_pred_ensemble))):\n",
    "        print(\"Torch model nan, returning random y pred vector\")\n",
    "        #zero_vector = [x for x in range(0, len(y_pred))]\n",
    "        #y_pred = zero_vector\n",
    "        random_y_pred_vector = (np.random.choice(a=[False, True], size=(len(y_test_ann,)))).astype(int)\n",
    "        y_pred = random_y_pred_vector\n",
    "        y_pred_ensemble = random_y_pred_vector\n",
    "    else:\n",
    "        #plot_auc(y_hat, f\"Deep ANN Torch {para_str}\")    \n",
    "        pass\n",
    "\n",
    "    auc_score_weighted = metrics.roc_auc_score(y_test_ann, y_pred_ensemble)\n",
    "\n",
    "    mccscore_weighted = matthews_corrcoef(y_test_ann, y_pred_ensemble)\n",
    "\n",
    "    auc_score_weighted = round(metrics.roc_auc_score(y_test_ann,y_pred_ensemble), 4)\n",
    "    print(\"ANN unweighted ensemble AUC: \", auc)\n",
    "    print(\"ANN weighted   ensemble AUC: \", auc_score_weighted)\n",
    "    print(\"ANN weighted   ensemble AUC difference: \", auc_score_weighted-auc)\n",
    "    print(\"ANN unweighted ensemble MCC: \", mccscore_unweighted)\n",
    "    print(\"ANN weighted   ensemble MCC: \", mccscore_weighted)\n",
    "\n",
    "\n",
    "    #score = (1-de.fun)\n",
    "    #optimal_weights = de.x\n",
    "\n",
    "\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "#     \n",
    "    #print(len(y_pred_ensemble))\n",
    "    torch.cuda.empty_cache() # exp\n",
    "    \n",
    "    return y_pred_ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "031f6fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_ann_weight(input_shape, batch_size, train_data, test_data):\n",
    "    free_gpu = str(get_free_gpu())\n",
    "\n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "    parameter_space = {\n",
    "        'column_length':[input_shape],\n",
    "       #'epochs': [50, 200],\n",
    "    'batch_size': [batch_size],#,int(X_train.shape[0]/100), int(X_train.shape[0]/200)],\n",
    "    #'learning_rate': lr_space,\n",
    "    #'learning_rate': [0.1, 0.001, 0.0005, 0.0001],\n",
    "    'deep_layers_1':[2],\n",
    "    'dropout_val':[0.001]\n",
    "    }\n",
    "\n",
    "    additional_grid = {\n",
    "        'epochs': [20],\n",
    "        #'epochs':[100],\n",
    "    'learning_rate': [0.0001]\n",
    "    }\n",
    "    size_test = []\n",
    "    # Loop over al grid search combinations\n",
    "    for values in itertools.product(*additional_grid.values()):\n",
    "        point = dict(zip(additional_grid.keys(), values))\n",
    "        # merge the general settings\n",
    "        settings = { **point}\n",
    "        #print(settings)\n",
    "        size_test.append(settings)\n",
    "\n",
    "    #print(len(size_test))\n",
    "\n",
    "    # Select a random sample from the global parameter space\n",
    "    sample_parameter_space = {}\n",
    "    for key in parameter_space.keys():\n",
    "        sample_parameter_space[key] = random.choice(parameter_space.get(key))\n",
    "\n",
    "    additional_param_sample = random.choice(size_test)\n",
    "\n",
    "    additional_param_sample = {}\n",
    "    for key in additional_grid.keys():\n",
    "        additional_param_sample[key] = random.choice(additional_grid.get(key))    \n",
    "\n",
    "    print(sample_parameter_space)\n",
    "\n",
    "    print(additional_param_sample)\n",
    "\n",
    "    #os.environ[\"CUDA_VISIBLE_DEVICES\"]=free_gpu\n",
    "\n",
    "    device = torch.device(f\"cuda:{free_gpu}\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(device)\n",
    "\n",
    "    train_loader = DataLoader(dataset=train_data, batch_size=sample_parameter_space['batch_size'], shuffle=True)\n",
    "    test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "\n",
    "    # fit model with random sample of global parameter space\n",
    "    model = BinaryClassification(**sample_parameter_space)\n",
    "    \n",
    "    model.to(device)\n",
    "    #print(model)    \n",
    "\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=additional_param_sample['learning_rate'])\n",
    "    model.train()\n",
    "    for e in range(1, additional_param_sample['epochs']+1):\n",
    "        epoch_loss = 0\n",
    "        epoch_acc = 0\n",
    "        for X_batch, y_batch in train_loader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            y_pred = model(X_batch)\n",
    "\n",
    "            loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "            acc = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        #print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f} | AUC: {torchmetrics.functional.auc(y_batch, y_pred, reorder=True)}')\n",
    "\n",
    "\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f} | AUC: {torchmetrics.functional.auc(y_batch, y_pred, reorder=True)}')\n",
    "\n",
    "    para_str = (str(settings).replace(\"'\", \"_\").replace(\":\", \"_\").replace(\",\", \"_\").replace(\"{\", \"_\").replace(\"}\", \"_\").replace(\" \", \"_\")).replace(\"__\", \"_\")    \n",
    "\n",
    "    y_pred_ensemble = model(test_data.X_data.to(device))\n",
    "\n",
    "    y_pred_ensemble = torch.round(torch.sigmoid(y_pred_ensemble)).cpu().detach().numpy().flatten()\n",
    "\n",
    "    return y_pred_ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bf0d405",
   "metadata": {},
   "outputs": [],
   "source": [
    "#res = get_y_pred_ann_torch_weighting(best, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "972b608e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#res = get_y_pred_log_reg_weighting(best, valid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d561a42",
   "metadata": {},
   "outputs": [],
   "source": [
    "#best[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6f79dfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_y_pred_log_reg_weighting(best, valid=False):\n",
    "\n",
    "    model_train_time_warning_threshold = 15\n",
    "    start = time.time()\n",
    "    \n",
    "    global y_test\n",
    "\n",
    "    target_ensemble = best[0]\n",
    "    #Get prediction matrix\n",
    "    print('Get prediction matrix')\n",
    "    if(valid):\n",
    "        print(f\"get_y_pred_log_reg_weighting {valid}\")\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test = y_test_orig.copy()\n",
    "\n",
    "        prediction_array = []\n",
    "\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            # For model i, predict it's x_test\n",
    "            feature_columns = list(target_ensemble[i][2]) #get features model was trained on\n",
    "\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                \n",
    "                model = target_ensemble[i][1]\n",
    "                \n",
    "                model.fit(X_train[feature_columns], y_train)\n",
    "                \n",
    "                prediction_array.append(\n",
    "                    model.predict(X_train[feature_columns])) #Use model to predict x train\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(X_train[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     \n",
    "                model.to(device) # Has this model been fitted??\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                                    \n",
    "                    )\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train #Store predictions from x_train into matrix\n",
    "        \n",
    "        X_prediction_matrix_raw_X_train = prediction_matrix_raw_X_train.T\n",
    "\n",
    "        #Produce test results for valid   \n",
    "        prediction_array = []\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            feature_columns = list(target_ensemble[i][2])\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                prediction_array.append(\n",
    "                    target_ensemble[i][1].predict(x_test[feature_columns])) #Generate predictions from stored models on validset\n",
    "\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     \n",
    "                model.to(device)\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                               \n",
    "                    )\n",
    "\n",
    "\n",
    "        prediction_matrix_X_test = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_test = prediction_matrix_X_test.astype(float)\n",
    "        prediction_matrix_raw_X_test = prediction_matrix_X_test\n",
    "\n",
    "        prediction_matrix_raw_X_test = prediction_matrix_raw_X_test.T #Transpose predictions into columns for each model. X >>y\n",
    "        test_data = TestData(torch.FloatTensor(prediction_matrix_raw_X_test))\n",
    "\n",
    "    elif valid==False:\n",
    "        #Make predictions on xtrain and y train, feed results into nn to learn weights to map ensemble to true. Apply nn to test ensemble preds\n",
    "        prediction_array = []\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "\n",
    "\n",
    "                prediction_array.append(\n",
    "                        target_ensemble[i][5]) #Get stored y_pred from x_test (non validation set)\n",
    "\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train\n",
    "\n",
    "        X_prediction_matrix_raw_X_train = prediction_matrix_raw_X_train.T # transpose to matrix, columns are each model yhat vector\n",
    "\n",
    "        test_data = X_prediction_matrix_raw_X_train #TestData(torch.FloatTensor(X_prediction_matrix_raw_X_train)) #set test data to train set, only learn weights from training\n",
    "        #y_test = y_train.copy()\n",
    "        prediction_matrix_raw_X_test = X_prediction_matrix_raw_X_train \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#     train_data = TrainData(torch.FloatTensor(X_prediction_matrix_raw_X_train), \n",
    "#                            torch.FloatTensor(np.array(y_train)))    #data set to learn weights for x_train model preds to y_train labels\n",
    "    train_data_X = X_prediction_matrix_raw_X_train.copy()\n",
    "    train_data_y = y_test.copy()\n",
    "    print(train_data_X.shape, train_data_y.shape)\n",
    "\n",
    "\n",
    "\n",
    "    y_pred_unweighted = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        y_pred_unweighted.append(round(np.mean(prediction_matrix_raw_X_test.T[:,i])))    \n",
    "\n",
    "\n",
    "\n",
    "    auc = metrics.roc_auc_score(y_test, y_pred_unweighted)\n",
    "\n",
    "    mccscore_unweighted = matthews_corrcoef(y_test, y_pred_unweighted)\n",
    "\n",
    "    \n",
    "    y_pred_ensemble = get_weighted_log_reg_ensemble(train_data_X, train_data_y, train_data_y)\n",
    "\n",
    "    \n",
    "    auc_score_weighted = metrics.roc_auc_score(y_test, y_pred_ensemble)\n",
    "\n",
    "    mccscore_weighted = matthews_corrcoef(y_test, y_pred_ensemble)\n",
    "\n",
    "    auc_score_weighted = round(metrics.roc_auc_score(y_test,y_pred_ensemble), 4)\n",
    "    print(\"LogReg unweighted ensemble AUC: \", auc)\n",
    "    print(\"LogReg weighted   ensemble AUC: \", auc_score_weighted)\n",
    "    print(\"LogReg weighted   ensemble AUC difference: \", auc_score_weighted-auc)\n",
    "    print(\"LogReg unweighted ensemble MCC: \", mccscore_unweighted)\n",
    "    print(\"LogReg weighted   ensemble MCC: \", mccscore_weighted)\n",
    "\n",
    "\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "\n",
    "    print(len(y_pred_ensemble))\n",
    "    return y_pred_ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "940a0c72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weighted_log_reg_ensemble(X_train, y_train, X_test):\n",
    "    \n",
    "    \n",
    "    clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n",
    "    \n",
    "    y_pred_ensemble = clf.predict(X_test)\n",
    "\n",
    "    \n",
    "    \n",
    "    return y_pred_ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb043711",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_y_pred_ann_torch_weighting_grid(best, sample_parameter_space, additional_param_sample, valid=False):\n",
    "    #dont split pred matrix. it is produced by train set...\n",
    "    \n",
    "    #just apply its results to test data results \n",
    "\n",
    "    \n",
    "    #%%prun\n",
    "    #X = allData_float3_imputed\n",
    "    #global get_weighted_ensemble_prediction\n",
    "    model_train_time_warning_threshold = 15\n",
    "    start = time.time()\n",
    "    # Get prediction matrix:\n",
    "    X_train, x_test, y_train, y_test = train_test_split( #Here we DONT effectively crossfold validate +1 ypred must be uniform across all base\n",
    "            X, y, test_size=0.25, random_state=1 \n",
    "        )\n",
    "    print(len(X_train),len(X_test),len(y_test),len(y_test),len(X_test_orig),len(y_test_orig))\n",
    "\n",
    "    #print(len(x_test), 'x_test')\n",
    "    target_ensemble = best[0]\n",
    "    #Get prediction matrix\n",
    "    #print('Get prediction matrix')\n",
    "    if(valid):\n",
    "        #print(f\"get_y_pred_ann_torch_weighting {valid}\")\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test = y_test_orig.copy()\n",
    "\n",
    "        #print(len(x_test), 'x_test')\n",
    "\n",
    "        prediction_array = []\n",
    "        \n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            # For model i, predict it's x_test\n",
    "            feature_columns = list(target_ensemble[i][2])#list(target_ensemble[i][3].columns)\n",
    "\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                prediction_array.append(\n",
    "                    target_ensemble[i][1].predict(X_train[feature_columns])) # Target for resolving GPU/torch model types\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(X_train[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "                model.to(device)\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                    )\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train\n",
    "\n",
    "\n",
    "\n",
    "    else:\n",
    "        #Make predictions on xtrain and y train, feed results into nn to learn weights to map ensemble to true. Apply nn to test ensemble preds\n",
    "        prediction_array = []\n",
    "        for i in tqdm(range(0, len(target_ensemble))):\n",
    "            feature_columns = list(target_ensemble[i][2])\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "                    prediction_array.append(\n",
    "                        target_ensemble[i][1].predict(X_train[feature_columns])) # Target for resolving GPU/torch model types\n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(X_train[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "                model.to(device)\n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                    )\n",
    "\n",
    "\n",
    "        prediction_matrix_X_train = np.matrix(prediction_array)\n",
    "        prediction_matrix_X_train = prediction_matrix_X_train.astype(float)\n",
    "        prediction_matrix_raw_X_train = prediction_matrix_X_train\n",
    "\n",
    "    #Produce test results    \n",
    "    prediction_array = []\n",
    "    for i in tqdm(range(0, len(target_ensemble))):\n",
    "        feature_columns = list(target_ensemble[i][2])\n",
    "        if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "            prediction_array.append(\n",
    "                target_ensemble[i][1].predict(x_test[feature_columns])) # Target for resolving GPU/torch model types\n",
    "\n",
    "        else:\n",
    "\n",
    "            test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "            test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "            device = torch.device(\"cpu\")\n",
    "            model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "            model.to(device)\n",
    "            y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "            y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "            y_hat = y_hat.astype(int).flatten()\n",
    "            prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                )\n",
    "\n",
    "\n",
    "    prediction_matrix_X_test = np.matrix(prediction_array)\n",
    "    prediction_matrix_X_test = prediction_matrix_X_test.astype(float)\n",
    "    prediction_matrix_raw_X_test = prediction_matrix_X_test\n",
    "\n",
    "\n",
    "    #With prediction matrix    \n",
    "    #print('With prediction matrix..')\n",
    "    X_prediction_matrix_raw_X_train = prediction_matrix_raw_X_train.T\n",
    "    prediction_matrix_raw_X_test = prediction_matrix_raw_X_test.T\n",
    "    # if(valid):\n",
    "\n",
    "\n",
    "    train_data = TrainData(torch.FloatTensor(X_prediction_matrix_raw_X_train), \n",
    "                           torch.FloatTensor(np.array(y_train)))    \n",
    "    test_data = TestData(torch.FloatTensor(prediction_matrix_raw_X_test))\n",
    "\n",
    "\n",
    "    #print(\"Dimensions of test_ensemble data for first calc\", X_test_ensemble.shape)\n",
    "\n",
    "    free_gpu = str(get_free_gpu())\n",
    "\n",
    "    # Initialise global parameter space----------------------------------------------------------------\n",
    "\n",
    "  \n",
    "\n",
    "    print(sample_parameter_space)\n",
    "\n",
    "    print(additional_param_sample)\n",
    "\n",
    "    #os.environ[\"CUDA_VISIBLE_DEVICES\"]=free_gpu\n",
    "\n",
    "    device = torch.device(f\"cuda:{free_gpu}\" if torch.cuda.is_available() else \"cpu\")\n",
    "    #print(device)\n",
    "\n",
    "    train_loader = DataLoader(dataset=train_data, batch_size=sample_parameter_space['batch_size'], shuffle=True)\n",
    "    test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "\n",
    "    # fit model with random sample of global parameter space\n",
    "    model = BinaryClassification(**sample_parameter_space)\n",
    "\n",
    "    model.to(device)\n",
    "    #print(model)    \n",
    "\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=additional_param_sample['learning_rate'])\n",
    "    model.train()\n",
    "    for e in range(1, additional_param_sample['epochs']+1):\n",
    "        epoch_loss = 0\n",
    "        epoch_acc = 0\n",
    "        for X_batch, y_batch in train_loader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            y_pred = model(X_batch)\n",
    "\n",
    "            loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "            acc = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        #print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f} | AUC: {torchmetrics.functional.auc(y_batch, y_pred, reorder=True)}')\n",
    "\n",
    "\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f} | AUC: {torchmetrics.functional.auc(y_batch, y_pred, reorder=True)}')\n",
    "\n",
    "    para_str = (str(settings).replace(\"'\", \"_\").replace(\":\", \"_\").replace(\",\", \"_\").replace(\"{\", \"_\").replace(\"}\", \"_\").replace(\" \", \"_\")).replace(\"__\", \"_\")    \n",
    "\n",
    "    y_pred_ensemble = model(test_data.X_data.to(device))\n",
    "\n",
    "    y_pred_ensemble = torch.round(torch.sigmoid(y_pred_ensemble)).cpu().detach().numpy().flatten()\n",
    "\n",
    "    #print(\"y_pred_ensemble_weights_shape:\", y_pred_ensemble.shape)\n",
    "\n",
    "    \n",
    "\n",
    "    y_pred_unweighted = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        y_pred_unweighted.append(round(np.mean(prediction_matrix_X_test[:,i])))    \n",
    "    auc = metrics.roc_auc_score(y_test, y_pred_unweighted)\n",
    "\n",
    "    mccscore_unweighted = matthews_corrcoef(y_test, y_pred_unweighted)\n",
    "\n",
    "\n",
    "    #print(\"Ensemble ANN weighting training AUC: \", auc_score_weighted)\n",
    "\n",
    "\n",
    "    if(any(np.isnan(y_pred_ensemble))):\n",
    "        print(\"Torch model nan, returning random y pred vector\")\n",
    "        #zero_vector = [x for x in range(0, len(y_pred))]\n",
    "        #y_pred = zero_vector\n",
    "        random_y_pred_vector = (np.random.choice(a=[False, True], size=(len(y_test,)))).astype(int)\n",
    "        y_pred = random_y_pred_vector\n",
    "        y_pred_ensemble = random_y_pred_vector\n",
    "    else:\n",
    "        #plot_auc(y_hat, f\"Deep ANN Torch {para_str}\")    \n",
    "        pass\n",
    "\n",
    "    auc_score_weighted = metrics.roc_auc_score(y_test, y_pred_ensemble)\n",
    "    \n",
    "    mccscore_weighted = matthews_corrcoef(y_test, y_pred_ensemble)\n",
    "\n",
    "    auc_score_weighted = round(metrics.roc_auc_score(y_test,y_pred_ensemble), 4)\n",
    "    print(\"ANN unweighted ensemble AUC: \", auc)\n",
    "    print(\"ANN weighted   ensemble AUC: \", auc_score_weighted)\n",
    "    print(\"ANN weighted   ensemble AUC difference: \", auc_score_weighted-auc)\n",
    "    print(\"ANN unweighted ensemble MCC: \", mccscore_unweighted)\n",
    "    print(\"ANN weighted   ensemble MCC: \", mccscore_weighted)\n",
    "\n",
    "\n",
    "    #score = (1-de.fun)\n",
    "    #optimal_weights = de.x\n",
    "\n",
    "\n",
    "    end = time.time()\n",
    "    model_train_time = int(end-start)\n",
    "    if(debug):\n",
    "        debug_base_learner(model, mccscore, X_train, auc_score, model_train_time)\n",
    "\n",
    "    with open(file_path + \"ann_weight_log.txt\",\"a\") as file:\n",
    "        file.write(\"\\n\")\n",
    "        file.write(str(sample_parameter_space))\n",
    "        file.write(\"\\n\")\n",
    "        file.write(str(additional_param_sample))\n",
    "        file.write(str(\"\\n\"+\n",
    "        str(X_train.shape)+\n",
    "       \"ANN weighted ensemble AUC: \"+ str(auc_score_weighted)+ \"_\"+\n",
    "       \"ANN weighted ensemble AUC difference: \" + str(auc_score_weighted-auc)\n",
    "       ))\n",
    "        file.write(\"\\n\")\n",
    "        file.write(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f}')\n",
    "    #print(len(y_pred_ensemble))\n",
    "    return auc_score_weighted-auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40122626",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Target function to append weighting function/neural network for weighting\n",
    "\n",
    "def get_best_y_pred(best):\n",
    "    X_train, x_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1 # should this be one...?\n",
    "    )\n",
    "    \n",
    "    x_test = X_test_orig.copy()\n",
    "    y_test = y_test_orig.copy()\n",
    "    \n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    for i in tqdm(range(0, len(target_ensemble))):\n",
    "        # For model i, predict it's x_test\n",
    "        feature_columns = list(target_ensemble[i][2])#list(target_ensemble[i][3].columns)\n",
    "        \n",
    "        if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "            prediction_array.append(\n",
    "                target_ensemble[i][1].predict(x_test)) # Target for resolving GPU/torch model types\n",
    "        \n",
    "        else:\n",
    "\n",
    "            test_data = TestData(torch.FloatTensor(x_test.values))\n",
    "            test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "        \n",
    "            device = torch.device(\"cpu\")\n",
    "            model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "            model.to(device)\n",
    "            y_hat = model(test_data.X_data.to(device))\n",
    "    \n",
    "            y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "            \n",
    "            y_hat = y_hat.astype(int).flatten()\n",
    "            prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                )\n",
    "        \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "\n",
    "    # collapse the mean of each models prediction for each case into a binary label returning a final y_pred composite score from each model\n",
    "    y_pred_best = []\n",
    "    for i in tqdm(range(0, len(prediction_array[0]))):\n",
    "        # y_pred_best.append(round(np.mean(np.matrix(prediction_array)[:,i])))\n",
    "        try:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i], keepdims=True)[0][0][0])\n",
    "        except:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i])[0][0][0])\n",
    "            \n",
    "    return y_pred_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e284484c",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "#Target function to append weighting function/neural network for weighting\n",
    "\n",
    "def get_best_y_pred_valid(best):\n",
    "    X_train, x_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1 # should this be one...?\n",
    "    )\n",
    "    #x_test = X_test_orig.copy()\n",
    "    #y_test = y_test_orig.copy()\n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    for i in tqdm(range(0, len(target_ensemble))):\n",
    "        # For model i, predict it's x_test\n",
    "        feature_columns = list(target_ensemble[i][2])#list(target_ensemble[i][3].columns)\n",
    "        \n",
    "        if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "            prediction_array.append(\n",
    "                target_ensemble[i][1].predict(X_test_orig[feature_columns])) # Target for resolving GPU/torch model types\n",
    "        \n",
    "        else:\n",
    "#             print(\"get_best_y_pred, handling torch model prediction\")\n",
    "#             scaler = StandardScaler()\n",
    "#             print(\"feature_columns:\")\n",
    "#             print(feature_columns)\n",
    "#             print(X_train)\n",
    "#             X_train = scaler.fit_transform(X_train[feature_columns])\n",
    "#             x_test = scaler.transform(x_test[feature_columns])\n",
    "\n",
    "#             train_data = TrainData(torch.FloatTensor(X_train), \n",
    "#                                    torch.FloatTensor(y_train))\n",
    "            test_data = TestData(torch.FloatTensor(X_test_orig[feature_columns].values))\n",
    "            test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "        \n",
    "            device = torch.device(\"cpu\")\n",
    "            model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "            model.to(device)\n",
    "            y_hat = model(test_data.X_data.to(device))\n",
    "    \n",
    "            y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "            \n",
    "            y_hat = y_hat.astype(int).flatten()\n",
    "            prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                )\n",
    "        \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "\n",
    "    # collapse the mean of each models prediction for each case into a binary label returning a final y_pred composite score from each model\n",
    "    y_pred_best = []\n",
    "    for i in tqdm(range(0, len(prediction_array[0]))):\n",
    "        # y_pred_best.append(round(np.mean(np.matrix(prediction_array)[:,i])))\n",
    "        y_pred_best.append(stats.mode(\n",
    "            #np.matrix(prediction_array)[:, i], keepdims=None)[0][0][0])\n",
    "            np.matrix(prediction_array)[:, i])[0][0][0])\n",
    "    return y_pred_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c28def81",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_y_pred_unweighted(best, valid=False):\n",
    "    X_train, x_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1 # should this be one...?\n",
    "    )\n",
    "    \n",
    "    if(valid):\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test = y_test_orig.copy()\n",
    "    \n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    for i in range(0, len(target_ensemble)):\n",
    "\n",
    "        feature_columns = list(target_ensemble[i][2])#list(target_ensemble[i][3].columns)\n",
    "        \n",
    "        if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "            prediction_array.append(\n",
    "                target_ensemble[i][1].predict(x_test[feature_columns])) # Target for resolving GPU/torch model types\n",
    "        \n",
    "        else:\n",
    "\n",
    "            test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "            test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "        \n",
    "            device = torch.device(\"cpu\")\n",
    "            model = target_ensemble[i][1]                                     #Has this model already been trained?\n",
    "            model.to(device)\n",
    "            y_hat = model(test_data.X_data.to(device))\n",
    "    \n",
    "            y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "            \n",
    "            y_hat = y_hat.astype(int).flatten()\n",
    "            prediction_array.append(y_hat                                    #Do we need to handle nans here?\n",
    "                )\n",
    "    \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "\n",
    "    # collapse the mean of each models prediction for each case into a binary label returning a final y_pred composite score from each model\n",
    "    y_pred_best = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        # y_pred_best.append(round(np.mean(np.matrix(prediction_array)[:,i])))\n",
    "        try:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i], keepdims=True)[0][0][0])\n",
    "        except:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i])[0][0][0])\n",
    "    return y_pred_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76825c1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_y_pred_unweighted(best, valid=False):\n",
    "    \n",
    "    if(valid):\n",
    "        x_test = X_test_orig.copy()\n",
    "        y_test = y_test_orig.copy()\n",
    "    \n",
    "    prediction_array = []\n",
    "    target_ensemble = best[0]\n",
    "    if(valid):\n",
    "        for i in range(0, len(target_ensemble)):\n",
    "\n",
    "            feature_columns = list(target_ensemble[i][2])#list(target_ensemble[i][3].columns)\n",
    "\n",
    "            if(type(target_ensemble[i][1]) is not BinaryClassification):\n",
    "\n",
    "                model = target_ensemble[i][1]\n",
    "                \n",
    "                model.fit(X_train[feature_columns], y_train)\n",
    "\n",
    "                prediction_array.append(\n",
    "                    model.predict(x_test[feature_columns])) \n",
    "\n",
    "            else:\n",
    "\n",
    "                test_data = TestData(torch.FloatTensor(x_test[feature_columns].values))\n",
    "                test_loader = DataLoader(dataset=test_data, batch_size=1)\n",
    "\n",
    "                device = torch.device(\"cpu\")\n",
    "                model = target_ensemble[i][1]                                #Has this model been fitted already?     \n",
    "                model.to(device)\n",
    "                \n",
    "                #model.fit(X_train, y_train)\n",
    "                \n",
    "                y_hat = model(test_data.X_data.to(device))\n",
    "\n",
    "                y_hat = torch.round(torch.sigmoid(y_hat)).cpu().detach().numpy()\n",
    "\n",
    "                y_hat = y_hat.astype(int).flatten()\n",
    "                prediction_array.append(y_hat                                    \n",
    "                    )\n",
    "                \n",
    "    else:\n",
    "        for i in range(0, len(target_ensemble)):\n",
    "            y_pred = target_ensemble[i][5]\n",
    "            prediction_array.append(y_pred                                    \n",
    "                    )\n",
    "    \n",
    "    \n",
    "    prediction_matrix = np.matrix(prediction_array)\n",
    "\n",
    "    # collapse the mean of each models prediction for each case into a binary label returning a final y_pred composite score from each model\n",
    "    y_pred_best = []\n",
    "    for i in range(0, len(prediction_array[0])):\n",
    "        # y_pred_best.append(round(np.mean(np.matrix(prediction_array)[:,i])))\n",
    "        try:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i], keepdims=True)[0][0][0])\n",
    "        except:\n",
    "            y_pred_best.append(stats.mode(\n",
    "                np.matrix(prediction_array)[:, i])[0][0][0])\n",
    "    return y_pred_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75c2e0f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext Cython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2755490e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%cython -a\n",
    "\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "\n",
    "round_v = np.vectorize(round) \n",
    "\n",
    "def normalize(weights):\n",
    "    # calculate l1 vector norm\n",
    "    result = norm(weights, 1)\n",
    "    # check for a vector of all zeros\n",
    "    if result == 0.0:\n",
    "        return weights\n",
    "    # return normalized vector (unit norm)\n",
    "    return weights / result\n",
    "\n",
    "def get_weighted_ensemble_prediction_de_cython(weights, prediction_matrix_raw, y_test):\n",
    "    \n",
    "        \"\"\"Function used by DE algo to search for optimal weights with scoring\"\"\"\n",
    "\n",
    "        clean_prediction_matrix = prediction_matrix_raw.copy()\n",
    "        weights = normalize(weights)\n",
    "\n",
    "        weighted_prediction_matrix_array = (np.array(clean_prediction_matrix) * weights[:, None])\n",
    "        collapsed_weighted_prediction_matrix_array = weighted_prediction_matrix_array.sum(axis=0)\n",
    "\n",
    "        y_pred_best = round_v(collapsed_weighted_prediction_matrix_array)\n",
    "        \n",
    "        auc = metrics.roc_auc_score(y_test, y_pred_best) \n",
    "        score = auc\n",
    "        \n",
    "        #mcc = metrics.matthews_corrcoef(y_test, y_pred_best)\n",
    "\n",
    "        return 1-score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "864b3b41",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def plot_auc(y_pred_best, algorithm_params):\n",
    "    plt.ion()\n",
    "    false_positive_rate1, true_positive_rate1, threshold1 = roc_curve(\n",
    "        y_test_orig, y_pred_best\n",
    "    ) #does this need to be recomputed to fit ypred best size or predbest passed for final\n",
    "    auc = metrics.roc_auc_score(y_test_orig, y_pred_best)\n",
    "    plt.subplots(1, figsize=(10, 10))\n",
    "    plt.title(\"Receiver Operating Characteristic - ensemble \\n\" + algorithm_params)\n",
    "    plt.plot(\n",
    "        false_positive_rate1, true_positive_rate1, label=\"AUC=\" +\n",
    "        str(round(auc, 2))\n",
    "    )\n",
    "    plt.legend(loc=4)\n",
    "    plt.plot([0, 1], ls=\"--\")\n",
    "    plt.plot([0, 0], [1, 0], c=\".7\"), plt.plot([1, 1], c=\".7\")\n",
    "    plt.ylabel(\"True Positive Rate\")\n",
    "    plt.xlabel(\"False Positive Rate\")\n",
    "    plt.savefig(\n",
    "        file_path\n",
    "        + \"figures/\"\n",
    "        + algorithm_params\n",
    "        + \"_AUC_:\"\n",
    "        + str(round(auc, 2))\n",
    "        + \"_.png\",\n",
    "        dpi=300,\n",
    "    )\n",
    "    plt.show(block=True)\n",
    "    plt.close(\"all\")\n",
    "    cf_matrix = confusion_matrix(y_test_orig, y_pred_best)\n",
    "    group_names = [\"True Neg\", \"False Pos\", \"False Neg\", \"True Pos\"]\n",
    "    group_counts = [\"{0:0.0f}\".format(value) for value in cf_matrix.flatten()]\n",
    "    group_percentages = [\n",
    "        \"{0:.2%}\".format(value) for value in cf_matrix.flatten() / np.sum(cf_matrix)\n",
    "    ]\n",
    "    labels = [\n",
    "        f\"{v1}\\n{v2}\\n{v3}\"\n",
    "        for v1, v2, v3 in zip(group_names, group_counts, group_percentages)\n",
    "    ]\n",
    "    labels = np.asarray(labels).reshape(2, 2)\n",
    "    sns.heatmap(cf_matrix, annot=labels, fmt=\"\", cmap=\"Blues\")\n",
    "    # fig = _.get_figure()\n",
    "    plt.savefig(\n",
    "        file_path\n",
    "        + \"figures/\"\n",
    "        + algorithm_params\n",
    "        + \"_AUC_:\"\n",
    "        + str(round(auc, 2))\n",
    "        + \"CM_.png\",\n",
    "        dpi=300,\n",
    "    )\n",
    "    plt.close(\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2d53823",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#question: should fitness min...be fitness max? to maximise mcc score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80c041c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "print(\"Creating fitnessMin, individual\")\n",
    "# creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0,))\n",
    "# creator.create(\"Individual\", list, fitness=creator.FitnessMin)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cf1771a",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# test\n",
    "creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
    "creator.create(\"Individual\", list, fitness=creator.FitnessMax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33732edb",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "toolbox = base.Toolbox()\n",
    "toolbox.register(\"evaluate\", evaluate_weighted_ensemble_auc) #evaluate #warning set evaluate below also\n",
    "# pool = multiprocessing.Pool()\n",
    "# toolbox.register(\"map\", pool.map)\n",
    "#toolbox.register(\"map\", futures.map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f27d738a",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def imputeMultipleDataFrame(dataFrame):\n",
    "    imputer = IterativeImputer( n_nearest_features=None, imputation_order='ascending', missing_values = np.nan) #If None, all features will be used.\n",
    "    imp = IterativeImputer(max_iter=10, random_state=0)\n",
    "    imp.fit(dataFrame)\n",
    "    trans = imp.transform(dataFrame)\n",
    "    \n",
    "    to_return = pd.DataFrame(trans, columns=dataFrame.columns)  \n",
    "    return to_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00e458ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools as it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd85c9e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = {\n",
    "    'weighted': ['ann', 'de', 'unweighted'],\n",
    "    'use_stored_base_learners':[False],\n",
    "    'store_base_learners':[False], \n",
    "    'resample' : ['undersample'],\n",
    "    'scale'    : [True],\n",
    "    'n_features': ['all'],\n",
    "    'param_space_size':['medium'],\n",
    "    'n_unique_out': [10],\n",
    "    'outcome_var_n':['0.5_0.5_0.5',\n",
    "                     '0.5_0.5_0.75',\n",
    "                     '0.5_0.5_0.8',\n",
    "                     '0.5_0.5_0.9',\n",
    "                     '0.8_0.8_0.8'],\n",
    "    'div_p':[0],\n",
    "    'percent_missing':[95, 90],  #n/100 ex 95 for 95%\n",
    "                     'corr':[0.8, 0.5],\n",
    "                      'cxpb':[0.5, 0.75, 0.25],\n",
    "                       'mutpb':[0.2, 0.4, 0.8],\n",
    "                        'indpb':[0.025, 0.05, 0.075],\n",
    "                        't_size':[3, 6, 9],\n",
    "                     'data':[{'age':[True],\n",
    "                            'sex':[True],\n",
    "                             'bmi':[True],\n",
    "                             'ethnicity':[True],\n",
    "                            'bloods':[True, False],\n",
    "                            'diagnostic_order':[True, False],\n",
    "                            'drug_order':[True, False],\n",
    "                            'annotation_n':[True, False],\n",
    "                            'meta_sp_annotation_n':[True, False],\n",
    "                             'annotation_mrc_n':[True, False],\n",
    "                             'meta_sp_annotation_mrc_n':[True, False],\n",
    "                              'core_02':[False],\n",
    "                            'bed':[False],\n",
    "                            'vte_status':[True],\n",
    "                            'hosp_site':[True],\n",
    "                            'core_resus':[False],\n",
    "                            'news':[False]}]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98facb27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def c_prod(d):\n",
    "    if isinstance(d, list):\n",
    "         for i in d:\n",
    "            yield from ([i] if not isinstance(i, (dict, list)) else c_prod(i))\n",
    "    else:\n",
    "         for i in it.product(*map(c_prod, d.values())):\n",
    "            yield dict(zip(d.keys(), i))\n",
    "\n",
    "print(len(list(c_prod(grid))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70933003",
   "metadata": {},
   "outputs": [],
   "source": [
    "settings_list = list(c_prod(grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a18ed6f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "settings_list = random.sample(settings_list, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "955a5f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(settings_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822b8d93",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_project_dir_global = 'HFE_ML_GA_experiments/'\n",
    "\n",
    "st_time = datetime.datetime.now().strftime(\"%Y-%m-%d_%I-%M-%S_%p\")\n",
    "base_project_dir_global = base_project_dir_global + st_time + \"/\"\n",
    "base_project_dir = 'HFE_ML_GA_experiments/' + st_time + \"/\"\n",
    "additional_naming = \"HFE_ML_GA_Grid_\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "748f52a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_csv_path = 'df_imputed_10084_90_m.csv'\n",
    "input_csv_path = '30163_to_16408_imputed_outcome_grid.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b84db6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pathlib.Path(base_project_dir_global).mkdir(parents=True, exist_ok=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd90784b",
   "metadata": {},
   "outputs": [],
   "source": [
    "outcome_var = 'Hemochromatosis (disorder)_count_subject_present'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d40699d",
   "metadata": {},
   "outputs": [],
   "source": [
    "outcome_var = 'outcome_var'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5914f640",
   "metadata": {},
   "outputs": [],
   "source": [
    "additional_naming = \"HFE_\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7f82146",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_substring_list(string, substr):\n",
    "    return [str for str in string if\n",
    "             any(sub in str for sub in substr) and 'bmi' not in str]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3420b78f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_generation_progress_fitness(generation_progress_list):\n",
    "    fig = plt.figure()\n",
    "    ax = plt.axes()\n",
    "\n",
    "\n",
    "    x = [x for x in range(0, len(generation_progress_list))]\n",
    "    ax.plot(x,generation_progress_list);\n",
    "    plt.savefig(file_path+\n",
    "                \"figures//\"\n",
    "                    + \"best_pop=\"\n",
    "                    + str(pop_val)\n",
    "                    + \"_g=\"\n",
    "                    + str(g_val)\n",
    "                    + \"_nb=\"\n",
    "                    + str(nb_val)\n",
    "                    + \".png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "251ad4a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_project_dir_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afae4bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_project_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a00e6cb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "#init final grid scores\n",
    "column_list = ['nb_size', 'f_list', 'auc','mcc','f1','precision','recall','accuracy', \n",
    "               'nb_val', 'pop_val', 'g_val', 'g', 'weighted', \n",
    "               'use_stored_base_learners', 'store_base_learners',\n",
    "               'resample', 'scale', 'n_features', 'param_space_size', 'n_unique_out',\n",
    "               'outcome_var_n', 'div_p', 'percent_missing', 'corr', \n",
    "                'age', 'sex', 'bmi','ethnicity', 'bloods', 'diagnostic_order',\n",
    "                'drug_order', 'annotation_n', 'meta_sp_annotation_n',\n",
    "                'meta_sp_annotation_mrc_n','annotation_mrc_n',\n",
    "                'core_02','bed','vte_status','hosp_site','core_resus','news',\n",
    "                'X_train_size', 'X_test_orig_size', 'X_test_size',\n",
    "                'run_time', 'cxpb', 'mutpb', 'indpb', 't_size']\n",
    "\n",
    "column_list = column_list +['BL_' + str(x) for x in range(0, 64)]\n",
    "\n",
    "df = pd.DataFrame( data = None, columns = column_list)\n",
    "\n",
    "df.to_csv(base_project_dir + 'final_grid_score_log.csv', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4521d822",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(settings_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c648c826",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(settings_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "589495ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_outcome_list = ['Hemochromatosis (disorder)_count',\n",
    " 'Hemochromatosis (disorder)_count_subject_present',\n",
    " 'Hemochromatosis (disorder)_count_relative_present',\n",
    " 'Bronze cirrhosis (disorder)_count',\n",
    " 'Bronze cirrhosis (disorder)_count_subject_present',\n",
    " 'Bronze cirrhosis (disorder)_count_relative_present',\n",
    " 'Bronze diabetes (disorder)_count',\n",
    " 'Bronze diabetes (disorder)_count_subject_present',\n",
    " 'Bronze diabetes (disorder)_count_relative_present',\n",
    " 'Hereditary hemochromatosis (disorder)_count',\n",
    " 'Hereditary hemochromatosis (disorder)_count_subject_present',\n",
    " 'Hereditary hemochromatosis (disorder)_count_relative_present',\n",
    " 'Autosomal dominant hereditary hemochromatosis (disorder)_count',\n",
    " 'Autosomal dominant hereditary hemochromatosis (disorder)_count_subject_present',\n",
    " 'Autosomal dominant hereditary hemochromatosis (disorder)_count_relative_present',\n",
    " '1186847009_count',\n",
    " '1186847009_count_subject_present',\n",
    " '1186847009_count_relative_present',\n",
    " 'Hemochromatosis type 3 (disorder)_count',\n",
    " 'Hemochromatosis type 3 (disorder)_count_subject_present',\n",
    " 'Hemochromatosis type 3 (disorder)_count_relative_present',\n",
    " 'Juvenile hemochromatosis (disorder)_count',\n",
    " 'Juvenile hemochromatosis (disorder)_count_subject_present',\n",
    " 'Juvenile hemochromatosis (disorder)_count_relative_present',\n",
    " '1186849007_count',\n",
    " '1186849007_count_subject_present',\n",
    " '1186849007_count_relative_present',\n",
    " '1186844002_count',\n",
    " '1186844002_count_subject_present',\n",
    " '1186844002_count_relative_present',\n",
    " 'Idiopathic hemochromatosis (disorder)_count',\n",
    " 'Idiopathic hemochromatosis (disorder)_count_subject_present',\n",
    " 'Idiopathic hemochromatosis (disorder)_count_relative_present',\n",
    " 'Neonatal hemochromatosis (disorder)_count',\n",
    " 'Neonatal hemochromatosis (disorder)_count_subject_present',\n",
    " 'Neonatal hemochromatosis (disorder)_count_relative_present',\n",
    " 'Primary hemochromatosis (disorder)_count',\n",
    " 'Primary hemochromatosis (disorder)_count_subject_present',\n",
    " 'Primary hemochromatosis (disorder)_count_relative_present',\n",
    " 'Secondary hemochromatosis (disorder)_count',\n",
    " 'Secondary hemochromatosis (disorder)_count_subject_present',\n",
    " 'Secondary hemochromatosis (disorder)_count_relative_present',\n",
    " 'African nutritional hemochromatosis (disorder)_count',\n",
    " 'African nutritional hemochromatosis (disorder)_count_subject_present',\n",
    " 'African nutritional hemochromatosis (disorder)_count_relative_present',\n",
    " 'Erythropoietic hemochromatosis (disorder)_count',\n",
    " 'Erythropoietic hemochromatosis (disorder)_count_subject_present',\n",
    " 'Erythropoietic hemochromatosis (disorder)_count_relative_present',\n",
    " 'Hemochromatosis following repeated red blood cell transfusion (disorder)_count',\n",
    " 'Hemochromatosis following repeated red blood cell transfusion (disorder)_count_subject_present',\n",
    " 'Hemochromatosis following repeated red blood cell transfusion (disorder)_count_relative_present',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count_subject_present',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count_relative_present',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count_subject_present',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count_relative_present',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count_subject_present',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count_relative_present',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count_subject_present',\n",
    " 'Homozygous For The C . 845G A P . Cys282Tyr Variant_count_relative_present',\n",
    " 'Homozygous For The C2828Y Mutation Of The Hfe Gene_count',\n",
    " 'Homozygous For The C2828Y Mutation Of The Hfe Gene_count_subject_present',\n",
    " 'Homozygous For The C2828Y Mutation Of The Hfe Gene_count_relative_present',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count_subject_present',\n",
    " 'Hfe Ph63D Heterozygous Normal Variant_count_relative_present',\n",
    " 'Homozygous For The H63D Mutation_count',\n",
    " 'Homozygous For The H63D Mutation_count_subject_present',\n",
    " 'Homozygous For The H63D Mutation_count_relative_present',\n",
    " 'This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 282 C Y This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 63 H D_count',\n",
    " 'This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 282 C Y This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 63 H D_count_subject_present',\n",
    " 'This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 282 C Y This Patient Is A Carrier For The Mutation In The Hfe Gene At Position 63 H D_count_relative_present',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count_subject_present',\n",
    " 'Homozygous For Haemochromatosis Gene Mutation_count_relative_present',\n",
    " 'Heterozygous For The Haemochromatosis Gene_count',\n",
    " 'Heterozygous For The Haemochromatosis Gene_count_subject_present',\n",
    " 'Heterozygous For The Haemochromatosis Gene_count_relative_present',\n",
    " 'Haemochromatosis gene screening test (observable entity)_count',\n",
    " 'Haemochromatosis gene screening test (observable entity)_count_subject_present',\n",
    " 'Haemochromatosis gene screening test (observable entity)_count_relative_present',\n",
    " 'Carrier of hemochromatosis (finding)_count',\n",
    " 'Carrier of hemochromatosis (finding)_count_subject_present',\n",
    " 'Carrier of hemochromatosis (finding)_count_relative_present',\n",
    " 'Hemochromatosis gene screening test (procedure)_count',\n",
    " 'Hemochromatosis gene screening test (procedure)_count_subject_present',\n",
    " 'Hemochromatosis gene screening test (procedure)_count_relative_present',\n",
    " 'Iron overload (disorder)_count',\n",
    " 'Iron overload (disorder)_count_subject_present',\n",
    " 'Iron overload (disorder)_count_relative_present',\n",
    " 'Family history of hemochromatosis (situation)_count',\n",
    " 'Family history of hemochromatosis (situation)_count_subject_present',\n",
    " 'Family history of hemochromatosis (situation)_count_relative_present']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff894bad",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for value in settings_list:\n",
    "\n",
    "\n",
    "    global_param_dict = value #settings_list[0]\n",
    "\n",
    "    print(f\"Starting... {global_param_dict}\")\n",
    "    #global_param_str = str(global_param_dict).replace(\"{\", \"\").replace(\"}\", \"\").replace(\":\", \"\").replace(\" \", \"\").replace(\",\", \"\").replace(\"'\", \"_\").replace(\"__\", \"_\").replace(\"'\",\"\").replace(\",\",\"\").replace(\": \", \"_\").replace(\"{\",\"\").replace(\"}\",\"\").replace(\"True\",\"T\").replace(\"False\", \"F\").replace(\" \",\"_\").replace(\"[\", \"\").replace(\"]\", \"\").replace(\"_\",\"\")\n",
    "    #print(global_param_str)\n",
    "    \n",
    "    use_stored_base_learners = global_param_dict.get('use_stored_base_learners')\n",
    "\n",
    "    store_base_learners = global_param_dict.get('store_base_learners')\n",
    "\n",
    "#     global_param_str = str(global_param_dict).replace(\"'\",\"\").replace(\",\",\"\").replace(\": \", \"_\").replace(\"{\",\"\").replace(\"}\",\"\").replace(\"True\",\"T\").replace(\"False\", \"F\").replace(\" \",\"_\")\n",
    "\n",
    "#     global_param_str = global_param_str.replace(\"_\",\"\")\n",
    "    \n",
    "    \n",
    "#     global_param_str = global_param_str.replace(\"ethnicity\", \"eth\").replace(\"percent\", \"pct\").replace(\"sample\", \"smp\").replace(\"diagnostic\", \"diag\")\n",
    "    \n",
    "#     global_param_str = global_param_str.replace(\"medium\", \"med\").replace(\"annotation\", \"anno\").replace(\"feature\", \"feat\").replace(\"baselearners\", \"bl\")\n",
    "    \n",
    "#     global_param_str = base_project_dir_global + global_param_str\n",
    "    \n",
    "    \n",
    "    str_b = ''\n",
    "    for key in global_param_dict.keys():\n",
    "        if(key != 'data'):\n",
    "            str_b = str_b + '_' + str(global_param_dict.get(key))\n",
    "        else:\n",
    "            for key in global_param_dict.get('data'):\n",
    "                str_b = str_b + str(int(global_param_dict.get('data').get(key)))\n",
    "    \n",
    "    \n",
    "    \n",
    "    global_param_str = str_b\n",
    "    \n",
    "    global_param_str = base_project_dir_global + global_param_str\n",
    "    \n",
    "    print(\"len(global_param_str)\", len(global_param_str))\n",
    "    \n",
    "    file_path = f\"{global_param_str + additional_naming}/\"\n",
    "    \n",
    "    \n",
    "    date_1 = datetime.datetime.now().strftime(\"%Y_%m_%d-%I:%M:%S_%p\")\n",
    "    current_log_scores_filename = f\"log_scores_{date}\"\n",
    "    file_path = file_path+f\"{date_1}/\"\n",
    "    #global_param_str = global_param_str + additional_naming + f\"/{date_1}/\" #added\n",
    "    \n",
    "    pathlib.Path(file_path).mkdir(parents=True, exist_ok=True) \n",
    "    \n",
    "    print(global_param_str)\n",
    "    \n",
    "    log_folder_path = f\"{file_path}/logs/\"\n",
    "    pathlib.Path(log_folder_path).mkdir(parents=True, exist_ok=True) \n",
    "    nblog = open(f\"{file_path}/logs/log.log\", \"w\")\n",
    "    nblog = open(f\"{file_path}/logs/log.log\", \"a+\")\n",
    "    sys.stdout.echo = nblog\n",
    "    sys.stderr.echo = nblog\n",
    "\n",
    "    get_ipython().log.handlers[0].stream = nblog\n",
    "    get_ipython().log.setLevel(logging.INFO)\n",
    "\n",
    "    get_ipython().run_line_magic('autosave', '5')\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "    pathlib.Path(file_path).mkdir(parents=True, exist_ok=True) \n",
    "    pathlib.Path(file_path+\"figures\").mkdir(parents=True, exist_ok=True) \n",
    "    pathlib.Path(file_path+\"/results_master_lists\").mkdir(parents=True, exist_ok=True) \n",
    "    pathlib.Path(file_path+\"/progress_logs\").mkdir(parents=True, exist_ok=True)\n",
    "    pathlib.Path(file_path+\"/progress_logs_scores\").mkdir(parents=True, exist_ok=True)\n",
    "    pathlib.Path(f\"{file_path}/\"+\"/torch\").mkdir(parents=True, exist_ok=True)\n",
    "    pathlib.Path(f\"{file_path}/\"+\"/xgb\").mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    model_store_path = f\"{file_path}/\"+\"/model_store.json\"\n",
    "\n",
    "    model_directory = {'models':{}}\n",
    "    jsonString = json.dumps(model_directory)\n",
    "    if(os.path.exists(model_store_path)==False):\n",
    "        jsonFile = open(model_store_path, \"w\")\n",
    "        jsonFile.write(jsonString)\n",
    "        jsonFile.close()\n",
    "        \n",
    "        \n",
    "        \n",
    "    log_store_dataframe_path = 'log_store_dataframe'   \n",
    "   \n",
    "    \n",
    "    #init log store\n",
    "    \n",
    "    column_list = ['nb_size', 'f_list', 'auc','mcc','f1','precision','recall','accuracy', 'nb_val',\n",
    "                   'pop_val', 'g_val', 'g', 'weighted', 'use_stored_base_learners', 'store_base_learners',\n",
    "                           'resample', 'scale', 'n_features', 'param_space_size', 'n_unique_out',\n",
    "                           'outcome_var_n', 'div_p', 'percent_missing', 'corr', \n",
    "                           'age', 'sex', 'bmi','ethnicity', 'bloods', 'diagnostic_order',\n",
    "                           'drug_order', 'annotation_n', 'meta_sp_annotation_n',\n",
    "                           'X_test_size','X_test_orig_size', \n",
    "                   'cxpb', 'mutpb', 'indpb', 't_size']\n",
    "    \n",
    "    df = pd.DataFrame(data = None, columns=column_list)\n",
    "    df.to_csv(f\"{file_path}/progress_logs_scores/{log_store_dataframe_path}.csv\", mode='w', index=True, header=True)\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "    #Set data\n",
    "    def filter_substring_list(string, substr):\n",
    "        return [str for str in string if\n",
    "             any(sub in str for sub in substr) and 'bmi' not in str]\n",
    "    #%%time\n",
    "    outcome_variable = f'outcome_var_{global_param_dict.get(\"outcome_var_n\")}'\n",
    "\n",
    "    df_raw = pd.read_csv(input_csv_path)\n",
    "    \n",
    "    orignal_feature_names = list(df_raw.columns)\n",
    "    \n",
    "    #df_raw = df_raw.sample(1000)\n",
    "\n",
    "    all_df_columns = list(df_raw.columns)\n",
    "\n",
    "    drop_list = []\n",
    "\n",
    "    index_level_list = list(filter(lambda k: '__index_level' in k, all_df_columns))\n",
    "    #print(f\"__index_level: {len(index_level_list)}\")\n",
    "    drop_list.extend(index_level_list)\n",
    "    #df_raw.drop(index_level_list, axis=1, inplace=True)\n",
    "\n",
    "    Unnamed_list = list(filter(lambda k: 'Unnamed' in k, all_df_columns))\n",
    "    #print(f\"Unnamed: {len(index_level_list)}\")\n",
    "    drop_list.extend(Unnamed_list)\n",
    "    #df_raw.drop(Unnamed_list, axis=1, inplace=True)\n",
    "\n",
    "    for elem in all_df_columns:\n",
    "        res = find_near_matches('hfe', elem.lower(), max_l_dist=0)\n",
    "        if(len(res)>0):\n",
    "            #df_raw.drop(elem, axis=1, inplace=True)\n",
    "            drop_list.append(elem)\n",
    "            #print(elem)\n",
    "\n",
    "    for elem in all_df_columns:\n",
    "        res = find_near_matches('client_idcode', elem.lower(), max_l_dist=0)\n",
    "        if(len(res)>0):\n",
    "            #df_raw.drop(elem, axis=1, inplace=True)\n",
    "            drop_list.append(elem)\n",
    "            #print(elem)\n",
    "\n",
    "    for col in df_raw.columns:\n",
    "        if(col.find(\"chrom\") !=-1):\n",
    "            #print(col)\n",
    "            drop_list.append(col)\n",
    "\n",
    "    print(\"------------------------\")\n",
    "\n",
    "\n",
    "    for elem in all_df_columns:\n",
    "        res = find_near_matches(outcome_variable.replace('_count_subject_present_', \"\"), elem.lower(), max_l_dist=0)\n",
    "        if(len(res)>0):\n",
    "            print(\"Outcome var found in df columns:\")\n",
    "            print(elem)\n",
    "\n",
    "\n",
    "    for elem in all_df_columns:\n",
    "        res = find_near_matches('chrom', elem.lower(), max_l_dist=0)\n",
    "        if(len(res)>0):\n",
    "            #f_raw.drop(elem, axis=1, inplace=True)\n",
    "            drop_list.append(elem)\n",
    "            #print(elem)\n",
    "\n",
    "\n",
    "    for elem in all_df_columns:\n",
    "        res = find_near_matches('hfe', elem.lower(), max_l_dist=0)\n",
    "        if(len(res)>0):\n",
    "            #df_raw.drop(elem, axis=1, inplace=True)\n",
    "            drop_list.append(elem)\n",
    "            #print(elem)\n",
    "\n",
    "    blood_test_substrings = ['_mean',\n",
    "                             '_median',\n",
    "                             '_mode',\n",
    "                             '_std',\n",
    "                             '_num-tests',\n",
    "                             '_days-since-last-test',\n",
    "                             '_max',\n",
    "                             '_min',\n",
    "                             '_most-recent',\n",
    "                             '_earliest-test',\n",
    "                             '_days-between-first-last',\n",
    "                             '_contains-extreme-low',\n",
    "                             '_contains-extreme-high']\n",
    "\n",
    "    diagnostic_test_substrings = ['_num-diagnostic-order',\n",
    "                                 '_days-since-last-diagnostic-order',\n",
    "                                 '_days-between-first-last-diagnostic']\n",
    "\n",
    "    drug_order_substrings = ['_num-drug-order',\n",
    "                             '_days-since-last-drug-order',\n",
    "                             '_days-between-first-last-drug']\n",
    "\n",
    "\n",
    "\n",
    "    meta_sp_annotation_count_list = list(filter(lambda k: '_count_subject_present' in k, all_df_columns))\n",
    "    print(f\"_count_subject_present: {len(meta_sp_annotation_count_list)}\")\n",
    "\n",
    "    annotation_count_list = list(filter(lambda k: '_count' in k and '_count_subject_present' not in k, all_df_columns))\n",
    "    print(f\"count_list:             {len(annotation_count_list)}\")\n",
    "\n",
    "    diagnostic_order_list =[]# list(filter(lambda k: '-diagnostic-order' in k, all_df_columns))\n",
    "    diagnostic_list = filter_substring_list(all_df_columns, diagnostic_test_substrings)\n",
    "    diagnostic_order_list.extend(diagnostic_list)\n",
    "    print(f\"diagnostic_order_list:  {len(diagnostic_order_list)}\")\n",
    "\n",
    "    drug_order_list = []#list(filter(lambda k: '-drug-order' in k, all_df_columns))\n",
    "    drug_list = filter_substring_list(all_df_columns, drug_order_substrings)\n",
    "    drug_order_list.extend(drug_list)\n",
    "    print(f\"drug_order_list:        {len(drug_order_list)}\")\n",
    "\n",
    "    bmi_list = list(filter(lambda k: 'bmi_' in k, all_df_columns))\n",
    "    print(f\"bmi_list:        {len(bmi_list)}\")\n",
    "\n",
    "    ethnicity_list = list(filter(lambda k: 'census_' in k, all_df_columns))\n",
    "    print(f\"ethnicity_list:        {len(ethnicity_list)}\")\n",
    "    \n",
    "    annotation_mrc_count_list = list(filter(lambda k: '_count_mrc_cs' in k, all_df_columns))\n",
    "    print(f\"_count_mrc_cs:        {len(annotation_mrc_count_list)}\")\n",
    "    \n",
    "    meta_sp_annotation_mrc_count_list = list(filter(lambda k: '_count_subject_present_mrc_cs' in k, all_df_columns))\n",
    "    print(f\"_count_subject_present_mrc_cs_list:        {len(meta_sp_annotation_mrc_count_list)}\")\n",
    "    \n",
    "    core_02_list = list(filter(lambda k: 'core_02_' in k, all_df_columns))\n",
    "    print(f\"core_02_list:        {len(core_02_list)}\")\n",
    "    \n",
    "    bed_list = list(filter(lambda k: 'bed_' in k, all_df_columns))\n",
    "    print(f\"bed_list:        {len(bed_list)}\")\n",
    "    \n",
    "    vte_status_list = list(filter(lambda k: 'vte_status_' in k, all_df_columns))\n",
    "    print(f\"vte_status_list:        {len(vte_status_list)}\")\n",
    "    \n",
    "    hosp_site_list = list(filter(lambda k: 'hosp_site_' in k, all_df_columns))\n",
    "    print(f\"hosp_site_list:        {len(hosp_site_list)}\")\n",
    "    \n",
    "    core_resus_list = list(filter(lambda k: 'core_resus_' in k, all_df_columns))\n",
    "    print(f\"core_resus_list:        {len(core_resus_list)}\")\n",
    "    \n",
    "    news_list = list(filter(lambda k: 'news_resus_' in k, all_df_columns))\n",
    "    print(f\"news_list:        {len(news_list)}\")\n",
    "\n",
    "\n",
    "    bloods_list = filter_substring_list(all_df_columns, blood_test_substrings)\n",
    "    print(f\"bloods_list: {len(bloods_list)}\")\n",
    "\n",
    "    candidate_feature_category_lists=[meta_sp_annotation_count_list, annotation_count_list, diagnostic_order_list, drug_order_list, bmi_list, ethnicity_list, bloods_list]\n",
    "\n",
    "    pertubation_columns = []\n",
    "\n",
    "    if(value.get('data').get('age') == True):\n",
    "        pertubation_columns.append('age')\n",
    "\n",
    "    if(value.get('data').get('sex') == True):\n",
    "        pertubation_columns.append('male')\n",
    "\n",
    "    if(value.get('data').get('bmi') == True):\n",
    "        pertubation_columns.extend(bmi_list)\n",
    "\n",
    "    if(value.get('data').get('ethnicity') == True):\n",
    "        pertubation_columns.extend(ethnicity_list)\n",
    "\n",
    "    if(value.get('data').get('bloods') == True):\n",
    "        pertubation_columns.extend(bloods_list)\n",
    "\n",
    "    if(value.get('data').get('diagnostic_order') == True):\n",
    "        pertubation_columns.extend(diagnostic_order_list)\n",
    "\n",
    "    if(value.get('data').get('drug_order') == True):\n",
    "        pertubation_columns.extend(drug_order_list)\n",
    "\n",
    "    if(value.get('data').get('annotation_n') == True):\n",
    "        pertubation_columns.extend(annotation_count_list)\n",
    "\n",
    "    if(value.get('data').get('meta_sp_annotation_n') == True):\n",
    "        pertubation_columns.extend(meta_sp_annotation_count_list)\n",
    "        \n",
    "    if(value.get('data').get('annotation_mrc_n') == True):\n",
    "        pertubation_columns.extend(annotation_mrc_count_list)\n",
    "            \n",
    "    if(value.get('data').get('meta_sp_annotation_mrc_n') == True):\n",
    "        pertubation_columns.extend(meta_sp_annotation_mrc_count_list)\n",
    "        \n",
    "    if(value.get('data').get('core_02') == True):\n",
    "        pertubation_columns.extend(core_02_list)\n",
    "        \n",
    "    if(value.get('data').get('bed') == True):\n",
    "        pertubation_columns.extend(bed_list)\n",
    "        \n",
    "    if(value.get('data').get('vte_status') == True):\n",
    "        pertubation_columns.extend(vte_status_list)\n",
    "        \n",
    "    if(value.get('data').get('hosp_site') == True):\n",
    "        pertubation_columns.extend(hosp_site_list)\n",
    "        \n",
    "    if(value.get('data').get('core_resus') == True):\n",
    "        pertubation_columns.extend(core_resus_list)\n",
    "        \n",
    "    if(value.get('data').get('news') == True):\n",
    "        pertubation_columns.extend(news_list)\n",
    "\n",
    "\n",
    "    display(global_param_dict.get('data'))\n",
    "\n",
    "    print(f\"Using {len(pertubation_columns)}/{len(all_df_columns)} columns\")\n",
    "\n",
    "    list_2 = df_raw.columns\n",
    "    list_1 = pertubation_columns.copy()\n",
    "\n",
    "    difference_list = list(set(list_2) - set(list_1))\n",
    "    print(f\"Omitting {len(difference_list)} :...\")\n",
    "    print(f\"{difference_list[0:5]}...\")\n",
    "\n",
    "    print(\"--------------------------\")\n",
    "    imputed_cols = list(df_raw.columns)\n",
    "    #     scaler = StandardScaler()\n",
    "\n",
    "    #     scaler.fit(df_raw)\n",
    "\n",
    "    #     scaled_imputed = scaler.transform(df_raw)\n",
    "\n",
    "    #     scaled_imputed_df = pd.DataFrame(data=scaled_imputed, columns = imputed_cols)\n",
    "\n",
    "    #corr_matrix = df_raw.corr().abs()\n",
    "   # temp_col_list = list(df_raw.columns)\n",
    "    temp_col_list = list(df_raw.select_dtypes(include=[float, int]).columns)\n",
    "\n",
    "    #corr_matrix = pd.DataFrame(np.corrcoef(df_raw.values, rowvar=False), columns=temp_col_list).abs()\n",
    "    \n",
    "    corr_matrix = pd.DataFrame(np.corrcoef(df_raw.select_dtypes(include=[float, int]).values, rowvar=False), columns=temp_col_list).abs()\n",
    "        \n",
    "    # Create a True/False mask and apply it\n",
    "    mask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n",
    "    tri_df = corr_matrix.mask(mask)\n",
    "    # List column names of highly correlated features (r >0.5 )\n",
    "    corr_to_drop = [c for c in tri_df.columns if any(tri_df[c] > value.get('corr'))]\n",
    "\n",
    "    #reduced_df = scaled_imputed_df.drop(corr_to_drop, axis=1)\n",
    "    print(f\"Identified {len(corr_to_drop)} correlated features to drop at >{value.get('corr')}\")\n",
    "    drop_list.extend(corr_to_drop)\n",
    "\n",
    "    print(\"------------------------\")\n",
    "    with open('percent_missing_dict.pickle', 'rb') as handle:\n",
    "        percent_missing_dict = pickle.load(handle)\n",
    "\n",
    "\n",
    "    percent_missing_threshold = value.get('percent_missing')\n",
    "    percent_missing_drop_list = []\n",
    "    for col in all_df_columns:\n",
    "        try:\n",
    "            if(percent_missing_dict.get(col)>percent_missing_threshold):\n",
    "                percent_missing_drop_list.append(col)\n",
    "        except:\n",
    "            pass\n",
    "    print(f\"Identified {len(percent_missing_drop_list)} at > {percent_missing_threshold} threshold\")\n",
    "    drop_list.extend(percent_missing_drop_list)\n",
    "\n",
    "\n",
    "    print(f\"Drop_list 0:5: \")\n",
    "    print('\\n'.join(map(str, drop_list[0:5])))\n",
    "\n",
    "    print(\"------------------------\")\n",
    "\n",
    "\n",
    "\n",
    "    try:\n",
    "        drop_list.remove(outcome_variable)\n",
    "    except: \n",
    "        print(outcome_variable + \"not in drop list\")\n",
    "\n",
    "    #df_raw.drop(drop_list, axis=1, inplace=True)\n",
    "\n",
    "    #df_raw.drop(['datetime', 'datetime.1'], axis=1, inplace=True)\n",
    "\n",
    "\n",
    "    final_column_list = [x for x in pertubation_columns if (x not in drop_list)]\n",
    "\n",
    "\n",
    "    df_imputed = df_raw[final_column_list].copy()\n",
    "    \n",
    "    df_imputed = df_imputed.loc[:,~df_imputed.columns.duplicated()].copy()\n",
    "\n",
    "    print(\"Screening for non float data types:\")\n",
    "    types = []\n",
    "    for col in df_imputed.columns:\n",
    "        if(df_imputed[col].dtype != int and df_imputed[col].dtype != float):\n",
    "            print(col)\n",
    "\n",
    "    y = df_raw[outcome_variable]\n",
    "\n",
    "    X = df_imputed\n",
    "\n",
    "    rename_cols = True\n",
    "    if(rename_cols):\n",
    "        regex = re.compile(r\"\\[|\\]|<\", re.IGNORECASE)\n",
    "        df_imputed.columns = [regex.sub(\"_\", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in df_imputed.columns.values]\n",
    "\n",
    "    #file_path = global_param_str + additional_naming\n",
    "    plot_path = file_path + '/figures/' #file_path\n",
    "    Path(plot_path).mkdir(parents=True, exist_ok=True)\n",
    "    Path(file_path).mkdir(parents=True, exist_ok=True) #added\n",
    "    #Path(plot_path + \"_scaled//\").mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    init_live_scores = []\n",
    "\n",
    "    with open(plot_path + \"live_classifier_grid_scores.pickle\", \"wb\") as handle:\n",
    "            pickle.dump(init_live_scores, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "\n",
    "\n",
    "    if(global_param_dict.get('resample') ==None):\n",
    "        X_train_orig, X_test_orig, y_train_orig, y_test_orig = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1\n",
    "        )\n",
    "\n",
    "\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X_train_orig, y_train_orig, test_size=0.25, random_state=1\n",
    "        )   \n",
    "\n",
    "\n",
    "    elif(global_param_dict.get('resample') =='undersample'):\n",
    "        from imblearn.under_sampling import RandomUnderSampler \n",
    "        rus = RandomUnderSampler(random_state=0)\n",
    "        X, y = rus.fit_resample(X, y)\n",
    "        #Create validation set\n",
    "        X_train_orig, X_test_orig, y_train_orig, y_test_orig = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1\n",
    "        )\n",
    "\n",
    "        #X = X_train_orig.copy()\n",
    "        #y = y_train_orig.copy()\n",
    "        #Resplit holding back _orig\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X_train_orig, y_train_orig, test_size=0.25, random_state=1\n",
    "        )\n",
    "        X = X_train_orig.copy()\n",
    "        y = y_train_orig.copy()\n",
    "\n",
    "    elif(global_param_dict.get('resample') =='oversample'):\n",
    "        #Train test split then oversample to avoid poision\n",
    "        X_train_orig, X_test_orig, y_train_orig, y_test_orig = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1\n",
    "        )\n",
    "\n",
    "        from imblearn.over_sampling import RandomOverSampler\n",
    "        sampling_strategy = 0.8\n",
    "        ros = RandomOverSampler(sampling_strategy=sampling_strategy)\n",
    "        X_res, y_res = ros.fit_resample(X_train_orig, y_train_orig)\n",
    "        print(y_res.value_counts())\n",
    "        X = X_res.copy()\n",
    "        y = y_res.copy()\n",
    "\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=0.25, random_state=1)\n",
    "\n",
    "    target_n_features = global_param_dict.get('n_features')\n",
    "\n",
    "    if(target_n_features != 'all'):\n",
    "\n",
    "        features = getNfeaturesANOVAF(target_n_features)\n",
    "\n",
    "        print(f\"target_n_features: {target_n_features}\")\n",
    "\n",
    "        X_train = X_train[features]\n",
    "        X_train_orig = X_train_orig[features]\n",
    "        X_test = X_test[features]\n",
    "        X_train_orig = X_train_orig[features]\n",
    "\n",
    "    x_train_col_val = len(X_train.columns)\n",
    "\n",
    "    length_x_data = len(X_train)\n",
    "\n",
    "    display(y.value_counts())\n",
    "\n",
    "    y.plot(kind=\"hist\")\n",
    "\n",
    "    scale = global_param_dict.get('scale')\n",
    "\n",
    "\n",
    "\n",
    "    col_names = X.columns\n",
    "    if scale == True:\n",
    "        scaler = ColumnTransformer(\n",
    "            [\n",
    "                (\n",
    "                    \"somename\",\n",
    "                    StandardScaler(),\n",
    "                    list(X.columns),\n",
    "                )\n",
    "            ],\n",
    "            remainder=\"passthrough\",\n",
    "        )\n",
    "        X = scaler.fit_transform(X)\n",
    "\n",
    "        X = pd.DataFrame(X, columns=col_names)\n",
    "        #plot_path = plot_path + \"_scaled//\"\n",
    "\n",
    "    if(global_param_dict.get('param_space_size') == 'medium'):\n",
    "        log_small = np.logspace(-1, -5, 3)\n",
    "        print(log_small)\n",
    "        bool_param = [True, False]\n",
    "        print(bool_param)\n",
    "        log_large = np.logspace(0.4, 2, 3).astype(int)\n",
    "        print(log_large)\n",
    "        log_large_long = np.floor(np.logspace(0.4, 2.8, 6)).astype(int)\n",
    "        print(log_large_long)\n",
    "        log_med_long = np.floor(np.logspace(0.4, 1.5, 5)).astype(int)\n",
    "        print(log_med_long)\n",
    "        log_med = np.floor(np.logspace(0.4, 1.5, 3)).astype(int)\n",
    "        print(log_med)\n",
    "        nstep = 3\n",
    "        log_zero_one = np.logspace(0.0, 1.0, nstep) / 10\n",
    "        print(log_zero_one)\n",
    "        lin_zero_one = np.linspace(0.1, 1.0, nstep) / 10\n",
    "        print(lin_zero_one)\n",
    "        \n",
    "    y = df_raw[outcome_variable]\n",
    "\n",
    "    X = df_imputed\n",
    "\n",
    "    rename_cols = False\n",
    "    if(rename_cols):\n",
    "        regex = re.compile(r\"\\[|\\]|<\", re.IGNORECASE)\n",
    "        df_imputed.columns = [regex.sub(\"_\", col) if any(x in str(col) for x in set(('[', ']', '<'))) else col for col in df_imputed.columns.values]\n",
    "\n",
    "    #Begin GA\n",
    "    all_curated_columns = list(df_imputed.columns)\n",
    "    \n",
    "    plot_path = file_path\n",
    "    fig_path = 'figures/'\n",
    "    #plot_path = \"ML_plots_HFE_Genetic_ensemble//\"\n",
    "    #plot_path = \"ML_plots_HFE_imputed_true_meta_demo//\"\n",
    "    Path(plot_path).mkdir(parents=True, exist_ok=True)\n",
    "    Path(plot_path + \"_scaled//\").mkdir(parents=True, exist_ok=True)\n",
    "    Path(plot_path+fig_path).mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    \n",
    "    y.value_counts().plot( kind='hist')\n",
    "    \n",
    "    y_train.value_counts().plot(kind='hist')\n",
    "    \n",
    "    y_test.value_counts().plot(kind='hist')\n",
    "    \n",
    "    \n",
    "    print(\"Double checking X columns for outcome variable and similar substrings, manually halt if contaminated!\")\n",
    "    print(outcome_variable in X.columns)\n",
    "    \n",
    "    \n",
    "    for elem in X.columns:\n",
    "        if(elem.lower().find('hemo') !=-1):\n",
    "            print(elem)\n",
    "            \n",
    "    for elem in X.columns:\n",
    "        if(elem.lower().find('hfe') !=-1):\n",
    "            print(elem)\n",
    "            \n",
    "    \n",
    "\n",
    "    allData_float3_imputed = X.copy()\n",
    "\n",
    "    start_val = 2\n",
    "    feature_parameter_vector = np.linspace(\n",
    "        2, len(X.columns), int(len(X.columns) + 1 - start_val)\n",
    "    ).astype(int)\n",
    "\n",
    "    # .drop('index', axis=1)\n",
    "    cv = KFold(n_splits=10, random_state=1, shuffle=True)\n",
    "\n",
    "    master_result_list = []\n",
    "    multi_processing = False\n",
    "    warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "    warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "    a = 1\n",
    "\n",
    "    # Check whether the specified path exists or not\n",
    "    isExist = os.path.exists(file_path)\n",
    "    if not isExist:\n",
    "\n",
    "        # Create a new directory because it does not exist\n",
    "        os.makedirs(file_path)\n",
    "        print(\"The new directory is created!\")\n",
    "\n",
    "    isExist = os.path.exists(file_path+\"/results_master_lists\")\n",
    "    if not isExist:\n",
    "\n",
    "        # Create a new directory because it does not exist\n",
    "        os.makedirs(file_path+\"/results_master_lists\")\n",
    "        print(\"The new directory is created!\")\n",
    "\n",
    "\n",
    "    #test space\n",
    "\n",
    "    nb_params = [4, 8, 16]\n",
    "    pop_params = [10, 20]\n",
    "    g_params = [10, 30]\n",
    "\n",
    "\n",
    "    nb_params = [4, 8, 16, 32]\n",
    "    pop_params = [32, 64, 128]\n",
    "    g_params = [128]\n",
    "\n",
    "    nb_params = [16]\n",
    "    pop_params = [32]\n",
    "    g_params = [128]\n",
    "    \n",
    "    print(\"Running...\")\n",
    "    \n",
    "    \n",
    "\n",
    "    modelFuncList = [\n",
    "#         dummy_model_gen,\n",
    "#         dummy_model_gen,\n",
    "       logModelGen,\n",
    "       perceptronModelGen,\n",
    "       extraTreesModelGen,\n",
    "       randomForestModelGen,\n",
    "       kNearestNeighborsModelGen,\n",
    "       XGBoostModelGen,\n",
    "       elasticNeuralNetworkModelGen,\n",
    "\n",
    "       DecisionTreeClassifier_ModelGen,\n",
    "       AdaBoostClassifier_ModelGen,\n",
    "       GaussianNB_ModelGen,\n",
    "       QuadraticDiscriminantAnalysis_ModelGen,\n",
    "       SVC_ModelGen,\n",
    "       GradientBoostingClassifier_ModelGen,\n",
    "       RandomForestClassifier_ModelGen,\n",
    "       MLPClassifier_ModelGen,\n",
    "        #KerasClassifier_ModelGen()\n",
    "       Pytorch_binary_class_ModelGen\n",
    "        ]\n",
    "    print(len(modelFuncList))\n",
    "    \n",
    "    \n",
    "    fitted_perceptron = perceptronModelGen_dummy()[1]\n",
    "    dummy_columns = perceptronModelGen_dummy()[2]\n",
    "    #pop = toolbox.population(n=pop_val)\n",
    "    \n",
    "    gen_eval_score_threshold_early_stopping = 5\n",
    "\n",
    "    #%%prun -s cumulative\n",
    "    if __name__ == \"__main__\":\n",
    "        grid = [nb_params, pop_params, g_params]\n",
    "        param_grid = list(itertools.product(*grid))\n",
    "        print(param_grid)\n",
    "        for elem in param_grid:\n",
    "            print(elem, elem[0] * elem[1], \"model generation space\")\n",
    "            print(elem, elem[0] * elem[2], \"individual evaluation space\")\n",
    "        print(len(param_grid))\n",
    "\n",
    "        prediction_array = None\n",
    "\n",
    "        #shared.setCont(creator)\n",
    "\n",
    "        #def main():\n",
    "\n",
    "        #small test:\n",
    "        #param_grid = [(4, 5, 4)]\n",
    "\n",
    "\n",
    "        #param_grid = [(5,5,5), (50,5,5), (5,50,5), (5,5,50)]\n",
    "\n",
    "\n",
    "        #random.shuffle(param_grid)\n",
    "        date = datetime.datetime.now().strftime(\"%Y_%m_%d-%I:%M:%S_%p\")\n",
    "        current_log_filename = f'log_{date}.txt'\n",
    "        for i in range(0, len(param_grid)):\n",
    "        #         best = evolve_best_ensemble(\n",
    "        #             param_grid[i][0], param_grid[i][1], param_grid[i][2])\n",
    "\n",
    "\n",
    "            try:\n",
    "                nb_val  = param_grid[i][0]\n",
    "                pop_val = param_grid[i][1]\n",
    "                g_val   = param_grid[i][2]\n",
    "                print(\"Evolving ensemble: \", \"nb_val:\", nb_val,\n",
    "                      \"pop_val:\", pop_val, \"g_val:\", g_val, \"...\")\n",
    "\n",
    "\n",
    "                generation_progress_list = []\n",
    "                print(\"Preparing log file\")\n",
    "                with open(file_path+\"/progress_logs/\"+current_log_filename, 'w') as file:\n",
    "                    pass \n",
    "\n",
    "                with open(file_path+\"/progress_logs/\"+current_log_filename, \"a\") as myfile:\n",
    "                    myfile.write(' '.join(str(i) for i in[[\"Evolving ensemble: \", \"nb_val:\", nb_val,\n",
    "                      \"pop_val:\", pop_val, \"g_val:\", g_val, \"...\"]]))\n",
    "                    myfile.write('\\n')\n",
    "                #global master_result_list\n",
    "                    myfile.close()\n",
    "                    \n",
    "                    \n",
    "                    \n",
    "                start = time.time()\n",
    "\n",
    "                # print(parameters)\n",
    "                #     nb_val = parameters[0]\n",
    "                #     pop_val = parameters[1]\n",
    "                #     g_val = parameters[2]\n",
    "                #         creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0,))\n",
    "                #         creator.create(\"Individual\", list, fitness=creator.FitnessMin)\n",
    "\n",
    "                # nb_val = 2\n",
    "                #toolbox = base.Toolbox()\n",
    "                print(\"Registering toolbox elements\")\n",
    "                toolbox.register(\"ensembleGenerator\", ensembleGenerator, nb_val=nb_val)\n",
    "                toolbox.register(\n",
    "                    \"individual\",\n",
    "                    tools.initRepeat,\n",
    "                    creator.Individual,\n",
    "                    toolbox.ensembleGenerator,\n",
    "                    n=1, #could potentially increase this to pass to multiprocessing? \n",
    "                )\n",
    "                toolbox.register(\"population\", tools.initRepeat, list, toolbox.individual)\n",
    "\n",
    "                toolbox.register(\"mate\", tools.cxTwoPoint)\n",
    "                toolbox.register(\"mutate\", tools.mutFlipBit, indpb=global_param_dict.get('indpb'))\n",
    "                toolbox.register(\"mutateFunction\", mutateEnsemble)\n",
    "                toolbox.register(\"mutateEnsemble\", toolbox.mutateFunction)\n",
    "                toolbox.register(\"select\", tools.selTournament, tournsize=global_param_dict.get('t_size'))\n",
    "\n",
    "                #pool = multiprocessing.Pool()\n",
    "                #toolbox.register(\"map\", pool.map)\n",
    "\n",
    "                # print(nb_val, pop_val, g_val)\n",
    "                # print(\"pop_val\", pop_val)\n",
    "                start = time.time()\n",
    "                #     nb_val = param_grid[k][0]\n",
    "                #     pop_val = param_grid[k][1]\n",
    "                #     g_val = param_grid[k][2]\n",
    "                print(f\"Generate intial population n=={pop_val}\")\n",
    "                pop = toolbox.population(n=pop_val)\n",
    "\n",
    "                # Evaluate the entire population\n",
    "                fitnesses = list(toolbox.map(toolbox.evaluate, pop))\n",
    "                for ind, fit in zip(pop, fitnesses):\n",
    "                    ind.fitness.values = fit\n",
    "\n",
    "                # CXPB  is the probability with which two individuals\n",
    "                #       are crossed\n",
    "                #\n",
    "                # MUTPB is the probability for mutating an individual\n",
    "                CXPB, MUTPB = global_param_dict.get('cxpb'), global_param_dict.get('mutpb')\n",
    "\n",
    "                # Extracting all the fitnesses of\n",
    "                fits = [ind.fitness.values[0] for ind in pop]\n",
    "\n",
    "                # Variable keeping track of the number of generations\n",
    "                g = 0\n",
    "\n",
    "                #testcrash\n",
    "\n",
    "                # Begin the evolution\n",
    "                chance_dummy_best_pred = [x for x in range(0, len(y_test))]\n",
    "\n",
    "                gen_eval_score = metrics.roc_auc_score(y_test, chance_dummy_best_pred)\n",
    "\n",
    "                gen_eval_score_counter = 0\n",
    "\n",
    "                pbar = tqdm(total = g_val+1)\n",
    "                #while currentData[0] <= runs:\n",
    "\n",
    "                stop_early = False\n",
    "\n",
    "                gen_eval_score_previous = gen_eval_score\n",
    "                gen_eval_score_gain = 0\n",
    "\n",
    "                highest_scoring_ensemble = (0, None)\n",
    "\n",
    "                while g < g_val and gen_eval_score < 0.999 and stop_early==False:\n",
    "\n",
    "\n",
    "                # while g < 50: alt ::  while g < g_val and  ?? eval some how measure AUC or mcc of ensemble? \n",
    "                #for i in tqdm(range(0, g_val)):\n",
    "                    # A new generation\n",
    "                    g = g + 1\n",
    "                    pbar.update(1)\n",
    "                    print(\"\\n -- Generation %i --\" % g)\n",
    "                    # Select the next generation individuals\n",
    "                    print(\"Selecting next generation individuals, \", len(pop))\n",
    "                    offspring = toolbox.select(pop, len(pop))\n",
    "                    # Clone the selected individuals\n",
    "                    print(\"Clone the selected individuals\")\n",
    "                    offspring = list(toolbox.map(toolbox.clone, offspring))\n",
    "                    print(\"Apply crossover and mutation on the offspring\")\n",
    "                    # Apply crossover and mutation on the offspring\n",
    "                    for child1, child2 in zip(offspring[::2], offspring[1::2]):\n",
    "                        if random.random() < CXPB:\n",
    "                            toolbox.mate(child1[0], child2[0])\n",
    "                            del child1.fitness.values\n",
    "                            del child2.fitness.values\n",
    "                    counter = 0\n",
    "                    print(\"mutate\")\n",
    "                    for mutant in offspring:\n",
    "                        if random.random() < MUTPB:\n",
    "                            # print(mutant[0][0])\n",
    "                            # mutant[0][0] = baseLearnerGenerator()#\n",
    "                            # print(offspring[counter])\n",
    "                            mutatedEnsemble = mutateEnsemble(offspring[counter])\n",
    "                            offspring[counter] = mutatedEnsemble\n",
    "                            # print(\"mutated into:\")\n",
    "                            # print(mutatedEnsemble)\n",
    "                            # toolbox.mutateEnsemble(mutant[0])\n",
    "                            # toolbox.mutate(mutant[0][0])\n",
    "                            del mutant.fitness.values\n",
    "                        counter = counter + 1\n",
    "                    print(\"Evaluate the individuals with an invalid fitness\")\n",
    "                    # Evaluate the individuals with an invalid fitness\n",
    "                    invalid_ind = [ind for ind in offspring if not ind.fitness.valid]\n",
    "                    #fitnesses = map(toolbox.evaluate, invalid_ind)\n",
    "                    fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)\n",
    "                    for ind, fit in zip(invalid_ind, fitnesses):\n",
    "                        ind.fitness.values = fit\n",
    "                    pop[:] = offspring\n",
    "                    print(\"Gather all the fitnesses in one list and print the stats\")\n",
    "                    # Gather all the fitnesses in one list and print the stats\n",
    "                    fits = [ind.fitness.values[0] for ind in pop]\n",
    "                    length = len(pop)\n",
    "                    mean = sum(fits) / length\n",
    "                    sum2 = sum(x * x for x in fits)\n",
    "                    std = abs(sum2 / length - mean**2) ** 0.5\n",
    "                    print(f\"min: {min(fits)}, max: {max(fits)} , mean: {mean}, std: {std}\")\n",
    "                    # pool.close() # experimental\n",
    "                    #Additional eval stage for generation truncation:\n",
    "                    #argmin... or argmax for auc\n",
    "\n",
    "                    #calculate the best individual from within the population by arg min or max on target metric\n",
    "                    best = pop[np.argmax([toolbox.evaluate(x) for x in pop])]\n",
    "                    #best_pred = get_best_y_pred(best)\n",
    "                    #gen_eval_score = metrics.roc_auc_score(y_test_orig, best_pred)\n",
    "\n",
    "                    #With best individual from population, evaluate their ensemble score on metric\n",
    "                    y_pred = get_y_pred_resolver(best, valid=False)\n",
    "\n",
    "                    gen_eval_score = metrics.roc_auc_score(y_test, y_pred)  \n",
    "                    print(f\"gen_eval_score == {gen_eval_score} Generation {g}\")\n",
    "                    generation_progress_list.append(gen_eval_score)\n",
    "\n",
    "\n",
    "\n",
    "                    if(gen_eval_score < highest_scoring_ensemble[0]):\n",
    "                        gen_eval_score_counter = gen_eval_score_counter + 1\n",
    "                        if(debug):\n",
    "                            print(f\"gen_eval_score_counter {gen_eval_score_counter}, highest so far: {highest_scoring_ensemble[0]}\")\n",
    "\n",
    "\n",
    "                        if(gen_eval_score_counter > gen_eval_score_threshold_early_stopping):\n",
    "                            stop_early = True\n",
    "                    elif(gen_eval_score > highest_scoring_ensemble[0]):\n",
    "                        if(debug):\n",
    "                            print(f\"gen_eval_score gain: {gen_eval_score-gen_eval_score_previous} rate: {(highest_scoring_ensemble[0]-0.5)/g} ETA: {round(((1-highest_scoring_ensemble[0])/(gen_eval_score_gain+1.00000000e-99)))}\")\n",
    "                        gen_eval_score_gain = gen_eval_score_gain + (gen_eval_score-gen_eval_score_previous)\n",
    "                        gen_eval_score_counter = 0\n",
    "\n",
    "                    if(gen_eval_score> highest_scoring_ensemble[0]):\n",
    "                        highest_scoring_ensemble = (gen_eval_score, best)  \n",
    "\n",
    "\n",
    "                    gen_eval_score_previous = gen_eval_score\n",
    "\n",
    "                pbar.close()  \n",
    "\n",
    "                #best = pop[np.argmax([toolbox.evaluate(x) for x in pop])] #was argmin\n",
    "\n",
    "                #Get stored highest ensemble\n",
    "                best = highest_scoring_ensemble[1]\n",
    "\n",
    "\n",
    "                print(\"\\n\")\n",
    "                print(\"Best Ensemble Model: \")\n",
    "                for i in range(0, len(best[0])):\n",
    "                    print(best[0][i][1], \"n features: \", len(best[0][i][2]))\n",
    "                    \n",
    "                print(f\"Best Ensemble diversity score: {measure_binary_vector_diversity(best)}\")\n",
    "                    \n",
    "                end = time.time()\n",
    "                print(end - start)\n",
    "\n",
    "\n",
    "                try:\n",
    "                    print(\"Getting final final best pred for plot with validation set, get weights from xtrain ytrain\")\n",
    "                    best_pred_orig = get_y_pred_resolver(best, valid=True)\n",
    "\n",
    "                    plot_auc(\n",
    "                        best_pred_orig,\n",
    "                        \"best_pop=\" + str(pop_val) + \"_g=\" + str(g_val) + \"_nb=\" + str(nb_val),\n",
    "                    )\n",
    "                    print(\"nb_val:\", nb_val, \"pop_val:\", pop_val, \"g_val:\", g_val,\n",
    "                          \"AUC: \", metrics.roc_auc_score(y_test_orig, best_pred_orig), \"g:\", g)\n",
    "                except Exception as e:\n",
    "                    print(\"Failed to get best y pred and plot auc\")\n",
    "                    print(e)\n",
    "                    pass\n",
    "                \n",
    "                \n",
    "\n",
    "\n",
    "                end = time.time()\n",
    "                print(\"nb_val:\", nb_val, \"pop_val:\", pop_val, \"g_val:\", g_val,\n",
    "                          \"AUC: \", metrics.roc_auc_score(y_test_orig, best_pred_orig), \"Run Time (min): \", round((end - start)/60, 3), \"g:\", g)\n",
    "                master_result_list.append([\"nb_val:\", nb_val, \"pop_val:\", pop_val, \"g_val:\", g_val,\n",
    "                          \"AUC: \", metrics.roc_auc_score(y_test_orig, best_pred_orig), \"Run Time (min): \", round((end - start)/60, 3), \"g:\", g])\n",
    "\n",
    "\n",
    "                with open(file_path+\"/progress_logs/\"+current_log_filename, \"a\") as myfile:\n",
    "                    myfile.write(' '.join([str(i) for i in [\"nb_val:\", nb_val, \"pop_val:\", pop_val, \"g_val:\", g_val,\n",
    "                          \"AUC: \", metrics.roc_auc_score(best_pred_orig, best_pred_orig), \"Run Time (min): \", round((end - start)/60, 3), \"g:\", g]]))\n",
    "                    myfile.write('\\n')\n",
    "                    myfile.close()\n",
    "                date = datetime.datetime.now().strftime(\"%Y_%m_%d-%I:%M:%S_%p\")\n",
    "\n",
    "                with open(f'{file_path}/results_master_lists/master_result_list_{date}.pkl', 'wb') as f:\n",
    "                    pickle.dump(master_result_list, f)\n",
    "\n",
    "                try:\n",
    "                    print(\"Writing grid perturbation to log\")\n",
    "                    #write line to best grid scores---------------------\n",
    "                    column_list = ['nb_size', 'f_list', 'auc','mcc','f1','precision','recall','accuracy', \n",
    "               'nb_val', 'pop_val', 'g_val', 'g', 'weighted', \n",
    "               'use_stored_base_learners', 'store_base_learners',\n",
    "               'resample', 'scale', 'n_features', 'param_space_size', 'n_unique_out',\n",
    "               'outcome_var_n', 'div_p', 'percent_missing', 'corr', \n",
    "                'age', 'sex', 'bmi','ethnicity', 'bloods', 'diagnostic_order',\n",
    "                'drug_order', 'annotation_n', 'meta_sp_annotation_n',\n",
    "                'meta_sp_annotation_mrc_n','annotation_mrc_n',\n",
    "                'core_02','bed','vte_status','hosp_site','core_resus','news',\n",
    "                'X_train_size', 'X_test_orig_size', 'X_test_size',\n",
    "                'run_time', 'cxpb', 'mutpb', 'indpb', 't_size']\n",
    "\n",
    "                    column_list = column_list +['BL_' + str(x) for x in range(0, 64)]\n",
    "\n",
    "                    line = pd.DataFrame( data = None, columns = column_list)\n",
    "\n",
    "\n",
    "                    auc = metrics.roc_auc_score(y_test_orig, best_pred_orig)\n",
    "                    mcc = matthews_corrcoef(y_test_orig, best_pred_orig)\n",
    "                    f1  = f1_score(y_test_orig, best_pred_orig, average='binary')\n",
    "                    precision = precision_score(y_test_orig, best_pred_orig, average='binary')\n",
    "                    recall = recall_score(y_test_orig, best_pred_orig, average='binary')\n",
    "                    accuracy = accuracy_score(y_test_orig, best_pred_orig)\n",
    "\n",
    "                    \n",
    "                    for key in global_param_dict:\n",
    "                        #print(key)\n",
    "                        if key != 'data':\n",
    "                            if(key in column_list):\n",
    "                                line[key] = [global_param_dict.get(key)]\n",
    "                        else:\n",
    "                            for key_1 in global_param_dict.get('data'):\n",
    "                                #print(key_1)\n",
    "                                if(key_1 in column_list):\n",
    "                                    line[key_1] = [global_param_dict.get('data').get(key_1)]\n",
    "                    \n",
    "                    \n",
    "\n",
    "                    for iii in range(0, len(best[0])):\n",
    "                        f_list = []\n",
    "                        current_f = best[0][iii][2]\n",
    "\n",
    "                        current_f_vector = []\n",
    "                        for elem in orignal_feature_names:\n",
    "                            if(elem in current_f):\n",
    "                                current_f_vector.append(1)\n",
    "                            else:\n",
    "                                current_f_vector.append(0)\n",
    "                        #f_list.append(np.array(current_f_vector))\n",
    "                        f_list.append(current_f_vector)\n",
    "                        \n",
    "                        line['BL_' + str(iii)] = [f_list]\n",
    "\n",
    "\n",
    "                    line['nb_size'] = len(best[0])\n",
    "                    #line['f_list'] = [f_list]\n",
    "\n",
    "\n",
    "                    line['auc'] = [auc]\n",
    "                    line['mcc'] = [mcc]\n",
    "                    line['f1'] = [f1]\n",
    "                    line['precision'] = [precision]\n",
    "                    line['recall'] = [recall]\n",
    "                    line['accuracy'] = [accuracy]\n",
    "\n",
    "                    line['nb_val'] = [nb_val]\n",
    "                    line['pop_val'] = [pop_val]\n",
    "                    line['g_val'] = [g_val]\n",
    "                    line['g'] = [g]\n",
    "                    line['X_train_size'] = [len(X_train)]\n",
    "                    line['X_test_orig_size'] = [len(X_test_orig)]\n",
    "                    line['X_test_size'] = [len(X_test)]\n",
    "\n",
    "                    end = time.time()\n",
    "\n",
    "                    line['run_time'] = int((end - start) / 60)\n",
    "                    line['cxpb'] = global_param_dict.get('cxpb')\n",
    "                    line['indpb'] = global_param_dict.get('indpb')\n",
    "                    line['mutpb'] = global_param_dict.get('mutpb')\n",
    "                    line['t_size'] = global_param_dict.get('t_size')\n",
    "                                    \n",
    "\n",
    "\n",
    "                    line[column_list].to_csv(base_project_dir + 'final_grid_score_log.csv' , mode='a', header=False, index=True)   \n",
    "                    #---------------------------    \n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "                    print(\"Failed to upgrade grid entry\")\n",
    "                    \n",
    "                    \n",
    "                    \n",
    "                    \n",
    "                    \n",
    "                #Convert model definition to string for low file size\n",
    "                best_str = best.copy()\n",
    "\n",
    "                for i in range(0, len(best_str[0])):\n",
    "                    best_str[0][i] = list(best_str[0][i])\n",
    "                    best_str[0][i][1] = str(best_str[0][i][1])\n",
    "                    best_str[0][i] = tuple(best_str[0][i])\n",
    "\n",
    "                plot_generation_progress_fitness(generation_progress_list)\n",
    "\n",
    "                with open(\n",
    "                    file_path\n",
    "                    + \"best_pop=\"\n",
    "                    + str(pop_val)\n",
    "                    + \"_g=\"\n",
    "                    + str(g_val)\n",
    "                    + \"_nb=\"\n",
    "                    + str(nb_val)\n",
    "                    + \".pkl\",\n",
    "                    \"wb\",\n",
    "                ) as file:\n",
    "                    # dump best_str instead of best which contains actual model. \n",
    "                    #Unnecessary since ensemble can be fitted in deployment\n",
    "                    #Can model always be stored as string during operation of GA?\n",
    "                    pickle.dump(best_str, file)\n",
    "\n",
    "                #Try to reset DEAP for second run. Should be seperate:\n",
    "                from deap import algorithms, base, creator, tools\n",
    "                del toolbox\n",
    "                gc.collect()\n",
    "                creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
    "                creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
    "                toolbox = base.Toolbox()\n",
    "                toolbox.register(\"evaluate\", evaluate_weighted_ensemble_auc)\n",
    "\n",
    "\n",
    "\n",
    "            except Exception as Argument:\n",
    "\n",
    "                 date = datetime.datetime.now().strftime(\"%Y_%m_%d-%I:%M:%S_%p\")\n",
    "                 # creating/opening a file\n",
    "                 f = open(f\"{file_path}/GEC_log_{date}.txt\", \"a\")\n",
    "                 # writing in the file\n",
    "                 f.write(str(Argument))\n",
    "                 f.write(str(traceback.format_exc()))\n",
    "                 # closing the file\n",
    "                 f.close()\n",
    "                 print(Argument)\n",
    "\n",
    "\n",
    "\n",
    "                 print(traceback.format_exc())\n",
    "\n",
    "                 continue\n",
    "\n",
    "        for line in (master_result_list):\n",
    "            print(line)\n",
    "            #print(master_result_list)\n",
    "\n",
    "            #best_weight_current = super_ensemble_weight_finder(best)\n",
    "        #         date = datetime.now().strftime(\"%Y_%m_%d-%I:%M:%S_%p\")\n",
    "        #         with open(f'ensemble_weights_found{date}.pkl', 'wb') as f:\n",
    "        #             pickle.dump(best_weight_current, f)\n",
    "\n",
    "\n",
    "        #main()\n",
    "\n",
    "        #del creator.Individual\n",
    "        #optuna or hyperopt integration for defining parameter space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f60c65b1",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "628ea44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9fc6232",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip freeze > requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfd6c302",
   "metadata": {},
   "outputs": [],
   "source": [
    "import session_info\n",
    "session_info.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7a5242",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "encoding": "# coding: utf-8",
   "executable": "/usr/bin/env python",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "ga_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}