Book_chapter_example_binary_classification_1280_dimension.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "95NTckuFZZzm"
      },
      "source": [
        "### requirements for the following codings\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UO71IBS6ZgZV"
      },
      "outputs": [],
      "source": [
        "### packages required\n",
        "!pip install fair-esm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DON02fGXgd8v"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import esm\n",
        "from keras.layers import Input, Dense, Activation, BatchNormalization, Flatten, Conv1D\n",
        "from keras.layers import Dropout, AveragePooling1D, MaxPooling1D\n",
        "from keras.models import Sequential,Model, load_model\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "import keras\n",
        "from keras import backend as K\n",
        "import tensorflow as tf\n",
        "if tf.test.gpu_device_name():\n",
        "    print('GPU found')\n",
        "    tf.config.experimental.set_visible_devices(tf.config.list_physical_devices('GPU')[0], 'GPU') # set the deep learning with GPU\n",
        "else:\n",
        "    print(\"No GPU found\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m91cA0H5w_eY"
      },
      "source": [
        "### peptide embeddings with esm2_t33_650M_UR50D pretrained models\n",
        "33 layers, 650M parameters, dataset: UR50/D 2021_04, embedding dimension: 1280\n",
        "\n",
        "original ESM github site is https://github.com/facebookresearch/esm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pl7XVx5HZsHf"
      },
      "outputs": [],
      "source": [
        "def esm_embeddings(esm2, esm2_alphabet, peptide_sequence_list):\n",
        "  # NOTICE: ESM for embeddings is quite RAM usage, if your sequence is too long,\n",
        "  #         or you have too many sequences for transformation in a single converting, (i personally suggest to do the embedding one by one if you are running locally or with T4 GPU at google colab)\n",
        "  #         you computer might automatically kill the job.\n",
        "  # load the python packages\n",
        "  import torch\n",
        "  import esm\n",
        "  import collections\n",
        "  import pandas as pd\n",
        "  import gc\n",
        "  # choose cuda for acceleration if available\n",
        "  if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "  else:\n",
        "    device = torch.device(\"cpu\")\n",
        "  esm2.eval()\n",
        "  esm2 = esm2.to(device)\n",
        "  batch_converter = esm2_alphabet.get_batch_converter() # this function is to tokenize your peptide sequences from amino acid sequence into numbers\n",
        "\n",
        "  # load the peptide sequence list into the bach_converter\n",
        "  batch_labels, batch_strs, batch_tokens = batch_converter(peptide_sequence_list)\n",
        "  batch_lens = (batch_tokens != alphabet.padding_idx).sum(1)\n",
        "  ## batch tokens are the tokenized results of the whole data set\n",
        "  batch_tokens = batch_tokens.to(device)\n",
        "  # Extract per-residue representations (on CPU)\n",
        "  with torch.no_grad():\n",
        "      # Here we export the last layer of the EMS model output as the representation of the peptides\n",
        "      # model'esm2_t6_8M_UR50D' only has 6 layers, and therefore repr_layers parameters is equal to 6\n",
        "      # if you want to change the esm model, for example: esm2_t12_35M_UR50D, which has 12 layers, they you will need to change the 6 to 12 at the following two line\n",
        "      results = esm2(batch_tokens, repr_layers=[30], return_contacts=False)\n",
        "  token_representations = results[\"representations\"][30].cpu() # the representation is generated for each amino acid residue; for example, if you have 3 residues, then the ourput shape is 3 * 320, 3 is the number of residues and the 320 is the vector for a single rediue\n",
        "\n",
        "  # Generate per-sequence representations via averaging\n",
        "  # NOTE: token 0 is always a beginning-of-sequence token, so we will discard it and start from token 1\n",
        "  sequence_representations = []\n",
        "  for i, tokens_len in enumerate(batch_lens):\n",
        "      sequence_representations.append(token_representations[i, 1 : tokens_len - 1].mean(0))\n",
        "  # save dataset\n",
        "  # sequence_representations is a list and each element is a tensor\n",
        "  embeddings_results = collections.defaultdict(list)\n",
        "  for i in range(len(sequence_representations)):\n",
        "      # tensor format can be transformed as numpy format sequence_representations[0].numpy() or sequence_representations[0].to_list\n",
        "      each_seq_rep = sequence_representations[i].tolist()\n",
        "      for each_element in each_seq_rep:\n",
        "          embeddings_results[i].append(each_element)\n",
        "  embeddings_results = pd.DataFrame(embeddings_results).T\n",
        "  del  batch_labels, batch_strs, batch_tokens, results, token_representations # delete those variables to save GPU memory\n",
        "  gc.collect() # release the GPU memory\n",
        "  return embeddings_results\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RddxugbsdR1Y"
      },
      "source": [
        "### data loading and embeddings\n",
        "assume you have already split your dataset as a train and a test dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IDPlUoVUgmmd"
      },
      "outputs": [],
      "source": [
        "# select the ESM model for embeddings (you can select you desired model from https://github.com/facebookresearch/esm)\n",
        "# NOTICE: if you choose other model, the following model architecture might not be very compitable\n",
        "#         bseides,please revise the correspdoning parameters in esm_embeddings function (layers for feature extraction)\n",
        "model, alphabet = esm.pretrained.esm2_t33_650M_UR50D()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LNlD8pvizH84"
      },
      "outputs": [],
      "source": [
        "# training dataset loading\n",
        "dataset = pd.read_excel('bitter_train.xlsx',header=0, index_col = None)\n",
        "# generate embedding for seqeunces\n",
        "sequence_list = dataset['sequence']\n",
        "embeddings_results = pd.DataFrame()\n",
        "for seq in sequence_list:\n",
        "    # the setting is just following the input format setting in ESM model, [name,sequence]\n",
        "    tuple_sequence = tuple([seq,seq])\n",
        "    peptide_sequence_list = []\n",
        "    peptide_sequence_list.append(tuple_sequence) # build a summarize list variable including all the sequence information\n",
        "    # employ ESM model for converting and save the converted data in csv format\n",
        "    one_seq_embeddings = esm_embeddings(model, alphabet, peptide_sequence_list)\n",
        "    embeddings_results= pd.concat([embeddings_results,one_seq_embeddings])\n",
        "embeddings_results.to_csv('train_main_esm2_t6_8M_UR50D_unified_320_dimension.csv')\n",
        "\n",
        "# test dataset loading\n",
        "dataset = pd.read_excel('bitter_test.xlsx',header=0, index_col = None)\n",
        "# generate embedding for seqeunces\n",
        "sequence_list = dataset['sequence']\n",
        "embeddings_results = pd.DataFrame()\n",
        "# embedding all the peptide one by one\n",
        "for seq in sequence_list:\n",
        "    # the setting is just following the input format setting in ESM model, [name,sequence]\n",
        "    tuple_sequence = tuple([seq,seq] )\n",
        "    peptide_sequence_list = []\n",
        "    peptide_sequence_list.append(tuple_sequence) # build a summarize list variable including all the sequence information\n",
        "    # employ ESM model for converting and save the converted data in csv format\n",
        "    one_seq_embeddings = esm_embeddings(model, alphabet, peptide_sequence_list)\n",
        "    embeddings_results= pd.concat([embeddings_results,one_seq_embeddings])\n",
        "embeddings_results.to_csv('test_main_esm2_t6_8M_UR50D_unified_320_dimension.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j7rGiqVLwSH4"
      },
      "source": [
        "#### dataset division\n",
        "ready for training and evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xk13-JbBXAph"
      },
      "outputs": [],
      "source": [
        "# training dataset loading\n",
        "dataset = pd.read_excel('bitter_train.xlsx',na_filter = False) # take care the NA sequence problem\n",
        "sequence_list = dataset['sequence']\n",
        "y_train = dataset['label']\n",
        "y_train = np.array(y_train) # transformed as np.array for CNN model\n",
        "\n",
        "# test dataset loading\n",
        "dataset = pd.read_excel('bitter_test.xlsx',na_filter = False) # take care the NA sequence problem\n",
        "sequence_list = dataset['sequence']\n",
        "y_test = dataset['label']\n",
        "y_test = np.array(y_test) # transformed as np.array for CNN model\n",
        "\n",
        "# loading the peptide embddings\n",
        "X_train = pd.read_csv('train_main_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "X_test = pd.read_csv('test_main_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "# generate the validation dataset for early stopping\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=123, shuffle= True, stratify =y_train )\n",
        "\n",
        "# transform the dataset into numpy format\n",
        "X_train = np.array(X_train)\n",
        "X_valid = np.array(X_valid)\n",
        "X_test = np.array(X_test)\n",
        "\n",
        "# normalize the X data range\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "scaler.fit(X_train)\n",
        "X_train = scaler.transform(X_train) # normalize X to 0-1 range\n",
        "X_valid = scaler.transform(X_valid) # normalize X to 0-1 range\n",
        "X_test = scaler.transform(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HubTATKXslKw"
      },
      "outputs": [],
      "source": [
        "# check the dimension of the dataset before model development\n",
        "print(X_train.shape)\n",
        "print(X_valid.shape)\n",
        "print(X_test.shape)\n",
        "\n",
        "print(y_train.shape)\n",
        "print(y_valid.shape)\n",
        "print(y_test.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1RGqyaijk-Kq"
      },
      "source": [
        "### dataset loading and embeddings\n",
        "(if you only have one dataset and need to split them)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4jpuP4v4tDZN"
      },
      "outputs": [],
      "source": [
        "# select the ESM model for embeddings (you can select you desired model from https://github.com/facebookresearch/esm)\n",
        "# NOTICE: if you choose other model, the following model architecture might not be very compitable\n",
        "#         bseides,please revise the correspdoning parameters in esm_embeddings function (layers for feature extraction)\n",
        "model, alphabet = esm.pretrained.esm2_t33_650M_UR50D()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "63DekwCSlGsh"
      },
      "outputs": [],
      "source": [
        "# whole dataset loading and dataset splitting\n",
        "dataset = pd.read_excel('bitter_single_dataset_example.xlsx',header=0, index_col = None)\n",
        "\n",
        "# generate the peptide embeddings\n",
        "sequence_list = dataset['sequence']\n",
        "embeddings_results = pd.DataFrame()\n",
        "for seq in sequence_list:\n",
        "    # the setting is just following the input format setting in ESM model, [name,sequence]\n",
        "    tuple_sequence = tuple([seq,seq])\n",
        "    peptide_sequence_list = []\n",
        "    peptide_sequence_list.append(tuple_sequence) # build a summarize list variable including all the sequence information\n",
        "    # employ ESM model for converting and save the converted data in csv format\n",
        "    one_seq_embeddings = esm_embeddings(model, alphabet, peptide_sequence_list)\n",
        "    embeddings_results= pd.concat([embeddings_results,one_seq_embeddings])\n",
        "embeddings_results.to_csv('whole_sample_dataset_esm2_t6_8M_UR50D_unified_320_dimension.csv')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yvuT95J2wXrq"
      },
      "source": [
        "#### dataset division\n",
        "ready for training and evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NcAG9Pg3qneT"
      },
      "outputs": [],
      "source": [
        "# loading the y dataset for model development\n",
        "y = dataset['label']\n",
        "y = np.array(y) # transformed as np.array for CNN model\n",
        "\n",
        "# read the peptide embeddings\n",
        "X_data = pd.read_csv('whole_sample_dataset_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "X = np.array(X_data)\n",
        "\n",
        "# split dataset as training and test dataset as ratio of 8:2\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123, shuffle= True, stratify =y)\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=234, shuffle= True, stratify =y_train)\n",
        "\n",
        "\n",
        "# normalize the X data range\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "scaler.fit(X_train)\n",
        "X_train = scaler.transform(X_train) # normalize X to 0-1 range\n",
        "X_valid = scaler.transform(X_valid)\n",
        "X_test = scaler.transform(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vijDuRdtldHJ"
      },
      "outputs": [],
      "source": [
        "# check the dimension of the dataset before model development\n",
        "print(X_train.shape)\n",
        "print(X_valid.shape)\n",
        "print(X_test.shape)\n",
        "\n",
        "print(y_train.shape)\n",
        "print(y_valid.shape)\n",
        "print(y_test.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U3Fagh9Iw83q"
      },
      "source": [
        "### Model architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "skIE-JAlKhPx"
      },
      "outputs": [],
      "source": [
        "def ESM_CNN(X_train, y_train, X_test, y_test):\n",
        "  inputShape=(1280,1) # input feature size\n",
        "  input = Input(inputShape)\n",
        "  x = Conv1D(32,(3),strides = (1),name='layer_conv1',padding='same')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = Activation('relu')(x)\n",
        "  x = MaxPooling1D((2), name='MaxPool1',padding=\"same\")(x)\n",
        "  x = Dropout(0.15)(x)\n",
        "  x = Flatten()(x)\n",
        "  x = Dense(64,activation = 'relu',name='fc1')(x)\n",
        "  x = Dropout(0.15)(x)\n",
        "  x = Dense(2,activation = 'softmax',name='fc2')(x)\n",
        "  model = Model(inputs = input,outputs = x,name='Predict')\n",
        "  adam = Adam(learning_rate=0.001)\n",
        "  # compile the model\n",
        "  model.compile(loss='sparse_categorical_crossentropy',optimizer=adam, metrics=['accuracy'])\n",
        "  # set checkpoint and save the best model\n",
        "  mc = ModelCheckpoint('best_model.keras',  monitor='val_accuracy', mode='max', verbose=1, save_best_only=True, save_weights_only=False)\n",
        "  # summary the callbacks_list\n",
        "  callbacks_list = [mc]\n",
        "  model_history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=200,callbacks=callbacks_list,batch_size = 8, verbose=1)\n",
        "  return model, model_history"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5JBlTA9shnQE"
      },
      "source": [
        "### model training and evaluation at test dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "LnelssIIAmWx"
      },
      "outputs": [],
      "source": [
        "# train the model & best model checkpoint will be save as 'best_model.keras'\n",
        "model, model_history = ESM_CNN(X_train, y_train, X_valid, y_valid)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "yIy3I8Sv9ra0"
      },
      "outputs": [],
      "source": [
        "# load the saved best model for performance evaluation at test dataset\n",
        "saved_model = load_model('best_model.keras')\n",
        "# result collection list\n",
        "ACC_collecton = []\n",
        "BACC_collecton = []\n",
        "Sn_collecton = []\n",
        "Sp_collecton = []\n",
        "recall_collecton = []\n",
        "precision_collecton = []\n",
        "f1_collecton = []\n",
        "MCC_collecton = []\n",
        "AUC_collecton = []\n",
        "# confusion matrix\n",
        "predicted_class= []\n",
        "predicted_protability = saved_model.predict(X_test,batch_size=1)\n",
        "for i in range(predicted_protability.shape[0]):\n",
        "  index = np.where(predicted_protability[i] == np.amax(predicted_protability[i]))[0][0] # the class with higher protability was treated as the prediction class\n",
        "  predicted_class.append(index)\n",
        "\n",
        "predicted_class = np.array(predicted_class) # transformed into a numpy for performance evaluation\n",
        "y_true = y_test\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import math\n",
        "# np.ravel() return a flatten 1D array\n",
        "TN, FP, FN, TP = confusion_matrix(y_true, predicted_class).ravel() # shape [ [True-Positive, False-positive], [False-negative, True-negative] ]\n",
        "ACC = (TP+TN)/(TP+TN+FP+FN)\n",
        "ACC_collecton.append(ACC)\n",
        "Sn_collecton.append(TP/(TP+FN))\n",
        "Sp_collecton.append(TN/(TN+FP))\n",
        "recall_collecton.append(TP/(TP+FN))\n",
        "precision_collecton.append(TP/(TP+FP))\n",
        "f1_collecton.append(2*TP/(TP+FP)*TP/(TP+FN)/ (TP/(TP+FP)+TP/(TP+FN)))\n",
        "MCC = (TP*TN-FP*FN)/math.pow(((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)),0.5)\n",
        "MCC_collecton.append(MCC)\n",
        "BACC_collecton.append(0.5*TP/(TP+FN)+0.5*TN/(TN+FP))\n",
        "# print the performance in the test dataset\n",
        "from sklearn.metrics import roc_auc_score\n",
        "AUC = roc_auc_score(y_test, predicted_protability[:,1])\n",
        "AUC_collecton.append(AUC)\n",
        "print(ACC_collecton[0])\n",
        "print(BACC_collecton[0])\n",
        "print(Sn_collecton[0])\n",
        "print(Sp_collecton[0])\n",
        "print(recall_collecton[0])\n",
        "print(precision_collecton[0])\n",
        "print(f1_collecton[0])\n",
        "print(MCC_collecton[0])\n",
        "print(AUC_collecton[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QEJgbbDfBATA"
      },
      "source": [
        "### model usage at new dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### generate embeddings for new dataset"
      ],
      "metadata": {
        "id": "vMQcMzXZBE7-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vyxGBEdbegW9"
      },
      "outputs": [],
      "source": [
        "model, alphabet = esm.pretrained.esm2_t6_8M_UR50D()\n",
        "# new dataset loading\n",
        "dataset = pd.read_excel('bitter_new_data.xlsx',header=0, index_col = None)\n",
        "# generate embedding for seqeunces\n",
        "sequence_list = dataset['sequence']\n",
        "embeddings_results = pd.DataFrame()\n",
        "for seq in sequence_list:\n",
        "    # the setting is just following the input format setting in ESM model, [name,sequence]\n",
        "    tuple_sequence = tuple([seq,seq])\n",
        "    peptide_sequence_list = []\n",
        "    peptide_sequence_list.append(tuple_sequence) # build a summarize list variable including all the sequence information\n",
        "    # employ ESM model for converting and save the converted data in csv format\n",
        "    one_seq_embeddings = esm_embeddings(model, alphabet, peptide_sequence_list)\n",
        "    embeddings_results= pd.concat([embeddings_results,one_seq_embeddings])\n",
        "embeddings_results.to_csv('new_data_esm2_t6_8M_UR50D_unified_320_dimension.csv')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### loading scaler (optional)\n",
        "you only need to run this section if you choose to use scaler\n",
        "\n",
        "assume you have already split your dataset as a train and a test dataset in the above methods"
      ],
      "metadata": {
        "id": "idLitI0S_oDf"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-xZ44bB3-9Fk"
      },
      "outputs": [],
      "source": [
        "# loading scaler from new dataset\n",
        "X_train = pd.read_csv('train_main_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "# transform the dataset into numpy format\n",
        "X_train = np.array(X_train)\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "scaler.fit(X_train)\n",
        "# loading the peptide embddings\n",
        "X_new_embeddings = pd.read_csv('new_data_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "# transform the dataset into numpy format\n",
        "X_new = np.array(X_new_embeddings)\n",
        "X_new = scaler.transform(X_new) # normalize X to 0-1 range"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### loading scaler (optional)\n",
        "you only need to run this section if you choose to use scaler\n",
        "\n",
        "if you only use one dataset above, run this cell instead of the above section for two datasets"
      ],
      "metadata": {
        "id": "tX24ufQ5_8UR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# read the peptide embeddings\n",
        "X_data = pd.read_csv('whole_sample_dataset_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "X = np.array(X_data)\n",
        "\n",
        "# please make sure the following numbers for random_state is consistant with the above data division section.\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123, shuffle= True, stratify =y)\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=234, shuffle= True, stratify =y_train)\n",
        "# loading the peptide embddings\n",
        "X_new_embeddings = pd.read_csv('new_data_esm2_t6_8M_UR50D_unified_320_dimension.csv',header=0, index_col = 0,delimiter=',')\n",
        "# normalize the X data range\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "scaler.fit(X_train)\n",
        "X_new = np.array(X_new_embeddings)\n",
        "X_new = scaler.transform(X_new) # normalize X to 0-1 range"
      ],
      "metadata": {
        "id": "6aWWbJiDADU1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### predict the dataset"
      ],
      "metadata": {
        "id": "dQ63hMXP_NAR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "usbXeMUDAjxv",
        "outputId": "90efde6f-0dd7-48eb-8cb5-89521c8d7de8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m6/6\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step  \n"
          ]
        }
      ],
      "source": [
        "# load the saved best model for performance evaluation at test dataset\n",
        "saved_model = load_model('best_model.keras')\n",
        "# confusion matrix\n",
        "predicted_class= []\n",
        "predicted_protability = saved_model.predict(X_new,batch_size=1)\n",
        "for i in range(predicted_protability.shape[0]):\n",
        "  index = np.where(predicted_protability[i] == np.amax(predicted_protability[i]))[0][0]\n",
        "  predicted_class.append(index)\n",
        "predicted_class = np.array(predicted_class)\n",
        "for i in range(dataset.shape[0]):\n",
        "  dataset.iloc[i,1] = predicted_class[i]\n",
        "dataset.to_excel('new_data_prediction_result.xlsx')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "95NTckuFZZzm",
        "m91cA0H5w_eY",
        "RddxugbsdR1Y",
        "j7rGiqVLwSH4",
        "1RGqyaijk-Kq",
        "yvuT95J2wXrq",
        "U3Fagh9Iw83q",
        "5JBlTA9shnQE",
        "QEJgbbDfBATA",
        "vMQcMzXZBE7-",
        "idLitI0S_oDf",
        "tX24ufQ5_8UR",
        "dQ63hMXP_NAR"
      ],
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}