notebooks/model_fine_tuning_Vit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: gdown in c:\\users\\rperera23\\appdata\\roaming\\python\\python39\\site-packages (4.7.1)\n",
      "Requirement already satisfied: tqdm in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from gdown) (4.64.1)\n",
      "Requirement already satisfied: requests[socks] in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from gdown) (2.28.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from gdown) (4.11.1)\n",
      "Requirement already satisfied: six in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from gdown) (1.16.0)\n",
      "Requirement already satisfied: filelock in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from gdown) (3.9.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from beautifulsoup4->gdown) (2.3.2.post1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from requests[socks]->gdown) (1.26.12)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from requests[socks]->gdown) (2022.9.24)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from requests[socks]->gdown) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from requests[socks]->gdown) (3.4)\n",
      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from requests[socks]->gdown) (1.7.1)\n",
      "Requirement already satisfied: colorama in c:\\program files\\arcgis\\pro\\bin\\python\\envs\\arcgispro-py3\\lib\\site-packages (from tqdm->gdown) (0.4.5)\n"
     ]
    }
   ],
   "source": [
    "!pip install gdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<contextlib.ExitStack at 0x1dd248bebb0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.backends.cudnn as cudnn\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "\n",
    "\n",
    "from src.models import VisionTransformerTimm\n",
    "from src.dataloader import EuroSatDownloader\n",
    "from src.training import train_fine_tuning, eval_method, eval_func\n",
    "from src.dataloader import MiniImageNetDataSet, createEuroSatDataLoaders, EuroSatDownloader\n",
    "from src.modelvis import visualize_models\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "'''\n",
    "The line cudnn.benchmark = True is typically used in deep learning projects that utilize the CUDA Deep Neural Network (cuDNN) library.\n",
    "When cudnn.benchmark is set to True, it enables cuDNN to automatically find the best algorithm configuration for the specific \n",
    "input sizes and hardware being used. This can result in improved performance during training and inference.By enabling benchmarking, \n",
    "cuDNN will run a short benchmarking phase during the first iteration of the model to determine the optimal algorithm configuration. \n",
    "This configuration is then cached and used for subsequent iterations, leading to faster execution times.\n",
    "It's important to note that enabling benchmarking may introduce some overhead during the initial benchmarking phase, so it is typically \n",
    "recommended to use it when the input sizes are consistent throughout the training process.\n",
    "Overall, setting cudnn.benchmark to True can help optimize the performance of deep learning models that use cuDNN.\n",
    "'''\n",
    "torch.backends.cudnn.benchmark = True\n",
    "\n",
    "'''\n",
    "The line plt.ion() is a function call that activates interactive mode in matplotlib.\n",
    "When interactive mode is enabled, any plot that is created will be displayed immediately \n",
    "and can be updated dynamically. This means that you can modify the plot after it is displayed, \n",
    "such as changing the data or adding annotations, and the changes will be reflected in real-time.\n",
    "'''\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9.2\n",
      "1.8.2\n"
     ]
    }
   ],
   "source": [
    "print(torchvision.__version__)\n",
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download EuroSat Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-07 05:54:02,937 - INFO - EuroSat database already exists.\n"
     ]
    }
   ],
   "source": [
    "extractDir = '..\\\\data'\n",
    "url = \"https://zenodo.org/records/7711810/files/EuroSAT_RGB.zip?download=1\"\n",
    "downloader = EuroSatDownloader(url, extractDir)\n",
    "dataDir = downloader.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "LEARNING_RATE = 0.01\n",
    "WEIGHT_DECAY = 0.001\n",
    "NUM_EPOCHS = 20\n",
    "BATCH_SIZE = 25\n",
    "MOMENTUM = 0.09\n",
    "STEP_SIZE = 5\n",
    "num_of_classes = 64\n",
    "EPISODES = 5\n",
    "GAMMA = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.Resize(224),\n",
    "        transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.GaussianBlur(3, sigma=(0.1, 2.0)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.Resize(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "    ])\n",
    "}\n",
    "\n",
    "# overall\n",
    "# Overall Mean of RGB channels: tensor([0.4076, 0.4147, 0.4355])\n",
    "# Overall Standard Deviation of RGB channels: tensor([0.1110, 0.0776, 0.0668])\n",
    "\n",
    "#pasture\n",
    "#tensor([0.1518, 0.2503, 0.2958])\n",
    "#tensor([0.0136, 0.0150, 0.0087])\n",
    "\n",
    "# highway\n",
    "#tensor([0.2684, 0.3432, 0.3769])\n",
    "#tensor([0.0997, 0.0726, 0.0556])\n",
    "\n",
    "dataDir = '..\\\\data\\\\EuroSAT_RGB'\n",
    "modelPath = '..\\\\data\\\\models\\\\best_model_VistionTransformerTimm_pc18_fixedTransformers.pth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please wait patiently, it may take some seconds...\n",
      "\n",
      "Training Episode: 1...\n",
      "{'train': 25, 'test': 75}\n",
      "epoch: 0, accuracy: 0.280000, avg. loss: 1.856061, test accuracy: 0.320000\n",
      "epoch: 1, accuracy: 0.600000, avg. loss: 1.218804, test accuracy: 0.560000\n",
      "epoch: 2, accuracy: 0.680000, avg. loss: 0.586799, test accuracy: 0.586667\n",
      "epoch: 3, accuracy: 1.000000, avg. loss: 0.142466, test accuracy: 0.653333\n",
      "epoch: 4, accuracy: 1.000000, avg. loss: 0.057376, test accuracy: 0.666667\n",
      "epoch: 5, accuracy: 1.000000, avg. loss: 0.042904, test accuracy: 0.680000\n",
      "epoch: 6, accuracy: 1.000000, avg. loss: 0.043208, test accuracy: 0.706667\n",
      "epoch: 7, accuracy: 1.000000, avg. loss: 0.038049, test accuracy: 0.706667\n",
      "epoch: 8, accuracy: 1.000000, avg. loss: 0.047638, test accuracy: 0.720000\n",
      "epoch: 9, accuracy: 1.000000, avg. loss: 0.024132, test accuracy: 0.720000\n",
      "epoch: 10, accuracy: 1.000000, avg. loss: 0.029144, test accuracy: 0.720000\n",
      "epoch: 11, accuracy: 1.000000, avg. loss: 0.028593, test accuracy: 0.720000\n",
      "epoch: 12, accuracy: 1.000000, avg. loss: 0.027308, test accuracy: 0.720000\n",
      "epoch: 13, accuracy: 1.000000, avg. loss: 0.031973, test accuracy: 0.720000\n",
      "epoch: 14, accuracy: 1.000000, avg. loss: 0.024434, test accuracy: 0.720000\n",
      "epoch: 15, accuracy: 1.000000, avg. loss: 0.020583, test accuracy: 0.720000\n",
      "epoch: 16, accuracy: 1.000000, avg. loss: 0.021452, test accuracy: 0.720000\n",
      "epoch: 17, accuracy: 1.000000, avg. loss: 0.025430, test accuracy: 0.720000\n",
      "epoch: 18, accuracy: 1.000000, avg. loss: 0.020639, test accuracy: 0.720000\n",
      "epoch: 19, accuracy: 1.000000, avg. loss: 0.024572, test accuracy: 0.720000\n",
      "{'River', 'PermanentCrop', 'Forest', 'Residential', 'Highway'}\n",
      "Episode 1: Test Accuracy: 0.7200\n",
      "\n",
      "Training Episode: 2...\n",
      "{'train': 25, 'test': 75}\n",
      "epoch: 0, accuracy: 0.160000, avg. loss: 1.816314, test accuracy: 0.520000\n",
      "epoch: 1, accuracy: 0.720000, avg. loss: 0.840154, test accuracy: 0.760000\n",
      "epoch: 2, accuracy: 0.920000, avg. loss: 0.346448, test accuracy: 0.906667\n",
      "epoch: 3, accuracy: 1.000000, avg. loss: 0.073241, test accuracy: 0.906667\n",
      "epoch: 4, accuracy: 1.000000, avg. loss: 0.079012, test accuracy: 0.920000\n",
      "epoch: 5, accuracy: 1.000000, avg. loss: 0.070209, test accuracy: 0.920000\n",
      "epoch: 6, accuracy: 1.000000, avg. loss: 0.044523, test accuracy: 0.920000\n",
      "epoch: 7, accuracy: 1.000000, avg. loss: 0.043791, test accuracy: 0.920000\n",
      "epoch: 8, accuracy: 1.000000, avg. loss: 0.048032, test accuracy: 0.920000\n",
      "epoch: 9, accuracy: 1.000000, avg. loss: 0.026938, test accuracy: 0.920000\n",
      "epoch: 10, accuracy: 1.000000, avg. loss: 0.029949, test accuracy: 0.920000\n",
      "epoch: 11, accuracy: 1.000000, avg. loss: 0.039978, test accuracy: 0.920000\n",
      "epoch: 12, accuracy: 1.000000, avg. loss: 0.034179, test accuracy: 0.920000\n",
      "epoch: 13, accuracy: 1.000000, avg. loss: 0.030022, test accuracy: 0.920000\n",
      "epoch: 14, accuracy: 1.000000, avg. loss: 0.026030, test accuracy: 0.920000\n",
      "epoch: 15, accuracy: 1.000000, avg. loss: 0.023854, test accuracy: 0.920000\n",
      "epoch: 16, accuracy: 1.000000, avg. loss: 0.028770, test accuracy: 0.920000\n",
      "epoch: 17, accuracy: 1.000000, avg. loss: 0.025063, test accuracy: 0.920000\n",
      "epoch: 18, accuracy: 1.000000, avg. loss: 0.031338, test accuracy: 0.920000\n",
      "epoch: 19, accuracy: 1.000000, avg. loss: 0.024491, test accuracy: 0.920000\n",
      "{'River', 'Industrial', 'Forest', 'Residential', 'AnnualCrop'}\n",
      "Episode 2: Test Accuracy: 0.9200\n",
      "\n",
      "Training Episode: 3...\n",
      "{'train': 25, 'test': 75}\n",
      "epoch: 0, accuracy: 0.040000, avg. loss: 2.216875, test accuracy: 0.453333\n",
      "epoch: 1, accuracy: 0.840000, avg. loss: 0.500065, test accuracy: 0.733333\n",
      "epoch: 2, accuracy: 1.000000, avg. loss: 0.194855, test accuracy: 0.813333\n",
      "epoch: 3, accuracy: 1.000000, avg. loss: 0.149855, test accuracy: 0.813333\n",
      "epoch: 4, accuracy: 0.960000, avg. loss: 0.109570, test accuracy: 0.800000\n",
      "epoch: 5, accuracy: 1.000000, avg. loss: 0.065600, test accuracy: 0.826667\n",
      "epoch: 6, accuracy: 1.000000, avg. loss: 0.047224, test accuracy: 0.826667\n",
      "epoch: 7, accuracy: 1.000000, avg. loss: 0.055614, test accuracy: 0.813333\n",
      "epoch: 8, accuracy: 1.000000, avg. loss: 0.043349, test accuracy: 0.813333\n",
      "epoch: 9, accuracy: 1.000000, avg. loss: 0.038484, test accuracy: 0.813333\n",
      "epoch: 10, accuracy: 1.000000, avg. loss: 0.033257, test accuracy: 0.813333\n",
      "epoch: 11, accuracy: 1.000000, avg. loss: 0.042967, test accuracy: 0.813333\n",
      "epoch: 12, accuracy: 1.000000, avg. loss: 0.030087, test accuracy: 0.813333\n",
      "epoch: 13, accuracy: 1.000000, avg. loss: 0.047110, test accuracy: 0.813333\n",
      "epoch: 14, accuracy: 1.000000, avg. loss: 0.033973, test accuracy: 0.813333\n",
      "epoch: 15, accuracy: 1.000000, avg. loss: 0.035353, test accuracy: 0.813333\n",
      "epoch: 16, accuracy: 1.000000, avg. loss: 0.030902, test accuracy: 0.813333\n",
      "epoch: 17, accuracy: 1.000000, avg. loss: 0.046500, test accuracy: 0.813333\n",
      "epoch: 18, accuracy: 1.000000, avg. loss: 0.033666, test accuracy: 0.813333\n",
      "epoch: 19, accuracy: 1.000000, avg. loss: 0.034700, test accuracy: 0.813333\n",
      "{'Residential', 'Pasture', 'AnnualCrop', 'SeaLake', 'HerbaceousVegetation'}\n",
      "Episode 3: Test Accuracy: 0.8133\n",
      "\n",
      "Training Episode: 4...\n",
      "{'train': 25, 'test': 75}\n",
      "epoch: 0, accuracy: 0.120000, avg. loss: 2.086181, test accuracy: 0.293333\n",
      "epoch: 1, accuracy: 0.680000, avg. loss: 0.663174, test accuracy: 0.626667\n",
      "epoch: 2, accuracy: 0.880000, avg. loss: 0.462442, test accuracy: 0.800000\n",
      "epoch: 3, accuracy: 1.000000, avg. loss: 0.092832, test accuracy: 0.853333\n",
      "epoch: 4, accuracy: 1.000000, avg. loss: 0.071619, test accuracy: 0.853333\n",
      "epoch: 5, accuracy: 1.000000, avg. loss: 0.033113, test accuracy: 0.853333\n",
      "epoch: 6, accuracy: 1.000000, avg. loss: 0.053376, test accuracy: 0.853333\n",
      "epoch: 7, accuracy: 1.000000, avg. loss: 0.043632, test accuracy: 0.853333\n",
      "epoch: 8, accuracy: 1.000000, avg. loss: 0.033649, test accuracy: 0.853333\n",
      "epoch: 9, accuracy: 1.000000, avg. loss: 0.029379, test accuracy: 0.853333\n",
      "epoch: 10, accuracy: 1.000000, avg. loss: 0.030109, test accuracy: 0.853333\n",
      "epoch: 11, accuracy: 1.000000, avg. loss: 0.029007, test accuracy: 0.853333\n",
      "epoch: 12, accuracy: 1.000000, avg. loss: 0.026385, test accuracy: 0.853333\n",
      "epoch: 13, accuracy: 1.000000, avg. loss: 0.030477, test accuracy: 0.853333\n",
      "epoch: 14, accuracy: 1.000000, avg. loss: 0.023502, test accuracy: 0.853333\n",
      "epoch: 15, accuracy: 1.000000, avg. loss: 0.028631, test accuracy: 0.853333\n",
      "epoch: 16, accuracy: 1.000000, avg. loss: 0.023847, test accuracy: 0.853333\n",
      "epoch: 17, accuracy: 1.000000, avg. loss: 0.021562, test accuracy: 0.853333\n",
      "epoch: 18, accuracy: 1.000000, avg. loss: 0.026469, test accuracy: 0.853333\n",
      "epoch: 19, accuracy: 1.000000, avg. loss: 0.024102, test accuracy: 0.853333\n",
      "{'River', 'Forest', 'Residential', 'AnnualCrop', 'SeaLake'}\n",
      "Episode 4: Test Accuracy: 0.8533\n",
      "\n",
      "Training Episode: 5...\n",
      "{'train': 25, 'test': 75}\n",
      "epoch: 0, accuracy: 0.080000, avg. loss: 2.137310, test accuracy: 0.373333\n",
      "epoch: 1, accuracy: 0.720000, avg. loss: 0.953224, test accuracy: 0.613333\n",
      "epoch: 2, accuracy: 0.880000, avg. loss: 0.404388, test accuracy: 0.666667\n",
      "epoch: 3, accuracy: 0.960000, avg. loss: 0.164864, test accuracy: 0.720000\n",
      "epoch: 4, accuracy: 1.000000, avg. loss: 0.099022, test accuracy: 0.706667\n",
      "epoch: 5, accuracy: 1.000000, avg. loss: 0.064457, test accuracy: 0.773333\n",
      "epoch: 6, accuracy: 1.000000, avg. loss: 0.058721, test accuracy: 0.760000\n",
      "epoch: 7, accuracy: 1.000000, avg. loss: 0.070502, test accuracy: 0.720000\n",
      "epoch: 8, accuracy: 1.000000, avg. loss: 0.056796, test accuracy: 0.746667\n",
      "epoch: 9, accuracy: 1.000000, avg. loss: 0.043664, test accuracy: 0.746667\n",
      "epoch: 10, accuracy: 1.000000, avg. loss: 0.036839, test accuracy: 0.746667\n",
      "epoch: 11, accuracy: 1.000000, avg. loss: 0.034372, test accuracy: 0.746667\n",
      "epoch: 12, accuracy: 1.000000, avg. loss: 0.031758, test accuracy: 0.760000\n",
      "epoch: 13, accuracy: 1.000000, avg. loss: 0.045625, test accuracy: 0.760000\n",
      "epoch: 14, accuracy: 1.000000, avg. loss: 0.031624, test accuracy: 0.760000\n",
      "epoch: 15, accuracy: 1.000000, avg. loss: 0.030048, test accuracy: 0.760000\n",
      "epoch: 16, accuracy: 1.000000, avg. loss: 0.035204, test accuracy: 0.760000\n",
      "epoch: 17, accuracy: 1.000000, avg. loss: 0.036185, test accuracy: 0.760000\n",
      "epoch: 18, accuracy: 1.000000, avg. loss: 0.031018, test accuracy: 0.760000\n",
      "epoch: 19, accuracy: 1.000000, avg. loss: 0.027822, test accuracy: 0.760000\n",
      "{'River', 'Forest', 'Residential', 'Highway', 'HerbaceousVegetation'}\n",
      "Episode 5: Test Accuracy: 0.7600\n",
      "\n",
      "Average Test Accuracy: 0.8133333333333335\n"
     ]
    }
   ],
   "source": [
    "print('Please wait patiently, it may take some seconds...')\n",
    "total_test_acc = 0.0\n",
    "\n",
    "good_classes = [set() for _ in range(EPISODES)]\n",
    "bad_classes = set()\n",
    "\n",
    "for episode in range(EPISODES):\n",
    "    print(f\"\\nTraining Episode: {episode + 1}...\")\n",
    "    \n",
    "    loaded_model = torch.load(modelPath, map_location=device)\n",
    "    model = VisionTransformerTimm(64)\n",
    "    model.load_state_dict(loaded_model)\n",
    "    model.model.head = nn.Linear(model.model.head.in_features, 5).to(device)\n",
    "\n",
    "    for param in model.parameters():\n",
    "        param.requires_grad = True\n",
    "    \n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(params = model.parameters(), lr = LEARNING_RATE, momentum = MOMENTUM, weight_decay = WEIGHT_DECAY)\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size = STEP_SIZE, gamma = GAMMA)\n",
    "    dataloaders, class_names, dataset_sizes = createEuroSatDataLoaders(data_transforms, dataDir, split=0.25, batch_size = BATCH_SIZE)\n",
    "    print(dataset_sizes)\n",
    "    \n",
    "    model, _, _ = train_fine_tuning(model, dataloaders, criterion, optimizer, scheduler, num_epochs = NUM_EPOCHS, learning_rate = LEARNING_RATE)\n",
    "    test_loader = dataloaders['test']\n",
    "    acc_test, test_loss = eval_method(net=model, data_loader=test_loader)\n",
    "    total_test_acc += acc_test\n",
    "    if acc_test < 0.6:\n",
    "        bad_classes.update(class_names)\n",
    "    elif acc_test > 0.7:\n",
    "        good_classes[episode].update(class_names)\n",
    "    print(class_names)\n",
    "    print(f\"Episode {episode + 1}: Test Accuracy: {acc_test:.4f}\")\n",
    "\n",
    "average_test_acc = total_test_acc / EPISODES\n",
    "print(\"\\nAverage Test Accuracy:\", average_test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set()\n",
      "set()\n",
      "set()\n",
      "set()\n",
      "set()\n",
      "set()\n"
     ]
    }
   ],
   "source": [
    "for good_set in good_classes:\n",
    "    print(bad_classes - good_set)\n",
    "print(bad_classes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}