inside_bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "inside_bert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "813fc032906f4a4b93b079b2ede46614": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31d16fd6185f410f9bb236b91db2be98",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0671db3ca6d14c88a748b8fc79edd4e2",
              "IPY_MODEL_7e8809272cb248a9b2810431c477da43"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "31d16fd6185f410f9bb236b91db2be98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0671db3ca6d14c88a748b8fc79edd4e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2d4d89d851c546b88ec94939ee8cb2cb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ef75b618147e43d5b0894fe3f10378bc"
          },
          "model_module_version": "1.5.0"
        },
        "7e8809272cb248a9b2810431c477da43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4eea6ab46cfe4d2bbeeca10d7ff7bf50",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:28&lt;00:00, 20.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_196bc9904eab4ae0b4c9b7f083304fb2"
          },
          "model_module_version": "1.5.0"
        },
        "2d4d89d851c546b88ec94939ee8cb2cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "ef75b618147e43d5b0894fe3f10378bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "4eea6ab46cfe4d2bbeeca10d7ff7bf50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "196bc9904eab4ae0b4c9b7f083304fb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "2dd09319e0df488c953ea1cdea62312f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8d3e5ee1b76e4d87aa790c2b9bf3d631",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d8f05c058ef44de7b2e3801a044a0de5",
              "IPY_MODEL_c1440c985ddb43388de623a4f18986b0"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "8d3e5ee1b76e4d87aa790c2b9bf3d631": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "d8f05c058ef44de7b2e3801a044a0de5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b68b1416cb5b4bc5a0cf0134271263c0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_786137ae3d904acd890329d9bc1edd1c"
          },
          "model_module_version": "1.5.0"
        },
        "c1440c985ddb43388de623a4f18986b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9cab05c431984bb890f214cb9b99a7f6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:14&lt;00:00, 30.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_91a79c4f86d3467bb6dba3ae22b35687"
          },
          "model_module_version": "1.5.0"
        },
        "b68b1416cb5b4bc5a0cf0134271263c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "786137ae3d904acd890329d9bc1edd1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "9cab05c431984bb890f214cb9b99a7f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "91a79c4f86d3467bb6dba3ae22b35687": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "5aac931e402d4e7f88a0fc39c142e6c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea469d87f3cd430ebdbcf85f732033d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a945116be64341ca9018fcf62d68045f",
              "IPY_MODEL_08cc4387a2a3443b8bca921afdc08e9b"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "ea469d87f3cd430ebdbcf85f732033d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "a945116be64341ca9018fcf62d68045f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6c460920bbe74319be927b7de6cec4b0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16b74057dabb448c91f9306a30f6d679"
          },
          "model_module_version": "1.5.0"
        },
        "08cc4387a2a3443b8bca921afdc08e9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d8e6692429974739b7b75c9f7d700a39",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:02&lt;00:00, 98.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30e5ed1261bd4cf7845f53c224241a07"
          },
          "model_module_version": "1.5.0"
        },
        "6c460920bbe74319be927b7de6cec4b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "16b74057dabb448c91f9306a30f6d679": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "d8e6692429974739b7b75c9f7d700a39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "30e5ed1261bd4cf7845f53c224241a07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "2d632da2e0b948cbb00c4ca0a0a55f60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b03f83f066b8476dbbf27003d74b880c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_91b78d64f1b949eeb3e4d25b8a2b8945",
              "IPY_MODEL_5d425d0f23ff41d9b45b6090bc15d986"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "b03f83f066b8476dbbf27003d74b880c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "91b78d64f1b949eeb3e4d25b8a2b8945": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_48922a2dc4f24aa5be81131c1f854f22",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7d76f16dcd8747dcaf62802d9513f87c"
          },
          "model_module_version": "1.5.0"
        },
        "5d425d0f23ff41d9b45b6090bc15d986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_326adbbe4d574f7cb93448b49c354534",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 36.4B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_74f2079b924e4ad4ac0501a4515165fe"
          },
          "model_module_version": "1.5.0"
        },
        "48922a2dc4f24aa5be81131c1f854f22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "7d76f16dcd8747dcaf62802d9513f87c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "326adbbe4d574f7cb93448b49c354534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "74f2079b924e4ad4ac0501a4515165fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0c14c23dfd144eb084083e3df9838d80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fd6157893f4c4a2ca121c32a97682a74",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0dd4efa8884f4e88918369bb02770d32",
              "IPY_MODEL_fe1519de8ee74584b7beb641e8e1e65c"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "fd6157893f4c4a2ca121c32a97682a74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0dd4efa8884f4e88918369bb02770d32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_54ed975ccbee4e168a2400b6e542c682",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4623404c9d664846a6183a9355fd5b1f"
          },
          "model_module_version": "1.5.0"
        },
        "fe1519de8ee74584b7beb641e8e1e65c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2d7340bf26294a4a865f63cd842ad0b9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 2.22MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d6ac1a7a8234d3fbe5e700d5ef30d20"
          },
          "model_module_version": "1.5.0"
        },
        "54ed975ccbee4e168a2400b6e542c682": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "4623404c9d664846a6183a9355fd5b1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "2d7340bf26294a4a865f63cd842ad0b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "3d6ac1a7a8234d3fbe5e700d5ef30d20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "956e735c28474c379038bb250b0c268c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a4ae947f220c4a79ba254915c4cf8ca4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_25ebe9caa03b471ea20f875965822373",
              "IPY_MODEL_f8d533e04e2b4e8e94f8485a0a75e1fc"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "a4ae947f220c4a79ba254915c4cf8ca4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "25ebe9caa03b471ea20f875965822373": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_98ad30ef82e94089a3f970380fece0a4",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1947,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1947,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c209f1e20d34b5d983521d45e03237b"
          },
          "model_module_version": "1.5.0"
        },
        "f8d533e04e2b4e8e94f8485a0a75e1fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_36dd3cf4b0994a788e6978f847d85485",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5.29k/? [00:02&lt;00:00, 1.88kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cf2ed7ee41ab45e58cdc002a2e579f2d"
          },
          "model_module_version": "1.5.0"
        },
        "98ad30ef82e94089a3f970380fece0a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "9c209f1e20d34b5d983521d45e03237b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "36dd3cf4b0994a788e6978f847d85485": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "cf2ed7ee41ab45e58cdc002a2e579f2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "20e7c233366f4960af358647fc55fb2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a3b4aa5ac25444979dff08b340bcd477",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9af7e55b7f7f43f2a62e13536a9ddfa0",
              "IPY_MODEL_51f139737b1b42aa9cf6bc7246cc757b"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "a3b4aa5ac25444979dff08b340bcd477": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "9af7e55b7f7f43f2a62e13536a9ddfa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7b0605b4683a4efda8a3a7b1b87e522b",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1021,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1021,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_732f79f9ba6a4a899de7d7fd3c5cecd3"
          },
          "model_module_version": "1.5.0"
        },
        "51f139737b1b42aa9cf6bc7246cc757b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c36f5698ca3c4bfb8ddee6185652b325",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.36k/? [00:00&lt;00:00, 47.6kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8fc4f18f2d241b5be0ea370f0f5cd89"
          },
          "model_module_version": "1.5.0"
        },
        "7b0605b4683a4efda8a3a7b1b87e522b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "732f79f9ba6a4a899de7d7fd3c5cecd3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "c36f5698ca3c4bfb8ddee6185652b325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "b8fc4f18f2d241b5be0ea370f0f5cd89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "e00915c03da84b4db4817a41b5a70c1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_11d7298002174fc9bf5868616db85ac3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_85073e02610c4af49fb9cacbed1974ec",
              "IPY_MODEL_c463f96b9ada4620b1d4e74ea5796f01"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "11d7298002174fc9bf5868616db85ac3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "85073e02610c4af49fb9cacbed1974ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c47706f18edf471fa1e3333814098b85",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8116577,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8116577,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f2c7d32992294b87b652a1d5e6b2ce67"
          },
          "model_module_version": "1.5.0"
        },
        "c463f96b9ada4620b1d4e74ea5796f01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f933a71a37824c0c94014cb630b33afd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30.3M/? [00:01&lt;00:00, 23.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32dd4f881c78447aa66a64ccf3e9d8b3"
          },
          "model_module_version": "1.5.0"
        },
        "c47706f18edf471fa1e3333814098b85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "f2c7d32992294b87b652a1d5e6b2ce67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "f933a71a37824c0c94014cb630b33afd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "32dd4f881c78447aa66a64ccf3e9d8b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "1e7b7253e8ae4f1896aacb4b025eff62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cc9bf90f4b3b4d2da6d2c62e326421a3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_df6111ffb7f04d5ba91d755cd415f831",
              "IPY_MODEL_713d8c1749c24169a42e74ce27319297"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "cc9bf90f4b3b4d2da6d2c62e326421a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "df6111ffb7f04d5ba91d755cd415f831": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_909028f5595344609707fda09138ddc3",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1054280,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1054280,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8c0d26c9aea4ac1bbe9f5588baaf97d"
          },
          "model_module_version": "1.5.0"
        },
        "713d8c1749c24169a42e74ce27319297": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_23e2e22aa628434188a9395cf2bac22d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.85M/? [00:00&lt;00:00, 14.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d19c3bb57814ae8887a374d450b50b1"
          },
          "model_module_version": "1.5.0"
        },
        "909028f5595344609707fda09138ddc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "b8c0d26c9aea4ac1bbe9f5588baaf97d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "23e2e22aa628434188a9395cf2bac22d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "6d19c3bb57814ae8887a374d450b50b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "3536460505d743da970e10052fdb0374": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1d8f8d53a45d4347a3063efe89d34640",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_23c8d9e8a9c94ae98bdc081a14b049db",
              "IPY_MODEL_57545c835a6a4384a95c62290a6d1612"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "1d8f8d53a45d4347a3063efe89d34640": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "23c8d9e8a9c94ae98bdc081a14b049db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e218ec4e93994b12976b772be77a2eaf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7b73f6f50c714a6a9c7bacc478a85595"
          },
          "model_module_version": "1.5.0"
        },
        "57545c835a6a4384a95c62290a6d1612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d92a82801e454c07a8bd435de51b28ff",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 87599/0 [00:07&lt;00:00, 8797.50 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80225e01fcd14fc5a3fe2839d5bfd659"
          },
          "model_module_version": "1.5.0"
        },
        "e218ec4e93994b12976b772be77a2eaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "7b73f6f50c714a6a9c7bacc478a85595": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "d92a82801e454c07a8bd435de51b28ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "80225e01fcd14fc5a3fe2839d5bfd659": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "4fdd0db7c47242789d29db7d4596450c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02d75e6ee1fc4d4894272d8dc82354a2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_eacc281c720f48dcac12e61e6effc5a2",
              "IPY_MODEL_69a9da26012948a2b0bb60907a2d04f2"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "02d75e6ee1fc4d4894272d8dc82354a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "eacc281c720f48dcac12e61e6effc5a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e222a7989689479c99d5eeeae8553e95",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a10cf8317181497fb329ee2c9dd88e15"
          },
          "model_module_version": "1.5.0"
        },
        "69a9da26012948a2b0bb60907a2d04f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ea386b7f33624dac885bfb75c59668c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10570/0 [00:00&lt;00:00, 8719.16 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7411ba313b3348489925f943bcc74785"
          },
          "model_module_version": "1.5.0"
        },
        "e222a7989689479c99d5eeeae8553e95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "a10cf8317181497fb329ee2c9dd88e15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ea386b7f33624dac885bfb75c59668c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "7411ba313b3348489925f943bcc74785": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "15e62ce495804af88f80f2f9e7f032c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c0bd8ac8c42845789aea8f58ddace4d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3075a65030f244ca8c02cb26666cf332",
              "IPY_MODEL_32a6faab02e94528b9bf64d40530e707"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "c0bd8ac8c42845789aea8f58ddace4d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "3075a65030f244ca8c02cb26666cf332": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1d879a06d7a44ac0a2bd16e65637a873",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 59,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 59,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd32131fc363401a9b2731ed4b22cb91"
          },
          "model_module_version": "1.5.0"
        },
        "32a6faab02e94528b9bf64d40530e707": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d04886221d1f4e7fa213c6429cf60d63",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 59/59 [00:32&lt;00:00,  1.82ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35b4fb0fce2445118b82867891414ee5"
          },
          "model_module_version": "1.5.0"
        },
        "1d879a06d7a44ac0a2bd16e65637a873": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "fd32131fc363401a9b2731ed4b22cb91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "d04886221d1f4e7fa213c6429cf60d63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "35b4fb0fce2445118b82867891414ee5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "bf27d5db4f5c41b5af5d07b167a7f8cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e0c87ba45fa84041919c77cd205a4e59",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f906513a8bad4a7d9f0bf6617312eae7",
              "IPY_MODEL_42c70a3119fa49d7a331c8d6b90e3070"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "e0c87ba45fa84041919c77cd205a4e59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "f906513a8bad4a7d9f0bf6617312eae7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7370f6dc45d24fcaa95607b5b93f136e",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 11,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 11,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1cfecd15308b4fdab7720d1ef5c09286"
          },
          "model_module_version": "1.5.0"
        },
        "42c70a3119fa49d7a331c8d6b90e3070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0aa8493b90d845b0bdcf010d4f5bfb24",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 11/11 [00:07&lt;00:00,  1.46ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b06933e172549dd865b18ba2642bbd4"
          },
          "model_module_version": "1.5.0"
        },
        "7370f6dc45d24fcaa95607b5b93f136e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "1cfecd15308b4fdab7720d1ef5c09286": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0aa8493b90d845b0bdcf010d4f5bfb24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "5b06933e172549dd865b18ba2642bbd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "1421fbd7041149d3a2f1ff9e9f2fc4aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5937d42f58be45beb3aebd5edbbef105",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ecc48df63fce411081ffcebdcc8bb065",
              "IPY_MODEL_4c64c064a688465d852fbc2a57ef087c"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "5937d42f58be45beb3aebd5edbbef105": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ecc48df63fce411081ffcebdcc8bb065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_40c630a6187c48cf9cd7011b4574a585",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_176318f0f80541acb0631eceedef9ef4"
          },
          "model_module_version": "1.5.0"
        },
        "4c64c064a688465d852fbc2a57ef087c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a311b116567a4d44a840cf97d9516838",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29/29 [2:07:35&lt;00:00, 263.99s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e8155fd6c4e4156928eb915f7cbb190"
          },
          "model_module_version": "1.5.0"
        },
        "40c630a6187c48cf9cd7011b4574a585": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "176318f0f80541acb0631eceedef9ef4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "a311b116567a4d44a840cf97d9516838": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "4e8155fd6c4e4156928eb915f7cbb190": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "b79f4abd93cf4694908f04f82100d8e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e5417262ec164e7c8afcc668810af42d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0555bca300d1466aad538965818b01e6",
              "IPY_MODEL_522fc8cf79b144c2a950c27523594972"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "e5417262ec164e7c8afcc668810af42d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0555bca300d1466aad538965818b01e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_30b34b78dd3f4904ad4174bc335aac7f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac6c58d2182c450b9797c1cafb70bedb"
          },
          "model_module_version": "1.5.0"
        },
        "522fc8cf79b144c2a950c27523594972": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4ca84699247f4878a4665d2cc42938ad",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29/29 [09:43&lt;00:00, 20.14s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ac76a04d4d54c90b0f80929030249fd"
          },
          "model_module_version": "1.5.0"
        },
        "30b34b78dd3f4904ad4174bc335aac7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "ac6c58d2182c450b9797c1cafb70bedb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "4ca84699247f4878a4665d2cc42938ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "6ac76a04d4d54c90b0f80929030249fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "9c420c3bfa424906bfa494b226e2907a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_20d6490579914d69b003a553cadf1644",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5acee8e2818646c18e6dc08d5dc3a94c",
              "IPY_MODEL_5e52156ead384641aff0366d800c935e"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "20d6490579914d69b003a553cadf1644": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "5acee8e2818646c18e6dc08d5dc3a94c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e4bdf4e8f4ee4eb19a0b5ac0ae4a2d96",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28908,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28908,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_237ffb1f0cfc4375ab4d6a3c42e8cfa6"
          },
          "model_module_version": "1.5.0"
        },
        "5e52156ead384641aff0366d800c935e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe4f75b3aa594325aeb80063c0bafb9d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28908/28908 [01:29&lt;00:00, 323.18it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_15d6ee18dbb74c75bb16c78e33484abc"
          },
          "model_module_version": "1.5.0"
        },
        "e4bdf4e8f4ee4eb19a0b5ac0ae4a2d96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "237ffb1f0cfc4375ab4d6a3c42e8cfa6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "fe4f75b3aa594325aeb80063c0bafb9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "15d6ee18dbb74c75bb16c78e33484abc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "83c5de0bd35c480fb448c90f2c16c51f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1a21f1dc66124e63a9b4952f30bd1166",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_807de3b18c7d41c9b11e45bdcfe11935",
              "IPY_MODEL_62a57b34b2fe4d1a8a2d128a8ba17cea"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "1a21f1dc66124e63a9b4952f30bd1166": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "807de3b18c7d41c9b11e45bdcfe11935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ad944b55b2a24c34a3dafb9852202857",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1726,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1726,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed80463428864e0fa11b67f034c08963"
          },
          "model_module_version": "1.5.0"
        },
        "62a57b34b2fe4d1a8a2d128a8ba17cea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba1bf85fa0644758aa6abe5b807ea69a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.51k/? [00:00&lt;00:00, 9.34kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e63fa7ac974e4d23aec70a65252f618d"
          },
          "model_module_version": "1.5.0"
        },
        "ad944b55b2a24c34a3dafb9852202857": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "ed80463428864e0fa11b67f034c08963": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ba1bf85fa0644758aa6abe5b807ea69a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "e63fa7ac974e4d23aec70a65252f618d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ce5ba1af33684a328f83a33d7888841f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cade3114c79e4bc692ca31e2eb1ee8ff",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dfeb105a607f49a59ffafdb08bdef022",
              "IPY_MODEL_58e29c1f24b04521bde60aa8c9f6441b"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "cade3114c79e4bc692ca31e2eb1ee8ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "dfeb105a607f49a59ffafdb08bdef022": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6b8fc7f02aad4816acdbe991251a052c",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1119,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1119,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_505d0af9bbda41598afdffddc36095e4"
          },
          "model_module_version": "1.5.0"
        },
        "58e29c1f24b04521bde60aa8c9f6441b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_70dc6ac663a54e718adb1315ff36da0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.31k/? [00:00&lt;00:00, 52.5kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_caf02f85fb02418a850e3776b0709188"
          },
          "model_module_version": "1.5.0"
        },
        "6b8fc7f02aad4816acdbe991251a052c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "505d0af9bbda41598afdffddc36095e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "70dc6ac663a54e718adb1315ff36da0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "caf02f85fb02418a850e3776b0709188": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "87096b907d67440da538eb7f797b7848": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c62b21898a04fc29511db53f032507e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ec16172829e24547a80dd04789d709f0",
              "IPY_MODEL_453876ce77004371987e12586fa6e410"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "8c62b21898a04fc29511db53f032507e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ec16172829e24547a80dd04789d709f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fc7ee193eae245f182a5a2b55c3cfd6f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_32df2363e0864fdaab1b3131a7e8204a"
          },
          "model_module_version": "1.5.0"
        },
        "453876ce77004371987e12586fa6e410": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7f609a3427ad4578bdcac0279f7f17d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  8.68ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c28646616f3409ab97b1e20736f8399"
          },
          "model_module_version": "1.5.0"
        },
        "fc7ee193eae245f182a5a2b55c3cfd6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "32df2363e0864fdaab1b3131a7e8204a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "7f609a3427ad4578bdcac0279f7f17d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "6c28646616f3409ab97b1e20736f8399": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "eeaf41eb4e194187bc1630b504a379ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_452da70ac94b4c97aae5421860abc59e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8b6b5ff3319e478d939d67359368aa20",
              "IPY_MODEL_bd9bf739e5584fd8891e7115ff80661e"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "452da70ac94b4c97aae5421860abc59e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "8b6b5ff3319e478d939d67359368aa20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8ac5a1e04cdf45f39739c5d706c09d96",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f765ad233eb49c7a742d032b6924e87"
          },
          "model_module_version": "1.5.0"
        },
        "bd9bf739e5584fd8891e7115ff80661e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_869509b4ddf74fefb7182836509b9f9f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  1.10it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5a2ddcd374ce4cceaf1291587a2dd2ab"
          },
          "model_module_version": "1.5.0"
        },
        "8ac5a1e04cdf45f39739c5d706c09d96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "1f765ad233eb49c7a742d032b6924e87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "869509b4ddf74fefb7182836509b9f9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "5a2ddcd374ce4cceaf1291587a2dd2ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ion-bueno/bert-from-inside/blob/main/inside_bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WgMKxMqIGXgD"
      },
      "source": [
        "# Installation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uo8tsaAo5a11"
      },
      "source": [
        "## Modules"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B9DPjwT0HBlN"
      },
      "source": [
        "During this notebook it is used Pytorch. It is installed running next cell."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2YcVR9diuNjr"
      },
      "source": [
        "# !pip3 install torch==1.9.0+cu102 torchvision==0.10.0+cu102 torchaudio===0.9.0 -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hDbShn3OWDj0"
      },
      "source": [
        "BERT was developed in TensorFlow in [github.com/google-research/bert](https://github.com/google-research/bert), but it is going to be used the libray of [HuggingFace](https://huggingface.co/transformers/index.html) to show the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Amub2TUVG_hD"
      },
      "source": [
        "!pip3 install datasets\n",
        "!pip3 install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oic_Lz0GZ6Pt"
      },
      "source": [
        "Import other modules."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s2LVsAsvOwqW"
      },
      "source": [
        "# Pytorch modules\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "import torch.nn.functional as F\n",
        "# Colab modules\n",
        "from google.colab import files\n",
        "# Usual modules used\n",
        "from pathlib import Path\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import math\n",
        "import random\n",
        "import os\n",
        "import json\n",
        "import shutil\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import datetime\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qmA7A_2yZ28N"
      },
      "source": [
        "Mount drive to load files from Google Drive. Useful to load trained models."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KatQ5N1QI_ie"
      },
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aKArDx-4DHLE"
      },
      "source": [
        "## Recurrent Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ejw1cJpDix2"
      },
      "source": [
        "Recurrent functions employed during the develoment of the NLP tasks."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PeinK47-Flnz"
      },
      "source": [
        "def info_device():\n",
        "    # If there's a GPU available\n",
        "    if torch.cuda.is_available():\n",
        "        print(f'\\nUsing GPU {torch.cuda.get_device_name(0)}.', '\\n')\n",
        "    # If not\n",
        "    else:\n",
        "        print('\\nNo GPU available, using the CPU instead.', '\\n')\n",
        "\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round(elapsed))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n",
        "\n",
        "\n",
        "def plot_loss_curves(df):\n",
        "    # Use plot styling from seaborn.\n",
        "    sns.set(style='darkgrid')\n",
        "\n",
        "    # Increase the plot size and font size.\n",
        "    sns.set(font_scale=1.5)\n",
        "    plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "    # Plot the learning curve.\n",
        "    plt.plot(df['Training Loss'], 'b-o', label=\"Training\")\n",
        "    plt.plot(df['Validation Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "    # Label the plot.\n",
        "    plt.title(\"Training & Validation Loss\")\n",
        "    plt.xlabel(\"Epoch\")\n",
        "    plt.ylabel(\"Loss\")\n",
        "    plt.legend()\n",
        "    plt.xticks(df.index.values)\n",
        "\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def save_json(path, file):\n",
        "    with open(path, \"w\") as outfile:\n",
        "        json.dump(file, outfile)\n",
        "\n",
        "\n",
        "def download_folder(dir): \n",
        "    !zip -r $dir\\.zip $dir\n",
        "    files.download(f'{dir}.zip')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ksCpEC_5vdb_"
      },
      "source": [
        "# 1 - Background\n",
        "\n",
        "Recurrent neural networks (RNN) were declared as state of the art in sequence modeling and transduction problems, in particular long short-term memory (LSTM) and gated recurrent neural networks (GRU).\n",
        "\n",
        "These models work respect the position of the symbols, input and output. They generate a sequence of hidden states $h_t$, function of previous hidden states $h_{t-1}$ and current input $x_t$.\n",
        "\n",
        "\\begin{equation}\n",
        "  h_t = f(x_t, h_{t-1})\n",
        "\\end{equation}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ByU_MM9YmQLq"
      },
      "source": [
        "![RNN](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/RNN.png)\n",
        "\n",
        "> RNN high level structure. Image from [Dive into Deep Learning](https://d2l.ai/)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EL9zstqnViDC"
      },
      "source": [
        "However, RNNs are really slow to train due to the sequential computation they employ. This operations cannot be optimized with modern hardware employed today (GPUs or TPUs).\n",
        "\n",
        "Another important issue is the difficulty to capture long-term dependencies. Gradient based learning algorithms face many problems as the sequence or spans increases. Gradients can exploit or vanish, resulting in many difficulties for gradient optimization."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHtLxnVeSryf"
      },
      "source": [
        "One novel mechanism employed was **Attention**. It is based on retaining the most significant information. It is applied between the elements of the input and the output. In translation for example, between the source sentence and the translation. Attention has been widely used for many tasks, specially in translation.\n",
        "\n",
        "In this work we are interested in one type of attention: **Self-attention**. Attention is only applied between the elements of the input sequence, relating themselves."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3fs7sOi1O53b"
      },
      "source": [
        "# 2 - Self Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NlzWfPuaNpyf"
      },
      "source": [
        "It is an attention mechanism relating different positions of a single sequence in order to compute a representation of the sequence. The goal is mapping sets to sets, regardless of the order in the sequence. Self-attention has been used successfully in a variety of tasks including reading comprehension, abstractive summarization, textual entailment and learning task-independent sentence representations. \\\\"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "snXFyJKGRZSP"
      },
      "source": [
        "## 2.1 - Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gddSt8MOQdP6"
      },
      "source": [
        "If we have a sentence with length $ n $, each word is usually represented with a word vector $ x_i \\in \\mathbb R^{d}, i \\in \\lbrace1, \\dots, n \\rbrace $ which dimension $ d $ depends on the size of embedding space used. This space tries to cluster similar words, classifying them according to a number $ d $ of features.\n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{The} & \\text{orange} & \\text{is} & \\text{very} & \\text{healthy} \\\\\n",
        "  x_1 & x_2 & x_3 & x_4 & x_5\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_ZPAha90o8p",
        "outputId": "2f5d08e8-73c6-4a90-f153-5b1bd51b78ba"
      },
      "source": [
        "#@title { vertical-output: true }\n",
        "sentence = \"The orange is very healthy\" #@param {type:\"string\"}\n",
        "list_sentence = sentence.split()\n",
        "#@markdown Dimension of the embedding vector\n",
        "d =  3#@param {type:\"number\"}\n",
        "vocab = set(list_sentence)\n",
        "word_to_ix = {word: i for i, word in enumerate(vocab)}\n",
        "embeds = nn.Embedding(len(word_to_ix), d)\n",
        "\n",
        "for word in list_sentence:\n",
        "  lookup_tensor = torch.tensor(word_to_ix[word], dtype=torch.long)\n",
        "  vector = embeds(lookup_tensor)\n",
        "  print(word)\n",
        "  print(vector.detach().numpy(), '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The\n",
            "[-1.4270844  0.3595118  0.1073881] \n",
            "\n",
            "orange\n",
            "[-0.3226558   0.84128296 -0.22574684] \n",
            "\n",
            "is\n",
            "[ 1.8625331  0.5788812 -1.896803 ] \n",
            "\n",
            "very\n",
            "[0.48085693 0.69332105 0.11034871] \n",
            "\n",
            "healthy\n",
            "[ 0.67152673 -0.9558171   0.3913486 ] \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FQiDwc4JybzS"
      },
      "source": [
        "These vectors do not provide context information, i.e. the relationship with the rest of the elements in the sequence. For example, $ x_2 $ does not determine if *orange* refers to the color or to the fruit, so it is not clear if it exists a clear relationship with *healthy*.\n",
        "\n",
        "Then it is needed another vector, formed by attention weigths, $ y_i \\in \\mathbb R^{n}, i \\in \\lbrace1,\\dots,n\\rbrace $, where $ n $ is the number of elements in the sequence. For all $ i, j \\in \\lbrace{1,\\dots,n}\\rbrace $.\n",
        "\n",
        "\\begin{equation}\n",
        "  y_i = \\sum^{n}_{j} x_j w_{ij}\n",
        "\\end{equation}\n",
        "\n",
        "The question is, how could we get these weigths? In order to illustrate the process, we are going to use the previous sentence and get the weights associated to the new embedding vector for *orange* ($ y_{2} $).\n",
        "\n",
        "> Remember that $ n $ is the number of words in the sentence and $ d $ the dimension of the word vectors."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fo04Vmks3Gq0"
      },
      "source": [
        "> The explanation is based in the slides of [A series of videos on transformers](https://www.youtube.com/playlist?list=PLDw5cZwIToCvXLVY2bSqt7F2gu8y-Rqje) by Lennart Svensson."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9BgDkBMJ2yxK"
      },
      "source": [
        "## 2.2 - Weighted Averages of Words"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2_6qnuArbEUh"
      },
      "source": [
        "The traditional idea was based in providing larger weigths to similar words making use of the **dot product**. Two similar vectors are going to get a bigget dot product than two very distant. Applying the dot product and softmax respect $ x_2 $"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dqoYDolEOJdg"
      },
      "source": [
        "\\begin{equation}\n",
        "  \\begin{matrix} z_{1} = x_1^T x_2 & \\dots & z_{n} = x_n^T x_2 \\end{matrix} \\\\\n",
        "  \\begin{bmatrix} w_{1} & \\dots & w_{n} \\end{bmatrix} = \\text{softmax} \\begin{pmatrix} z_{1} & \\dots & z_{n} \\end{pmatrix} \\\\\n",
        "  y_2 = \\sum_{j=1}^{n} x_j w_{j}\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IRMYg52-XN6d"
      },
      "source": [
        "![Weighted Averaged of Words](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/weight_ave_words.png)\n",
        "\n",
        "> Observations and weights respect time. Image from [A series of videos on transformers](https://www.youtube.com/playlist?list=PLDw5cZwIToCvXLVY2bSqt7F2gu8y-Rqje)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P7WPCEYKYbh_"
      },
      "source": [
        "Intuitively, it is a method which could obtain good results. Nevertheless, several times correlated words are not necessary close. Therefore it is not an accurate and general method to extract the information between one element and the rest of the sequence."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2s8JJyH4MNy0"
      },
      "source": [
        "## 2.3 - Weights in Self-Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "biJITJoQXosT"
      },
      "source": [
        "It is necessary to introduce three vectors: **query**, **keys** and **values**. It is also included three new parameters: $ W_Q, W_K, W_V $, which are going to be trained. For all $ i \\in \\lbrace{1,\\dots,n}\\rbrace $.\n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{Query} & \\rightarrow & q_i = W^Q x_2 \\\\\n",
        "  \\text{Keys} & \\rightarrow & k_i = W^K x_i \\\\\n",
        "  \\text{Values} & \\rightarrow & v_i = W^V x_i \n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2outRvCKMV-Z"
      },
      "source": [
        "Instead of using directly the inputs $ x_i $, it is employed the **query** $ q_i $ and **keys** $ k_i \\in \\lbrace{1,\\dots,n}\\rbrace $ to calculate the weights:\n",
        "\\begin{equation}\n",
        "    \\begin{matrix} z_{1} = \\frac{k_1^T q_2}{\\sqrt{d}} & \\dots & z_{n} = \\frac{k_n^T q_2}{\\sqrt{d}} \\end{matrix} \\\\\n",
        "  \\begin{bmatrix} w_1 & \\dots & w_n \\end{bmatrix} = \\text{softmax} \\begin{pmatrix} z_1 & \\dots & z_n \\end{pmatrix}\n",
        "\\end{equation}\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2AAWQFphXsty"
      },
      "source": [
        "To get the final embedding vector it is used the **values** $ v_i \\in \\lbrace{1,\\dots,n}\\rbrace $ instead of the inputs $ x_i $:\n",
        "\n",
        "\\begin{equation}\n",
        "  y_2 = \\sum_{j=1}^{n} v_j w_{j}\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sLDkf7xi-hsw"
      },
      "source": [
        "Intuitively,the query attends the word for which we are going to get the attention weights. Keys corresponds with the rest of the elements in the dot product operation. Values are used in the final combination with the weights. Thanks to learnable matrixes, these combinations are optimized to find combinations and correlations in the own sentence."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VsBg0CcUAqvi"
      },
      "source": [
        "![Self-attention process](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/self-attention_process.png)\n",
        "\n",
        "> Self-attention process. Image from [Illustrated: Self-Attention](https://towardsdatascience.com/illustrated-self-attention-2d627e33b20a)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4PLlEXM0YpG2"
      },
      "source": [
        "### 2.3.1 - Matrix Notation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lnvZVIwwDn5W"
      },
      "source": [
        "To represent the whole process in matrix notation:\n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{Input} & \\rightarrow & X = \\begin{bmatrix} x_1 & \\dots & x_n \\end{bmatrix}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Ca6iaIA1EjQ",
        "outputId": "cbceb572-2b07-4d00-9a99-554f5c89e932"
      },
      "source": [
        "lookup_tensor = torch.tensor([word_to_ix[word] for word in list_sentence], dtype=torch.long)\n",
        "X = embeds(lookup_tensor).T\n",
        "print('Input: word per column\\n')\n",
        "print(X.detach().numpy())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: word per column\n",
            "\n",
            "[[-1.4270844  -0.3226558   1.8625331   0.48085693  0.67152673]\n",
            " [ 0.3595118   0.84128296  0.5788812   0.69332105 -0.9558171 ]\n",
            " [ 0.1073881  -0.22574684 -1.896803    0.11034871  0.3913486 ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yME-uGRP24l_"
      },
      "source": [
        "To calculate **query**, **keys** and **values**:\n",
        "\n",
        "\\begin{matrix}\n",
        "  Q = W^Q X = \\begin{bmatrix} q_1 & \\dots & q_n \\end{bmatrix}  \\\\\n",
        "  K = W^K X = \\begin{bmatrix} k_1 & \\dots & k_n \\end{bmatrix} \\\\\n",
        "  V = W^V X = \\begin{bmatrix} v_1 & \\dots & v_n \\end{bmatrix}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MEMuOjMq4XU0",
        "outputId": "eb392c0f-b8fe-49f9-97f3-e19052d464f0"
      },
      "source": [
        "# initialize weight matrixes\n",
        "dim = (d, d)\n",
        "W_Q = torch.randn(dim)\n",
        "W_K = torch.randn(dim)\n",
        "W_V = torch.randn(dim)\n",
        "\n",
        "# calculate query, keys, values\n",
        "Q = W_Q @ X\n",
        "K = W_K @ X\n",
        "V = W_V @ X\n",
        "\n",
        "print('Query\\n')\n",
        "print(Q.detach().numpy(), '\\n')\n",
        "print('Keys\\n')\n",
        "print(K.detach().numpy(), '\\n')\n",
        "print('Values\\n')\n",
        "print(V.detach().numpy(), '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Query\n",
            "\n",
            "[[ 0.66137916 -0.31311184 -1.8683293  -0.61087376  0.21525767]\n",
            " [ 4.272571    1.2250328  -4.875417   -1.2629364  -2.323167  ]\n",
            " [-0.64698535  0.33287638  3.119269    0.25764674 -0.4008254 ]] \n",
            "\n",
            "Keys\n",
            "\n",
            "[[ 1.3502357  -0.27039123 -5.01392    -0.3503492   0.27921772]\n",
            " [ 0.274171   -0.14648661  0.53703576 -0.66290426 -0.06655848]\n",
            " [-0.545273   -0.15799947 -0.52057207  0.49421638  0.44610292]] \n",
            "\n",
            "Values\n",
            "\n",
            "[[ 0.11682092  0.2756145   0.7931453   0.0509825  -0.39097068]\n",
            " [ 1.0154781   0.9634653  -0.3460012   0.4539004  -1.2331994 ]\n",
            " [ 1.1838771   0.14729327 -3.344792   -0.04831012 -0.22063345]] \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IJhYMfJBhMV_"
      },
      "source": [
        "And the **weights**:\n",
        "\n",
        "\\begin{matrix}\n",
        "  Z = \\frac{K^T Q}{\\sqrt{d}} \\\\\n",
        "  W = \\text{softmax}(Z)\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r9Iir7Nn1NCp",
        "outputId": "8f3250ca-b0ff-4916-e0c5-b2f925098051"
      },
      "source": [
        "Z = (K.T @ Q) / (d)\n",
        "# dim=0 since we want to apply it per column\n",
        "W = nn.LogSoftmax(dim=0)(Z)\n",
        "print('Weigths\\n')\n",
        "print(W.detach().numpy(), '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Weigths\n",
            "\n",
            "[[-0.80924207 -1.8262454  -4.5308046  -2.2876265  -1.6740227 ]\n",
            " [-1.8491417  -1.7859013  -2.4352193  -1.7472793  -1.5162972 ]\n",
            " [-1.8432397  -1.0519345  -0.9688676  -1.1002672  -2.337527  ]\n",
            " [-2.7429042  -1.9160632  -0.8680274  -1.4575832  -1.2092679 ]\n",
            " [-1.744424   -1.7435957  -2.2792778  -1.8409595  -1.61947   ]] \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eYvPf694-192"
      },
      "source": [
        "Finally, we get the **output** matrix:\n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{Output} & \\rightarrow & Y = VW = \\begin{bmatrix} y_1 & \\dots & y_n \\end{bmatrix}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qg6YXnOC_e4w",
        "outputId": "0fe85d3e-4e31-43ce-a7e7-769c59eb6b19"
      },
      "source": [
        "Y = V @ W\n",
        "print('Output\\n')\n",
        "print(Y.detach().numpy(), '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Output\n",
            "\n",
            "[[-1.523965   -0.9558918  -1.1220506  -0.97603995 -1.8959593 ]\n",
            " [-1.0593709  -1.9306962  -4.195147   -2.0171115  -0.9038039 ]\n",
            " [ 5.4522324   1.5706611  -1.9371274   1.191127    6.029093  ]] \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0yR7j8HDIaw"
      },
      "source": [
        "As it is normal, we have not obtained a meaningful result, since $ W^Q, W^K, W^V $ have not been trained. At the end, we are looking for a rectangular matrix which relates every word in the sentence. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qkH-EJ8jlh8O"
      },
      "source": [
        "![Intuition self-attention](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/intuition_att.png)\n",
        "\n",
        "> Intuition about attention. Image from [exBERT](https://huggingface.co/exbert/?model=bert-base-cased&modelKind=bidirectional&sentence=The%20orange%20is%20very%20healthy&layer=0&heads=..0,1,2,3,4,5,6,7,8,9,10,11&threshold=0.7&tokenInd=null&tokenSide=null&maskInds=..&hideClsSep=true) tool."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rx2d2n9VycK7"
      },
      "source": [
        "# 3 - The Transformer\n",
        "\n",
        "In 2017 was presented the Transformer in the paper [Attention is All You Need](https://arxiv.org/abs/1706.03762). It is the first transduction model relying entirely on self-attention to compute representations of its input and output without using sequence aligned RNNs or convolution. The Transformer reached a new state of the art in translation quality, the original goal for the model. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A_xD9N3umEyu"
      },
      "source": [
        "![Transformer architecture](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/transformer.png)\n",
        "\n",
        "> Transformer's architecture. Image from [Attention is All You Need](https://arxiv.org/abs/1706.03762)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78bu--O6DXAu"
      },
      "source": [
        "In order to understand how the Transformer works, it is going to be explained the main parts.\n",
        "\n",
        "> The code is based mainly in the implementation of [The Annotated Transformer](http://nlp.seas.harvard.edu/2018/04/03/attention.html), which implements the model step by step as it is discussed in the original paper."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9XTCL8IJ9Ma"
      },
      "source": [
        "## 3.1 - Model Architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BIW801LBKBh5"
      },
      "source": [
        "The model is based in an encoder-decoder structure, as other competitive neural sequence transduction models. It receives as input a complete sentence and generates the probabilities for the next word as output. Usually it is selected the higher one and feed it into the decoder until it is selected the token which indicates the end of sequence. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nofb2wJqO9TP"
      },
      "source": [
        "![High level transformer architecture](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/hl_trans_archi.png)\n",
        "\n",
        "> Transformer's diagram. Image from [A series of videos on transformers](https://www.youtube.com/watch?v=0SmNEp4zTpc&list=PLDw5cZwIToCvXLVY2bSqt7F2gu8y-Rqje).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jopZSHTl6sk2"
      },
      "source": [
        "The Transformer generates one symbol at a time, consuming the previously generated symbols as additional input, so it is an example of auto-regressive model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxiXBozxPpWI"
      },
      "source": [
        "## 3.2 - Input Embedding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UOKfYGq8R90U"
      },
      "source": [
        "Before going into the encoder and decoder stacks, mention how the sentences are preprocessed. The word vectors used as input in the encoder and the decoder are formed by the sum of other two vectors: **word embedding** and **positional encoding**."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rE33YK_we8uE"
      },
      "source": [
        "> Let's define some variables:\n",
        "\n",
        "\n",
        "\n",
        "*   Number of sentences in a batch: $ m $\n",
        "*   Number of words per sentence: $ n $\n",
        "*   Embedding dimension: $ d_{model} $\n",
        "*   Size of vocabulary: $ vocab $\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "np5nmAyUbPgV"
      },
      "source": [
        "![Transformer input embedding](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/in_emb.png)\n",
        "\n",
        "> Transformer's input embedding."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fsV9N-zJYnGg"
      },
      "source": [
        "### 3.2.1 - Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CLsRZqxRTL4i"
      },
      "source": [
        "It is used learned embeddings to convert the input tokens and output tokens to vectors of dimension $ d_{model} $. It is shared the same weight matrix between the two embbeding layers, similar to [Using the Output Embedding to Improve Language Models](https://arxiv.org/abs/1608.05859). Also mention those weights are multiplied by $ \\sqrt{d_{model}} $."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kav4S0GhU3Jd"
      },
      "source": [
        "class Embeddings(nn.Module):\n",
        "  def __init__(self, d_model, vocab):\n",
        "    super(Embeddings, self).__init__()\n",
        "    self.lut = nn.Embedding(vocab, d_model)\n",
        "    self.d_model = d_model\n",
        "\n",
        "  def forward(self, x):\n",
        "    return self.lut(x) * math.sqrt(self.d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mmnQ3x8IYrRV"
      },
      "source": [
        "### 3.2.2 - Positional Encoding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xuY7cXIoTnaj"
      },
      "source": [
        "The attention blocks only map sets to sets, due to that, there is not any information about the position of the word in the sentence. Then it is injected some information about the relative or absolute position. In the original paper, it is used sine and cosine functions of different frequencies:\n",
        "\n",
        "\\begin{equation}\n",
        "  PE_{(pos, 2i)} = sin(pos/10000^{2i/d_{model}}) \\\\\n",
        "  PE_{(pos, 2i+1)} = cos(pos/10000^{2i/d_{model}})\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9s3UgPQ0Wdck"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "  \"\"\"Implement the PE function\n",
        "\n",
        "  \"\"\"\n",
        "  def __init__(self, d_model, dropout, max_len=5000):\n",
        "    super(PositionalEncoding, self).__init__()\n",
        "    self.dropout = nn.Dropout(p=dropout)\n",
        "    pe = torch.zeros(max_len, d_model)\n",
        "    position = torch.arange(0, max_len).unsqueeze(1)\n",
        "    div_term = torch.exp(torch.arange(0, d_model, 2) * \n",
        "                         -(math.log(10000.0) / d_model))\n",
        "    pe[:, 0::2] = torch.sin(position * div_term)\n",
        "    pe[:, 1::2] = torch.cos(position * div_term)\n",
        "    pe = pe.unsqueeze(0)\n",
        "    self.register_buffer('pe', pe)\n",
        "\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = x + Variable(self.pe[:, :x.size(1)], requires_grad=False)\n",
        "    return self.dropout(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rhcQ8jbyyzlp"
      },
      "source": [
        "The output is the addition of both embeddings. \n",
        "\n",
        "\\begin{matrix}\n",
        "  x_i = & \\underbrace{E_i}_\\text{Word Embedding} & + & \\underbrace{P_i}_\\text{Positional Encoding}\n",
        "\\end{matrix}\n",
        "\n",
        "Let's see how is the input and how is transformed after this process"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpnPSaICY6OA"
      },
      "source": [
        "Function to mask Transformer inputs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lzuoKnVa03e"
      },
      "source": [
        "def subsequent_mask(size):\n",
        "  \"\"\"Mask out subsequent positions\n",
        "\n",
        "  \"\"\"\n",
        "  attn_shape = (1, size, size)\n",
        "  subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')\n",
        "  return torch.from_numpy(subsequent_mask) == 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5oSCMJUEY2pU"
      },
      "source": [
        "Class to manage one batch of data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eCLpgj6fZ4UW"
      },
      "source": [
        "class Batch:\n",
        "  \"\"\"Object for holding a batch of data with mask during training\n",
        "\n",
        "  \"\"\"\n",
        "  def __init__(self, src, trg=None, pad=0):\n",
        "    self.src = src\n",
        "    self.src_mask = (src != pad).unsqueeze(-2)\n",
        "    if trg is not None:\n",
        "      self.trg = trg[:, :-1]\n",
        "      self.trg_y = trg[:, 1:]\n",
        "      self.trg_mask = self.make_std_mask(self.trg, pad)\n",
        "      self.ntokens = (self.trg_y != pad).data.sum()\n",
        "\n",
        "  @staticmethod\n",
        "  def make_std_mask(tgt, pad):\n",
        "    \"\"\"Create a mask to hide padding and future words\n",
        "\n",
        "    \"\"\"\n",
        "    tgt_mask = (tgt != pad).unsqueeze(-2)\n",
        "    tgt_mask = tgt_mask & Variable(subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))\n",
        "    return tgt_mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vS_sxWBgY_wJ"
      },
      "source": [
        "Function to generate a random batch of data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1f_dsjQNVVfT"
      },
      "source": [
        "def batch_gen(V, batch, words):\n",
        "  \"\"\"Generate random data for a src-tgt copy task.\n",
        "\n",
        "  Args:\n",
        "  V (int): until number created for random int\n",
        "  batch (int): number of rows\n",
        "  words (int): number of columns\n",
        "\n",
        "  Returns:\n",
        "  Batch\n",
        "\n",
        "  \"\"\"\n",
        "  data = torch.from_numpy(np.random.randint(1, V, size=(batch, words)))\n",
        "  data[:, 0] = 1\n",
        "  src = Variable(data, requires_grad=False)\n",
        "  tgt = Variable(data, requires_grad=False)\n",
        "  return Batch(src, tgt, 0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0cyPzIoPVu_A"
      },
      "source": [
        "We can check the Transformer input"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZAKlqt0iSBb",
        "outputId": "5a4e6b58-7e84-4518-d4df-fbfaf6ca4a7f"
      },
      "source": [
        "#@title { vertical-output: true }\n",
        "#@markdown Number of sentences in a batch\n",
        "m = 5 #@param {type:\"integer\"}\n",
        "\n",
        "#@markdown Number of words in a sentence\n",
        "n = 10 #@param {type:\"integer\"}\n",
        "\n",
        "#@markdown Embedding dimension\n",
        "d_model = 512 #@param {type:\"integer\"}\n",
        "\n",
        "#@markdown Vocabulary size\n",
        "vocab = 11 #@param {type:\"integer\"}\n",
        "\n",
        "batch = batch_gen(vocab, m, n)\n",
        "emb = Embeddings(d_model, vocab)\n",
        "pos = PositionalEncoding(d_model, 0.01)\n",
        "\n",
        "emb_w = emb.lut.weight\n",
        "print('Embedding matrix with dimension: (vocab x d_model)')\n",
        "print(tuple(emb_w.shape), '\\n')\n",
        "#print(emb_w.detach().numpy(), '\\n')\n",
        "\n",
        "enc_in = batch.src\n",
        "print('Encoder input before embedding with dimension: (m x n) ')\n",
        "print(tuple(enc_in.shape), '\\n')\n",
        "\n",
        "enc_out = nn.Sequential(emb, pos)(enc_in)\n",
        "print('Encoder input with dimension: (m x n x d_model)')\n",
        "print(tuple(enc_out.shape), '\\n')\n",
        "\n",
        "dec_in = batch.trg\n",
        "print('Decoder input before embedding  with dimension: (m x n-1) ')\n",
        "print(tuple(dec_in.shape), '\\n')\n",
        "\n",
        "dec_out = nn.Sequential(emb, pos)(dec_in)\n",
        "print('Decoder input with dimension: (m x n-1 x d_model)')\n",
        "print(tuple(dec_out.shape), '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Embedding matrix with dimension: (vocab x d_model)\n",
            "(11, 512) \n",
            "\n",
            "Encoder input before embedding with dimension: (m x n) \n",
            "(5, 10) \n",
            "\n",
            "Encoder input with dimension: (m x n x d_model)\n",
            "(5, 10, 512) \n",
            "\n",
            "Decoder input before embedding  with dimension: (m x n-1) \n",
            "(5, 9) \n",
            "\n",
            "Decoder input with dimension: (m x n-1 x d_model)\n",
            "(5, 9, 512) \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l9ofzwczProp"
      },
      "source": [
        "## 3.3 - Encoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7PP1EybI5CV6"
      },
      "source": [
        "The encoder stacks $ N $ encoder blocks which map sets to sets, what means the output size is the same than input: $ ( m \\ \\text{x} \\ n \\ \\text{x} \\ d_{model} ) $. In the original implementation, they employ $ N=6 $ encoders."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ms1C-CsrbIuR"
      },
      "source": [
        "![Transformer's encoder](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/encoder.png)\n",
        "\n",
        "> Encoder's architecture."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eAibzJp2sMNL"
      },
      "source": [
        "Each block is formed by two sub-layers: **Multi-Head Attention** and **Feed Forward**. Both are followed by an **Add \\& Norm** layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nuOR0vIaZeH6"
      },
      "source": [
        "### 3.3.1 - Multi-Head Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MSCGGrr5Uof3"
      },
      "source": [
        "The Transformer is differentiated for using exclusively self-attention. This model employs a type of self-attention called **Scaled Dot-Product Attention**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3njTSJhVEfur"
      },
      "source": [
        "#### 3.3.1.1 - Scaled Dot-Product Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RdUPRWV1u9Ao"
      },
      "source": [
        "It is fast and space-efficient in practice, since it can be implemented using highly optimized matrix multiplication code.\n",
        "\n",
        "We have queries $ Q $ and keys $ K $ of dimension $ d_k $ and values $ V $ of dimension $ d_v $.\n",
        "\n",
        "\\begin{equation}\n",
        "  \\text{Attention}(Q, K, V) = \\text{softmax}(\\frac{QK^T}{\\sqrt{d_k}})V\n",
        "\\end{equation}\n",
        "\n",
        "For large values of $ d_k $ the softmax function could have extremely small gradients. For this reason, the dot product is scaled by $ \\frac{1}{\\sqrt{d_k}} $."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MLtjORVBnZ1D"
      },
      "source": [
        "![Scaled dot product attention](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/sdp_attention.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YeDeFHvgWprw"
      },
      "source": [
        "def sdp_attention(query, key, value, mask=None, dropout=None):\n",
        "  d_k = query.size(-1)\n",
        "  scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n",
        "  if mask is not None:\n",
        "    scores = scores.masked_fill(mask == 0, -1e9)\n",
        "  p_attn = F.softmax(scores, dim=-1)\n",
        "  if dropout is not None:\n",
        "    p_attn = dropout(p_attn)\n",
        "  return torch.matmul(p_attn, value), p_attn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_cxAhy-RZxHi"
      },
      "source": [
        "> To remember variables:\n",
        "\n",
        "*   Number of sentences in a batch: $ m $\n",
        "*   Number of words per sentence: $ n $\n",
        "*   Embedding dimension: $ d_{model} $\n",
        "\n",
        "> And let's introduce new ones\n",
        "\n",
        "\n",
        "*   Attention blocks: $ h $\n",
        "*   Dimension attention layers: $ d_k = \\frac{d_{model}}{h} $"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R64X60ZyWAQz"
      },
      "source": [
        "Multi-Head Attention is based on performing attention in parallel. There are $ h $ different blocks, with identical structure but different parameterers: $ W^Q $, $ W^K $ and $ W^V $. The output vectors from different heads are concatenated for each word using the parameter $ W^O $.\n",
        "\n",
        "\\begin{equation}\n",
        "  \\text{MultiHead}(Q,K,V) = \\text{Concat}(\\text{head}_1, \\dots, \\text{head}_h) W_O \\\\\n",
        "  \\text{where} \\ \\text{head}_i = \\text{Attention}(QW^Q_i, KW^K_i, VW^V_i) \\\\\n",
        "  W^Q_i \\in \\mathbb R^{d_{model}\\ x \\ d_k}, \\ W^K_i \\in \\mathbb R^{d_{model}\\ x \\ d_k}, \\ W^V_i \\in \\mathbb R^{d_{model}\\ x \\ d_v} \\\\\n",
        "  W^O \\in \\mathbb R^{h d_v\\ x \\ d_{model}}\n",
        "\\end{equation}\n",
        "\n",
        "The idea is to get information from different representation subspaces at different positions. To get an intuition, we could see it as one head is understanding the sentence gramatically, another one from the point of spelling, etc."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IB3nhzM6dyOk"
      },
      "source": [
        "> In the papers [Attention is All You Need](https://arxiv.org/abs/1706.03762) and [The Annotated Transformer](http://nlp.seas.harvard.edu/2018/04/03/attention.html), it is used $ h = 8 $ and they assume that $ d_v = d_k $."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KcZ1ULyDnpXm"
      },
      "source": [
        "![Multi-Head Attention](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/multi__head_attention.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vKpHHa4vZ0Lc"
      },
      "source": [
        "To generate $ N $ identical layers:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wIptwAwCj7wB"
      },
      "source": [
        "import copy\n",
        "\n",
        "def clones(module, N):\n",
        "  return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SCpFzMs6Z6yr"
      },
      "source": [
        "A class to represent Multi-Head Attention:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OGabuvuHebXG"
      },
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "  \n",
        "  def __init__(self, h, d_model, dropout=0.1):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    assert d_model % h == 0\n",
        "    # To assume d_v always equals d_k\n",
        "    self.d_k = d_model // h\n",
        "    self.h = h\n",
        "    self.linears = clones(nn.Linear(d_model, d_model), 4)\n",
        "    self.attn = None\n",
        "    self.dropout = nn.Dropout(p=dropout)\n",
        "\n",
        "\n",
        "  def forward(self, query, key, value, mask=None):\n",
        "    if mask is not None:\n",
        "      # Same mask applied to all h heads.\n",
        "      mask = mask.unsqueeze(1)\n",
        "    nbatches = query.size(0)\n",
        "\n",
        "    # 1) Do all the linear projections in batch from d_model => h x d_k\n",
        "    query, key, value = \\\n",
        "      [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "      for l, x in zip(self.linears, (query, key, value))]\n",
        "\n",
        "    # 2) Apply attention on all the projected vectors in batch.\n",
        "    att_out, self.attn = sdp_attention(query, key, value, mask=mask, \n",
        "                                       dropout=self.dropout)\n",
        "\n",
        "    # 3) \"Concat\" using a view and apply a final linear.\n",
        "    concat_out = att_out.transpose(1, 2).contiguous().view(nbatches, -1, \n",
        "                                                           self.h * self.d_k)\n",
        "\n",
        "    # 4) Last linear\n",
        "    return query, key, value, att_out, concat_out, self.linears[-1](concat_out)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KbId20t_aBu4"
      },
      "source": [
        "We can see how the tensor is divided into the $ h $ heads and then contactenated to recover its original shape."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "se0tXhR_fCks",
        "outputId": "78e05c02-c2ed-438e-fbdc-cfdfefa15283"
      },
      "source": [
        "#@title  { vertical-output: true }\n",
        "#@markdown Attention blocks\n",
        "h =  8#@param {type:\"integer\"}\n",
        "\n",
        "# get previous output as input\n",
        "input = enc_out\n",
        "\n",
        "multi = MultiHeadAttention(h , d_model)\n",
        "q, k, v, att, concat, output = multi(input, input, input)\n",
        "\n",
        "print('Input with dimension: (m x n)')\n",
        "print(tuple(input.shape), '\\n')\n",
        "\n",
        "print('Query, keys and values with dimension: (m x h x n x d_k)')\n",
        "print(tuple(q.shape), '\\n')\n",
        "\n",
        "print('Output with dimension: (m x n x d_model)')\n",
        "print(tuple(output.shape), '\\n')\n",
        "\n",
        "for w, layer in zip(['W_Q', 'W_K', 'W_V', 'W_O'], multi.linears): \n",
        "  print(f'{w} with dimension: {tuple(layer.weight.shape)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input with dimension: (m x n)\n",
            "(5, 10, 512) \n",
            "\n",
            "Query, keys and values with dimension: (m x h x n x d_k)\n",
            "(5, 8, 10, 64) \n",
            "\n",
            "Output with dimension: (m x n x d_model)\n",
            "(5, 10, 512) \n",
            "\n",
            "W_Q with dimension: (512, 512)\n",
            "W_K with dimension: (512, 512)\n",
            "W_V with dimension: (512, 512)\n",
            "W_O with dimension: (512, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pyScS4mWaIGG"
      },
      "source": [
        "### 3.3.2 - Feed-Forward"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "leKQ5Bk1M6fW"
      },
      "source": [
        "Each layer contains a fully connected feed-forward network. It is used different parameters between blocks.\n",
        "\n",
        "\\begin{equation}\n",
        "  \\text{FFN}(x) = \\text{max}(0,\\ xW_1 \\ + \\ b_1)W_2 \\ + \\ b_2\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lAS0XzeuaNSt"
      },
      "source": [
        "> In the original paper the input and output have dimensionality $ d_{model} = 512 $ and the inner-layer $ d_{ff} = 2048 $."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMlushNuv3BV"
      },
      "source": [
        "class PositionwiseFeedForward(nn.Module):\n",
        "\n",
        "  def __init__(self, d_model, d_ff, dropout=0.1):\n",
        "    super(PositionwiseFeedForward, self).__init__()\n",
        "    self.w_1 = nn.Linear(d_model, d_ff)\n",
        "    self.w_2 = nn.Linear(d_ff, d_model)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, x):\n",
        "    return self.w_2(self.dropout(F.relu(self.w_1(x))))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "98jLXRaxnDJv"
      },
      "source": [
        "### 3.3.3 Add & Norm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5rgP8tPIVuS7"
      },
      "source": [
        "This layer corresponds to a [residual connection](https://openaccess.thecvf.com/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html) followed by [layer normalization](https://arxiv.org/abs/1607.06450) in both sub-layers. The goal of a residual connection is easing the training, with layers learning functions with reference to the layer inputs, instead of learning unreferenced ones. The layer normalization is a normalization which is applied per feature."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-PEU842En4eM"
      },
      "source": [
        "![Layer Normalization](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/layer_norm.png)\n",
        "\n",
        "> Layer normalization. Image from [Transformer video](https://www.youtube.com/watch?v=TQQlZhbC5ps&list=PLTl9hO2Oobd_bzXUpzKMKA3liq2kj6LfE) by CodeEmporium."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yDoWEwSxtlCi"
      },
      "source": [
        "Then, the output of each sub-layer is:\n",
        "\n",
        "\\begin{equation}\n",
        "    \\text{LayerNorm}(x \\ + \\ \\text{Sublayer}(x))\n",
        "\\end{equation}\n",
        "\n",
        "where $ \\text{Sublayer}(x) $ is the function implemented by the corresponding sub-layer. \n",
        "\n",
        "In order to reduce overfitting, it is applied dropout to the output of each sub-layer, before it is added to the sub-layer input and normalized."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yWCRO3n8w6GK"
      },
      "source": [
        "class LayerNorm(nn.Module):\n",
        "\n",
        "  def __init__(self, features, eps=1e-6):\n",
        "    super(LayerNorm, self).__init__()\n",
        "    self.a_2 = nn.Parameter(torch.ones(features))\n",
        "    self.b_2 = nn.Parameter(torch.zeros(features))\n",
        "    self.eps = eps\n",
        "\n",
        "  def forward(self, x):\n",
        "    mean = x.mean(-1, keepdim=True)\n",
        "    std = x.std(-1, keepdim=True)\n",
        "    return self.a_2 * (x - mean) / (std + self.eps) + self.b_2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m0riUa70aU8b"
      },
      "source": [
        "To show normalization process in a layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nsnnp7boyMI5",
        "outputId": "af2e26e3-6be8-4e00-b458-f6d42547eebb"
      },
      "source": [
        "#@title { vertical-output: true }\n",
        "#@markdown This variable would correspond with d_model in the Transformer\n",
        "features = 5 #@param {type:\"integer\"}\n",
        "rows = 2 #@param {type:\"integer\"}\n",
        "\n",
        "batch_norm = batch_gen(5, rows, features)\n",
        "input = batch_norm.src.float()\n",
        "print('Input')\n",
        "print(input.detach().numpy(), '\\n')\n",
        "\n",
        "output = LayerNorm(features)(input)\n",
        "print('Output')\n",
        "print(output.detach().numpy())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input\n",
            "[[1. 1. 4. 4. 2.]\n",
            " [1. 4. 1. 1. 2.]] \n",
            "\n",
            "Output\n",
            "[[-0.9231321  -0.9231321   1.055008    1.055008   -0.26375207]\n",
            " [-0.61357147  1.6873218  -0.61357147 -0.61357147  0.15339291]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "il6ellS9aas_"
      },
      "source": [
        "Class to represent Sublayer Connection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "klJw2QYEw8js"
      },
      "source": [
        "class SublayerConnection(nn.Module):\n",
        "  \"\"\"A residual connection followed by a layer norm.\n",
        "  Note for code simplicity the norm is first as opposed to last\n",
        "  \n",
        "  \"\"\"\n",
        "  def __init__(self, size, dropout):\n",
        "    super(SublayerConnection, self).__init__()\n",
        "    self.norm = LayerNorm(size)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "  def forward(self, x, sublayer):\n",
        "    \"\"\"Apply residual connection to any sublayer with the same size\n",
        "\n",
        "    \"\"\"\n",
        "    return x + self.dropout(sublayer(self.norm(x)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aofgg0qifWRd"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oB-x7Ye92U_A"
      },
      "source": [
        "Now that all layers are explained, we can implement the whole encoder. First, it is defined the class for an instance in the stack."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHgrd2822bir"
      },
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "\n",
        "  def __init__(self, size, self_attn, feed_forward, dropout):\n",
        "    super(EncoderLayer, self).__init__()\n",
        "    self.self_attn = self_attn\n",
        "    self.feed_forward = feed_forward\n",
        "    self.sublayer = clones(SublayerConnection(size, dropout), 2)\n",
        "    self.size = size\n",
        "\n",
        "  def forward(self, x, mask):\n",
        "    x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n",
        "    return self.sublayer[1](x, self.feed_forward)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V9_hBtxp3Rhm"
      },
      "source": [
        "And the whole class for the encoder\n",
        "\n",
        "> In the original paper they used a stack of $ N=6 $"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NF0BUBYQ3UhK"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    \n",
        "  def __init__(self, layer, N):\n",
        "    super(Encoder, self).__init__()\n",
        "    self.layers = clones(layer, N)\n",
        "    self.norm = LayerNorm(layer.size)\n",
        "\n",
        "  def forward(self, x, mask):\n",
        "    for layer in self.layers: \n",
        "      x = layer(x, mask)\n",
        "    return self.norm(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s7qGfrVEaoXA"
      },
      "source": [
        "To show encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L3fL2fnkMQKl",
        "outputId": "eb2010cb-dbc7-4be7-ae89-955d718df74d"
      },
      "source": [
        "#@title \n",
        "#@markdown Attention blocks\n",
        "h =  8#@param {type:\"integer\"}\n",
        "#@markdown Embedding dimension\n",
        "d_model = 512 #@param {type:\"integer\"}\n",
        "#@markdown Dimension inner layer\n",
        "d_ff = 2048 #@param {type:\"integer\"}\n",
        "#@markdown Dropout\n",
        "dropout = 0.1 #@param {type:\"number\"}\n",
        "#@markdown Encoder blocks\n",
        "N = 6 #@param {type:\"integer\"}\n",
        "\n",
        "c = copy.deepcopy\n",
        "attn = MultiHeadAttention(h, d_model)\n",
        "ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
        "\n",
        "enc_block = EncoderLayer(d_model, c(attn), c(ff), dropout)\n",
        "encoder = Encoder(enc_block, N)\n",
        "\n",
        "print(f'One encoder block, there are {N} like this\\n')\n",
        "print(encoder.layers[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "One encoder block, there are 6 like this\n",
            "\n",
            "EncoderLayer(\n",
            "  (self_attn): MultiHeadAttention(\n",
            "    (linears): ModuleList(\n",
            "      (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "    )\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (feed_forward): PositionwiseFeedForward(\n",
            "    (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "    (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (sublayer): ModuleList(\n",
            "    (0): SublayerConnection(\n",
            "      (norm): LayerNorm()\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (1): SublayerConnection(\n",
            "      (norm): LayerNorm()\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i6apm_esRlPP"
      },
      "source": [
        "## 3.4 - Decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A5WY0_QqyvBF"
      },
      "source": [
        "It is also composed of a stack of $ N $ decoder blocks. Using the output from the encoder its function is also mapping sets to sets, maintaning the shape: number of vectors $ m $ and length $ n $. However, the dimensionality in the number of words differs one position from the encoder, being the size: $ ( m \\ \\text{x} \\ n-1 \\ \\text{x} \\ d_{model} ) $. In the original implementation, they employ $ N=6 $ decoders."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ivW20glBbTXN"
      },
      "source": [
        "![Transformer's decoder](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/decoder.png)\n",
        "\n",
        "> Decoder architecture. Image from [Attention is All You Need](https://arxiv.org/abs/1706.03762)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2XLa9Wa8U7Wj"
      },
      "source": [
        "The architecture is almost equal to the encoder. Appart from the dimension, there are other two differences:\n",
        "\n",
        "-   The mask for the first multi-attention.\n",
        "-   The encoder's output used as input in the second multi-attention.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W5Ocr6ZOa_fk"
      },
      "source": [
        "### 3.4.1 - Masked Multi-Head Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XutrVAx7Vk_H"
      },
      "source": [
        "The use of a mask is already implemented in the self-attention function and in the generation of a batch. This prevents attending to subsequent positions and ensures that the predictions only depend on known outputs previous to the current one."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MzScbuiZbYh6"
      },
      "source": [
        "![Masked self-attention](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/masked_attn.png)\n",
        "\n",
        "> Self-attention intuition in encoder and decoder. Image from [Transformer video](https://www.youtube.com/watch?v=TQQlZhbC5ps&list=PLTl9hO2Oobd_bzXUpzKMKA3liq2kj6LfE) by CodeEmporium."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hZgY-1yCaphw",
        "outputId": "09228477-129d-4111-eed4-5dc7235ce320"
      },
      "source": [
        "enc_mask = batch.src_mask.detach().numpy()\n",
        "dec_mask = batch.trg_mask[0].detach().numpy()\n",
        "\n",
        "print('No mask used in encoder\\n')\n",
        "print(enc_mask, '\\n\\n')\n",
        "\n",
        "print('Mask used in decoder\\n')\n",
        "print(dec_mask)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No mask used in encoder\n",
            "\n",
            "[[[ True  True  True  True  True  True  True  True  True  True]]\n",
            "\n",
            " [[ True  True  True  True  True  True  True  True  True  True]]\n",
            "\n",
            " [[ True  True  True  True  True  True  True  True  True  True]]\n",
            "\n",
            " [[ True  True  True  True  True  True  True  True  True  True]]\n",
            "\n",
            " [[ True  True  True  True  True  True  True  True  True  True]]] \n",
            "\n",
            "\n",
            "Mask used in decoder\n",
            "\n",
            "[[ True False False False False False False False False]\n",
            " [ True  True False False False False False False False]\n",
            " [ True  True  True False False False False False False]\n",
            " [ True  True  True  True False False False False False]\n",
            " [ True  True  True  True  True False False False False]\n",
            " [ True  True  True  True  True  True False False False]\n",
            " [ True  True  True  True  True  True  True False False]\n",
            " [ True  True  True  True  True  True  True  True False]\n",
            " [ True  True  True  True  True  True  True  True  True]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zvvlsaTyb7BS"
      },
      "source": [
        "### 3.4.2 - Encoder-Decoder Self-Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AFncrjt8eTiR"
      },
      "source": [
        "The second sublayer in the decoder is a multi-head attention, without mask in this case, but using the encoder's output as input in the block.\n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{Query} & \\rightarrow & Q^D  & \\text{from decoder's input embedding} \\\\\n",
        "  \\text{Keys} & \\rightarrow & K^E & \\text{from encoder's output} \\\\\n",
        "  \\text{Values} & \\rightarrow & V^E & \\text{from encoder's output}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o8E3Qhk1h22R"
      },
      "source": [
        "Once these two differences are explained, it can be implemented the decoder's blocks."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K3vUE7fUzZi4"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVuMeBPJh-tN"
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "\n",
        "  def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n",
        "    super(DecoderLayer, self).__init__()\n",
        "    self.size = size\n",
        "    self.self_attn = self_attn\n",
        "    self.src_attn = src_attn\n",
        "    self.feed_forward = feed_forward\n",
        "    self.sublayer = clones(SublayerConnection(size, dropout), 3)\n",
        "\n",
        "  def forward(self, x, memory, src_mask, tgt_mask):\n",
        "    \n",
        "    x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n",
        "    x = self.sublayer[1](x, lambda x: self.src_attn(x, memory, memory, src_mask))\n",
        "    return self.sublayer[2](x, self.feed_forward)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vNn7e0ZiiK1G"
      },
      "source": [
        "And the complete decoder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F8l37wzGiMPw"
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "\n",
        "  def __init__(self, layer, N):\n",
        "    super(Decoder, self).__init__()\n",
        "    self.layers = clones(layer, N)\n",
        "    self.norm = LayerNorm(layer.size)\n",
        "\n",
        "  def forward(self, x, memory, src_mask, tgt_mask):\n",
        "    for layer in self.layers:\n",
        "      x = layer(x, memory, src_mask, tgt_mask)\n",
        "    return self.norm(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jCnMNMjUcBQo"
      },
      "source": [
        "To show decoder architecture."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K8cZH1NvB2Ml",
        "outputId": "df66e0ba-0876-48a6-deeb-16f123eb3f33"
      },
      "source": [
        "#@title \n",
        "#@markdown Attention blocks\n",
        "h =  8#@param {type:\"integer\"}\n",
        "#@markdown Embedding dimension\n",
        "d_model = 512 #@param {type:\"integer\"}\n",
        "#@markdown Dimension inner layer\n",
        "d_ff = 2048 #@param {type:\"integer\"}\n",
        "#@markdown Dropout\n",
        "dropout = 0.1 #@param {type:\"number\"}\n",
        "#@markdown Encoder blocks\n",
        "N = 6 #@param {type:\"integer\"}\n",
        "\n",
        "c = copy.deepcopy\n",
        "attn = MultiHeadAttention(h, d_model)\n",
        "ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
        "\n",
        "dec_block = DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout)\n",
        "decoder = Decoder(dec_block, N)\n",
        "\n",
        "print(f'One decoder block, there are {N} like this\\n')\n",
        "print(decoder.layers[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "One decoder block, there are 6 like this\n",
            "\n",
            "DecoderLayer(\n",
            "  (self_attn): MultiHeadAttention(\n",
            "    (linears): ModuleList(\n",
            "      (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "    )\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (src_attn): MultiHeadAttention(\n",
            "    (linears): ModuleList(\n",
            "      (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "      (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "    )\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (feed_forward): PositionwiseFeedForward(\n",
            "    (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "    (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (sublayer): ModuleList(\n",
            "    (0): SublayerConnection(\n",
            "      (norm): LayerNorm()\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (1): SublayerConnection(\n",
            "      (norm): LayerNorm()\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (2): SublayerConnection(\n",
            "      (norm): LayerNorm()\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zxEz0m8hW9y2"
      },
      "source": [
        "## 3.5 - Generator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tBFIP1RX2KwQ"
      },
      "source": [
        "The final block is a combination of a feed-forward network and a softmax in order to get the probabilities for next word."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z_qOaBC1cK13"
      },
      "source": [
        "![Transformer's generator](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/generator.png)\n",
        "\n",
        "> Final module, generator. Modified image from [Attention is All You Need](https://arxiv.org/abs/1706.03762)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R1XNpbMhBHh6"
      },
      "source": [
        "class Generator(nn.Module):\n",
        "   \n",
        "  def __init__(self, d_model, vocab):\n",
        "    super(Generator, self).__init__()\n",
        "    self.proj = nn.Linear(d_model, vocab)\n",
        "\n",
        "  def forward(self, x):\n",
        "    return F.log_softmax(self.proj(x), dim=-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UyjjFoHV4Z9H"
      },
      "source": [
        "## 3.6 - Final Visualization\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q-Csm3wmI6Vm"
      },
      "source": [
        "Finally, we can implement the whole transformer architecture."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A6pCQJOKDOpR"
      },
      "source": [
        "class EncoderDecoder(nn.Module):\n",
        "\n",
        "  def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):\n",
        "    super(EncoderDecoder, self).__init__()\n",
        "    self.encoder = encoder\n",
        "    self.decoder = decoder\n",
        "    self.src_embed = src_embed\n",
        "    self.tgt_embed = tgt_embed\n",
        "    self.generator = generator\n",
        "\n",
        "  def forward(self, src, tgt, src_mask, tgt_mask):\n",
        "    return self.decode(self.encode(src, src_mask), src_mask, tgt, tgt_mask)\n",
        "\n",
        "  def encode(self, src, src_mask):\n",
        "    return self.encoder(self.src_embed(src), src_mask)\n",
        "\n",
        "  def decode(self, memory, src_mask, tgt, tgt_mask):\n",
        "    return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ypAq7A13cUIm"
      },
      "source": [
        "Function to get a transforme by just setting the parameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbLhOCWTchCh"
      },
      "source": [
        "def make_model(src_vocab, tgt_vocab, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1):\n",
        "    c = copy.deepcopy\n",
        "    attn = MultiHeadAttention(h, d_model)\n",
        "    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
        "    position = PositionalEncoding(d_model, dropout)\n",
        "    model = EncoderDecoder(\n",
        "            Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),\n",
        "            Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),\n",
        "            nn.Sequential(Embeddings(d_model, src_vocab), c(position)),\n",
        "            nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),\n",
        "            Generator(d_model, tgt_vocab))\n",
        "\n",
        "    # This was important from their code.\n",
        "    # Initialize parameters with Glorot / fan_avg.\n",
        "    for p in model.parameters():\n",
        "        if p.dim() > 1:\n",
        "            nn.init.xavier_uniform_(p)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9Zt7OVFcq6e"
      },
      "source": [
        "Set the Transformer parameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKdpBa-aFIR_"
      },
      "source": [
        "#@markdown Size of source vocabulary\n",
        "src_vocab = 11#@param {type:\"integer\"}\n",
        "#@markdown Size of target vocabulary\n",
        "tgt_vocab = 11#@param {type:\"integer\"}\n",
        "#@markdown Number of encoder and decoder blocks\n",
        "N = 6#@param {type:\"integer\"}\n",
        "#@markdown Inner Transformer size\n",
        "d_model = 512#@param {type:\"integer\"}\n",
        "#@markdown Hidden feed forward size\n",
        "d_ff = 2048#@param {type:\"integer\"}\n",
        "#@markdown Number of attention heads\n",
        "h = 8#@param {type:\"integer\"}\n",
        "#@markdown Dropout probability\n",
        "dropout = 0.1#@param {type:\"number\"}\n",
        "\n",
        "model = make_model(src_vocab, tgt_vocab, N, d_model, d_ff, h, dropout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CH-9hBJudSnF"
      },
      "source": [
        "To visualize the whole architecture or per block."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rf9xw--dFRnk",
        "outputId": "236b8e5e-e100-4bc4-fee1-986a3a4cdf58"
      },
      "source": [
        "#@markdown Select a block in the Transformer to visualize it\n",
        "block = \"transformer\" #@param [\"transformer\", \"encoder\", \"decoder\", \"src_embed\", \"tgt_embed\", \"generator\"]\n",
        "\n",
        "if block == 'transformer':\n",
        "    print(model)\n",
        "else:\n",
        "    print(getattr(model, block))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EncoderDecoder(\n",
            "  (encoder): Encoder(\n",
            "    (layers): ModuleList(\n",
            "      (0): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): EncoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (norm): LayerNorm()\n",
            "  )\n",
            "  (decoder): Decoder(\n",
            "    (layers): ModuleList(\n",
            "      (0): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (5): DecoderLayer(\n",
            "        (self_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (src_attn): MultiHeadAttention(\n",
            "          (linears): ModuleList(\n",
            "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
            "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
            "          )\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (sublayer): ModuleList(\n",
            "          (0): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (1): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (2): SublayerConnection(\n",
            "            (norm): LayerNorm()\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (norm): LayerNorm()\n",
            "  )\n",
            "  (src_embed): Sequential(\n",
            "    (0): Embeddings(\n",
            "      (lut): Embedding(11, 512)\n",
            "    )\n",
            "    (1): PositionalEncoding(\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "  )\n",
            "  (tgt_embed): Sequential(\n",
            "    (0): Embeddings(\n",
            "      (lut): Embedding(11, 512)\n",
            "    )\n",
            "    (1): PositionalEncoding(\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "  )\n",
            "  (generator): Generator(\n",
            "    (proj): Linear(in_features=512, out_features=11, bias=True)\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vcmcatefeCb3"
      },
      "source": [
        "# 4 - BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KexD7CXsOLxz"
      },
      "source": [
        "**B**idirectional **E**ncoder **R**epresentations from **T**ransformer, released in 2019 in paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805), obtained new state-of-the-art results on eleven NLP tasks. \n",
        "\n",
        "BERT was designed to pretrain deep bidirecional representations from unlabeled text by using self-attention to get the left and right context. This approach overcomes current pre-training techniques, where they use unidirectional language models to learn general language representations. Two examples which were competitive models before BERT are [ELMo](https://arxiv.org/abs/1802.05365) and [GPT](https://openai.com/blog/language-unsupervised/).\n",
        "\n",
        "This pretrained model can be fine-tuned adding one output layer and used for a wide range of NLP tasks."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4-Qe3tZVePqX"
      },
      "source": [
        "## 4.1 - Model Architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GR6BHy1eOAez"
      },
      "source": [
        "It is a multi-layer bidirectional Transformer encoder, based on [Attention is All You Need](https://arxiv.org/abs/1706.03762) and implemented as [The Annotated Transformer](http://nlp.seas.harvard.edu/2018/04/03/attention.html). The different hyperparameters and the original notation are:\n",
        "\n",
        "\n",
        "\n",
        "*   Number of layers: $ L $, number of encoder blocks which compose the whole stack. Previously defined as $ N $ in the Encoder part.\n",
        "\n",
        "*   Hidden size: $ H $, embedding dimension of the model. Previously deined as $ d_{model} $.\n",
        "\n",
        "*   Self-attention heads: $ A $, number of heads in self-attention blocks. Previously defined as $ h $.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DXQPhg66eVM5"
      },
      "source": [
        "There are defined two model sizes: \n",
        "\n",
        "\\begin{matrix}\n",
        "  \\text{BERT}_{\\text{BASE}} & \\rightarrow & L=12, \\ H=768, \\ A=12 & \\text{Total parameters}:\\ 110\\text{M} \\\\\n",
        "  \\text{BERT}_{\\text{LARGE}} & \\rightarrow & L=24, \\ H=1024, \\ A=16 & \\text{Total parameters}:\\ 340\\text{M}\n",
        "\\end{matrix}\n",
        "\n",
        "In both cases the inner layer for the last feed-forward block is defined as $ 4H $, being in the BASE 3072 and in the LARGE 4096."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6EBnC4w7eYxG"
      },
      "source": [
        "BERT has been already trained, so we can easily instantiate one of the base model classes of the [HuggingFace](https://huggingface.co/transformers/index.html) library from a pretrained model. To load the required model is used the class [`BertModel`](https://huggingface.co/transformers/model_doc/bert.html#transformers.BertModel).\n",
        "\n",
        "> [HuggingFace](https://huggingface.co/transformers/index.html) has the the possibility to differentiate between *cased* and *uncased* models. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bt7Oq3-r3PmA"
      },
      "source": [
        "from transformers import BertModel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186,
          "referenced_widgets": [
            "813fc032906f4a4b93b079b2ede46614",
            "31d16fd6185f410f9bb236b91db2be98",
            "0671db3ca6d14c88a748b8fc79edd4e2",
            "7e8809272cb248a9b2810431c477da43",
            "2d4d89d851c546b88ec94939ee8cb2cb",
            "ef75b618147e43d5b0894fe3f10378bc",
            "4eea6ab46cfe4d2bbeeca10d7ff7bf50",
            "196bc9904eab4ae0b4c9b7f083304fb2",
            "2dd09319e0df488c953ea1cdea62312f",
            "8d3e5ee1b76e4d87aa790c2b9bf3d631",
            "d8f05c058ef44de7b2e3801a044a0de5",
            "c1440c985ddb43388de623a4f18986b0",
            "b68b1416cb5b4bc5a0cf0134271263c0",
            "786137ae3d904acd890329d9bc1edd1c",
            "9cab05c431984bb890f214cb9b99a7f6",
            "91a79c4f86d3467bb6dba3ae22b35687"
          ]
        },
        "id": "yp0Vn4p1R4EH",
        "outputId": "df67fd33-6f91-491a-bef7-d28e3892b686"
      },
      "source": [
        "#@markdown Select one configuration\n",
        "model_name = \"bert-base-uncased\" #@param [\"bert-base-uncased\", \"bert-base-cased\", \"bert-large-uncased\", \"bert-large-cased\"]\n",
        "# We need to create the model and tokenizer\n",
        "model = BertModel.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "813fc032906f4a4b93b079b2ede46614",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2dd09319e0df488c953ea1cdea62312f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M1qpQYvHS8Rj"
      },
      "source": [
        "The warning is telling us we are throwing away some weights and randomly initializing some other. That is because we are removing the head used to pretrain the model on a masked language modeling objective and replacing it with a new head for which we don't have pretrained weights. The library warns us we should fine-tune this model before using it for inference, which is exactly what we are going to do in next sections."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SvyTaI6sh-oB",
        "outputId": "df1236f7-32fc-40c3-cf10-bfd5fbf64385"
      },
      "source": [
        "#@markdown Select whole model or a block to visualize it\n",
        "block = \"bert\" #@param [\"bert\", \"embeddings\", \"encoder\", \"pooler\"]\n",
        "if block == 'bert':\n",
        "    print(model)\n",
        "else:\n",
        "    print(getattr(model, block))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertModel(\n",
            "  (embeddings): BertEmbeddings(\n",
            "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "    (position_embeddings): Embedding(512, 768)\n",
            "    (token_type_embeddings): Embedding(2, 768)\n",
            "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (encoder): BertEncoder(\n",
            "    (layer): ModuleList(\n",
            "      (0): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (1): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (2): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (3): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (4): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (5): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (6): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (7): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (8): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (9): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (10): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (11): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (pooler): BertPooler(\n",
            "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "    (activation): Tanh()\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4F7MxCNyfPeo"
      },
      "source": [
        "### 4.1.1 - Input-Output Representations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xsn3fTeGbFdL"
      },
      "source": [
        "In order to handle a variety of tasks, the input can be a single sentence or a pair of sentences in one token sequence, which is certainly the name given to the input. The input representation of a word is the addition of three different embeddings:\n",
        "\n",
        "*   **Token Embedding**: normal word embedding. It is used WordPiece with a 30.000 token vocabulary.\n",
        "\n",
        "*   **Segment Embedding**: to differentiate the sentence the word belongs to. It is used the special token [SEP] to separate both sentences. \n",
        "\n",
        "*   **Position Embedding**: to indicate the position in the whole sequence.\n",
        "\n",
        "The first token is always a special classification token [CLS], whose final hidden state layer is used in classification tasks.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3hBoDdzBAIvD"
      },
      "source": [
        "![BERT's input](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/in_BERT.png)\n",
        "\n",
        "> Input Embedding. Image from [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WSmOADeOfOiY",
        "outputId": "a5449702-b0e3-4c38-c1c6-455fb852f420"
      },
      "source": [
        "print(model.embeddings)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertEmbeddings(\n",
            "  (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "  (position_embeddings): Embedding(512, 768)\n",
            "  (token_type_embeddings): Embedding(2, 768)\n",
            "  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CGc2QVvs6vDl"
      },
      "source": [
        "As output we get the corresponding sequence embedding, as it is explained in the encoder, and *NSP* for classification. These values are going to be explained better in next section."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9g6fHh4mfHXX"
      },
      "source": [
        "## 4.2 - Pre-training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i2M14_bCgkfM"
      },
      "source": [
        "This step is in charge of the model to understand the language and its context. It is used two unsupervised tasks instead of the traditional left-to-right or right-to-left."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_3AH5O6uALAE"
      },
      "source": [
        "![Pre-training BERT](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/pretrain_BERT.png)\n",
        "\n",
        "> Pre-training process in BERT. Image from [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Isq7F8XppTO"
      },
      "source": [
        "They used as pre-training data the BooksCorpus and English Wikipedia (2,500M words). The authors recommended to use a document-level corpus rather than a shuffled sentence-level corpus in order to extract long contiguous sequences.\n",
        "\n",
        "Training of $ \\text{BERT}_{\\text{BASE}} $ was performed on 4 Cloud TPUs (16 TPU chips total). While training of $ \\text{BERT}_{\\text{LARGE}} $ eployed 16 Cloud TPUs (64 TPU chips total). Each pre-training took 4 days to complete."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6zmfGdJHfFC1"
      },
      "source": [
        "### 4.2.1 - Masked Language Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swuvqUT5jALw"
      },
      "source": [
        "Some tokens are masked at random and then predicted. The final $ T_i $ vectors have the same size and are generated simultaneously. They are fed into an output softmax over the vocabulary (30k) to get a distribution and compare with cross entropy loss in order to train the model.\n",
        "\n",
        "However, as the output have all the worlds, included the ones were not masked, it is only considered the masked ones and ignores the others to calculate the cross entropy loss. This ensures more focus into predict the masked values and increases the context awareness."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SkGOlDsqpShP"
      },
      "source": [
        "![MLM](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/cross_entro_BERT.png)\n",
        "\n",
        "> Prediction of masked word in BERT. Image from [BERT Neural Network - EXPLAINED!](https://www.youtube.com/watch?v=xI0HHN5XKDo&t=348s) by CodeEmporium."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0QIZVHpfbJA"
      },
      "source": [
        "### 4.2.2 - Next Sentence Prediction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kuQTTjvOjdp8"
      },
      "source": [
        "Many important tasks as question and answering are based on understanding relationship between two sentences, which is not captured by language modeling. For that reason, while the model is being trained in MLM, is also trained in next sentence prediction.\n",
        "\n",
        "For this purpose it is used the special token [CLS] in the input and C in the output to indicate if *Sentence B* could follow *Sentence A*."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ul_LRGnUpb-j"
      },
      "source": [
        "![Pre-train BERT](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/pretrain_BERT_methods.png)\n",
        "\n",
        "> MLM and NSP in pre-training BERT. Image from [BERT Neural Network - EXPLAINED!](https://www.youtube.com/watch?v=xI0HHN5XKDo&t=348s) by CodeEmporium."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9ky2Sp-N5Ut"
      },
      "source": [
        "## 4.3 - Fine-Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7DJIu80m4QdU"
      },
      "source": [
        "Thanks to self-attention, which is in charge of applying bidirectional cross attention between the two sentences, is straightforward to model many downstream tasks. In addition to, the two sentences as input in pretraining are equivalent to different trainings in NLP as sentence pairs in paraphrasing, hypothesis-premise pairs, question-passage pairs, etc.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NwtnjUvpwFh"
      },
      "source": [
        "![Fine-tune BERT](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/finetune_examples.png)\n",
        "\n",
        "> Different NLP tasks implemented by fine-tuning BERT. Image from [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dASIeoBig8mC"
      },
      "source": [
        "Output token representations and $ C $ are fed into an output layer. Only new parameters are learned from sratch, while the rest are slightly fine-tuned. \n",
        "\n",
        "That is the reason fine-tune is relatively inexpensive in computation cost compared with pretraining. In the original paper, authors claim that all of the results can be replicated in at most 1 hour on a single Cloud TPU, or a few hours on a GPU, starting from the exact same pre-trained model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dRQ7TcbAp9Nl"
      },
      "source": [
        "![Fine-tune BERT details](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/finetune_BERT_details.png)\n",
        "\n",
        "> Output layer added in fine-tuning. Image from [BERT Neural Network - EXPLAINED!](https://www.youtube.com/watch?v=xI0HHN5XKDo&t=348s) by CodeEmporium."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OzxZgfiLw-1_"
      },
      "source": [
        "The original experiments in the paper correspond with text classification on [GLUE](https://gluebenchmark.com/) and question answering on [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/). It is also evaluated on [SWAG](https://paperswithcode.com/sota/common-sense-reasoning-on-swag), where given a sentence, the task is to choose the most plausible continuation among four choices."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2qEQi_BDffW1"
      },
      "source": [
        "## 4.4 - Self-Attention Visualization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fm8NzovuqEqC"
      },
      "source": [
        "As it has been mentioned, the model contains $ L $ blocks and $ A $ attention heads. Many models which come from BERT acquired the encoder to perform self-attention. Self-attention operations are not straightforward, even further if we have different blocks and many heads into them.\n",
        "\n",
        "To get an intuition for BERT (and other similar models) respect the attention weights between elements, we can use [exBERT](https://huggingface.co/exbert/?model=bert-base-cased&modelKind=bidirectional&sentence=This%20model%20has%20revolutionized%20the%20world!&layer=0&heads=..0,1,2,3,4,5,6,7,8,9,10,11&threshold=0.7&tokenInd=null&tokenSide=null&maskInds=..&hideClsSep=true). Following next steps:\n",
        "\n",
        "1. First we select a model and introduce the input sentence.\n",
        "2. We can choose if hide special tokens and the percentage of attention shown.\n",
        "3. We select an encoder block (as *Layer*).\n",
        "4. Lastly the attention heads we want to visualize."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NyG_JpNiG299"
      },
      "source": [
        "![exBERT](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/exbert.png)\n",
        "\n",
        "> Example of [exBERT](https://huggingface.co/exbert/?model=bert-base-cased&modelKind=bidirectional&sentence=This%20model%20has%20revolutionized%20the%20world!&layer=0&heads=..0,1,2,3,4,5,6,7,8,9,10,11&threshold=0.7&tokenInd=null&tokenSide=null&maskInds=..&hideClsSep=true) visualization."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fhsnqD1UKfE4"
      },
      "source": [
        "# 5 - Sentiment Analysis\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ISgoHueqCeEg"
      },
      "source": [
        "A tradiditonal task in NLP is **Sentiment Analysis**. The target is to determine if the sentence has a *positive* or *negative* sentiment, corresponding with a binary classification problem. As it is normal, these two labels depend on the context of our text. In spite of being one of the most simple applications, it is really useful and currently demanded in many applications, for example in reviews of films and series.\n",
        "\n",
        "In addition to be a standard assignment, it compounds one of the task of [GLUE](https://gluebenchmark.com/) (General Language Understanding Evaluation), a group of nine classification tasks on sentences or pairs of sentences, which is commonly used to evaluate state-of-the-art models. Sentiment analysis corresponds with [SST-2](https://nlp.stanford.edu/sentiment/index.html) (Stanford Sentiment Treebank)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oEm7PX1G3luN"
      },
      "source": [
        "![Sentiment Analysis](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/sentiment_analysis.PNG)\n",
        "\n",
        "> Example of sentiment analysis in one of the [Hugging Face's models](https://huggingface.co/models)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q7sSjqazGNDT"
      },
      "source": [
        "> For this task, two tutorials [Fine tuning BERT for Sentiment Analysis](https://skimai.com/fine-tuning-bert-for-sentiment-analysis/) and [BERT Fine-Tuning Tutorial with PyTorch](http://mccormickml.com/2019/07/22/BERT-fine-tuning/), have been used as guide and support to the development of the application."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C72t07tkgGGf"
      },
      "source": [
        "## 5.1 - Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Sy2wgHzAwRJ"
      },
      "source": [
        "In spite of being much faster than pre-training, fine-tune a BERT model for any class could be slow for a CPU. Due to that, we are going to try using a GPU."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DYpkzpTFBEIY"
      },
      "source": [
        "> In case of running it from Colab, we could make use of Google's GPUs. Make sure to select GPU from *Runtime > Change runtime type > Hardware accelerator > GPU*."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i_x4k_DmO5M8",
        "outputId": "fafc4878-2616-4402-c061-6115616f8aee"
      },
      "source": [
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UWOX4s-agL1m"
      },
      "source": [
        "## 5.2 - Load SST-2 Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mSzeqGUCIPCh"
      },
      "source": [
        "First of all we need to download the dataset. We’ll use the `wget` package to download the dataset to the Colab instance’s file system."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I_EKeJwIMOjb"
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O1QkRBVoI6G0"
      },
      "source": [
        "In this [repository](https://github.com/nyu-mll/GLUE-baselines) is explained how to manage GLUE datasets, altough in this case we only need the corresponding URL. We donwload the dataset and then unzip it to access it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DJ0uAJ2WNE1t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9c1586d6-789f-4afe-b8fa-1a9ae6510f94"
      },
      "source": [
        "import wget\n",
        "\n",
        "sentiment_dir = Path('sentiment_analysis')\n",
        "if not os.path.exists(sentiment_dir):\n",
        "    !mkdir $sentiment_dir\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "url = 'https://dl.fbaipublicfiles.com/glue/data/SST-2.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "if not os.path.exists('./SST-2.zip'):\n",
        "    print('Downloading dataset...')\n",
        "    wget.download(url, './SST-2.zip')\n",
        "\n",
        "# Unzip the dataset (if we haven't already)\n",
        "sst2_dataset = Path('sst-2_dataset')\n",
        "if not os.path.exists(sst2_dataset):\n",
        "    !unzip SST-2.zip\n",
        "    shutil.move('SST-2', sst2_dataset)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading dataset...\n",
            "Archive:  SST-2.zip\n",
            "   creating: SST-2/\n",
            "  inflating: SST-2/dev.tsv           \n",
            "   creating: SST-2/original/\n",
            "  inflating: SST-2/original/README.txt  \n",
            "  inflating: SST-2/original/SOStr.txt  \n",
            "  inflating: SST-2/original/STree.txt  \n",
            "  inflating: SST-2/original/datasetSentences.txt  \n",
            "  inflating: SST-2/original/datasetSplit.txt  \n",
            "  inflating: SST-2/original/dictionary.txt  \n",
            "  inflating: SST-2/original/original_rt_snippets.txt  \n",
            "  inflating: SST-2/original/sentiment_labels.txt  \n",
            "  inflating: SST-2/test.tsv          \n",
            "  inflating: SST-2/train.tsv         \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KyXCF0E3fW8o"
      },
      "source": [
        "There are three different datasets: *train.csv*, *test.csv*, *dev.csv*. As its own name indicates, they are used for training, testing and validate respectively, the studied model. As there is a [GLUE leaderboard](https://gluebenchmark.com/leaderboard), the test dataset is not labeled. In order to study the performance of our model, we are going to divide the train dataset, which has enough samples, into train and test. Test set is going to be one third of the original train test."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMYvqVOQfIg8"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train_df = pd.read_csv(sst2_dataset/'train.tsv', delimiter='\\t', header=1, names=['sentence', 'label'])\n",
        "val_df = pd.read_csv(sst2_dataset/'dev.tsv', delimiter='\\t', header=1, names=['sentence', 'label'])\n",
        "train_df, test_df = train_test_split(train_df, test_size=0.33, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TaVAVVZgJPID"
      },
      "source": [
        "They are saved into a dictionay to allow an easy access."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JO9lvi3_T_oA"
      },
      "source": [
        "# Load the datasets into a pandas dataframe.\n",
        "original_datasets = {\n",
        "    'train':    train_df, \n",
        "    'val':      val_df, \n",
        "    'test':     test_df\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8sUA4BM7LO3C"
      },
      "source": [
        "To get a fast intuition on the dataset, we can plot some used sentences for the three splits. Mention the label 1 corresponds with positive sentiment while label 0 with negative.\n",
        "\n",
        "\\begin{matrix}\n",
        "    \\text{Class 0} & \\rightarrow & \\text{Negative sentiment} \\\\\n",
        "    \\text{Class 1} & \\rightarrow & \\text{Positive sentiment}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "_1s1Y3T5RS-z",
        "outputId": "1331ba7d-0d13-4940-b613-c17d08a14795"
      },
      "source": [
        "#@markdown Select a split\n",
        "split = \"train\" #@param [\"train\", \"val\", \"test\"]\n",
        "\n",
        "#@markdown Select number of examples to visualize it\n",
        "num_examples = 5 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "\n",
        "print(f'Some sentences of {split} split, which has a length of {len(original_datasets[split])}\\n')\n",
        "pd.set_option(\"max_colwidth\", 150)\n",
        "original_datasets[split].sample(num_examples)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some sentences of train split, which has a length of 45123\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6839</th>\n",
              "      <td>( and probably should have )</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43072</th>\n",
              "      <td>clear-cut</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62472</th>\n",
              "      <td>a vivid , sometimes surreal ,</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33700</th>\n",
              "      <td>is never satisfactory .</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58915</th>\n",
              "      <td>is impossible</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                             sentence  label\n",
              "6839    ( and probably should have )       0\n",
              "43072                      clear-cut       1\n",
              "62472  a vivid , sometimes surreal ,       1\n",
              "33700        is never satisfactory .       0\n",
              "58915                  is impossible       0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o03dUBH1KxIp"
      },
      "source": [
        "Depending on the computation velocity, training process could be very slow. For a fast intuition on the model, we can remove some data, since only the training set has more than 45K samples. With `perc` we select the percentage of data we are going to use. Removed sample are selected randomly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3d-eYHQgegp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "44de4ea0-0c71-439c-dae1-ed6423b76bdd"
      },
      "source": [
        "#@title 5.2.1 Resize Dataset { vertical-output: true }\n",
        "#@markdown Percentage of dataset to use %\n",
        "perc = 100 #@param {type:\"slider\", min:1, max:100, step:1}\n",
        "datasets = {}\n",
        "\n",
        "for key, df in original_datasets.items():\n",
        "    new_df = df.sample(n=round(len(df)*(perc/100)), random_state=np.random.RandomState())\n",
        "    datasets[key] = new_df.reset_index(drop=True)\n",
        "    if perc != 100:\n",
        "        print(f'{key:<10}', f'From: {len(original_datasets[key]):<10}', f'To: {len(datasets[key]):<10}')\n",
        "    else:\n",
        "        print(f'{key:<10}', f'size: {len(original_datasets[key]):<10}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train      size: 45123     \n",
            "val        size: 871       \n",
            "test       size: 22225     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxgbVHrorSqy"
      },
      "source": [
        "It is important to know how many samples we have per class in our training dataset, in case we are dealing with an imbalanced classification problem. The number of samples is going to depend on how the original train datatset has been splitted to get the test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "chVHT0EOrcW3",
        "outputId": "30fb0d06-8371-4061-9c05-a8dfb048df8d"
      },
      "source": [
        "train_labels = datasets['train'].label\n",
        "\n",
        "# Print resulted samples\n",
        "print(f\"{'Total samples':^15} | {'Positive':^10} | {'Negative':^10}\")\n",
        "print(42*'-')\n",
        "print(f\"{len(train_labels):^15} | {len(train_labels[train_labels==1]):^10} | {len(train_labels[train_labels==0]):^10}\")\n",
        "print(42*'-')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Total samples  |  Positive  |  Negative \n",
            "------------------------------------------\n",
            "     45123      |   25208    |   19915   \n",
            "------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z9MMh57I1PmH"
      },
      "source": [
        "As it can be seen, the number of samples per class is balanced."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iPwweniNgTvl"
      },
      "source": [
        "## 5.3 -  Preprocessing SST-2 Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "krEPtT0qTpQ9"
      },
      "source": [
        "Before input sequences into BERT, it is necessary to preprocess data samples. The steps to perform depend on the model, but mainly the guide to follow is:\n",
        "\n",
        "*   Tokenize sequences, converting text into integers according to embedding space used.\n",
        "\n",
        "*   Pad each sentence to the maximum length which is in your batch.\n",
        "\n",
        "*   Truncate each sentence to the maximum length the model can accept (if applicable). \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MRbrb0iDqkZX"
      },
      "source": [
        "![Tokenization](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/tokenization_types.png)\n",
        "\n",
        "> Tokenization example."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iePZgKqDVcFD"
      },
      "source": [
        "The whole text must also be divided into sentences. Each one can select the more suitable approach depending on the origin of the text. In this case, the text is already divided into sentences.\n",
        "\n",
        "BERT employs [WordPiece](https://research.google/pubs/pub37842/) subword tokenization algorithm. Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller subwords, but rare words should be decomposed into meaningful subwords.\n",
        "\t\t\n",
        "WordPiece is very similar to [Byte Pair Encoding (BPE)](https://arxiv.org/abs/1508.07909). Every character present in training data is included to learn the corresponding merge rules. In contrast to BPE that chooses the most frequent symbol pair, this algorithm maximizes the likelihood of the training data when it is added to the vocabulary. This approach evaluates if it is worthy to merge two symbols, instead of separated."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JjbGx7_xMfdH"
      },
      "source": [
        "Remember that BERT input follows next scheme:\n",
        "\n",
        "\\begin{matrix}\n",
        "    \\text{[CLS]} & \\text{Sentence A} & \\text{[SEP]} & \\text{Sentence B} & \\text{[SEP]}\n",
        "\\end{matrix}\n",
        "\n",
        "or\n",
        "\n",
        "\\begin{matrix}\n",
        "    \\text{[CLS]} & \\text{Sentence} & \\text{[SEP]}\n",
        "\\end{matrix} \\\\\n",
        "\n",
        "depending on the final task. For classification of one sentence, it corresponds with second option.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dSdChhNAOfLC"
      },
      "source": [
        "In order to preprocess according to the model, it is going to be used the [`BertTokenizer`](https://huggingface.co/transformers/model_doc/bert.html?highlight=berttokenizer#transformers.BertTokenizer) from Hugging Face, which ensures geting a tokenizer that corresponds to the desired model architecture. Besides that, it downloads the vocabulary used when pretraining this specific checkpoint.\n",
        "\n",
        "\n",
        "It is necessary to pass to the tokenizer the name of the pre-trained model we are going to use. It is selected [`bert-base-uncased`](https://huggingface.co/bert-base-uncased), a $ \\text{BERT}_{\\text{BASE}} $ which does not distinguish between upper and lower case."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_QEvK9RzQdln"
      },
      "source": [
        "from transformers import BertTokenizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VdAmTkmzO6D_"
      },
      "source": [
        "MODEL_NAME = 'bert-base-uncased'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198,
          "referenced_widgets": [
            "5aac931e402d4e7f88a0fc39c142e6c5",
            "ea469d87f3cd430ebdbcf85f732033d5",
            "a945116be64341ca9018fcf62d68045f",
            "08cc4387a2a3443b8bca921afdc08e9b",
            "6c460920bbe74319be927b7de6cec4b0",
            "16b74057dabb448c91f9306a30f6d679",
            "d8e6692429974739b7b75c9f7d700a39",
            "30e5ed1261bd4cf7845f53c224241a07",
            "2d632da2e0b948cbb00c4ca0a0a55f60",
            "b03f83f066b8476dbbf27003d74b880c",
            "91b78d64f1b949eeb3e4d25b8a2b8945",
            "5d425d0f23ff41d9b45b6090bc15d986",
            "48922a2dc4f24aa5be81131c1f854f22",
            "7d76f16dcd8747dcaf62802d9513f87c",
            "326adbbe4d574f7cb93448b49c354534",
            "74f2079b924e4ad4ac0501a4515165fe",
            "0c14c23dfd144eb084083e3df9838d80",
            "fd6157893f4c4a2ca121c32a97682a74",
            "0dd4efa8884f4e88918369bb02770d32",
            "fe1519de8ee74584b7beb641e8e1e65c",
            "54ed975ccbee4e168a2400b6e542c682",
            "4623404c9d664846a6183a9355fd5b1f",
            "2d7340bf26294a4a865f63cd842ad0b9",
            "3d6ac1a7a8234d3fbe5e700d5ef30d20"
          ]
        },
        "id": "lp5EPjvGOngp",
        "outputId": "ac115467-f221-4319-9858-5b3f7f67a432"
      },
      "source": [
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...\\n')\n",
        "tokenizer = BertTokenizer.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5aac931e402d4e7f88a0fc39c142e6c5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d632da2e0b948cbb00c4ca0a0a55f60",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c14c23dfd144eb084083e3df9838d80",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Esjs9IJyQ-mU"
      },
      "source": [
        "From tokenizer's instance it can be obtained information about the model:\n",
        "\n",
        "- Dimension of the vocabulary used\n",
        "- Maximum allowed length of the model\n",
        "- In which size is applied padding\n",
        "- Special used tokens for separation token, padding token, classification token, mask token and other unknown tokens.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MbJCgRpQYODR",
        "outputId": "fd17c30c-51bc-48ab-95dd-cf62a746aa28"
      },
      "source": [
        "#@markdown Select an option to visualize it\n",
        "attribute = \"vocab_size\" #@param [\"vocab_size\", \"model_max_length\", \"padding_side\", \"sep_token\", \"pad_token\", \"cls_token\", \"mask_token\", \"unk_token\"]\n",
        "\n",
        "print(getattr(tokenizer, attribute))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30522\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_1hLzeZHRJHb"
      },
      "source": [
        "BERT works with sentences of same dimension. Due to that, it is required to pad (add padding tokens until a specified size) or truncate (separate a sentence if it is longer than the maximum model length). For this reason it is very useful to know which is the maximum length in our sentences, and limit the input size to this value."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MLbnuF07maXq",
        "outputId": "fc9e3161-8c25-4825-af7a-3d8ce3fa997c"
      },
      "source": [
        "# Concatenate sentences in datasets\n",
        "all_sentences = np.concatenate([datasets['train'].sentence.values, \n",
        "                                datasets['val'].sentence.values, \n",
        "                                datasets['test'].sentence.values])\n",
        "\n",
        "# Encode our concatenated data\n",
        "encoded_sentences = [tokenizer.encode(sent, add_special_tokens=True) for sent in all_sentences]\n",
        "\n",
        "# Find the maximum length\n",
        "max_len = max([len(sent) for sent in encoded_sentences])\n",
        "print(f'Max length: {max_len}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max length: 66\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "re7NomrbSU4B"
      },
      "source": [
        "In addition to, it is necessary to add the special tokens: [CLS] at the beginning of a sentence and [SEP] between them, altough in this task it will be only at the end of each one."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqTTwCtTSwNw"
      },
      "source": [
        "In next function, we tokenize each sentence in addition to add special tokens and perform the required padding or truncating operations."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q2uqvgtbrZSV"
      },
      "source": [
        "def preprocessing_for_sa(data, tokenizer, max_len=512):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            sent,                           # Tokenize sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=max_len,             # Max length to truncate/pad\n",
        "            padding='max_length',           # Pad sentence to max length\n",
        "            truncation=True,                # Truncate to max_length\n",
        "            return_attention_mask=True      # Return attention mask\n",
        "            )\n",
        "        \n",
        "        # Get output and attention mask\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0oMD7QcTUH_"
      },
      "source": [
        "From this function we will get the corresponding token IDs as `input_ids`. Also, the second output `attention_masks`, indicates which tokens are useful and which ones are padding tokens. This requirement in the input is necessary for BERT."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ggrf-hV_3Qmr"
      },
      "source": [
        "encoded_datasets = dict.fromkeys(datasets)\n",
        "\n",
        "for split, df in datasets.items(): \n",
        "    encoded_datasets[split] = dict()\n",
        "    encoded_datasets[split]['input_ids'], encoded_datasets[split]['attention_mask'] = preprocessing_for_sa(df.sentence, tokenizer, max_len)\n",
        "    # Add the corresponding labels\n",
        "    encoded_datasets[split]['label'] = torch.tensor(df.label)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CS872sOtUW60"
      },
      "source": [
        "To get an intutiton about the whole process, we can print a random sentence as well as the corresponding token IDs and the result after tokenize it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0uMO0Akt4VSR",
        "outputId": "5ca808ba-edb3-40e0-d4ab-b9277f7d46f5"
      },
      "source": [
        "# Random index\n",
        "rnd_index = random.randint(0, len(datasets[split])-1)\n",
        "# Original sentence\n",
        "sentence = datasets[split].sentence[rnd_index]\n",
        "print('Original sentence:')\n",
        "print(sentence, '\\n')\n",
        "\n",
        "# IDs\n",
        "ids = encoded_datasets[split]['input_ids'][rnd_index]\n",
        "print('Token IDs:')\n",
        "print(ids.detach().numpy(), '\\n')\n",
        "\n",
        "# Token IDs\n",
        "#tokens = tokenizer.convert_ids_to_tokens(ids)\n",
        "print('After tokenization:')\n",
        "print(np.array(tokenizer.convert_ids_to_tokens(ids)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original sentence:\n",
            "i 've never seen or heard anything quite like this film , and i recommend it for its originality alone  \n",
            "\n",
            "Token IDs:\n",
            "[  101  1045  1005  2310  2196  2464  2030  2657  2505  3243  2066  2023\n",
            "  2143  1010  1998  1045 16755  2009  2005  2049  2434  3012  2894   102\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0] \n",
            "\n",
            "After tokenization:\n",
            "['[CLS]' 'i' \"'\" 've' 'never' 'seen' 'or' 'heard' 'anything' 'quite'\n",
            " 'like' 'this' 'film' ',' 'and' 'i' 'recommend' 'it' 'for' 'its'\n",
            " 'original' '##ity' 'alone' '[SEP]' '[PAD]' '[PAD]' '[PAD]' '[PAD]'\n",
            " '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]'\n",
            " '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]'\n",
            " '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]'\n",
            " '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]' '[PAD]'\n",
            " '[PAD]' '[PAD]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C0qzQm6Ag2fG"
      },
      "source": [
        "We will create an iterator for our dataset using the torch [data loader](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html) class. This will help save on memory during training and boost the training speed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kmgor3yXeUsW"
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "def generate_data_loaders(batch_size, test=False):\n",
        "    # Train and val dataloaders\n",
        "    if test is False:\n",
        "        dataloaders = list()\n",
        "        for dataset in (encoded_datasets['train'], \n",
        "                        encoded_datasets['val']):\n",
        "            \n",
        "            data = TensorDataset(dataset['input_ids'], \n",
        "                                dataset['attention_mask'], \n",
        "                                dataset['label'])\n",
        "            sampler = RandomSampler(data)\n",
        "            dataloader = DataLoader(data, sampler=sampler, batch_size=batch_size)\n",
        "            dataloaders.append(dataloader)\n",
        "        return tuple(dataloaders)\n",
        "\n",
        "    # Test dataloader\n",
        "    else:\n",
        "        dataset = encoded_datasets['test']\n",
        "        data = TensorDataset(dataset['input_ids'], \n",
        "                                dataset['attention_mask'], \n",
        "                                dataset['label'])\n",
        "        sampler = SequentialSampler(data)\n",
        "        return DataLoader(data, sampler=sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gPQVNkKcgfQp"
      },
      "source": [
        "## 5.4 - Generation of the Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7jqdXV0CFsG4"
      },
      "source": [
        "To fine-tune on GLUE, it is introduced classification layer weigths $ W \\in \\mathbb R^{K \\ x \\ H} $, where $ K $ is the number of labels. Then it is computed a standard classification loss with $ C $ and $ W $:\n",
        "\n",
        "\\begin{equation}\n",
        "    \\text{log}(\\text{softmax}(CW^T))\n",
        "\\end{equation}\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUkyebe_q0cZ"
      },
      "source": [
        "![Fine tune sentiment analysis](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/sentiment_analysis_fine_tune.png)\n",
        "\n",
        "> Fine-tune BERT on sentiment analysis. Image from [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhpgGfl9heot"
      },
      "source": [
        "In **Sentiment Analysis** task, the final model is just a combination of BERT's encoder and a classifier to get the final probabilities."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5bV6gHgCqrRP"
      },
      "source": [
        "![BERT Classifier](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/bert-classifier.png)\n",
        "\n",
        "> Diagram of BERT Classifier. Image from [The Illustrated BERT, ELMo, and co.](http://jalammar.github.io/illustrated-bert/)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vUAMrvRQYa_a"
      },
      "source": [
        "Hugging Face provides [`BertForSequenceClassification`](https://huggingface.co/transformers/model_doc/bert.html?highlight=berttokenizer#bertforsequenceclassification) to perform this task. However, in order to show the simplicity of fine-tune process, it is going to be created an own model. At the end, this is the combination of pre-trained [`BertModel`](https://huggingface.co/transformers/model_doc/bert.html#transformers.BertModel) and a classifier, composed by a simple feed-forward neural netword with only one hidden layer, plus a softmax to get the probabilities of each class."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujfnQXfrhwcB"
      },
      "source": [
        "# Create the BertClassfier class\n",
        "class BertClassifier(nn.Module):\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "    def __init__(self, in_classifier=50, dropout=0.1, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained(MODEL_NAME)\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        H = in_classifier\n",
        "        D_in = self.bert.config.hidden_size\n",
        "\n",
        "        self.classifier = nn.Sequential(\n",
        "                nn.Linear(D_in, H),\n",
        "                nn.ReLU(),\n",
        "                nn.Dropout(dropout),\n",
        "                nn.Linear(H, 2)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "        \n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "        \n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        # outputs[0] = last_hidden_state\n",
        "        # last_hidden_state dimension: (n_batches x words x inner_dim_BERT)\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_1NLY5sbc097"
      },
      "source": [
        "Finally, we have to decide the classifier's parameters. In our case the number of neurons in the hidden layer of the neural network as `in_classifier` and a dropout probability as `dropout_classifier`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r__KXCK8-9jI",
        "outputId": "1a74bdbb-6a3b-474e-dc31-83efb847f854"
      },
      "source": [
        "#@markdown Size classifier layer\n",
        "in_classifier = 50 #@param {type:\"integer\"}\n",
        "#@markdown Classifier droput probability\n",
        "dropout_classifier = 0.01 #@param {type:\"number\"}\n",
        "model = BertClassifier(in_classifier, dropout_classifier)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_LVmrGBOeB15"
      },
      "source": [
        "With the model created, we can visualize each part: the whole model, only pretrained BERT or the classifier."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cyXhITJRm0Pq",
        "outputId": "1c68fddd-3276-48d7-e815-cdf816bfb7af"
      },
      "source": [
        "#@markdown Select whole model, only pretrained BERT or the classifier\n",
        "block = \"classifier\" #@param [\"model\", \"bert\", \"classifier\"]\n",
        "if block == 'model':\n",
        "    print(model)\n",
        "else:\n",
        "    print(getattr(model, block))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=768, out_features=50, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Dropout(p=0.01, inplace=False)\n",
            "  (3): Linear(in_features=50, out_features=2, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V6iOXiN8-8gc"
      },
      "source": [
        "To fine-tune our Bert Classifier, we need to create an optimizer. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "code",
        "id": "D4RYJWzH0wYH"
      },
      "source": [
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "def init_model(train_dataloader,\n",
        "               in_classifier=50, dropout_classifier=0.1, \n",
        "               learning_rate=5e-5, epsilon=1e-8, \n",
        "               epochs=4):\n",
        "    # Model\n",
        "    bert_classifier = BertClassifier(in_classifier, dropout_classifier)\n",
        "    # Run model in device\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(\n",
        "        bert_classifier.parameters(),\n",
        "        lr=learning_rate,    \n",
        "        eps=epsilon    \n",
        "    )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0, # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beqNViLcz0es"
      },
      "source": [
        "## 5.5 - Fine-Tune on Sentiment Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bf1km6oGwsat"
      },
      "source": [
        "Before going into detail about how to fine-tune the model, a recap about binary classification problems. There are 4 possible combinations in a binary classification problem (predict a class between two possible ones).\n",
        "\n",
        "- **True Positive (TP)**: if both predicted and actual values are *Positive*.\n",
        "- **False Positive (FP)**: if predicted value is *Positive* and actual one is *Negative*.\n",
        "- **False Negative (FN)**: if predicted value is *Negative* and actual one is *Positive*.\n",
        "- **True Negative (NP)**: if both predicted and actual values are *Negative*.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wSiDusMqx_Nx"
      },
      "source": [
        "![Confussion matrix](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/conf_matrix.png)\n",
        "\n",
        "> Confusion matrix. Predicted vs. real values."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GOQkdtq70lGH"
      },
      "source": [
        "One method to measure the performance of the model is the **accuracy**. This is defined as the coefficient between the number of correct predictions and the total number of predictions. Then, with previous classification possibilities defined we have:\n",
        "\n",
        "\\begin{equation}\n",
        "    \\text{Accuracy} = \\frac{\\text{Number of correct predictions}}{\\text{Total number of predictions}} = \\frac{TP \\ + \\ TN}{TP \\ + \\ TN \\ + \\ FP \\ + \\ FN}\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "za8pBoDYzw5B"
      },
      "source": [
        "In addition to the accuracy, we use the [cross-entropy loss](https://machinelearningmastery.com/cross-entropy-for-machine-learning/). Each sample has a known class label with a probability of 1.0, and 0.0 for the other label. The model can estimate the probability of an example belonging to each class label. Cross-entropy can then be used to calculate the difference between the two probability distributions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mo_Ew7bvz74W"
      },
      "source": [
        "loss_fn = nn.CrossEntropyLoss()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zb7K4dyOestG"
      },
      "source": [
        "To train the model, as in other architectures, we are going to iterate through batches. At the end of each epoch the model is evaluated respect the validation dataset. We use a [fixed seed](https://machinelearningmastery.com/reproducible-results-neural-networks-keras/) for the random number generator to ensure the same result in two consecutive executions with same parameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jBwRZLp97bTB"
      },
      "source": [
        "class BertTrainer():\n",
        "\n",
        "    def __init__(self, params):\n",
        "        self.epochs = params['epochs']\n",
        "        self.train_dataloader, self.val_dataloader = generate_data_loaders(params['batch_size'])\n",
        "        self.training_stats = []\n",
        "        self.model, self.optimizer, self.scheduler = init_model(self.train_dataloader,\n",
        "                                                                params['in_classifier'], params['dropout'],\n",
        "                                                                params['learning_rate'], params['epsilon'],\n",
        "                                                                self.epochs)\n",
        "        \n",
        "    def set_seed(self, seed_value=42):\n",
        "        \"\"\"Set seed for reproducibility.\n",
        "        \"\"\"\n",
        "        random.seed(seed_value)\n",
        "        np.random.seed(seed_value)\n",
        "        torch.manual_seed(seed_value)\n",
        "        torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "    def train(self, evaluation=True):\n",
        "        self.set_seed(42)\n",
        "        info_device()\n",
        "        for epoch_i in range(self.epochs):\n",
        "\n",
        "            # =======================================\n",
        "            #               Training\n",
        "            # =======================================\n",
        "            print(f'======== Epoch {epoch_i+1} / {epochs} ========\\n')\n",
        "\n",
        "            # Print the header of the result table\n",
        "            print(f'Training: {len(self.train_dataloader)} batches\\n')\n",
        "            print(f\"{'Batch':^7} | {'Train Loss':^12} | {'Elapsed':^9}\")\n",
        "            print(\"-\"*35)\n",
        "\n",
        "            # Measure the elapsed time of each epoch\n",
        "            training_time = 0\n",
        "            t0_batch = time.time()\n",
        "\n",
        "            # Reset tracking variables at the beginning of each epoch\n",
        "            total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "            # Put the model into the training mode\n",
        "            self.model.train()\n",
        "\n",
        "            # For each batch of training data...\n",
        "            for step, batch in enumerate(self.train_dataloader):\n",
        "\n",
        "                batch_counts +=1\n",
        "                # Load batch to device\n",
        "                b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "                # Zero out any previously calculated gradients\n",
        "                self.model.zero_grad()\n",
        "\n",
        "                # Perform a forward pass. This will return logits.\n",
        "                logits = self.model(b_input_ids, b_attn_mask)\n",
        "\n",
        "                # Compute loss and accumulate the loss values\n",
        "                loss = loss_fn(logits, b_labels)\n",
        "                batch_loss += loss.item()\n",
        "                total_loss += loss.item()\n",
        "\n",
        "                # Perform a backward pass to calculate gradients\n",
        "                loss.backward()\n",
        "\n",
        "                # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "                torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)\n",
        "\n",
        "                # Update parameters and the learning rate\n",
        "                self.optimizer.step()\n",
        "                self.scheduler.step()\n",
        "\n",
        "                # Print the loss values and time elapsed for every 20 batches\n",
        "                if (step % 20 == 0 and step != 0) or (step == len(self.train_dataloader) - 1):\n",
        "                    # Calculate time elapsed for 20 batches\n",
        "                    time_elapsed = time.time() - t0_batch\n",
        "                    training_time += time_elapsed\n",
        "                    time_elapsed = format_time(time_elapsed)\n",
        "\n",
        "                    # Print training results\n",
        "                    print(f\"{step:^7} | {batch_loss / batch_counts:^12.6f} | {time_elapsed:^9}\")\n",
        "\n",
        "                    # Reset batch tracking variables\n",
        "                    batch_loss, batch_counts = 0, 0\n",
        "                    t0_batch = time.time()\n",
        "\n",
        "            # Calculate the average loss over the entire training data\n",
        "            avg_train_loss = total_loss / len(self.train_dataloader)\n",
        "            # Format training_time\n",
        "            training_time = format_time(training_time)\n",
        "\n",
        "            print(\"-\"*35)\n",
        "            print(f\"{'-':^7} | {avg_train_loss:^12.6f} | {training_time:^9}\")\n",
        "            print(\"-\"*35)\n",
        "            print(\"\\n\")\n",
        "\n",
        "            # =======================================\n",
        "            #               Evaluation\n",
        "            # =======================================\n",
        "\n",
        "            if evaluation:\n",
        "                # After the completion of each training epoch, measure the model's performance\n",
        "                # on our validation set.\n",
        "                val_loss, val_accuracy, validation_time = self.evaluate()\n",
        "\n",
        "            print(\"\\n\")\n",
        "\n",
        "            # Record all statistics from this epoch.\n",
        "            self.training_stats.append(\n",
        "                {\n",
        "                    'epoch': epoch_i + 1,\n",
        "                    'Training Loss': avg_train_loss,\n",
        "                    'Validation Loss': val_loss,\n",
        "                    'Accuracy': val_accuracy,\n",
        "                    'Training Time': training_time,\n",
        "                    'Validation Time': validation_time\n",
        "                }\n",
        "            )\n",
        "\n",
        "        # Finished all epochs\n",
        "        print(\"Training completed!\")\n",
        "\n",
        "\n",
        "    def evaluate(self):\n",
        "        \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "        on our validation set.\n",
        "        \"\"\"\n",
        "        # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "        # the test time.\n",
        "        self.model.eval()\n",
        "\n",
        "        # Set time\n",
        "        t0_val = time.time()\n",
        "\n",
        "        # Tracking variables\n",
        "        val_accuracy = []\n",
        "        val_loss = []\n",
        "\n",
        "        # For each batch in our validation set...\n",
        "        for batch in self.val_dataloader:\n",
        "            # Load batch to device\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "            # Compute logits\n",
        "            with torch.no_grad(): \n",
        "                logits = self.model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            val_loss.append(loss.item())\n",
        "\n",
        "            # Get the predictions\n",
        "            preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "            # Calculate the accuracy rate\n",
        "            accuracy = (preds == b_labels).cpu().numpy().mean()\n",
        "            val_accuracy.append(accuracy)\n",
        "\n",
        "        # Compute the average accuracy and loss over the validation set.\n",
        "        val_loss = np.mean(val_loss)\n",
        "        val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "        # Compute elapsed time\n",
        "        time_elapsed = format_time(time.time() - t0_val)\n",
        "\n",
        "        print('Evaluation\\n')\n",
        "        print(f\"{'Val Loss':^10} | {'Accuracy':^10} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*35)\n",
        "        print(f\"{val_loss:^10.6f} | {val_accuracy:^10.6f} | {time_elapsed:^9}\")\n",
        "        print(\"-\"*35)\n",
        "\n",
        "        return val_loss, val_accuracy, time_elapsed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Owa1c6DDge93"
      },
      "source": [
        "Once we have declared the `BertTrainer` class, it is only necessary to specify the parameters to train the model. The authors recommend following hyper-parameters:\n",
        "\n",
        "- Batch size: 16 or 32\n",
        "- Learning rate for AdamW optimizer: 5e-5, 3e-5 or 2e-5\n",
        "- Number of epochs: 2, 3, 4"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yX0SIOS7AtWH"
      },
      "source": [
        "#@title 5.5.1 - Hyperparameters Selection\n",
        "in_classifier = 50 #@param {type:\"integer\"}\n",
        "dropout_classifier = 0.1 #@param {type:\"number\"}\n",
        "batch_size =  32#@param {type:\"integer\"}\n",
        "learning_rate = 5e-5 #@param {type:\"number\"}\n",
        "epochs =  3#@param {type:\"integer\"}\n",
        "epsilon = 1e-8 #@param {type:\"number\"}\n",
        "\n",
        "args = {\n",
        "    'batch_size': batch_size,\n",
        "    'in_classifier': in_classifier,\n",
        "    'dropout': dropout_classifier,\n",
        "    'learning_rate': learning_rate,\n",
        "    'epsilon': epsilon,\n",
        "    'epochs': epochs,\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4vkGIBQ9jIrY"
      },
      "source": [
        "And now we only need to train the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "THVUZrz-jIKy",
        "outputId": "442f2c04-44b5-4585-e539-5f20eaeb6547"
      },
      "source": [
        "trainer = BertTrainer(args)\n",
        "bert_classifier = trainer.model\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Using GPU Tesla P100-PCIE-16GB. \n",
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "\n",
            "Training: 1411 batches\n",
            "\n",
            " Batch  |  Train Loss  |  Elapsed \n",
            "-----------------------------------\n",
            "  20    |   0.594877   |  0:00:05 \n",
            "  40    |   0.403328   |  0:00:04 \n",
            "  60    |   0.372488   |  0:00:04 \n",
            "  80    |   0.300714   |  0:00:04 \n",
            "  100   |   0.271782   |  0:00:04 \n",
            "  120   |   0.288423   |  0:00:04 \n",
            "  140   |   0.294095   |  0:00:04 \n",
            "  160   |   0.276833   |  0:00:04 \n",
            "  180   |   0.245413   |  0:00:04 \n",
            "  200   |   0.297411   |  0:00:04 \n",
            "  220   |   0.268379   |  0:00:04 \n",
            "  240   |   0.228694   |  0:00:04 \n",
            "  260   |   0.250261   |  0:00:04 \n",
            "  280   |   0.266757   |  0:00:04 \n",
            "  300   |   0.260484   |  0:00:04 \n",
            "  320   |   0.235156   |  0:00:04 \n",
            "  340   |   0.268404   |  0:00:04 \n",
            "  360   |   0.257511   |  0:00:04 \n",
            "  380   |   0.262482   |  0:00:04 \n",
            "  400   |   0.260302   |  0:00:04 \n",
            "  420   |   0.260245   |  0:00:04 \n",
            "  440   |   0.306050   |  0:00:04 \n",
            "  460   |   0.266214   |  0:00:04 \n",
            "  480   |   0.232383   |  0:00:04 \n",
            "  500   |   0.254883   |  0:00:04 \n",
            "  520   |   0.213762   |  0:00:04 \n",
            "  540   |   0.194550   |  0:00:04 \n",
            "  560   |   0.261667   |  0:00:04 \n",
            "  580   |   0.212399   |  0:00:04 \n",
            "  600   |   0.219349   |  0:00:04 \n",
            "  620   |   0.223114   |  0:00:04 \n",
            "  640   |   0.242093   |  0:00:04 \n",
            "  660   |   0.208020   |  0:00:04 \n",
            "  680   |   0.216071   |  0:00:04 \n",
            "  700   |   0.184852   |  0:00:04 \n",
            "  720   |   0.242806   |  0:00:04 \n",
            "  740   |   0.213228   |  0:00:04 \n",
            "  760   |   0.210928   |  0:00:04 \n",
            "  780   |   0.193333   |  0:00:04 \n",
            "  800   |   0.188394   |  0:00:04 \n",
            "  820   |   0.195721   |  0:00:04 \n",
            "  840   |   0.202991   |  0:00:04 \n",
            "  860   |   0.183897   |  0:00:04 \n",
            "  880   |   0.187544   |  0:00:04 \n",
            "  900   |   0.198881   |  0:00:04 \n",
            "  920   |   0.171337   |  0:00:04 \n",
            "  940   |   0.196235   |  0:00:04 \n",
            "  960   |   0.191563   |  0:00:04 \n",
            "  980   |   0.216839   |  0:00:04 \n",
            " 1000   |   0.220146   |  0:00:04 \n",
            " 1020   |   0.192785   |  0:00:04 \n",
            " 1040   |   0.194340   |  0:00:04 \n",
            " 1060   |   0.238898   |  0:00:04 \n",
            " 1080   |   0.177734   |  0:00:04 \n",
            " 1100   |   0.156712   |  0:00:04 \n",
            " 1120   |   0.166886   |  0:00:04 \n",
            " 1140   |   0.191542   |  0:00:04 \n",
            " 1160   |   0.202513   |  0:00:04 \n",
            " 1180   |   0.201319   |  0:00:04 \n",
            " 1200   |   0.182120   |  0:00:04 \n",
            " 1220   |   0.174201   |  0:00:04 \n",
            " 1240   |   0.179056   |  0:00:04 \n",
            " 1260   |   0.172330   |  0:00:04 \n",
            " 1280   |   0.176040   |  0:00:04 \n",
            " 1300   |   0.174482   |  0:00:04 \n",
            " 1320   |   0.189948   |  0:00:04 \n",
            " 1340   |   0.136250   |  0:00:04 \n",
            " 1360   |   0.184669   |  0:00:04 \n",
            " 1380   |   0.185888   |  0:00:04 \n",
            " 1400   |   0.191364   |  0:00:04 \n",
            " 1410   |   0.154066   |  0:00:02 \n",
            "-----------------------------------\n",
            "   -    |   0.229471   |  0:05:14 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "Evaluation\n",
            "\n",
            " Val Loss  |  Accuracy  |  Elapsed \n",
            "-----------------------------------\n",
            " 0.198681  |  0.921875  |  0:00:02 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "\n",
            "Training: 1411 batches\n",
            "\n",
            " Batch  |  Train Loss  |  Elapsed \n",
            "-----------------------------------\n",
            "  20    |   0.095619   |  0:00:05 \n",
            "  40    |   0.056857   |  0:00:04 \n",
            "  60    |   0.138405   |  0:00:04 \n",
            "  80    |   0.137155   |  0:00:04 \n",
            "  100   |   0.117811   |  0:00:04 \n",
            "  120   |   0.114282   |  0:00:04 \n",
            "  140   |   0.135754   |  0:00:04 \n",
            "  160   |   0.108871   |  0:00:04 \n",
            "  180   |   0.129877   |  0:00:04 \n",
            "  200   |   0.094528   |  0:00:04 \n",
            "  220   |   0.080868   |  0:00:04 \n",
            "  240   |   0.085807   |  0:00:04 \n",
            "  260   |   0.102981   |  0:00:04 \n",
            "  280   |   0.112261   |  0:00:04 \n",
            "  300   |   0.108653   |  0:00:04 \n",
            "  320   |   0.139558   |  0:00:04 \n",
            "  340   |   0.076917   |  0:00:04 \n",
            "  360   |   0.119643   |  0:00:04 \n",
            "  380   |   0.136395   |  0:00:04 \n",
            "  400   |   0.144276   |  0:00:04 \n",
            "  420   |   0.086453   |  0:00:04 \n",
            "  440   |   0.090190   |  0:00:04 \n",
            "  460   |   0.125222   |  0:00:04 \n",
            "  480   |   0.121619   |  0:00:04 \n",
            "  500   |   0.132149   |  0:00:04 \n",
            "  520   |   0.131476   |  0:00:04 \n",
            "  540   |   0.109485   |  0:00:04 \n",
            "  560   |   0.082315   |  0:00:04 \n",
            "  580   |   0.151981   |  0:00:04 \n",
            "  600   |   0.120436   |  0:00:04 \n",
            "  620   |   0.095242   |  0:00:04 \n",
            "  640   |   0.088653   |  0:00:04 \n",
            "  660   |   0.101809   |  0:00:04 \n",
            "  680   |   0.139515   |  0:00:04 \n",
            "  700   |   0.110699   |  0:00:04 \n",
            "  720   |   0.129305   |  0:00:04 \n",
            "  740   |   0.094139   |  0:00:04 \n",
            "  760   |   0.103645   |  0:00:04 \n",
            "  780   |   0.081371   |  0:00:04 \n",
            "  800   |   0.115043   |  0:00:04 \n",
            "  820   |   0.061173   |  0:00:04 \n",
            "  840   |   0.118043   |  0:00:04 \n",
            "  860   |   0.077380   |  0:00:04 \n",
            "  880   |   0.132251   |  0:00:04 \n",
            "  900   |   0.130737   |  0:00:04 \n",
            "  920   |   0.088729   |  0:00:04 \n",
            "  940   |   0.112240   |  0:00:04 \n",
            "  960   |   0.105244   |  0:00:04 \n",
            "  980   |   0.105561   |  0:00:04 \n",
            " 1000   |   0.077572   |  0:00:04 \n",
            " 1020   |   0.087891   |  0:00:04 \n",
            " 1040   |   0.101912   |  0:00:04 \n",
            " 1060   |   0.126772   |  0:00:04 \n",
            " 1080   |   0.110038   |  0:00:04 \n",
            " 1100   |   0.115434   |  0:00:04 \n",
            " 1120   |   0.081633   |  0:00:04 \n",
            " 1140   |   0.098614   |  0:00:04 \n",
            " 1160   |   0.095972   |  0:00:04 \n",
            " 1180   |   0.076473   |  0:00:04 \n",
            " 1200   |   0.128595   |  0:00:04 \n",
            " 1220   |   0.105103   |  0:00:04 \n",
            " 1240   |   0.103116   |  0:00:04 \n",
            " 1260   |   0.096922   |  0:00:04 \n",
            " 1280   |   0.082862   |  0:00:04 \n",
            " 1300   |   0.082383   |  0:00:04 \n",
            " 1320   |   0.117683   |  0:00:04 \n",
            " 1340   |   0.155244   |  0:00:04 \n",
            " 1360   |   0.115328   |  0:00:04 \n",
            " 1380   |   0.055669   |  0:00:04 \n",
            " 1400   |   0.115340   |  0:00:04 \n",
            " 1410   |   0.069533   |  0:00:02 \n",
            "-----------------------------------\n",
            "   -    |   0.106998   |  0:05:13 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "Evaluation\n",
            "\n",
            " Val Loss  |  Accuracy  |  Elapsed \n",
            "-----------------------------------\n",
            " 0.316442  |  0.916773  |  0:00:02 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "\n",
            "Training: 1411 batches\n",
            "\n",
            " Batch  |  Train Loss  |  Elapsed \n",
            "-----------------------------------\n",
            "  20    |   0.070164   |  0:00:05 \n",
            "  40    |   0.052032   |  0:00:04 \n",
            "  60    |   0.071900   |  0:00:04 \n",
            "  80    |   0.040711   |  0:00:04 \n",
            "  100   |   0.027677   |  0:00:04 \n",
            "  120   |   0.047961   |  0:00:04 \n",
            "  140   |   0.109828   |  0:00:04 \n",
            "  160   |   0.035246   |  0:00:04 \n",
            "  180   |   0.080793   |  0:00:04 \n",
            "  200   |   0.080250   |  0:00:04 \n",
            "  220   |   0.058560   |  0:00:04 \n",
            "  240   |   0.057623   |  0:00:04 \n",
            "  260   |   0.056009   |  0:00:04 \n",
            "  280   |   0.059698   |  0:00:04 \n",
            "  300   |   0.050147   |  0:00:04 \n",
            "  320   |   0.067714   |  0:00:04 \n",
            "  340   |   0.076660   |  0:00:04 \n",
            "  360   |   0.053613   |  0:00:04 \n",
            "  380   |   0.061405   |  0:00:04 \n",
            "  400   |   0.080451   |  0:00:04 \n",
            "  420   |   0.059583   |  0:00:04 \n",
            "  440   |   0.045141   |  0:00:04 \n",
            "  460   |   0.052592   |  0:00:04 \n",
            "  480   |   0.056022   |  0:00:04 \n",
            "  500   |   0.060556   |  0:00:04 \n",
            "  520   |   0.072512   |  0:00:04 \n",
            "  540   |   0.040129   |  0:00:04 \n",
            "  560   |   0.027290   |  0:00:04 \n",
            "  580   |   0.046979   |  0:00:04 \n",
            "  600   |   0.068145   |  0:00:04 \n",
            "  620   |   0.085336   |  0:00:04 \n",
            "  640   |   0.100750   |  0:00:04 \n",
            "  660   |   0.061067   |  0:00:04 \n",
            "  680   |   0.061293   |  0:00:04 \n",
            "  700   |   0.056629   |  0:00:04 \n",
            "  720   |   0.063599   |  0:00:04 \n",
            "  740   |   0.054522   |  0:00:04 \n",
            "  760   |   0.044392   |  0:00:04 \n",
            "  780   |   0.068211   |  0:00:04 \n",
            "  800   |   0.032039   |  0:00:04 \n",
            "  820   |   0.068314   |  0:00:04 \n",
            "  840   |   0.070409   |  0:00:04 \n",
            "  860   |   0.044320   |  0:00:04 \n",
            "  880   |   0.058187   |  0:00:04 \n",
            "  900   |   0.059834   |  0:00:04 \n",
            "  920   |   0.062197   |  0:00:04 \n",
            "  940   |   0.118746   |  0:00:04 \n",
            "  960   |   0.051381   |  0:00:04 \n",
            "  980   |   0.042083   |  0:00:04 \n",
            " 1000   |   0.060976   |  0:00:04 \n",
            " 1020   |   0.042508   |  0:00:04 \n",
            " 1040   |   0.048518   |  0:00:04 \n",
            " 1060   |   0.056130   |  0:00:04 \n",
            " 1080   |   0.054160   |  0:00:04 \n",
            " 1100   |   0.047458   |  0:00:04 \n",
            " 1120   |   0.061985   |  0:00:04 \n",
            " 1140   |   0.055338   |  0:00:04 \n",
            " 1160   |   0.059186   |  0:00:04 \n",
            " 1180   |   0.067080   |  0:00:04 \n",
            " 1200   |   0.080768   |  0:00:04 \n",
            " 1220   |   0.060998   |  0:00:04 \n",
            " 1240   |   0.057772   |  0:00:04 \n",
            " 1260   |   0.040410   |  0:00:04 \n",
            " 1280   |   0.051326   |  0:00:04 \n",
            " 1300   |   0.068485   |  0:00:04 \n",
            " 1320   |   0.044764   |  0:00:04 \n",
            " 1340   |   0.042807   |  0:00:04 \n",
            " 1360   |   0.062017   |  0:00:04 \n",
            " 1380   |   0.060097   |  0:00:04 \n",
            " 1400   |   0.073291   |  0:00:04 \n",
            " 1410   |   0.024193   |  0:00:02 \n",
            "-----------------------------------\n",
            "   -    |   0.059282   |  0:05:12 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "Evaluation\n",
            "\n",
            " Val Loss  |  Accuracy  |  Elapsed \n",
            "-----------------------------------\n",
            " 0.263314  |  0.929688  |  0:00:02 \n",
            "-----------------------------------\n",
            "\n",
            "\n",
            "Training completed!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZBJw6_Kufca"
      },
      "source": [
        "After training the model, we can easily print its statistics in a data frame and save it as an excel table."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "d9cqHoFsscUh",
        "outputId": "1eef61af-579d-413b-8561-e280b9f43550"
      },
      "source": [
        "# Create a DataFrame from our training statistics.\n",
        "df_stats = pd.DataFrame(data=trainer.training_stats)\n",
        "\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "\n",
        "# Save data frame\n",
        "df_stats.to_excel(sentiment_dir/'training_stats.xlsx')\n",
        "\n",
        "df_stats"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.229471</td>\n",
              "      <td>0.198681</td>\n",
              "      <td>0.921875</td>\n",
              "      <td>0:05:14</td>\n",
              "      <td>0:00:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.106998</td>\n",
              "      <td>0.316442</td>\n",
              "      <td>0.916773</td>\n",
              "      <td>0:05:13</td>\n",
              "      <td>0:00:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.059282</td>\n",
              "      <td>0.263314</td>\n",
              "      <td>0.929688</td>\n",
              "      <td>0:05:12</td>\n",
              "      <td>0:00:02</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Validation Loss  Accuracy Training Time Validation Time\n",
              "epoch                                                                        \n",
              "1           0.229471         0.198681  0.921875       0:05:14         0:00:02\n",
              "2           0.106998         0.316442  0.916773       0:05:13         0:00:02\n",
              "3           0.059282         0.263314  0.929688       0:05:12         0:00:02"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0FlBfCt1vzj9"
      },
      "source": [
        "We can print how both (training and validation) losses evolve during epochs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "54tP7ZpO-5Ui",
        "outputId": "fd45cde2-ed74-4e22-8830-db34369a4b1f"
      },
      "source": [
        "plot_loss_curves(df_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3iUZdr38e9MMpM66b0CgSSUEIo0E3oLRVEBYWHBtTfUVx/XspZV93F3H9TFzq6ouy6CKE1AUdEESOhSBOkCmsykkzqpM5O53z9CBsYESCRhknB+jmOPY3Pd7ZwxTH65ct7XrVIURUEIIYQQQgjRIagdXYAQQgghhBCi+STACyGEEEII0YFIgBdCCCGEEKIDkQAvhBBCCCFEByIBXgghhBBCiA5EArwQQgghhBAdiAR4IcQ1z2AwEBcXx1tvvfWbz/HUU08RFxfXilV1Xhd7v+Pi4njqqaeadY633nqLuLg4DAZDq9e3Zs0a4uLi2L17d6ufWwghWoOzowsQQohfa0kQTk1NJSIiog2r6Xiqqqr45z//ycaNGykoKMDPz4+BAwfywAMPEBMT06xzPPzww3zzzTd8/vnn9OzZs8l9FEVh7NixlJeXs23bNlxdXVvzZbSp3bt3s2fPHm677Ta8vLwcXU4jBoOBsWPHMnfuXJ5//nlHlyOEaGckwAsh2p2FCxfafb1v3z4+/fRTZs2axcCBA+22+fn5XfH1wsPDOXToEE5OTr/5HH/5y1948cUXr7iW1vDss8/y5ZdfMnXqVAYPHkxhYSFpaWkcPHiw2QF+xowZfPPNN6xevZpnn322yX127dpFdnY2s2bNapXwfujQIdTqq/OH4T179vD2229z8803Nwrw06ZNY8qUKWg0mqtSixBCtJQEeCFEuzNt2jS7r+vq6vj000/p169fo22/VlFRgaenZ4uup1KpcHFxaXGdF2ovYa+6upqvv/6a5ORkXnvtNdv4ggULMJlMzT5PcnIyoaGhbNiwgSeeeAKtVttonzVr1gD1Yb81XOl/g9bi5OR0Rb/MCSFEW5MeeCFEhzVmzBjmzZvH0aNHufPOOxk4cCA33ngjUB/kFy1axMyZMxkyZAh9+vRh/PjxvPrqq1RXV9udp6me7AvHNm/ezPTp00lISCA5OZn/+7//w2Kx2J2jqR74hjGj0cif//xnhg0bRkJCArNnz+bgwYONXk9JSQlPP/00Q4YMoX///syfP5+jR48yb948xowZ06z3RKVSoVKpmvyFoqkQfjFqtZqbb76Z0tJS0tLSGm2vqKhg06ZNxMbG0rdv3xa93xfTVA+81WrlX//6F2PGjCEhIYGpU6eyfv36Jo8/ffo0L7zwAlOmTKF///4kJiZyyy23sHLlSrv9nnrqKd5++20Axo4dS1xcnN1//4v1wBcXF/Piiy8ycuRI+vTpw8iRI3nxxRcpKSmx26/h+J07d/LBBx8wbtw4+vTpw8SJE1m7dm2z3ouWOH78OA8++CBDhgwhISGByZMns2TJEurq6uz2y83N5emnn2b06NH06dOHYcOGMXv2bLuarFYr//nPf7jhhhvo378/AwYMYOLEifzpT3/CbDa3eu1CiN9GZuCFEB1aTk4Ot912GykpKUyYMIGqqioA8vPzWbVqFRMmTGDq1Kk4OzuzZ88e3n//fY4dO8YHH3zQrPNv3bqV5cuXM3v2bKZPn05qaioffvgh3t7e3Hfffc06x5133omfnx8PPvggpaWl/Pvf/+aee+4hNTXV9tcCk8nE7bffzrFjx7jllltISEjgxIkT3H777Xh7ezf7/XB1deWmm25i9erVfPHFF0ydOrXZx/7aLbfcwuLFi1mzZg0pKSl227788ktqamqYPn060Hrv96/97W9/47///S+DBg3iD3/4A0VFRbz00ktERkY22nfPnj3s3buXUaNGERERYftrxLPPPktxcTH33nsvALNmzaKiooJvv/2Wp59+Gl9fX+DS914YjUZ+97vfkZmZyfTp0+nVqxfHjh3jk08+YdeuXaxcubLRX34WLVpETU0Ns2bNQqvV8sknn/DUU08RFRXVqBXst/rxxx+ZN28ezs7OzJ07l4CAADZv3syrr77K8ePHbX+FsVgs3H777eTn5zNnzhy6dOlCRUUFJ06cYO/evdx8880ALF68mDfffJPRo0cze/ZsnJycMBgMpKWlYTKZ2s1fmoS45ilCCNHOrV69WomNjVVWr15tNz569GglNjZW+eyzzxodU1tbq5hMpkbjixYtUmJjY5WDBw/axvR6vRIbG6u8+eabjcYSExMVvV5vG7darcqUKVOUpKQku/M++eSTSmxsbJNjf/7zn+3GN27cqMTGxiqffPKJbezjjz9WYmNjlXfffddu34bx0aNHN3otTTEajcrdd9+t9OnTR+nVq5fy5ZdfNuu4i5k/f77Ss2dPJT8/32781ltvVXr37q0UFRUpinLl77eiKEpsbKzy5JNP2r4+ffq0EhcXp8yfP1+xWCy28cOHDytxcXFKbGys3X+bysrKRtevq6tTfv/73ysDBgywq+/NN99sdHyDhu+3Xbt22cb+8Y9/KLGxscrHH39st2/Df59FixY1On7atGlKbW2tbTwvL0/p3bu38uijjza65q81vEcvvvjiJfebNWuW0rNnT+XYsWO2MavVqjz88MNKbGyssmPHDkVRFOXYsWNKbGys8t57713yfDfddJMyadKky9YnhHAsaaERQnRoPj4+3HLLLY3GtVqtbbbQYrFQVlZGcXEx119/PUCTLSxNGTt2rN0qNyqViiFDhlBYWEhlZWWzzvGHP/zB7uuhQ4cCkJmZaRvbvHkzTk5OzJ8/327fmTNnotPpmnUdq9XKI488wvHjx/nqq68YMWIEjz/+OBs2bLDb77nnnqN3797N6omfMWMGdXV1fP7557ax06dP88MPPzBmzBjbTcSt9X5fKDU1FUVRuP322+160nv37k1SUlKj/d3d3W3/v7a2lpKSEkpLS0lKSqKiooIzZ860uIYG3377LX5+fsyaNctufNasWfj5+fHdd981OmbOnDl2bUvBwcF07dqVX3755TfXcaGioiIOHDjAmDFjiI+Pt42rVCruv/9+W92A7Xto9+7dFBUVXfScnp6e5Ofns3fv3lapUQjRNqSFRgjRoUVGRl70hsNly5axYsUKTp06hdVqtdtWVlbW7PP/mo+PDwClpaV4eHi0+BwNLRulpaW2MYPBQFBQUKPzabVaIiIiKC8vv+x1UlNT2bZtG6+88goRERG88cYbLFiwgCeeeAKLxWJrkzhx4gQJCQnN6omfMGECXl5erFmzhnvuuQeA1atXA9jaZxq0xvt9Ib1eD0C3bt0abYuJiWHbtm12Y5WVlbz99tt89dVX5ObmNjqmOe/hxRgMBvr06YOzs/2PTWdnZ7p06cLRo0cbHXOx753s7OzfXMevawLo3r17o23dunVDrVbb3sPw8HDuu+8+3nvvPZKTk+nZsydDhw4lJSWFvn372o577LHHePDBB5k7dy5BQUEMHjyYUaNGMXHixBbdQyGEaFsS4IUQHZqbm1uT4//+97/5+9//TnJyMvPnzycoKAiNRkN+fj5PPfUUiqI06/yXWo3kSs/R3OObq+Gmy0GDBgH14f/tt9/m/vvv5+mnn8ZisRAfH8/Bgwd5+eWXm3VOFxcXpk6dyvLly9m/fz+JiYmsX7+ekJAQhg8fbtuvtd7vK/E///M/bNmyhVtvvZVBgwbh4+ODk5MTW7du5T//+U+jXyra2tVaErO5Hn30UWbMmMGWLVvYu3cvq1at4oMPPuCuu+7ij3/8IwD9+/fn22+/Zdu2bezevZvdu3fzxRdfsHjxYpYvX2775VUI4VgS4IUQndK6desIDw9nyZIldkEqPT3dgVVdXHh4ODt37qSystJuFt5sNmMwGJr1sKGG15mdnU1oaChQH+Lfffdd7rvvPp577jnCw8OJjY3lpptuanZtM2bMYPny5axZs4aysjIKCwu577777N7Xtni/G2awz5w5Q1RUlN2206dP231dXl7Oli1bmDZtGi+99JLdth07djQ6t0qlanEtP//8MxaLxW4W3mKx8MsvvzQ5297WGlq7Tp061WjbmTNnsFqtjeqKjIxk3rx5zJs3j9raWu68807ef/997rjjDvz9/QHw8PBg4sSJTJw4Eaj/y8pLL73EqlWruOuuu9r4VQkhmqN9TQ8IIUQrUavVqFQqu5lfi8XCkiVLHFjVxY0ZM4a6ujr++9//2o1/9tlnGI3GZp1j5MiRQP3qJxf2t7u4uPCPf/wDLy8vDAYDEydObNQKcim9e/emZ8+ebNy4kWXLlqFSqRqt/d4W7/eYMWNQqVT8+9//tlsS8ciRI41CecMvDb+e6S8oKGi0jCSc75dvbmvPuHHjKC4ubnSuzz77jOLiYsaNG9es87Qmf39/+vfvz+bNmzl58qRtXFEU3nvvPQDGjx8P1K+i8+tlIF1cXGztSQ3vQ3FxcaPr9O7d224fIYTjyQy8EKJTSklJ4bXXXuPuu+9m/PjxVFRU8MUXX7QouF5NM2fOZMWKFbz++utkZWXZlpH8+uuviY6ObrTufFOSkpKYMWMGq1atYsqUKUybNo2QkBD0ej3r1q0D6sPYO++8Q0xMDJMmTWp2fTNmzOAvf/kLGRkZDB48uNHMblu83zExMcydO5ePP/6Y2267jQkTJlBUVMSyZcuIj4+36zv39PQkKSmJ9evX4+rqSkJCAtnZ2Xz66adERETY3W8AkJiYCMCrr77KDTfcgIuLCz169CA2NrbJWu666y6+/vprXnrpJY4ePUrPnj05duwYq1atomvXrm02M3348GHefffdRuPOzs7cc889PPPMM8ybN4+5c+cyZ84cAgMD2bx5M9u2bWPq1KkMGzYMqG+veu6555gwYQJdu3bFw8ODw4cPs2rVKhITE21BfvLkyfTr14++ffsSFBREYWEhn332GRqNhilTprTJaxRCtFz7/EkmhBBX6M4770RRFFatWsXLL79MYGAgkyZNYvr06UyePNnR5TWi1Wr56KOPWLhwIampqXz11Vf07duX//znPzzzzDPU1NQ06zwvv/wygwcPZsWKFXzwwQeYzWbCw8NJSUnhjjvuQKvVMmvWLP74xz+i0+lITk5u1nlvuOEGFi5cSG1tbaObV6Ht3u9nnnmGgIAAPvvsMxYuXEiXLl14/vnnyczMbHTj6CuvvMJrr71GWloaa9eupUuXLjz66KM4Ozvz9NNP2+07cOBAHn/8cVasWMFzzz2HxWJhwYIFFw3wOp2OTz75hDfffJO0tDTWrFmDv78/s2fP5qGHHmrx03+b6+DBg02u4KPVarnnnntISEhgxYoVvPnmm3zyySdUVVURGRnJ448/zh133GHbPy4ujvHjx7Nnzx42bNiA1WolNDSUe++9126/O+64g61bt7J06VKMRiP+/v4kJiZy77332q10I4RwLJVyNe4sEkII8ZvU1dUxdOhQ+vbt+5sfhiSEEKJzkR54IYRoJ5qaZV+xYgXl5eVNrnsuhBDi2iQtNEII0U48++yzmEwm+vfvj1ar5cCBA3zxxRdER0dz6623Oro8IYQQ7YS00AghRDvx+eefs2zZMn755Reqqqrw9/dn5MiRPPLIIwQEBDi6PCGEEO2EBHghhBBCCCE6EOmBF0IIIYQQogORAC+EEEIIIUQHIjextlBJSSVW69XvOvL396SoqOKqX1cIIa6UfH4JIToyR3yGqdUqfH09LrpdAnwLWa2KQwJ8w7WFEKIjks8vIURH1t4+w6SFRgghhBBCiA5EArwQQgghhBAdiAR4IYQQQgghOhAJ8EIIIYQQQnQgEuCFEEIIIYToQGQVmjZQXV1JRUUZdXXmVjtnQYEaq9XaaucTjuXkpMHT0xs3t4svESWEEEII0RQJ8K3MbDZhNJbg4xOARuOCSqVqlfM6O6uxWCTAdwaKomA211JaehZnZw0ajdbRJQkhhBCiA5EWmlZmNJbi6emNVuvaauFddC4qlQqt1hUPD28qKkodXY4QQgghOhgJ8K3MYjHh4uLm6DJEB+Dq6obZbHJ0GUIIIYToYKSFppVZrXWo1U6OLkN0AGq1E1ZrnaPLEKLN7Mnbz/rTX1NaW4qPiw83xqQwOGSAo8sSQogOTwJ8G5DWGdEc8n0iOrM9eftZfnw1Zmv9zfwltaUsP74aQEK8EEJcIWmhEUII0erWn/7aFt4bmK1mPj+1EasiN+QLIcSVkBl40S4sWHAPAG+//d5VPVYI0fr0xhxKapu+QbvMVM4f018gUhdGlFcEUbr6/wW6+ctfpYQQopkkwItLSk6+rln7rVy5ntDQsDauRgjRXlkVK0eLTpCqz+BkyamL7ufu7M51wYlkGg1sNezAYrUA4ObsSqQugihduC3UB7j5SagXQogmqBRFURxdREdSVFSB1XrxtywvL5OQkOhWv66j1oH/5puNdl9/9tkn5Ofn8tBDj9mNjxgxGje33776jtlc/6d2jUZzVY91tLb6fhHiajHVmdmdt4/N+gzyqwrxcfFmdGQyrk4urPppg10bjUatYU78dFsPfJ21jpzKfLKMerKM2WSVG8ipyMWi1N/c7e7sRmRDoD83W+/v6iuhXghxVQUG6igsNF7Va6rVKvz9PS+6XWbgxSVNnDjZ7ustW1IpKyttNP5rNTU1uLq6Nvs6VxK+O2JwF6KjK6s1kpG9g4zsXVSYK4nShXN7r9/RP6gvTudW4tI6aS+5Co2T2olIXRiRujCSzo1ZrBZyKvPQl2eTaTSgNxpI02dQdy7Uezi714f6C9pv/Fx9JNQLIa4pEuDFFVuw4B4qKip44ok/8dZbizhx4jhz587nzjvvJSNjC+vXr+XkyROUl5cRGBjE5Mk3MG/e7Tg5OdmdA873se/fv5eHH76Pl19eyM8/n+Hzz1dTXl5GQkIif/zjn4iIiGyVYwFWr/6MFSuWUVR0lpiYGBYseJQlSxbbnVMIUS+nIo9UfTp78w5Qp1jpG9CLMVEjiPHu0ihEDw4ZwOCQAS2avXJWO9uCeRJDADBbLeRW5NkCfVa5ge+yttpuhvXQuNuOqQ/24fi6SKgXQnReEuA7gJ1H8liTfoaishr8vVy4ZWQMw3qHOLosO6WlJTzxxKNMmJBCSsoUgoPr69u48Qvc3NyZNWsu7u5u7Nu3l/ff/yeVlZU8+OAjlz3vRx99gFrtxJw58zEay/nkk6W8+OKzLFnyUascu3btKhYtWki/fgOYNet35Obm8vTTj6PT6QgMDPrtb4gQnYiiKBwrPkmaPoNjxSfRqjVcHzaE0ZFJBLkHtvn1NWrn+mDuFWEbM9eZyanMI+tcoM8yZvNt1hZbqPfUeNgF+ihdBD4u3hLqhRCdggT4dm7nkTw++uo4pnP970XltXz01XGAdhXiz54t5KmnnmPq1Gl24y+88L+4uJxvpbnpphm88spfWbt2JXfffT9arfaS57VYLHz44Uc4O9d/q3p5efPGG69y5swpunXrfkXHms1m3n9/Mb17J/D66+/a9uvevQcvv/yCBHhxzTPXmfk+/wBp+gxyK/Px1noxrdskksKH4KFxd2htGicN0V6RRHtFQvj5erMrc8kqN5ybrc9mU+ZmW6jXaTztAn2UV32oF0KIjkYC/FWw/cdcth3K/U3Hns4pw1Jnf9OsyWLl3xuPkf5DTovOldw3lKSE0N9Ux+W4urqSkjKl0fiF4b2qqhKTyUxiYn/WrVtDZuYv9OgRe8nzTplyoy1YAyQm9gMgJyf7sgH+csceP36UsrIyHnjgZrv9xo9P4c03/3HJcwvRmRlNFWRk7yTdsBOjuYIIzzDm95zFwOBEnNXt98eGxklDF68ounhF2cZMdWayK3LqA315NllGA0eLTqBQ/7nqpdWda78531fv7eLlqJcghBDN0n4/iQVAo/B+uXFHCQwMsgvBDc6cOc2SJYvZv/97Kisr7bZVVlZc9rwNrTgNdLr6H6xG4+X7aS93bF5e/S9Vv+6Jd3Z2JjS0bX7REaI9y6vMJ02fwe68/VisFvr492Rs1HB6+MR02NYTrZOGrt7RdPU+v9qTqc6EoSKHrHOBPsto4EjRcVuo99Z6EeUVbtdX76XVOeolCCFEIxLgr4KkhN8+8/3Hd7dTVF7baNzfy4Un57afx5FfONPewGg08tBD9+Du7smdd95HeHgEWq2WkyePs3jxW1itl18WU612anK8OaufXsmxQlwrFEXhRMkpUvXpHC06gUbtzNCQgYyOHE6IR+dsI9M6aenm3YVu3l1sY7V1JgzGHFugzyo3cPjs+VDv4+LdaKZep734Em9CCNGWJMC3c7eMjLHrgQfQOqu5ZWSMA6tqngMH9lFWVsbLL79Cv37nf9nIzW1Z609bCQmp/6XKYNCTmNjfNm6xWMjNzSUm5tItOkJ0ZGarhX35P5CmzyC7Ihed1pOpXScyPHwonloPR5d31bk4aYnx6UKMTxfbWI2lBkNFrt2Nsj+ePWoL9b4uPrZA3/AQKgn1QoirQQJ8O9dwo2p7X4WmKWq1GrCf8Tabzaxdu9JRJdmJj++Ft7c369evZeLEybYWoG+//RqjsdzB1QnRNirMlWzL3k26YTtlJiNhHiH8Pn4m14X0R9OO+9sdwdXZle4+Xenu09U2Vm2pwWDMrn/w1LnZ+oNnj9i2+7r4EH3BGvWRXuF4aq69X4iEEG1LPq07gGG9QxieGOaQJ7FeiYSEvuh0Xrz88gvMmDELlUrFN99spL10sGg0Gu644x4WLXqF//f/HmD06LHk5uby1VcbCA+P6LA9v0I0Jb+qkM36bezK3YvZaqaXXxzzooYT79tDvtdbwM3ZlR6+MfTwPf9X0GpLNfqG9pvy+lD/Q+Fh23Z/V1+7fvpIXbjDV/ERQnRsDg3wJpOJN954g3Xr1lFeXk58fDyPPvoow4YNu+Rx69evZ9WqVZw+fZqysjKCgoIYMmQICxYsIDw8vNH+K1eu5MMPP8RgMBAWFsb8+fOZO3duW70scY63tw8LFy7i7bdfZ8mSxeh0XkyYMInrrhvMY48tcHR5AEyfPgtFUVixYhnvvPMGMTE9+Pvf/8Hrr7+KVuvi6PKEuCKKonCq9Ayp+nQOnz2Ok9qJwcH9GR05nDDP9v9XvI7CzdmNWN8YYi8I9VXmavQXzNJnGbM5UPijbXuAqx+RXhFE6+oDfZQuHHcJ9UKIZlIpDryj77HHHmPTpk3Mnz+f6Oho1q5dy+HDh1m6dCn9+/e/6HELFy6ksLCQ+Ph4vL29ycnJ4bPPPqOuro7169cTGHj+wSIrVqzgz3/+MykpKSQlJbF3717WrVvHk08+yR133NHimouKKrBaL/6W5eVlEhISfdHtv5Wzs7rDzcB3VFarlalTxzNy5GiefPLZNr1WW32/iGtbnbWOfQUHSdNnoDdm46nxYET4MIZHDHPIaioteRJrZ1Zlrrqg9SabrHIDRTXFtu0Bbv62QB99bqbezdnNgRULIcAxn2FqtQp//4vfU+OwAH/o0CFmzpzJ008/zR/+8AcAamtrmTp1KkFBQSxbtqxF5zty5Ai33HILTzzxBHfeeScANTU1jBw5koEDB/Luu+/a9n388cdJS0tj69at6HQt+2EmAb5zqa2txcXFfqZ948YN/PWvL/L8839hwoRJbXp9CfCiNVWZq9iWs5uthh2U1pYR4h7EmKjhDAoegNZJ47C6JMBfXIW5Er0xG315NpnnZuuLa0ps24PcAupn6M/11deH+sarfgkh2k57DPAOa6H5+uuv0Wg0zJw50zbm4uLCjBkzWLRoEQUFBQQFNX8Js7CwMADKy8/ffLh7925KS0uZM2eO3b5z585lw4YNpKenM2VK44cPiWvHoUM/sHjxW4waNQYvL29OnjzOl1+up1u3GEaPHufo8oRolsKqIjYbtrEz93tMdSbifXswJ346Pf1iUavUji5PXIKnxoOefrH09Dv/ULsKU+X5WXqjgTNlmewrOGjbHuweWD9Lr6tf/SZSF4arhHohrikOC/DHjh2ja9eueHjY353ft29fFEXh2LFjlw3wpaWl1NXVkZOTwzvvvANg1z9/9OhRAPr06WN3XO/evVGr1Rw9elQC/DUuLCycgIBAVq36lPLyMry8vElJmcJ99y1Ao3HcjKUQl6MoCmfKMknVp3Oo8AhqlZpBwf0ZEzWccE95EFlH5qn1oJd/HL3842xjRlOFre1GbzRwqvRn9ub/AIAKFUHugedukq1/AFWEZxiuznIfjxCdlcMCfGFhIcHBwY3GG/rXCwoKLnuOiRMnUlpaCoCPjw/PP/88Q4cOtbuGVqvFx8fH7riGseZcQ3Ru4eERLFy4yNFlCNFsddY6fij8kVR9Bpnlejyc3ZkYPZoREdfj7eLl6PJEG9FpPentH0fvC0J9ucl4LtDXt9+cLDnF9/n7gfpQH+wRVL9OvS6CaK/6UK910jrqJQghWpHDAnxNTU2TM5wN/ci1tY2fPvprb7/9NlVVVfz888+sX7+eysrKZl2j4TrNucavXaofCaCgQI2zc9v8ybqtziscR61WExgoj2gXl1dlqib1zHa++mkzZ6uKCdUFcdfA3zGyy1BcnNt/KJPv89YXiI6Y8DC7sZLqMs6UZHGmOJPTJVmcLD7FnrxzoV6lIsIrlG6+UcT4RdPNN4pon4gO8f0jhKO1t88whwV4V1dXzGZzo/GGUP3rGwubMmjQIABGjhzJ2LFjueGGG3B3d+f3v/+97Romk6nJY5u6ebE5LncTq9VqbZObTeUm1s7JarXKzX3ikoqqi9li2M6OnD3U1NXSw6cbM7rfSG//eNQqNeUltUDLJyOuJrmJ9WpSE6XpQlRwF0ad+yN3aW1Z/Sz9ufab/TmH2frLrvq9VWpC3INsN8lG6SII9wx16E3PQrQ3chPrBQIDA5tsYSksLARo0Q2sAJGRkfTu3ZsNGzbYAnxgYCBms5nS0lK7NhqTyURpaWmLryGEEFfLz2WZpOoz+KHgR1QqFQOD+jEmKpkoXYSjSxMdjI+LNz4u3iQE9ALq758oM5XbAn2m0cDhs8fYlbsXqA/1oR7BFzx8Kpxwj1A0EuqFaLczOqEAACAASURBVDccFuDj4+NZunQplZWVdjeyHjx40La9pWpqaqiurrZ93bNnTwAOHz5McnKybfzw4cNYrVbbdiGEaA+sipWDhUdI06dzpiwTN2c3xkWNZFRkEj4u3o4uT3QSKpWqPtQHepMY2BuoD/WltWUXPE02mx/PHmVn7vdAfagP8wixu1E2zDMUjVoe6C6EIzjsX15KSgoffvghK1eutK0DbzKZWLNmDQMGDLDd4JqTk0N1dTUxMeefcFdcXIyfn5/d+Q4fPszx48eZPHmybWzo0KH4+PiwfPlyuwD/ySef4O7uzogRI9rwFQohRPPUWGrYmbuXzfptFNUUE+Dmz8zYaQwNuU5WEhFXhUqlwtfVB19XHxID61duUxSF4ppS2yy93pjNwcLD7MjdA4CTyokwz3Oh/txa9WEeIThLqBeizTnsX1liYiIpKSm8+uqrFBYWEhUVxdq1a8nJyeFvf/ubbb8nn3ySPXv2cOLECdvY6NGjmTRpErGxsbi7u3Pq1ClWr16Nh4cHDzzwgG0/V1dXHn74YV566SUeeeQRkpOT2bt3L+vXr+fxxx/Hy0tWbBBCOE5JTSlbDNvZnrObaksNMd5duKXHVPoG9JL124XDqVQq/N188XfzpV9QAtAQ6ktsgT6r3MCBgkNsz9kNgLPKiTDPUFugj9LVh3ontZMjX4oQnY5Df01euHAhr7/+OuvWraOsrIy4uDjee+89Bg4ceMnj5syZw86dO/nuu++oqakhMDCQlJQUHnjgASIjI+32nTt3LhqNhg8//JDU1FRCQ0N55plnmD9/flu+NCGEuKjMcj1p+gz2FxwCoH9gAmOihtPFK8rBlQlxafWh3g9/Nz8GBPUF6kN9UU2xbZ36LKOBfQUH2dYQ6tXOhHuEngv09e03oR7BEuqFuAIqRVEuvqSKaORyq9Dk5WUSEhLd6tftTKvQbNy4gb/+9UVWrlxPaGj9EmgzZtxA//4DeeaZF1p87JXav38vDz98H2+++U8GDLiuVc7ZXG31/SLaH6ti5cezR0nNyuB02c+4OrmSFDaYUZFJ+Ln6Orq8NiWr0Fx7FEXhbHUxWUb9BcE+m5q6GqA+1Ed4htkCfZRXBCHuQRLqRbskq9CIDumJJx5l//7v2bDhW9zc3Jrc57HHFnDkyI+sX7/pNy3PeTV89903FBcXceutcxxdiriG1NaZ2JW7l836DAqri/Bz9WV6jxsYFjoIN2dXR5cnRJtQqVQEuvsT6O7PwOB+QP0vsWeri+xm6vfk7Sc9eycAGrWGCM/6mfpIXQTRugiC3QMl1AvRBAnw4rLGj5/Ijh0ZbNu2lfHjUxptLykpZt++75kwYdJvDu/Ll69GrW7bnt/U1E389NPJRgG+X78BpKZuv+hDv4T4LUpry9hq2MG27F1UWarp6hXFjTGTSAzoLYFEXJPUKjVB7oEEuQdy3QWhvrDqbH2oN9aH+l25e9lq2AGAVq0hQhd2wZKW9aFe7hER1zoJ8OKyhg8fhZubO999902TAT4t7Tvq6uqYMKHxtubSah33JEC1Wt1u/2ogOh69MYc0fTr78g9iVaz0C+zDmKgRdPOWVikhfk2tUhPsEUSwRxCDQvoD9aG+oOqsLdBnlRvYkfs9WwzbAdA6aYn0DLMF+ihdOEES6sU1RgK8uCxXV1eGDx/J5s3fUV5e3mj1nu+++wZ/f38iI6N59dW/s2/fHvLz83F1dWXAgOt48MFHLtuv3lQP/Jkzp3n99Vc4fPhHvL29mTbtFgICAhsdm5GxhfXr13Ly5AnKy8sIDAxi8uQbmDfvdpyc6mc6Fyy4hx9+qH+ceHJyfZ97SEgoq1ZtuGgPfGrqJj7++D9kZv6Cu7sHSUnDuf/+h+0eCrZgwT1UVFTw/PMv8Y9/LOTYsSPodF7MnDmbuXNva9kbLTosq2LlSNFx0rIyOFl6GhcnLSPChzEqMpkAN7/Ln0AIYaNWqQnxCCLEI4jBIQOA+n9j+VWFttabLGM223N2s9mwDQAXJy0RnuFEe0UQqQsnWhdBoHuAhHrRaUmA7wD25O1nw5mvKa4pxdfFhxtjUmwfalfL+PEpbNr0FVu2pHLjjTfbxvPycjl8+BAzZszm2LEjHD58iHHjJhIYGERubg6ff76ahx66l48/Xomra/P7fYuKzvLww/dhtVr5/e9vw9XVjfXr1zY5U75x4xe4ubkza9Zc3N3d2LdvL++//08qKyt58MFHALjttjuorq4mPz+Xhx56DAA3N/eLXr/hZtnevRO4//6HKSjIZ/XqTzl27AhLlvzXro7y8jL+538eZvTosYwdO4HNm79j8eK36NatO8OGJTX7NYuOx1RnYnfefjbrM8ivKsTHxZubu0/h+tDBuGuavl9ECNFyDU+HDfUIZkho/Up1VsVKXmWBLdBnlRvIyN6J2WoBwNXJhUhduC3QR3pFEOjmL6FedAoS4Nu5PXn7WX58NWarGYCS2lKWH18NcFVD/KBBQ/Dx8eW7776xC/DfffcNiqIwfvxEYmK6M3r0OLvjkpJGcN99t7NlSyopKVOafb1lyz6irKyU999fSlxc/VN5J02ayu9+d3OjfV944X9xcTn/y8FNN83glVf+ytq1K7n77vvRarUMGjSUNWtWUlZWysSJkxud40IWi4XFi9+ie/dY3nrrX7b2nri4eF544Rk2bFjLjBmzbfsXFOTz5z//r629aOrUacyYMZUvv1wnAb6TKqs1kp69g4zsnVSaq4jSRXB77zn0D0yQ/nYhrhK1Sk2YZwhhniEMDa3/62mdtY68qgLbqjdZxvpQn2YL9a5E6sKI8qq/STZSVx/qVSqVI1+KEC0mAf4q2J27z/Y46pb6uSwLi2KxGzNbzSw7toodOXtadK5hoYNsMxct5ezszJgx4/j889WcPXuWgIAAAL77bhMREZH06tXHbn+LxUJlZQUREZF4euo4efJ4iwL8zp3bSUhItIV3AF9fX8aPn8TatSvt9r0wvFdVVWIymUlM7M+6dWvIzPyFHj1iW/Rajx8/SklJsS38NxgzZjzvvPMGO3Zstwvwnp6ejBs30fa1RqOhZ8/e5ORkt+i6ov3LrsglTZ/B3rwD1ClW+gb0YkzUCGK8u0gAEKIdcFI7Ee4ZSrhnKMMYBNSH+tzKfLsbZbcadmA5F+rdnN3Oz9Lr6ttw/F395N+0aNckwLdzvw7vlxtvS+PHp7BmzUrS0jZx661z+OWXnzl16iS33343ALW1NSxd+h82btxAYWEBFz5ioKKiokXXys/PIyEhsdF4VFTjGwHPnDnNkiWL2b//eyorK+22VVa27LpQ3xbU1LXUajUREZHk5+fajQcFBTf6oNfpvDh9+lSLry3aH0VROFp8krSsdI6X/IRWrSEpfAijIpIJcg9wdHlCiMtwUjsRoQsjQhfG9ReE+pzKfLt16jfrM7AodQC4O7sRdS7QNzxR1t/VV0K9aDckwF8FQ0IH/uaZ72e3/5WS2tJG474uPvy/AfddaWktkpCQSGhoON9++zW33jqHb7/9GsDWOrJo0Sts3LiBmTN/R58+CXh6egIqXnjhT7TV88KMRiMPPXQP7u6e3HnnfYSHR6DVajl58jiLF7+F1dr2D79SX6RlQp6R1rGZ68x8n3+AVH0GeZX5eGu9mNZtEknhQ/DQXPz+CSFE++ekdiJSF0akLoyGRkeL1UJOZR768mwyjQb0RgNp+gzqzoV6D2d3u0AfpYvAz9VHQr1wCAnw7dyNMSl2PfBQ/7CLG2N++5KNV2LcuAksXfpvDAY9qambiIvraZupbuhzf+ihR23719bWtnj2HSA4OASDQd9oPCsr0+7rAwf2UVZWxssvv0K/fufvCcjNzWnirM37kA0JCbVd68JzKoqCwaCna9eYZp1HdExGUwXp2TtJN+ygwlxJhGcYt/WazYCgvjir5SNTiM7KWe1sC+ZJDAHAbLWQW5FH5rnlLPVGA99lbcWq1E8OeWjc7daoj9KF4+sioV60Pflp1M413Kjq6FVoGkyYMImlS//N228vwmDQ24X1pmaiV6/+lLq6uhZfZ9iwJFauXMGJE8dtffAlJSV8++1Xdvs1PPzpwtlus9ncqE8ewM3NrVm/TMTH98LX14/PP1/FpElTbQ942rw5lcLCAubOnd/i1yPav9zKfDbrM9idtx+L1UIf/56MjRpBD59u8sNYiGuURu1cH8y9IiC8fsxcZyanMo/Mc4E+02jg26wttlDvqfGwm6WP0oXj4+ItnyOiVUmA7wAGhwzg+ojrsFjavh3kcrp27Ub37rFs25aOWq1m7NjzN29ef30y33yzEQ8PT7p06cqRIz+yd+8evL29W3ydOXNu45tvNvLYYw8yY8ZsXFxcWb9+LcHBoVRU/GTbLyGhLzqdFy+//AIzZsxCpVLxzTcbaap7JS4unk2bvuKtt/5BfHwv3NzcSU4e0Wg/Z2dn7r//If761xd56KF7GTduAgUF+axa9SndusVwww2NV8IRHZOiKJwoOUWqPp2jRSfQqJ0ZGnodYyKSCfYIcnR5Qoh2SOOkIdorkmivSNuYqc5MdkWuLdDrjdlsKt5sC/U6raddoI/yisDHpeU/G4VoIAFetNiECSmcOnWS/v0H2lajAXjkkcdRq9V8++1X1NaaSEhI5PXX3+Gxxx5q8TUCAgJ4881/sWjRQpYu/Y/dg5z+/ve/2Pbz9vZh4cJFvP326yxZshidzosJEyZx3XWDeeyxBXbnnDZtOidPHmfjxi/49NPlhISENhngASZPvgGtVsuyZR/xzjtv4OHhwfjxKdx330Py1NZOwGy1sDf/B9Ky0smpzEOn9WRq14kMDx+Kp9bD0eUJIToYrZOGrt5RdPWOso2Z6kxkV+Re0H6TzdGiEyjUzzB5aXV2gT5KF4G3i9fFLiGEHZUid9q1SFFRBVbrxd+yvLxMQkJa/5Hpzs7qdjEDL1pXW32/iKZVmCvZlr2LrYYdlJuMhHmEMCZqBNcF90Mj/e1tJjBQR2Gh0dFlCOFwtXUmsityzrXf1N8sm19ZYAv13lovWy99Q1+9l1bn4KqFIz7D1GoV/v6eF90uP7GEEJ1eflUhm/Xb2JW7F7PVTC//OMZGjiDOt7v0pQohrhoXJy3dvLvQzbuLbazGUouhIqc+0J/rqz989pgt1Pu4eF9wo2x9sNdpLx7sxLVBArwQolNSFIWfSs+Qpk/nx7PHcFY7Mzh4AKMjkwnzDHF0eUIIAYCrswvdfbrS3aerbazGUoOhIpescr3tAVQ/nj1qC/W+Lj72M/W6CGn/u8ZIgBdCdCoWq4X9BYdIy0pHX5GDp8aDyV3HMyJ8mMxaCSE6BFdn10ahvtpSg8GYff6JsuUGDhYetm33c/W1C/SRXuF4aiTUd1YS4IUQnUKVuYptObvZathBaW0ZIe5BzImfzqDgAWidNI4uTwghroibsys9fGPo4Xv+WSTVlmr0DaG+3ECW0cAPF4R6f1ffC9aor5+xd5cH0XUKEuCFEB1aQdVZthi2sTPne0xWM/G+PZgTP4Oefj1Qq9SOLk8IIdqMm7Mbsb7difXtbhurMjeEeoNtpv5A4Y+27QGufnbr1EfqwnHXuDmifHEFJMALITocRVE4XfYLaVnpHDp7FLVKzaDg/oyJGk64Z6ijyxNCCIdx17gR59edOL/zob7SXHU+1JcbyCw3sL/gkG17oJu/3dNkI3XhuDlLqG/PJMALITqMOmsdBwp/JC0rg0yjHg+NOxO7jGFE+DBZP1kIIS7CQ+NOvF8P4v162MYqzJX1ob7cQJYxm5/Ls9hXcNC2PcgtgCiv+hn6aF0EEbpw3JxdHVG+aIIE+DagKIosTScuSx7B0HzVlmq25+xhi347JbWlBLkHMDvuZoaEDETrpHV0eUII0eF4ajzo6RdLT79Y21iFqfJ8640xm9Olv7A3/wcAVKgIcg+wBfpIXQSRujBcJdQ7hAT4Vubk5IzZbEKrlad1ikszm004Ock/wUs5W13MFsM2duTsobbORKxPDLPibqK3f7z0twshRCvz1HrQyz+OXv5xtjGjqeJc6019C86p0p/tQn2weyCRugiiz83WR3iG4eosGaitSXpoZZ6ePpSWFuLjE4hGo5WZeNGIoiiYzSZKSwvR6XwdXU679HNZJqlZ6fxQeBiVSsXAoH6MiUomShfh6NKEEOKaotN60ts/nt7+8baxcpPRtupNljGbkyWn+D5/P3Au1HsEnZulDyfaK4IIzzD5a2krUynyd/wWKSqqwGq99FtWXV1JRUUpdXWWVruuWq3GarW22vmEYzk5OePp6YObm6zR26DOWsfBs0dIy8rg5/JM3JzdGB4+lJER1+Pj4u3o8sQVcMRjyIUQV1dZbbkt0DeE+3JT/b97FSpCPYKJ1IXbVsCJ8AztMKHeEZ9harUKf/+LP7tEAnwLNSfAtwX5ASg6qxpLDTtyv2eLfhtFNSUEuPkzOjKZoSHXyZ9hOwn5/BLi2lRaW4bemE1muQG90UCm0YDRVAGAWqUmxD3IbknLcM/QdvncjvYY4KWFRgjhEMU1JWwxbGd79h5q6mqI8e7K9B43kBDQS/rbhRCiE/Bx8cbHxZuEgF5AfQtpaW3Z+afJGg0cPnuMXbl7gfpQH+oRbAv00V4RhHmEoGmHod7RJMALIa6qzHI9afoM2xrEA4L6MiZyONFekQ6uTAghRFtSqVT4uvrg6+pDYmBv4HyozzQa0JfXz9L/ePYoO3O/B+pDfbhHCJEXrFMf5hmKRn1tR9hr+9ULIa4Kq2Llx7NHSc3K4HTZz7g6uTI6MplREUn4ucqNvEIIca26MNT3C+wD1If64ppSW9uN3pjNwcLD7MjdA4CTyokwz5BzM/X1ffVhHiE4X0Oh/tp5pUKIq662zsTO3O/ZrN/G2eoi/F19mdHjRoaFXidrBwshhGiSSqXC380Xfzdf+gUlAPWhvqimxPY0Wb0xm/0Fh9iesxsAZ5UTYZ6htkAfpasP9U5qJ0e+lDYjAV4I0epKa8vYatjBtuxdVFmq6eoVzbSYSSQG9O60H6ZCCCHajkqlIsDNjwA3PwYE9QUaQn3xuZtks8k0GthXcJBtDaFe7Uy4R6it9SZKF0GoR3Cn+Dkkq9C0kKxCI8TF6Y3ZpGZlsK/gBxRFoV9QAmMih9PNO9rRpQkHks8vIcTVoigKhdVF59tvyrPJMmZTU1cDgEbtTLhnmF37TYh7UJOhfk/eftaf/prS2lJ8XHy4MSaFwSEDrsrrkGUkW5kEeCHsWRUrR4qOk5qVzk+lZ3Bx0nJ92GBGRSQT4Obn6PJEOyCfX0IIR7IqVs5WF51bn75+BRy9MZuauloANGoNEZ5hRHmF21bA0Ruz+eTEGsxWs+08GrWGOfHTr0qIlwDfyiTAC1HPVGdid94+0vQZFFSdxdfFh1GRSSSFDcbN2c3R5Yl2RD6/hBDtjVWxUlh11naTbGa5AUNFNrV1pkse5+viw/8m/anN65N14IUQraqs1kh69g4ysndSaa4iWhfJHb3n0C8woVP0FQohhOj81Co1wR5BBHsE2WbUrYqVgqpCsozZfHR0RZPHldSWXs0yL0oCvBCiWbIrcknLymBv/gHqFCt9A3szJnI4Md5dUKlUji5PCCGEuCJqlZoQj2BCPIJZf/rrJsO6r4uPAyprTAK8EOKiFEXhaPFJ0rLSOV7yE1q1hqTwIYyKSCbIPcDR5QkhhBBt4saYFJYfX92oB/7GmBQHVnWeBHghRCPmOjN78veTpt9GXmU+3lovpsVMIjlsCO4ad0eXJ4QQQrSphrYaR61CczkS4IUQNkZTBemGHaRn76TCXEmkZxi39ZrNgKC+19QT7oQQQojBIQMYHDKgXd6ILz+RhRDkVuaTlpXBnvz9WKwWEgJ6MiZyBD18ukl/uxBCCNHOSIAX4hqlKAonSk6RmpXO0eITaNQahoUOYnREEsEeQY4uTwghhBAXIQFeiGuM2Wphb/4PpGWlk1OZh5dWxw3dJpIcNhRPrYejyxNCCCHEZUiAF+IaUWGqJCN7F1uzt2M0VRDmEcK8nrcyMLgfGulvF0IIIToM+aktRCeXX1lAmmEbu3P3Ybaa6eUfx9jIEcT5dpf+diGEEKIDcmiAN5lMvPHGG6xbt47y8nLi4+N59NFHGTZs2CWP27RpExs3buTQoUMUFRURGhrK6NGjeeCBB9DpdHb7xsXFNXmOF154gd/97net9lqEaE8UReGn0tOkZmVwuOgYzmpnhoQMYHTkcEI9gh1dnhBCCCGugEMD/FNPPcWmTZuYP38+0dHRrF27lrvvvpulS5fSv3//ix733HPPERQUxLRp0wgLC+PEiRMsXbqUjIwMVq9ejYuLi93+ycnJ3HjjjXZjiYmJbfKahHAki9XC/oJDpGWlo6/IwVPjweSu4xkRPgyd1tPR5QkhhBCiFTgswB86dIgvv/ySp59+mj/84Q8A3HTTTUydOpVXX32VZcuWXfTYN998kyFDhtiN9enThyeffJIvv/ySW265xW5bt27dmDZtWqu/BiHaiypzFduyd7PFsJ0yUzkhHsHMjZ/BoOD+aJw0ji5PCCGEEK3IYQH+66+/RqPRMHPmTNuYi4sLM2bMYNGiRRQUFBAU1PRSdr8O7wDjxo0D4PTp000eU1NTg0qlajQ7L0RHVlB1li2GbezM+R6T1Uy8bw9+33MmPf1ipb9dCCGE6KQcFuCPHTtG165d8fCwX7aub9++KIrCsWPHLhrgm3L27FkAfH19G21btWoVS5cuRVEUYmNjefjhhxk/fvyVvQAhHERRFE6X/UJaVjqHzh7FSaXmupD+jIkcTrhnqKPLE0IIIUQbc1iALywsJDi48c10gYGBABQUFLTofEuWLMHJyYkJEybYjffv35/JkycTERFBbm4u//3vf1mwYAGvvfYaU6dO/e0vQIirrM5ax4HCH0nNSifLaMBD487ELmMYEX493i66y59ACCGEEJ2CwwJ8TU0NGk3j3tyGFpfa2tpmn2vDhg2sWrWKe++9l6ioKLttK1assPv65ptvZurUqbzyyitMmTKlxW0G/v6OuxEwMFBC2rWo0lRF6pntfPXTZoqqSgjTBXP3wDmM6DIEF2eto8sTolnk80sI0ZG1t88whwV4V1dXzGZzo/GG4N7cXvW9e/fyzDPPMGrUKB555JHL7u/u7s7s2bN57bXXOHPmDDExMS2qu6ioAqtVadExrSEwUEdhofGqX1c4ztnqYrbot7Ejdw+1dSZifWK4tftN9PKPQ61SU15SCzT/F10hHEU+v4QQHZkjPsPUatUlJ40dFuADAwObbJMpLCwEaFb/+/Hjx7n//vuJi4tj0aJFODk5NevaoaH1fcJlZWUtqFiIq+NMWSZpWen8UHgYlUrFdcH9GBM5nEhduKNLE0IIIUQ74LAAHx8fz9KlS6msrLS7kfXgwYO27ZeSlZXFXXfdhZ+fH//6179wd3dv9rX1ej0Afn5+v6FyIVpfnbWOg2ePkJaVzs/lWbg7uzE+ehQjI67Hx8Xb0eUJIYQQoh1xWIBPSUnhww8/ZOXKlbZ14E0mE2vWrGHAgAG2G1xzcnKorq62a3UpLCzkjjvuQKVS8cEHH1w0iBcXFzfaVlJSwvLly4mIiKBLly5t8tqEaK5qSw07c79ni34bRTUlBLj5c2vsTQwNvQ4XJ+lvF0IIIURjDgvwiYmJpKSk8Oqrr1JYWEhUVBRr164lJyeHv/3tb7b9nnzySfbs2cOJEydsY3fddRd6vZ677rqLffv2sW/fPtu2qKgo21Ncly1bRmpqKqNGjSIsLIz8/Hw+/fRTiouLeeedd67eixXiV4prStii3872nD3U1NUQ492V6T1uJCGgJ2qV2tHlCSGEEKIdc1iAB1i4cCGvv/4669ato6ysjLi4ON577z0GDhx4yeOOHz8OwPvvv99o280332wL8P3792f//v2sXLmSsrIy3N3d6devH/fee+9lryFEW8gs15Oalc6Bwh8BGBDUlzGRw4n2inRwZUIIIYToKFSKolz9JVU6MFmFRrSUVbFy6OxR0rLSOV32C27OriSFDWFURBK+rj6OLk+INiefX0KIjkxWoRHiGlJjqWVX3l4267dxtroIf1dfZvS4kWGh1+Hq7Oro8oQQQgjRQUmAF6KVldSUstWwg205u6m2VNPNO5qbYiaTGNhb+tuFEEIIccUkwAvRSrKMBtKytrGv4AcURaFfUAJjI4fT1Tva0aUJIYQQohORAC/EFbAqVo4UHSc1K52fSs/g6uTCqIgkRkUk4e8mzxkQQgghROuTAC/Eb2CqM7Erdx+bDRkUVJ3F18WHm7tPISlsMG7Obo4uTwghhBCdmAR4IVqgrLacdMMOMrJ3UWmpIloXyR2959AvMAEntZOjyxNCCCHENUACvBDNkF2RS2pWOnvzf8CqWOkb2JuxkSPo5h2NSqVydHlCCCGEuIZIgBfiIqyKlWPFJ0nLyuB4yU9onbQkhw9ldEQyge7+ji5PCCGEENcoCfBC/Iqpzsz3+ftJy8ogr6oAHxdvboqZTFLYYNw17o4uTwghhBDXOAnwQpxjNFWQbthBevZOKsyVRHqGcVuv2QwMSpT+diGEEEK0GxLgxTUvpyKPzfoM9uQfwGK1kBDQk7GRI+ju003624UQQgjR7kiAF9ckRVE4XvITaVkZHC0+gUatYVjoIEZHJhPsHujo8oQQQgghLkoCfDu380gea7aepri8Fj8vF24ZGcOw3iGOLqvDMlst7M07QJo+g5zKPLy0Om7olkJy+BA8NR6OLk8IIYQQ4rIkwLdjO4/k8dFXxzFZrAAUldfy0VfHASTEt1CFqZKM7F1szd6O0VRBuGco83reysDgfmjU8s9ACCGEEB2HJJd2bM3W07bw3sBksbJm62kJ8M2UV1nAZn0Gu/P2YbZa6O0fz5jI4cT5dpf+diGEEEJ0SBLg27Gi8toWjYt6xVdQXwAAIABJREFUiqLwU+lpUrMyOFx0DGe1M0NCBjAmcjghHsGOLk8IIYQQ4opIgG/H/L1cmgzrahVs2pPFiH5huGrlP2EDi9XCvvyDpOkzMFTkoNN4MqXreIaHD0On9XR0eUIIIYQQrUKlKIri6CI6kqKiCqzWq/OW/boHHsDZSUWAtyt5xdV4uDozekA44wZG4uWhvSo1tUeV5iq2Ze9iq2E7ZSYjIR7BjI0czqDg/micNI4uT4hrXmCgjsJCo6PLEEKI38QRn2FqtQp//4tPPsr0bTvW0Ofe1Co0p3PK+HpXFl/uyOSbPXqSE0KZODiSIN9r50mhBVWFbNZvZ1fu95isZnr6xfL7yFvp6Rcr/e1CCCGE6LRkBr6FruYM/IUu9ttfblEl3+zJYsfhPOqsCtfFBTF5aDTRIbqrXuPVoCgKp8t+ITUrnR/PHsVJpWZQyABGRyYT7hnq6PKEEE2QGXghREfWHmfgJcC3UHsL8A1KK2r59ns9W37Iprq2jl5dfJk0NJpe0b6dYja6zlrHgYJDpOozyDIa8NC4MyJ8GMPDr8fbpXP+siJEZyEBXgjRkUmA7wTaa4BvUFVjYesP2Wzaq6eswkR0sI5JQ6MYGBeIk1p9FSptXVXmarbn7GaLYTultWUEuwcyJnI4g0MGopX+diE6BAnwQoiOTAJ8J9DeA3wDs8XKziN5fLU7i/ziKgJ9XEkZHEVSQihajVMbVto6zlYXs0W/jR25e6itMxHr252xkcPp5R+HWtXxfhER4lomAV4I0ZFJgO8EOkqAb2BVFA6cPMtXuzM5k1OOzl3DuIER/5+9Ow9v6rzTh39rlyV5t+RVEsaAzWZsNtvse8yStaFpFmiSJj+apDNZ2t+bZnJNZ6Zb8jZpliszeZsSMikMCQkUwpBiIGDCErCN2QzY7GBJ3iTvuyVbev+wLVBsCAKbI8n357r4w4/O0XlO2jy5/eU534O5ExOgCfK9CvblhqvYYzqAk7bTEIvEmBydhrn6mdAHxwk9NSK6TQzwROTPGOADgL8F+F4ulwvnzfXYnmfCqcs1UMgkmJ0Wh0VT9IgIUQ7gTL3X5ezCyeozyDXtx5VGE1TSIMyIz8TshGkIU4QKOjciunMM8ETkz3wxwLON5BAhEomQbAhHsiEcZmszduSXYnehBXuOWpA5JhrZGQbEa+/uy47aOttxuLwAey3foba9DtqgSDwy6gFkxE6GQjJ0+9oTERER3Qwr8F7y1wp8f6ob2rCrwIz9ReWwO5yYkBSJxZlGjNKHDeh1vq+2vQ7fmr/Dd+UFaO9qx4iwRMzTz8L4qNHc304UgFiBJyJ/5osVeAZ4LwVSgO/V3OZA7lELdh+1oLnNgRHxoVicYcCEkVEQD2ALyquNJuSaDuC47RQAYKIuFfP0M2EM0Q/YNYjI9zDAE5E/Y4APAIEY4Ht1OLpwsKgCOwtMqG5oR2ykCtkZBmSNjYFUcnuVcafLiaLqYuSa9uNSw1UESZWYHpeBOQnTEa4c3Eo/EfkGBngi8mcM8AEgkAN8ry6nE0fOWpGTZ4LZ2ozwYAUWTtZjdlocghS39thEe2cH8ioKsdd8ANXttYhURmCufgayYidDKRX2oVkiursY4InInzHAB4ChEOB7uVwunLlSi+15pThrqkeQQop5E+OxYLIeoer+HzKta6/HPsshHCzPR1tnG4aHGjFPPwsTtGO5v51oiGKAJyJ/5osBnl1o6IZEIhHGDY/EuOGRuFLRiJy8Umw/XIqdBWZMHx+D7KkGREeoAACmJgtyTQdw1HoSLpcL6brxmKeficRQo8B3QURERBRYWIH30lCqwPenqrYVOwtMOHiqEl1dXRg1tgMu7RWYW0uhlCgwLW4q5iRMR2RQhNBTJSIf4SvrFxHR7WAFnvxedIQKP1k4HNEjbfim9DuY0ABnnRKRHWl4cNwcTEyKhWgAO9cQERERkScGeLplDR2N3fvby/LQ0tkKY4geM2MXo84cgd2FFvzX2bMw6CzIzjRgSooOEjH3vBMRERENNG6h8dJQ3EJjaSpHrvkACqtOwOlyYoJ2LObpZ2F4qNFdbXd0OpFXXIkd+SZU1LQiKlSJe6YaMCM1FgqZRJB5E5Fv4BYaIvJnvriFhgHeS0MlwDtdThTXnEOu+QDO1V2EXCLHtNgpmJMwA1pV5E3Oc+HkhWpszy/FpbJGaIJkWDApAfMmJUATJLtr8yci38EAT0T+jAE+AAR6gLd3OXCk8hhyzQdQ2WpFmCIUcxKmY3rcVKhkKq++67y5Hjl5pTh5qQZymRizUuOwaKoeUaFBgzR7IvJFDPBE5M98McBzDzwBABrtTdhvOYwDZYfR7GiBPjgeT455FBN1qZCIb28LzCh9GEbpw2CxNWNHvgl7j5ch91gZMsbosDjDiATdjf+PSURERET9YwXeS4FWgS9vrsRe8wEUVB1Hl7ML46JGY75+FkaEJQ54N5mahnZ8U2jGvhPl6HB0ITUpEoszDBilD2PnGqIAxgo8EfkzX6zAM8B7KRACvMvlwtnaC9hj3o+S2vOQiWXIip2MOfoZiFZpB+QaN9Pc5sDeYxbsPmpBU6sDw+NCsDjDiPRRURAzyBMFHAZ4IvJnDPABwJ8DvMPZicLK48g1H0B5SyVC5MGYnTAdM+IzoJGpB2imt87u6MJ3pyqwo8AEW307YiJUyM4wIGtsDGRStqAkChQM8ETkzxjgA4A/BvgmezMOluVhX9khNNmbEa+JxXz9LEyMngCZWPjHILqcThw9Z8P2vFKYqpoRqpFj0WQ9ZqfFQ6UUfn5EdGcY4InInzHAf4/dbsf777+PrVu3orGxESkpKXj55ZeRlZV10/N27dqF7du3o6ioCDU1NYiNjcXcuXPx/PPPIzg4uM/xGzduxCeffAKLxYK4uDisXLkSjz/++G3N2Z8CfGWLFbnmAyioPAqHsxNjI1MwXz8Lo8KTfHLPucvlQvHVOuTkl6L4ah2CFBLMSY/Hwsl6hGkUQk+PiG4TAzwR+TMG+O955ZVXsGvXLqxcuRJGoxFbtmzB6dOnsW7dOqSnp9/wvIyMDOh0OixYsABxcXE4d+4cNmzYgGHDhuHvf/87FIprYW/Dhg34t3/7N2RnZ2P69OkoLCzE1q1b8eqrr+Lpp5/2es6+HuBdLhfO111Crnk/TtechUwsxdSYSZinn4EYdfRdmOnAuFrZiJw8EwrPWSERizBtXAyyM4yIifCulSURCY8Bnoj8GQP8dYqKirB8+XK89tprePLJJwEAHR0dWLZsGXQ6HdavX3/Dc/Pz85GRkeEx9tVXX+HVV1/FG2+8gYceeggA0N7ejtmzZ2PSpEn48MMP3cf+6le/Qm5uLvbt29dvxf5mfDXAdzo7cbTqJHLNB2BpLkewTIPZCdMwIz4TwXL/bddorWvFzgIzDp6qQGenExNHaZGdaUBSXKjQUyOiW8QAT0T+zBcDvGAbjHfs2AGZTIbly5e7xxQKBR5++GG8++67sFqt0Ol0/Z77/fAOAAsWLAAAXLp0yT2Wn5+P+vp6PPbYYx7HPv7449i2bRv279+PpUuXDsTtCKbF0dq9v93yHRrsTYhVR+PxlOWYEp0GmcT/33yqC1dhxT3JuH9GInYfNSP3aBmOnrchWR+GxZlGjB8e4ZPbgYiIiIgGy4AE+M7OTuzZswcNDQ2YO3cutNofbkVYUlKCxMREqNWe3U9SU1PhcrlQUlJywwDfn+rqagBAeHi4e6y4uBgAMG7cOI9jx44dC7FYjOLiYr8N8NZWG/aaDyKvohB2pwOjI0Zhhf4RpESMDMhAG6KW46FZSVicYcSBk+XYecSM9zaeRIJWg8WZBkxJ0UEqYecaIiIiCnxeB/g//elPyM/Px9///ncA3Xuun3rqKRQWFsLlciEsLAxffvklDAbDTb/HZrMhOrrvnuze8G+1Wr2a1+rVqyGRSLBo0SKPa8jlcoSFhXkc2zvm7TWEUFB5DP97aQfqO+oRpghDRswklLdU4lR1MSQiMabETMQ8/UzEaWKEnupdEaSQYtFUA+ZNSkB+cRVy8k1Yva0Ym/ddxqKpesxKjYNCfntvjiUiIiLyB14H+AMHDmDatGnun3Nzc3HkyBE888wzGD16NH73u9/hr3/9K37/+9/f9Hva29shk/Xd4tH7AGpHR8ctz2nbtm3YtGkTVq1a5fGLw42u0Xsdb67R62b7kQbagdICfH5uM+xddgBAXUc9dpTugUKiwI/GLsaiEbMRpgy5a/PxNQ/EhOK+OSNReLYKf8+9gM93X8DXh0qxbEYilk5PRCg71xD5DK3Wu+eNiIh8ia+tYV4H+MrKShiNRvfPe/fuRUJCAn71q18BAC5cuIBt27b94PcolUo4HI4+472h+vpOMjdTWFiI119/HXPmzMGLL77Y5xp2u73f8zo6Om75Gte7mw+x/s/xLe7wfj2VNAhzo+fA0QTYmvhgWKJWjV89koaLlgZszyvF57vO4e+5FzAzNQ73TNUjKixI6CkSDWl8iJWI/FlAPMTqcDgglV47LT8/36Mir9frYbPZfvB7tFptv1tYes+9lf3vZ8+exXPPPYfk5GS8++67kEg8t05otVo4HA7U19d7bKOx2+2or6/3ao+9EOo66r0aH+pGJITinx9ORXl1C3bkm/DtiTLsPV6GqaN1yM4wwBDtW789ExEREd0Or5/6i4mJwfHjxwF0V9vNZjOmTJni/rympgYq1Q/36k5JScGVK1fQ0tLiMX7y5En35zdjMpnwzDPPICIiAh999FG/1xw9ejQA4PTp0x7jp0+fhtPpdH/uq8IVYV6NU7e4KDWeXjoa/+/Ps7BwSgKOX6zGv//3EbzzxQmUlNaBLx8mIiIif+Z1gF+6dCm++uorrFq1CqtWrYJGo8Hs2bPdn5eUlPzgA6wAkJ2dDYfDgY0bN7rH7HY7Nm/ejIkTJ7ofcC0vL/doDQl0V+mffvppiEQirFmzBhEREf1eIzMzE2FhYfjss888xj///HOoVCrMmjXrlu9bCPclZUMm9tzDLxPLcF9StkAz8i8RIUo8Mm8k3n5+Gn40ezhM1ma89flx/H5tIQrPWgXp509ERER0pyT//u///u/enJCeno6qqiocPnwYCoUCv/3tb92V7KamJvzrv/4r7r33XmRlZd30e2JiYnDx4kWsX78eLS0tsFgseOONN3Dp0iW89dZbiIuLAwA8//zz+NOf/oR/+qd/cp/72GOP4fLly3j00Udht9tx7tw595+2tjbExsYCAKRSKVQqFT799FNcvHgRzc3NWLt2LbZu3YoXX3zRY+vPrWprs+NuFXDjNbGIUIbD1GhBR1c7whVheHjUfZgaM/HuTCBAyKUSjNKHYf6keEQEK1FSWodvT5Qjv7gKUqkY8VFqSMRsQUk0WNRqBVpb+38eiYjI1wmxholEIqhU8ht/PpBvYnU6nWhpaYFSqbxh95frdXR04L333sO2bdvQ0NCA5ORkvPLKKx7BesWKFSgoKMC5c+fcY8nJyTf8zgcffBBvvvmmx9iXX36JTz75BBaLBbGxsVixYgVWrlx5G3fou29ipVvndLpw7LwN2/NKcbWyCaFqORZMTsDc9HiolP7/8isiX8P1i4j8mS8+xDqgAd5ut0Muv/FvC4GAAT5wuFwunC2tw/Z8E85cqYVSLsGctHgsnKJHeDBbUBINFK5fROTPfDHAe71vYN++ffjggw88xtavX4+JEyciLS0Nv/zlL/ttD0nka0QiEUYPi8AvH0nDvz05BRNGRGHnERP+n//vED7ZXoKKmpYf/hIiIiKiu8zrNpJr1qxBZGSk++dLly7hj3/8I/R6PRISErB9+3aMHz8eTz755EDOk2hQGWOCseq+sXho1nDsLDDhYFEFDhZVIH1kFBZnGjEiPlToKRIREREBuI0Af/nyZY+uM9u3b4dCocCmTZug0Wjwy1/+El999RUDPPklbVgQnliUjPtmJCL3qAV7jlpw/EI1RiWEYnGmEalJkRCJREJPk4iIiIYwrwN8Q0MDwsPD3T8fOnQImZmZ0Gi69+lMnToV+/btG7gZEgkgRCXHAzOHIzvDgAMnK7DziAnvbypCvFaN7KkGZIyJhlTCzjVERER093mdQMLDw1FeXg4AaG5uxqlTpzB58mT3552dnejq6hq4GRIJSCmXYuEUPd5clYVnlnW3S13zjxL8+qPD2HXEjHZ7p8AzJCIioqHG6wp8WloaNmzYgBEjRmD//v3o6uryeCFSaWkpdDrdgE6SSGhSiRjTxsUia2wMTl2uwfY8EzbsuYBt313B3IkJWDApASHqwO7ARERERL7B6wD/z//8z1i5ciVeeuklAN1910eMGAGguy3f7t27kZGRMbCzJPIRIpEIqUlRSE2KwqWyBuTkm/CPQ1exs8CEGamxuGeqAbqwIKGnSURERAHstvrA19fX49ixYwgODsaUKVPc4w0NDfjqq6+QkZGBlJSUAZ2or2AfePq+ipoW7Mg34fCZSnQ5XZiSosPiDCOMMcFCT43IJ3D9IiJ/5ot94Af0RU5DAQM83UhdUwd2F5qx93gZ2u1dGDssHIszjRhtDGfnGhrSuH4RkT8LqABvMpmwZ88emM1mAIBer8f8+fNhMBhub6Z+ggGefkhreye+PVGGb46Y0dBihzEmGIszDJicrINYzCBPQw/XLyLyZwET4N977z2sXr26T7cZsViMVatW4cUXX/R+pn6CAZ5ulaOzC4dOV2JHvglVdW3QhQXhngwDpo+LgVwmEXp6RHcN1y8i8me+GOC9foh106ZN+Mtf/oL09HQ888wzGDlyJADgwoULWLNmDf7yl79Ar9fjoYceuv1ZEwUAmVSC2WnxmJkah+MXbNieZ8K6neew9cBlLJisx9yJ8VArZUJPk4iIiPyM1xX4hx56CDKZDOvXr4dU6pn/Ozs78fjjj8PhcGDz5s0DOlFfwQo83S6Xy4Vzpnpszy/F6cu1UMglmD0hDoum6BERohR6ekSDhusXEfmzgKjAX7p0Ca+88kqf8A4AUqkUS5YswTvvvOPt1xIFPJFIhBRjOFKM4TBbm5GTX4rdhRbsOWpB5thoZGcYER+lFnqaRERE5OO8DvAymQytra03/LylpQUyGbcFEN2MXqfB/7l3LB6aORy7jpixv6gc352qRNqIKCzONGBkQpjQUyQiIiIfJfb2hPHjx+OLL75AdXV1n89qamrw5ZdfYsKECQMyOaJAFxUWhMcWjsJbz03D/TMScbGsAW/8zzH88X+O4sSFajjZ5ZWIiIi+x+s98EeOHMGTTz4JtVqNH/3oR+63sF68eBGbN29GS0sLPv30U0yePHlQJiw07oGnwdRh78LBUxXYWWBCdUM74qLUyJ5qQObYaEglXv++TeQTuH4RkT/zxT3wt9VGMjc3F7/73e9QUVHhMR4XF4ff/OY3mDNnjtcT9RcM8HQ3dDmdOFJixfY8Eyy2ZoQHK7Boih6zJsQhSOH1zjciQXH9IiJ/FjABHgCcTidOnz4Ni8UCoPtFTmPHjsWXX36JtWvXYvv27bc3Yx/HAE93k8vlwukrtcjJK8VZUz1UCinmTozHgsl6hKrlQk+P6JZw/SIif+aLAf62S3lisRipqalITU31GK+rq8OVK1du92uJ6DoikQjjh0di/PBIXC5vRE5+KbYfLsXOAjNmjI/BPRkGRIerhJ4mERER3UX8u3giPzE8LgQvPDgelbWt2FlgwsFTFdh3shyTknVYkmnAsJgQoadIREREdwEDPJGfiYlQ4afZKXhgRiK+KbRg73ELCs9aMdoYjiWZRowZFg6RSCT0NImIiGiQMMAT+alQjQIPz0nC0iwjvj1Rhl1HzPjzFydgiNZgcYYRk1O0kIjZuYaIiCjQMMAT+bkghRSLM4xYMEmPvDOVyMk34aP/PYO/71MiO8OAGeNjIZdJhJ4mERERDZBbCvD//d//fctfeOzYsdueDBHdPplUjJkT4jA9NRYnLlQjJ68U/7PrPLYevIL5kxIwb2ICNEF8SzIREZG/u6U2kikpKd59qUiEkpKS256UL2MbSfIXLpcLFywN2J5XiqJLNVDIJJg1IQ6LpugRGaoUeno0hHD9IiJ/5rdtJNeuXTtgEyKiu0MkEmGUPgyj9GGw2JqRk2dC7jELco9ZMHV0NBZnGpCgvfHiQERERL7ptl/kNFSxAk/+rKahHbuOmLH/ZDk6HF1ITYrEkkwjRiaEsnMNDRquX0Tkz3yxAs8A7yUGeAoEzW0O5B6zYHehBc1tDiTFh2BxhhFpI6MgZpCnAcb1i4j8GQN8AGCAp0DS4ejCd6cqsCPfhOqGdsRGqpA91YDMsTGQSdmCkgYG1y8i8mcM8AGAAZ4CUZfTicKzNuTklcJkbUaYRo6FU/SYkxaPIAW7zdKd4fpFRP6MAT4AMMBTIHO5XCi+WofteaUoKa1DkEKKuenxWDA5AWEahdDTIz/F9YuI/JkvBniW1ojITSQSYWxiBMYmRuBKRSNy8k3IyS/FriMmTBsXi8UZBkRHqISeJhER0ZDGAE9E/UqMDcHzD4xDVV0rdhaYcbCoAgdOlmNishZLMo1IjA0ReopERERDErfQeIlbaGioamixY3ehGXuPlaG1oxMphjAszjRiXGIEW1DSTXH9IiJ/5otbaBjgvcQAT0NdW0cn9p8sx64jZtQ1dUCv02BxhgFTRusgEbNzDfXF9YuI/BkDfABggCfq1tnlRN6ZKuTkl6KiphWRIUrcM1WPmRPioJBJhJ4e+RCuX0TkzxjgAwADPJEnp8uFoos12J5fiouWBmiCZJg/KQHzJyVAEyQTenrkA7h+EZE/88UAz4dYieiOiEUipI2MQtrIKFyw1CMnz4StB68gJ78UM1PjcM9UPaJCg4SeJhERUcBggCeiATMyIQwjHw5DWXULduSX4tvjZdh7rAxTx+iwOMMIve7G1QQiIiK6NdxC4yVuoSG6dbWN7dh1xIx9J8vRYe/CuOERWJJhRLIhjJ1rhhCuX0Tkz3xxCw0DvJcY4Im819LuwN5jZdhdaEZjqwOJsSFYkmlA+kgtxGIG+UDH9YuI/BkDfABggCe6fXZHF747XYmd+SZY69sQHaFC9lQ9po2LhUzKFpSBiusXEfkzBvgAwABPdOecTheOnrdhe14pSiubEKqWY+EUPeakxUOl5KM5gYbrFxH5Mwb477Hb7Xj//fexdetWNDY2IiUlBS+//DKysrJuel5RURE2b96MoqIinD9/Hg6HA+fOnetznMViwfz58/v9jtWrV2PWrFlez5kBnmjguFwulJTWISevFGeu1kEpl2BOejwWTtYjPFgh9PRogHD9IiJ/5osBXtBS169//Wvs2rULK1euhNFoxJYtW/Dss89i3bp1SE9Pv+F5+/btw8aNG5GcnAy9Xo/Lly/f9Dr33XcfZsyY4TGWkpIyIPdARLdPJBJhzLAIjBkWgdLKJuTkl2JngQm7C83IGhuD7AwDYiPVQk+TiIjIpwhWgS8qKsLy5cvx2muv4cknnwQAdHR0YNmyZdDpdFi/fv0Nz62uroZGo4FSqcQf/vAHrF279qYV+OuvcadYgScaXNb6NuwsMOFgUQU6O51IH6XF4gwDkuJDhZ4a3SauX0Tkz1iBv86OHTsgk8mwfPly95hCocDDDz+Md999F1arFTqdrt9zo6KivL5ea2srpFIp5HL5bc+ZiAafLiwIKxYl4/4ZidhTaEHuMQuOnbdhlD4MSzINGD88ki0oiYhoSBOs7UNJSQkSExOhVnv+9Xhqamr3vtiSkgG71vvvv4/09HSkpqbikUcewZEjRwbsu4locISo5Hhw1nC89fw0/GT+SFQ3tOG9jUX4zScFOHS6Ap1dTqGnSEREJAjBKvA2mw3R0dF9xrVaLQDAarXe8TXEYjFmzJiBhQsXQqfTobS0FGvWrMFTTz2FTz/9FJMnT77jaxDR4FLKpVg0RY95E+ORX1yFHfkmfPx1Cbbsv4xFUwyYNSEOCrlE6GkSERHdNYIF+Pb2dshksj7jCkV354mOjo47vkZcXBzWrFnjMbZkyRIsXboUb7/9NjZs2OD1d95sP9Jg02qDBbs2kS94ICYU988dicKSKvx970V8vucCvj58FUunD8eyGYkI1bBzja/i+kVE/szX1jDBArxSqYTD4egz3hvce4P8QIuOjsbSpUvx5Zdfoq2tDUFBQV6dz4dYiYQ3TKvGL388ARfLGpCTV4oN35zD5r0XMCM1FvdMNUAb5t2/1zS4uH4RkT/jQ6zX0Wq1/W6TsdlsAHDDB1gHQmxsLJxOJxobG70O8ETkO0bEh+KffpSKipoW5OSbsO9EOb49Xo4po3VYnGGAIdq3KiZEREQDQbAAn5KSgnXr1qGlpcXjQdaTJ0+6Px8sZrMZEokEoaFsS0cUCGIj1Xh6yWg8OHM4vik049vjZcgvrsLYxAgsyTAgxRjOzjVERBQwBOtCk52dDYfDgY0bN7rH7HY7Nm/ejIkTJ7ofcC0vL8elS5du6xq1tbV9xkpLS/GPf/wDkydPhlKpvL3JE5FPCg9W4MdzR+Dt56fhR7OHw2xtxlsbTuB3fytE4VmrINvfiIiIBppgFfgJEyYgOzsbb7/9Nmw2GwwGA7Zs2YLy8nK88cYb7uNeffVVFBQUeLyoqaysDFu3bgUAnDp1CgDw4YcfAuiu3M+bNw8A8NZbb8FsNiMzMxM6nQ4mk8n94Oqrr756V+6TiO4+lVKGpVnDsGiKHodOV2JHvgkffnUauvAgZE81YPr4GMik7FxDRET+SbA3sQLdD6y+99572LZtGxoaGpCcnIxXXnkF06ZNcx+zYsWKPgE+Pz8fK1eu7Pc7H3zwQbz55psAgK+//hobNmzAxYsX0dTUhJCQEEydOhW/+MUvMHLkyNuaMx9iJfI/TqcLx87bkJNfiisVTQhRy7FwcgLmpsdDpezbDYsGFtcvIvJnvvgQq6C+iFPjAAAgAElEQVQB3h8xwBP5L5fLhbOmeuTkl+L05Voo5BLMSYvDoikGhAezBeVg4fpFRP7MFwO8YFtoiIjuNpFIhNHGcIw2hsNU1YQd+SZ8c8SC3YUWZI2NQXaGAXFR6h/+IiIiIgGxAu8lVuCJAkt1fRt2HjHjwMly2DudSBsRhSWZRoxIYJeqgcL1i4j8mS9W4BngvcQATxSYmlrt2HPUgj1HLWhp78TIhFAszjQiNSkSYragvCNcv4jInzHABwAGeKLA1mHvwoGicuwsMKOmsR3xUWpkZxiQMSYaUolgnXf9GtcvIvJnDPABgAGeaGjo7HLiyFkrcvJKYbG1IDxYgXum6DErLQ5KOR8f8gbXLyLyZwzwAYABnmhocblcOHW5FjvyS3HWVA+1Uoq5E+OxYJIeIWq50NPzC1y/iMif+WKAZxmJiOgmRCIRUpMikZoUiUvlDdiRZ8I/DpViZ4EZM8bH4p6peujCVUJPk4iIhhBW4L3ECjwRVdS0YGeBGYdOV6DL6cLkZB2WZBphjAkWemo+iesXEfkzX6zAM8B7iQGeiHrVN3fgm0Izvj1ehraOLowZFo7FmUaMMYZDxM41bly/iMifMcAHAAZ4Ivq+1vZO7DtRhl2FZjQ022GMDsbiTAMmJWshEbNzDdcvIvJnDPABgAGeiG7E0enE4TOV2JFvQmVtK7RhSmRPNWD6+FjIZRKhpycYrl9E5M8Y4AMAAzwR/RCny4Xj56uRk1+Ky+WNCFbJsGBSAuZOTIAmSCb09O46rl9E5M98McCzCw0R0QATi0SYlKzFxFFROG+uR06+CVsOXMH2PBNmp8Vh0RQ9IkKUQk+TiIj8FAM8EdEgEYlESDaEI9kQDou1GTn5pdhdaMGeoxZkjolGdoYB8dobV1iIiIj6wy00XuIWGiK6E9UNbdh1xIz9J8thdzgxISkSizONGKUPE3pqg4brFxH5M1/cQsMA7yUGeCIaCM1tDuQetWD3UQua2xwYER+KxRkGTBgZBXGAtaDk+kVE/owBPgAwwBPRQOpwdOFgUQV2FphQ3dCO2EgVsjMMyBobA6kkMFpQcv0iIn/GAB8AGOCJaDB0OZ04ctaKnDwTzNZmhAcrsHCyHrPT4hCk8O/Hlbh+EZE/Y4APAAzwRDSYXC4XzlytRU6eCSWldQhSSDFvYjwWTNYjVC0Xenq3hesXEfkzXwzw/l3WISIKMCKRCOMSIzEuMRJXKhqRk1eK7YdLsbPAjOnjY5A91YDoCJXQ0yQiIgExwBMR+ajE2BA8/+B4VNW1Yme+CQdPVWL/iXJMStZicaYRibEhQk+RiIgEwC00XuIWGiISSkNzB3YftWDvsTK0dnQixRCGJZlGjE2MgMiHO9dw/SIif+aLW2gY4L3EAE9EQmvr6MS+E+X4ptCMuqYOGHQaZGcaMCVFB4nY9zrXcP0iIn/GAB8AGOCJyFd0djlx+EwlduSbUFHTiqhQJe6ZasCM1FgoZBKhp+fG9YuI/BkDfABggCciX+N0uXDyYjVy8ky4WNYATZAMCyYlYN6kBGiCZEJPj+sXEfk1XwzwfIiViMjPiUUipI/UIn2kFufN9diRb8JXB69ge34pZqXGYdFUPaJCg4SeJhERDRAGeCKiADJKH4ZR+jCU2ZqxI9+EvcfLkHusDBljdFicYUSC7sYVHSIi8g/cQuMlbqEhIn9S29iOXUfM2HeiHB2OLqQmRWJxhgGj9GF3rXMN1y8i8me+uIWGAd5LDPBE5I+a2xzYe7wMuwvNaGp1YHhcCBZnGJE+KgriQQ7yXL+IyJ8xwAcABngi8md2Rxe+O1WBHQUm2OrbEROhQnaGAVljYyCTDk4LSq5fROTPGOADAAM8EQUCp9OFwnNW5OSZUFrVhFCNHIsm6zE7LR4q5cA+HsX1i4j8GQN8AGCAJ6JA4nK5UFxah5y8UhRfrUOQQoI56fFYOFmPMI1iQK7B9YuI/JkvBnh2oSEiGsJEIhHGDovA2GERKK1sQk5+KXbkm/DNETOmjYtBdoYRMREqoadJRETXYQXeS6zAE1Ggs9a1YmeBGQdPVaCz04mJo7TIzjQgKS70tr6P6xcR+TNfrMAzwHuJAZ6IhorGFjt2H7Vg7zELWto7kawPw+JMI8YPj/CqBSXXLyLyZwzwAYABnoiGmnZ7J/afKMfOI2bUNXUgQavB4kwDpqToIJX8cOcarl9E5M8Y4AMAAzwRDVWdXU7kF1dhR74JZdUtiAxRYtFUPWalxkEhl9zwPK5fROTPGOADAAM8EQ11TpcLRZdqkJNXiguWBmiCZJg3MR7zJyUgWCXvczzXLyLyZwzwAYABnojomouWBuTkl+L4hWrIpWLMTI3DPVP1iAoLwuEzldi87xJqGzsQEaLAQ7OTkDU2RugpExF5hQE+ADDAExH1VV7dgh35Jhw+UwmXC0iMDYapqhmOLqf7GLlUjJ8uTmGIJyK/4osBfnDem01ERENKXJQaTy8djT89Nw2LpuhxqbzRI7wDgL3Tic37Lgk0QyKiwMEAT0REAyY8WIEfzxtxw89rGjuwce9FHD5TCYu1GZ3fC/lERPTD+CZWIiIacJEhCtQ0dvQZl4hF+KbQjM4ul/vnuCg1ErQa6HXdfxJ0GoSq+z4MS0RE3RjgiYhowD00Owl/yzkLe2ffPfBTUnSoqm2F2dYMi7UFZmszzprqcPhMpfvYEJUMCb2Bvifcx0aqIZPyL46JiAQN8Ha7He+//z62bt2KxsZGpKSk4OWXX0ZWVtZNzysqKsLmzZtRVFSE8+fPw+Fw4Ny5c/0e63Q6sWbNGnz++eew2WwYNmwYnnvuOSxZsmQwbomIiAD3g6o36kITr9UgXqsBxlw7p7nNAYu1GWZrc0+4b0busTI4en4JkIhFiIlUQa/VeIT7MI3cqzfDEhH5O0ED/K9//Wvs2rULK1euhNFoxJYtW/Dss89i3bp1SE9Pv+F5+/btw8aNG5GcnAy9Xo/Lly/f8Nh3330Xf/3rX/HII49g3Lhx2LNnD15++WWIxWJkZ2cPxm0RERG6Q3zW2Jhb7uCgCZIhxRiOFGO4e8zpdKGqrhVmazMstmaYq5pxwVKPvOIqj/MStGrodcFI0Kmh12kQH6WGTHrjl0sREfkzwdpIFhUVYfny5Xjttdfw5JNPAgA6OjqwbNky6HQ6rF+//obnVldXQ6PRQKlU4g9/+APWrl3bbwW+qqoK8+fPx6OPPorXX38dAOByufDEE0+goqICu3fvhljs3V/Hso0kEZF3BmP9am13wGLr3n7TG+4ttmbYHd3VepEIiIlQeWzB0es0CA9WsFpPRF7xxTaSglXgd+zYAZlMhuXLl7vHFAoFHn74Ybz77ruwWq3Q6XT9nhsVFXVL19i9ezccDgcee+wx95hIJMKjjz6KX/7ylygqKkJaWtqd3QgREd11KqUMo/RhGKUPc485nS7Y6tuuVeutzbhc3oiCEuu18xRS9/ab3nAfH6WGQs5qPRH5D8ECfElJCRITE6FWqz3GU1NT4XK5UFJScsMA7801NBoNEhMT+1wDAIqLixngiYgChFgsQnSECtERKkxOufbfj7aOTpTZWmC2NsFsa4HF2oyDpyrQYe8CAIgA6CJU0GvV18K9VoPIUCWr9UTkkwQL8DabDdHR0X3GtVotAMBqtfb57Hau0V+1fiCvQUREvi1IIcWIhFCMSAh1jzldLlQ3tMNibXY/OGuyNuPoORtc7vMkiNdeC/QJOg0StGoo5WzgRkTCEmwVam9vh0wm6zOuUCgAdO+HH4hryOV9ewnfyTVuth9psGm1wYJdm4joTvji+hWtC8HYkZ5/09vW0YnSykZcLW/E1YruPwXFVdjbXuY+JjZSjWFxIRgWG4LEuBAMiw1FdIQKYjGr9USBytfWMMECvFKphMPh6DPeG6p7Q/adXsNutw/oNfgQKxGRd/xt/YpUyRA5IhKTRkQC6G5+UNPY3t2zvmdv/ZWyBuSdrkBvGwiFTNLTCae3Ut9duQ9SsFpP5O/4EOt1tFptv1tYbDYbANzx/vfeaxQWFg7qNYiIKLCJRCJEhQYhKjQIaSOvbcvscHShvLq7E46l58HZI2et+PZEufuYqFAlEnq23xh6wr0uLIjVeiK6I4IF+JSUFKxbtw4tLS0eD7KePHnS/fmdGj16NDZu3IgrV654PMjae43Ro0ff8TWIiGhoUsgkSIwNQWJsiHvM5XKhrqnD3QWnuyNOC4ou1cDZU66Xy8SIj9JAr1O7K/UJOg3Uyr7bSomI+iNYgM/OzsYnn3yCjRs3uvvA2+12bN68GRMnTnQ/4FpeXo62tjYkJSV5fY358+fjjTfewGeffebRB37Dhg2Ii4vDhAkTBux+iIiIRCIRIkKUiAhRIjXpWrXe0dmF8upWjxaXx85XY//JCvcxESEKj571CVoNoiOCIPHyfSVEFPgEC/ATJkxAdnY23n77bdhsNhgMBmzZsgXl5eV444033Me9+uqrKCgo8HhRU1lZGbZu3QoAOHXqFADgww8/BNBduZ83bx4AICYmBitXrsQnn3yCjo4OjB8/Hrt370ZhYSHeffddr1/iREREdDtkUgmMMcEwxlx7EM7lcqGhxd7dBacn1FuszThzpRZdPc9ayaRixEWp3V1weltdBqv6NmggoqFDsDexAt0Pk7733nvYtm0bGhoakJycjFdeeQXTpk1zH7NixYo+AT4/Px8rV67s9zsffPBBvPnmm+6fnU4nVq9ejS+++AJWqxWJiYlYtWoVli1bdltz5kOsRETe4frlnc4uJypqWmG2Nnk8ONvYcq0pQ5hG3hPor23BiYlQQSphYYpooPniQ6yCBnh/xABPROQdrl8Do6HFDovtWt96i7UZ5TUt6Ozq/m+SVCJCXKTaowuOXqdBiJrVeqI74YsBnv2tiIiI/ECoWo5QdQTGDotwj3V2OVFZ2+rehmOxtqD4ai0Ona50HxOilkOvVUOvC0ZCz4OzcVFqVuuJ/BgDPBERkZ+SSsTdbSq1GmReN97UaofF1uKu1pttzdh91ILOLicAQCIWITZS5d6Gk9BTrQ9VyyESscUlka9jgCciIgowwSo5RhvlGG0Md491OZ2oqm1zd8GxWJtx3lyPvDNV7mM0QTJ3B5zeLThxUSrIpBIhboOIboABnoiIaAiQiLs72sRFqTF1dLR7vKXd0fMiqhaYrU0wW1uw70QZ7J3d1XqxSISYSNW1N832hPvwYAWr9UQCYYAnIiIawtRKGZIN4Ug2XKvWO50uWOvbrj0wa2vG5fJGFJRYrztP6vEiqu5qvRoKGav1RIONAZ6IiIg8iMUixESoEBOhwuQUnXu8tb0TZdXNHnvrD5yqQIe9CwAgEgHR4SqPnvV6nQaRIUpW64kGEAM8ERER3RKVUoqRCWEYmRDmHnO6XKiub4PZ2uJuc2mqbELh2WvV+iCFpPth255Ar9dqEK9VQylnDCG6Hfw3h4iIiG6bWCSCLlwFXbgKk5K17vF2eyfKbC3uSr3F2oy8M5XYe6ynWg9AGx7k7oKToNVAH61BVKgSYlbriW6KAZ6IiIgGnFIuRVJ8KJLiQ91jLpcLNQ3t7kBv7gn4x87b0PuKRIVc0vPAbLB7G06CVoMgBSMLUS/+20BERER3hUgkQlRYEKLCgpA+8lq1vsPRhfLqnmp9T4vLguIqfNvR6T4mKlTZp8WlNiwIYjGr9TT0MMATERGRoBQyCRJjQ5AYG+Iec7lcqGvqcHfB6Q33Jy5Ww9VTrpfLxIiPuhboE3oq9mqlTKA7Ibo7GOCJiIjI54hEIkSEKBERosSEEVHucbujCxU1rTBZm2DpeXD22Hkb9p8sdx8TGaLwfGhWp4EuPAgSsViIWyEacAzwRERE5DfkMgmMMcEwxgS7x1wuF+qb7e4uOOaeiv3pK7XocnaX62XS7hdZ9XbB6Q33miBW68n/MMATERGRXxOJRAgPViA8WIHxwyPd445OJypqWtxbcCzWZhRdqsHBogr3MeHBvdX6a+E+OkIFqYTVevJdDPBEREQUkGRSMQzRwTBEB3uMN7TYPd4ya7Y2o/jqtWq9VCJCXKTa/ZbZhJ5gH6KWC3EbRH0wwBMREdGQEqqWIzQxAmMTI9xjnV1OVNa2erxl9szVWnx3utLjvN4w3xvuYyNZrae7jwGeiIiIhjypRNy9lUarQebYa+NNrXZ3z/recL/7qAWdXU4AgEQsQmykyh3oe/fXh6rlEPGFVDRIGOCJiIiIbiBYJcfoYREYPexatb7L6URVbZvHFpxz5nocPlN13Xkyj571CVoN4qJUkEklQtwGBRgGeCIiIiIvSMTdHW3iotTIQLR7vLnNgTKb5976b4+Xwd7ZXa0Xi0SI6a3Wa9XuYB8erGC1nrzCAE9EREQ0ADRBMiQbwpFsCHePOZ0uWOvbPN4ye6msAfnF16r1aqXU4y2zCToN4qPUkMtYraf+McATERERDRKxWISYCBViIlSYkqJzj7e2d3b3re/tXW9txoGiCnQ4ugAAIhEQHe65t16v0yAihNV6YoAnIiIiuutUSilG6cMwSh/mHnO6XKi+vlpva0FpZROOnLW6jwlSSKHXqt0vokrQaZAQpYFCzmr9UMIAT0REROQDxCIRdOEq6MJVmJR8rVrf1tGJsuoWjxaXh05Xot3eU60HoA0P8njLbIJOg6hQJcSs1gckBngiIiIiHxakkGJEfChGxIe6x1wuF2oa2t2BvrfV5bFzNrh6jlHKJT1vmdW4w328Vo0gBeOfv+P/gkRERER+RiQSISosCFFhQUgfpXWPd9i7uqv1PV1wzNZmFBRX4dvjZe5jokKVHu0t9dEaaMOCWK33IwzwRERERAFCIZdgeFwIhseFuMdcLhfqmjpg6umC0xvuT1yshqunXC+XXXuR1bVwr4ZKKRPoTuhmGOCJiIiIAphIJEJEiBIRIUqkjYhyj9sdXSivaelpb9kCs7UJR89Zsf9kufuYyBBlz8Oyane4jw5XQSxmtV5IDPBEREREQ5BcJsGwmBAMi/Gs1tc3290vo7L07LE/dbkGXc7ucr1MKkZ8lNpjb32CTgNNEKv1dwsDPBEREREB6K7WhwcrEB6sQGpSpHvc0elERW+1vmcLzsmL1ThYVOE+JjxY4d5Xn6BTQ68LRkxEECRisRC3EtAY4ImIiIjopmRSMQzRwTBEB3uMN7TYYbY29WzB6Q73Z67Uuqv1UokYcVEq94uoeltchqjkQtxGwGCAJyIiIqLbEqqWIzQxEuMSr1XrO7ucqKxpvdbe0tqM01dr8d3pymvnaeTurTe94T4mUgWphNX6W8EAT0REREQDRioRuyvtGHttvLHV3t0Fx927vgW7Cy3o7HICACRiEWIj1dD3bL9J0Kmh12oQqlEIdCe+iwGeiIiIiAZdiEqOMcMiMGZYhHusy+lEZW2bu1JvsTXjrKkeh89UXXeerPsXgutaXMZGqiGTDt1qPQM8EREREQlCIu7uaBMfpUbGmGj3eHOb47pKfXew//Z4Geyd3dV6sUiE2EiVuxNOb7gP08ghGgIvpGKAJyIiIiKfogmSIcUYjhRjuHvM6XShqq4VFluL+8HZi5YG5BdXeZyXoFVf21sfrUFcpBpymUSI2xg0DPBERERE5PPEPXvkYyPVmJKic4+3tjt6Qv213vUHTlagw9EFABCJgJgI1XUtLrvDfUSI4qbV+sNnKrF53yXUNnYgIkSBh2YnIWtszKDf561ggCciIiIiv6VSyjBKH4ZR+jD3mNPlgq3+2t56s7UZVyoaUVBivXaeQuoO87196+Oj1FDIJTh8phJ/yznr3rJT09iBv+WcBQCfCPEM8EREREQUUMQiEaLDVYgOV2FS8rVqfVtHJ8psLR4tLg+erkCHvadaD0AXHoS6pg53eO9l73Ri875LDPBERERERHdLkEKKEQmhGJEQ6h5zulyoaWi/Vq23NaOqrq3f82saO+7WVG+KAZ6IiIiIhiyxSARtWBC0YUFIH6UFAPzfD7/rN6xHhvhGT/qh20CTiIiIiKgfD81Ogvx7feblUjEemp0k0Iw8sQJPRERERHSd3n3u7EJDREREROQnssbGIGtsDLTaYNhsTUJPx4OgW2jsdjveeustzJgxA6mpqfjxj3+Mw4cP39K5VVVVePHFFzF58mRMnDgRzz//PMxmc5/jkpOT+/3z+eefD/TtEBERERENOkEr8L/+9a+xa9curFy5EkajEVu2bMGzzz6LdevWIT09/YbntbS0YOXKlWhpacHPf/5zSKVSfPrpp1i5ciW++uorhIaGehw/Y8YM3HfffR5jEyZMGJR7IiIiIiIaTIIF+KKiIvzjH//Aa6+9hieffBIA8MADD2DZsmV4++23sX79+hue+9lnn6G0tBSbN2/GmDFjAAAzZ87Evffei08//RQvvviix/HDhw/H/fffP2j3QkRERER0twi2hWbHjh2QyWRYvny5e0yhUODhhx/G0aNHYbVab3juzp07kZaW5g7vAJCUlISsrCzk5OT0e057ezs6OnyjdycRERER0e0SLMCXlJQgMTERarXaYzw1NRUulwslJSX9nud0OnHu3DmMGzeuz2fjx4/H1atX0dbm2Xx/06ZNSEtLQ2pqKu6991588803A3cjRERERER3kWAB3mazQafT9RnXarsb6N+oAl9fXw+73e4+7vvnulwu2Gw291h6ejpefvllfPjhh/jNb34Du92OX/ziF/j6668H6E6IiIiIiO4ewfbAt7e3QyaT9RlXKLrfcHWj7S6943K5/Ibntre3u8c2bNjgccyDDz6IZcuW4a233sLSpUshEom8mndkpMar4weSVhss2LWJiO4E1y8i8me+toYJFuCVSiUcDkef8d6A3hvGv6933G633/BcpVJ5w+uqVCr85Cc/wZ///GdcvnwZSUnevVGrpqYZTqfLq3MGgi/2ICUiuhVcv4jInwmxhonFopsWjQXbQqPVavvdJtO7/aW/7TUAEBYWBrlc7rFN5vpzRSJRv9trrhcbGwsAaGho8HbaRERERESCEqwCn5KSgnXr1qGlpcXjQdaTJ0+6P++PWCzGqFGjcPr06T6fFRUVwWg0Iigo6KbX7n3hU0REhNfzFou923IzkIS8NhHRneD6RUT+7G6vYT90PcECfHZ2Nj755BNs3LjR3Qfebrdj8+bNmDhxIqKjowEA5eXlaGtr89jqcs899+Cdd95BcXGxu5Xk5cuXkZeXh2effdZ9XG1tbZ+QXldXh88++wwJCQkYNmyY1/MOD1f/8EGDRMj990REd4LrFxH5M19bw0Qul+vub+ju8eKLL2LPnj346U9/CoPBgC1btuD06dP429/+hkmTJgEAVqxYgYKCApw7d859XnNzMx588EG0tbXhqaeegkQiwaeffgqXy4WvvvoK4eHhAIAPPvgAe/bswZw5cxAXF4eqqip88cUXqK2txX/9139h7ty5gtw3EREREdHtEjTAd3R04L333sO2bdvQ0NCA5ORkvPLKK5g2bZr7mP4CPABUVlbij3/8I7777js4nU5kZGTg9ddfh16vdx9z8OBBrFmzBufPn0dDQwNUKhXS0tKwatUq9y8IRERERET+RNAAT0RERERE3hGsCw0REREREXmPAZ6IiIiIyI8wwBMRERER+REGeCIiIiIiP8IAT0RERETkRxjgiYiIiIj8CAM8EREREZEfkQo9Aeqf1WrF2rVrcfLkSZw+fRqtra1Yu3YtMjIyhJ4aEdFNFRUVYcuWLcjPz0d5eTnCwsKQnp6Ol156CUajUejpERHd1KlTp/CXv/wFxcXFqKmpQXBwMFJSUvDCCy9g4sSJQk8PAAO8z7py5QpWr14No9GI5ORkHD9+XOgpERHdko8//hjHjh1DdnY2kpOTYbPZsH79ejzwwAPYtGkTkpKShJ4iEdENmc1mdHV1Yfny5dBqtWhqasK2bdvwxBNPYPXq1Zg+fbrQU+SbWH1Vc3MzHA4HwsPDsXv3brzwwguswBORXzh27BjGjRsHuVzuHrt69SruvfdeLF26FG+++aaAsyMi8l5bWxsWLFiAcePG4aOPPhJ6OqzA+yqNRiP0FIiIbkt/f8U8bNgwjBw5EpcuXRJgRkREdyYoKAgRERFobGwUeioA+BArERHdBS6XC9XV1QgPDxd6KkREt6S5uRm1tbW4fPky3nnnHZw/fx5ZWVlCTwsAK/BERHQX/O///i+qqqrw8ssvCz0VIqJb8i//8i/YuXMnAEAmk+EnP/kJfv7znws8q24M8ERENKguXbqE3/72t5g0aRLuv/9+oadDRHRLXnjhBTzyyCOorKzE1q1bYbfb4XA4PJ7vEQq30BAR0aCx2WxYtWoVQkND8f7770Ms5n92iMg/JCcnY/r06fjRj36ENWvW4MyZM3jttdeEnhYABngiIhokTU1NePbZZ9HU1ISPP/4YWq1W6CkREd0WmUyG+fPnY9euXWhvbxd6OgzwREQ08Do6OvDzn/8cV69exUcffYThw4cLPSUiojvS3t4Ol8uFlpYWoafCAE9ERAOrq6sLL730Ek6cOIH3338faWlpQk+JiOiW1dbW9hlrbm7Gzp07ERsbi8jISAFm5YkPsfqwDz/8EADcfZO3bt2Ko0ePIiQkBE888YSQUyMiuqE333wTubm5mDt3Lurr67F161b3Z2q1GgsWLBBwdkREN/fSSy9BoVAgPT0dWq0WFRUV2Lx5MyorK/HOO+8IPT0AfBOrT0tOTu53PD4+Hrm5uXd5NkREt2bFihUoKCjo9zOuX0Tk6zZt2oStW7fi4sWLaGxsRHBwMNLS0vD0009j6tSpQk8PAAM8EREREZFf4R54IiIiIiI/wgBPRERERORHGOCJiIiIiPwIAzwRERERkR9hgCciIiIi8iMM8EREREREfoQBnoiIiIjIjzDAExGRz1uxYgXmzZsn9DSIiHyCVOgJEBGRMPLz87Fy5cobfhP9i+YAAASqSURBVC6RSFBcXHwXZ0RERLeCAZ6IaIhbtmwZZs2a1WdcLOZf0hIR+SIGeCKiIW7MmDG4//77hZ4GERHdIpZXiIjopiwWC5KTk/HBBx/g66+/xr333ovx48djzpw5+OCDD9DZ2dnnnLNnz+KFF15ARkYGxo8fjyVLlmD16tXo6urqc6zNZsPvf/97zJ8/H+PGjUNWVhaeeuopfPfdd32OraqqwiuvvIIpU6ZgwoQJ+NnPfoYrV64Myn0TEfkqVuCJiIa4trY21NbW9hmXy+XQaDTun3Nzc2E2m/H4448jKioKubm5+M///E+Ul5fjjTfecB936tQprFixAlKp1H3s3r178fbbb+Ps2bP485//7D7WYrHg0UcfRU1NDe6//36MGzcObW1tOHnyJA4dOoTp06e7j21tbcUTTzyBCRMm4OWXX4bFYsHatWvx/PPP4+uvv4ZEIhmkf0JERL6FAZ6IaIj74IMP8MEHH/QZnzNnDj766CP3z2fPnsWmTZswduxYAMATTzyBX/ziF9i8eTMeeeQRpKWlAQD+8Ic/wG63Y8OGDUhJSXEf+9JLL+Hrr7/Gww8/jKysLADAf/zHf8BqteLjjz/GzJkzPa7vdDo9fq6rq8PPfvYzPPvss+6xiIgIvPXWWzh06FCf84mIAhUDPBHREPfII48gOzu7z3hERITHz9OmTXOHdwAQiUR45plnsHv3bnzzzTdIS0tDTU0Njh8/joULF7rDe++xzz33HHbs2IFvvvkGWVlZqK+vx4EDBzBz5sx+w/f3H6IVi8V9uuZkZmYCAEpLSxngiWjIYIAnIhrijEYjpk2b9oPHJSUl9RkbMWIEAMBsNgPo3hJz/fj1hg8fDrFY7D7WZDLB5XJhzJgxtzRPnU4HhULhMRYWFgYAqK+vv6XvICIKBHyIlYiI/MLN9ri7XK67OBMiImExwBMR0S25dOlSn7GLFy8CAPR6PQAgISHBY/x6ly9fhtPpdB9rMBggEolQUlIyWFMmIgpIDPBERHRLDh06hDNnzrh/drlc+PjjjwEACxYsAABERkYiPT0de/fuxfnz5z2O/etf/woAWLhwIYDu7S+zZs3C/v37cejQoT7XY1WdiKh/3ANPRDTEFRcXY+vWrf1+1hvMASAlJQU//elP8fjjj0Or1WLPnj04dOgQ7r//fqSnp7uPe/3117FixQo8/vjjeOyxx6DVarF3714c/P/buUNVRaI4juO/fQGLUSbYBPEltBpsdsEyaNPoSwgGn0CLQZhklPsCFqvFZxBMu+3ChWXbZTncz6eeGTinfTnzZz4+Mh6PP/9AkySbzSb3+z3z+TyTyST9fj/v9zu32y2dTifr9fr7Dg5QKAEP8MM1TZOmaf66drlcPmfPh8Nhut1u9vt9Ho9H2u126rpOXddf3hkMBjkej9lutzkcDnm9XqmqKqvVKrPZ7MuzVVXldDplt9vler3mfD6n1Wql1+tlOp1+z4EBCvfrt2+UAPzD8/nMaDTKYrHIcrn839sB+PHMwAMAQEEEPAAAFETAAwBAQczAAwBAQdzAAwBAQQQ8AAAURMADAEBBBDwAABREwAMAQEEEPAAAFOQPFt+CMewFDvEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C--5u9xt5du0"
      },
      "source": [
        "While the the training loss is going down with each epoch, the validation loss is increasing. This suggests that we are training our model too long, and it is over-fitting on the training data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MkNh9tB_-efC"
      },
      "source": [
        "### 5.5.2 - Save Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B7mkkuXoGVZ_"
      },
      "source": [
        "Last step is to save the fine-tuned model as well as the training arguments."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gNPJgkmbYwO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd9ba363-3a56-4870-f5cd-c9dceb08cfa1"
      },
      "source": [
        "print('Saving model...')\n",
        "torch.save(bert_classifier.state_dict(), sentiment_dir/'pytorch_model.pth')\n",
        "\n",
        "print('Saving tokenizer data...')\n",
        "tokenizer.save_pretrained(sentiment_dir)\n",
        "\n",
        "print('Saving training arguments...')\n",
        "save_json(sentiment_dir/'training_args.json', args)\n",
        "\n",
        "print('Saving training statistics..')\n",
        "save_json(sentiment_dir/'training_stats.json', trainer.training_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving model...\n",
            "Saving tokenizer data...\n",
            "Saving training arguments...\n",
            "Saving training statistics..\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UDi8plm1iAp6"
      },
      "source": [
        "## 5.6 - Evaluation of the Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F2WZqXed64Bu"
      },
      "source": [
        "Once the model is fine-tuned, it is evaluated respect the test dataset. It is needed to define the prediction function which is going to be almost equal to the previous evaluation function, but applying a **softmax** to return the probabilities per class."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4beq2q02L13"
      },
      "source": [
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Predictions\n",
        "    all_logits = []\n",
        "\n",
        "    # Set time\n",
        "    t0_test = time.time()\n",
        "\n",
        "    # Tracking variables\n",
        "    test_accuracy = []\n",
        "    test_loss = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    info_device()\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to device\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        test_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean()\n",
        "        test_accuracy.append(accuracy)\n",
        "    \n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    test_loss = np.mean(test_loss)\n",
        "    test_accuracy = np.mean(test_accuracy)\n",
        "\n",
        "    # Compute elapsed time\n",
        "    time_elapsed = format_time(time.time() - t0_test)\n",
        "\n",
        "    # Calculate probabilities\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    test_stats = {\n",
        "        'Test Loss': test_loss,\n",
        "        'Accuracy': test_accuracy,\n",
        "        'Test time': time_elapsed\n",
        "    }\n",
        "\n",
        "    print(f\"{'Test Loss':^12} | {'Accuracy':^10} | {'Test time':^11}\")\n",
        "    print(\"-\"*40)\n",
        "    print(f\"{test_loss:^12.6f} | {test_accuracy:^10.6f} | {time_elapsed:^11}\")\n",
        "    print(\"-\"*40)\n",
        "  \n",
        "    return probs, test_stats"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UgDYuFTztwfN"
      },
      "source": [
        "From `BertTrainer` we get the `model` and the `test_dataloader`. Using last function, we get the corresponding probabilities for each class."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jy4fJdpU2ep1",
        "outputId": "bbd3bbbe-b006-415d-8efa-d3fafc7a0f1d"
      },
      "source": [
        "# Compute probabilities in test set\n",
        "probs, test_stats= bert_predict(bert_classifier, generate_data_loaders(32, test=True))\n",
        "\n",
        "# Compute predictions\n",
        "preds = np.argmax(probs, axis=1)\n",
        "\n",
        "# Test labels\n",
        "test_df = datasets['test']\n",
        "y_true = test_df.label"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Using GPU Tesla P100-PCIE-16GB. \n",
            "\n",
            " Test Loss   |  Accuracy  |  Test time \n",
            "----------------------------------------\n",
            "  0.226938   |  0.947077  |   0:00:46  \n",
            "----------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "14_4pt8LdERy",
        "outputId": "ad6f676d-92b0-4dc9-81fd-de6438b0ae79"
      },
      "source": [
        "print('Saving test statistics..')\n",
        "save_json(sentiment_dir/'test_stats.json', test_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving test statistics..\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AsP8WSWyaoW9"
      },
      "source": [
        "To get a fast intuition on the result, we can plot some predicted sentences. Remember that the label 1 corresponds with positive sentiment while label 0 with negative."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "Jfc4r_BWMoGm",
        "outputId": "b2e06b6f-0853-47b5-ac17-76721ce78de8"
      },
      "source": [
        "num_examples = 5 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "df = test_df.sample(num_examples)\n",
        "df['real'] = np.where(df.label==0, 'negative', 'positive')\n",
        "df['predicted'] = np.where(preds[df.index]==0, 'negative', 'positive')\n",
        "df['probability'] = np.amax(probs, axis=1)[df.index]\n",
        "pd.set_option(\"max_colwidth\", 150)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "      <th>real</th>\n",
              "      <th>predicted</th>\n",
              "      <th>probability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1891</th>\n",
              "      <td>of things that elevate `` glory '' above most of its ilk , most notably the mere presence of duvall</td>\n",
              "      <td>1</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.999101</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20706</th>\n",
              "      <td>muddled and derivative that few will bother thinking it all through</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.997816</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14030</th>\n",
              "      <td>its empty head</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.997687</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15419</th>\n",
              "      <td>betters it</td>\n",
              "      <td>1</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.986578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17873</th>\n",
              "      <td>after collateral damage , you might imagine that most every aggrieved father cliché has been unturned .</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.996029</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                       sentence  ...  probability\n",
              "1891       of things that elevate `` glory '' above most of its ilk , most notably the mere presence of duvall   ...     0.999101\n",
              "20706                                      muddled and derivative that few will bother thinking it all through   ...     0.997816\n",
              "14030                                                                                           its empty head   ...     0.997687\n",
              "15419                                                                                               betters it   ...     0.986578\n",
              "17873  after collateral damage , you might imagine that most every aggrieved father cliché has been unturned .   ...     0.996029\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3HkKh0_YJoK"
      },
      "source": [
        "### 5.6.1 - Variable Threshold"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YsSlCGCAajmD"
      },
      "source": [
        "Sentiment classification task is even difficult for human. Therefore, define a threshold is going to provide a safer margin to classify one class. \n",
        "\n",
        "For example, if we define that positive sentiment sample are the only ones where probability of being *positive* is bigger than 0.9, samples classified as positive are going to have surely positive sentiment. The main drawback here is all samples wich are positive and are classified as negative."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8yvLZpyAYp1A"
      },
      "source": [
        "To get a better intuiton about the performance, we can plot the confusion matrix."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8-eUICKC5ks"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "import itertools\n",
        "\n",
        "def plot_confusion_matrix(cm, classes, normalize=False):\n",
        "    if normalize:\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "        title = \"Normalized confusion matrix\"\n",
        "    else:\n",
        "        title = 'Confusion matrix, without normalization'\n",
        "\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
        "    plt.title(title)\n",
        "    plt.colorbar()\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "\n",
        "    fmt = '.2f' if normalize else 'd'\n",
        "    thresh = cm.min() + ((cm.max() - cm.min()) / 2)\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment=\"center\", color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.grid(False)\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 446
        },
        "id": "saeayM_oD1ng",
        "outputId": "26a23927-e18e-4fb2-dfa9-a441254175ec"
      },
      "source": [
        "plot_confusion_matrix(confusion_matrix(preds, y_true), ('positive', 'negative'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGtCAYAAADtf4sDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxUVf8H8M+wyyagoAauKZuAuCGgaSqJoKZmihtEaqihmaWGqb9KzQzRDLVcMlc0tTAqFRM1V9RwX0ADV1wAZZMdnPn9YXMfhxlwZrioY593r3k9z5xz7rln7szgd852JTKZTAYiIiIiEug97wYQERERvWgYIBERERFVwgCJiIiIqBIGSERERESVMEAiIiIiqoQBEhEREVElDJBII8nJyXjnnXfQsWNHODk5YcmSJbVyntjYWDg5OeH48eO1Uv/LyMnJCREREc+7GU+laTt15XX9Vx0/fhxOTk6IjY0V0tLT02v174M6lixZAicnJ6Snpz+3NpBuM3jeDSD1FBcXY8uWLfjzzz+RmpqKwsJC1K1bF61bt0ZAQADefPNNGBjU7ttZUVGBiRMnoqKiApMmTYKFhQWcnJxq9Zz/NQkJCUhOTsbEiROfd1OeqSVLlsDFxQV+fn7PuykqpaenY/v27fDz84OLi8vzbg7967/6faFngwGSDrhx4wbCwsJw/fp1+Pr6IiwsDNbW1njw4AESExMxffp0pKamYtq0abXajlu3buHWrVuIiIjAyJEja/Vc/fv3R58+fWBoaFir53nRJCQkYPv27Vr9wT937hz09F78TmFV7Vy6dCkGDhz4wgZIt2/fxtKlS2Fvb88ASQ329vY4d+4c9PX1a/U81X1fxo8fj7CwMBgZGdVqG+jlxQDpBVdSUoKxY8ciPT0dS5YsQa9evRTyw8LCcO7cOZw/f77W23L//n0AQN26dWv9XPr6+rX+x/VlUFJSAgMDAxgYGMDY2Ph5N0ctutJOXVNQUABzc/Pn3QwAgEQiee7vs/x7QaStF//n5n/ctm3bcO3aNbz77rtKwZGch4cHRowYoZCWkJCAoUOHwtPTE23btsXQoUORkJCgdGyPHj0QHByMtLQ0hIWFoW3btmjfvj0++OADZGVlCeWCg4OFXqPp06fDyclJGN+vbr5QcHAwevTooZB26tQpjBkzBp07d4a7uztee+01vPfeezhz5oxQpqo6s7Oz8cUXX6Bbt25wc3NDt27d8MUXXyAnJ0ehnPz4xMRErF69Gn5+fnBzc4O/vz+2b9+u8jpW9uTcipiYGPj7+8Pd3R39+vXD/v37AQCXL1/G6NGj0a5dO3Tq1Alz585FeXm5Qj3nzp1DREQE/P390aZNG+H92LNnj9K1krdNfn2fnNsREREBJycnZGdnY/r06fD19YWnpyfu3bsnHPPkXJ2YmBg4OTlh2bJlCufJyMiAt7c3AgICUFRUpNa1eNL06dPh7u6O0tJSIe306dNwcnKCl5cXpFKpkH7gwAE4OTlh586dQtqT7ZTPVQGA7du3K7zuyk6fPo2RI0fC09MTnTp1wowZM1BYWKhULiUlBeHh4ejUqRPc3d0RGBiIVatW4dGjRwrlVH02n2yTfP5MbGwsQkJChNcub19wcHC110k+B+bq1atYtGgRunbtCjc3N7z55ps4cOCAUvmKigqsXLkSgYGBcHd3R6dOnRAeHo7Lly9X2b6dO3firbfegoeHB+bOnatwfRMTExEUFIQ2bdqga9euWLlyJQAgLy8Pn376KXx8fNCmTRuMHTsWGRkZCufIyMjA/Pnz0b9/f3Ts2FG4jitXrlS6jqqomoMUHBys8P4++XjyfRDr+1LVHKT09HRMnToVvr6+cHNzg5+fHxYtWoTi4uIavX/08mF4/YLbvXs3ACAoKEjtY2JiYjB79my0aNEC77//PoDH//iEh4dj9uzZSnVlZGQgJCQEfn5+mDZtGlJSUrBlyxYUFBTgxx9/BACMGzcO7dq1w/LlyxEUFIT27dsDAGxsbDR6PVevXsWoUaNQv359hISEoF69enjw4AFOnjyJlJQUeHp6Vnnsw4cPMWzYMNy4cQODBg2Cq6srkpOTsXnzZhw7dgzbtm1T+gX9zTffoKSkBEFBQTAyMsLmzZsRERGBJk2aCK/haWJiYpCfn4/BgwfDyMgIGzZswIQJE/Dtt99i5syZ6Nu3L/z8/HDkyBFs2LABNjY2wnUHgD179uDq1avo3bs37O3tkZubi+3bt2PChAmIiopCv379hGsslUqRlJSEyMhI4fh27doptOfdd99F/fr18f7776OoqAimpqYq2z1ixAgcO3YMy5YtQ6dOndChQwdIpVJMmTIFhYWFWLt2bZXHVqdTp06IjY3FqVOn4OPjAwBITEyEnp4e8vLycOnSJbi5uQEAjh07BolEgk6dOqmsy8bGBpGRkZg2bRo6dOiAIUOGqCyXnJyMcePG4a233kLfvn1x4sQJ/Pzzz9DT08OcOXOEcufPn0dwcDAMDAwwYsQI1K9fH/v370dUVBRSUlKwcOFCjV9vx44dMW7cOKXPfv369dU6PiIiAgYGBhg1ahTKy8uxbt06hIeHIz4+Hg4ODkK5KVOmYNeuXejcuTOGDRuG+/fvIyYmBkOHDkVMTAxcXV0V6k1ISMCGDRswbNgwDB06VOGzf+nSJezfvx9DhgxB//79sWvXLixcuBDGxsb49ddfYW9vjwkTJuDmzZvYsGEDPvnkE6xdu1Y4/vLly/jzzz/xxhtvoEmTJigvL8ehQ4ewcOFCpKenY/bs2Rpfx3HjxuHtt99WSLt16xaWLFmCevXqCWlif1+edPv2bQwePBgPHz7E8OHD0bRpU5w4cQIrVqzAqVOnsHbtWqVeJ3XfP3oJyeiF5uXlJWvXrp3a5XNzc2Wenp4yPz8/2cOHD4X0hw8fynr27Cnz9PSU5eXlCendu3eXOTo6ynbs2KFQz+effy5zdHSUpaWlCWnHjh2TOTo6yn755ReFsr/88ovM0dFRduzYMaX2jBw5Uta9e3fh+bp162SOjo6ys2fPVvs6VNW5aNEimaOjo2zjxo0KZTdu3ChzdHSUffPNN0rH9+/fX1ZaWiqk37t3T9a6dWvZ5MmTqz3/k6+3S5cusvz8fCE9OTlZ5ujoKHNycpLt3r1b4ZiBAwfKOnfurJBWWFioVHdRUZGsV69esoCAAIX0Tz75RObo6KiyPfK8jz/+WGW+o6Oj7JNPPlFIy83NlXXv3l3WrVs3WW5urmzp0qUyR0dH2YYNG6p+4U9x7949maOjo2zRokVCWnBwsGzcuHGytm3bylauXCmkDxw4UNa3b9+ntlNV2pN5Tk5OsjNnziikv/feezJXV1dZQUGBkBYUFCRzcXGRJScnC2lSqVT2wQcfyBwdHWVHjx4V0it/NuVu3bolc3R0lEVHRwtpVX32qxMdHS1zdHSUhYWFyaRSqZB+9uxZmaOjoywqKkpIO3z4sMzR0VE2adIkhbLJyckyFxcX2bBhw5Ta5+rqKktNTVU6r6rrVVpaKuvcubPMyclJNmfOHIXy8+bNU/quFxcXK7RDbsqUKTJnZ2dZRkaGkKbq2qi6hpXl5ubK/P39ZV5eXrIbN24I6WJ9X+TX/9atW0LaRx99JHN0dJT99ddfCmXnz58vc3R0lG3dulXpeHXeP3o5cYjtBVdQUAAzMzO1yx85cgRFRUUIDg5W+EVpbm6O4OBgFBUV4ejRowrH2NnZITAwUCHN29sbwOMJ4mKysLAAAOzdu1dhiEYde/bsgY2NjVIPWFBQEGxsbFQOIQ4fPlxhkmaDBg3QvHlzXL9+Xe3zvvXWW0K7AcDZ2Rnm5uaws7NTGvZs164dsrKyFIZ+nuylKS4uRk5ODoqLi+Ht7Y20tDQUFBSo3RYAGD16tNpl69ati6ioKGRlZeG9997DsmXL0KNHjxpNsm/QoAGaNWuGY8eOAQBKS0tx5swZdOnSBV5eXkJ6fn4+kpOTq+w90oSnpyfatGmjkObt7Y2Kigrcvn0bAPDgwQOcPn0aPXr0gLOzs1BOIpFg/PjxAKA0TPMshISEQCKRCM89PDxgamqq8N2St2vcuHEKZZ2dndG9e3ecPHkS2dnZCvV269YNr776qspzVr5eRkZGcHd3h0wmUxoa7NChAwDF77qJiYnQjrKyMuTm5iI7OxtdunSBVCrFhQsXNLoGlZWXl2PixIlIT0/HsmXL0KRJEyFP7O+LnFQqxb59++Dq6opu3bop5I0dOxZ6enoq/4ao8/7Ry4lDbC84c3NzlfMsqiIfb2/VqpVSnjzt1q1bCumNGzdWKmtlZQUAyM3NVfvc6ujTpw9+++03LF++HGvXrkWbNm3QpUsX9OnTB/b29tUem56eDjc3N6UucAMDAzRr1gyXLl1SOqaq1yb/R1UdqrrR69ati4YNG6pMBx5fN3lg++DBAyxevBh79+7FgwcPlI7Jz8/XaHJts2bN1C4LPA7axowZg+XLl8PW1hbz5s3T6HhVvL298fPPP6OgoADnz59HaWkpvL29UV5ejsWLF6OsrAwnTpyAVCoVgu2aUOczKv/st2zZUqlsixYtoKenp/TZfxZUtd3a2lph3lx6ejr09PRUBjwtW7ZEQkIC0tPTFYa0q/scqDqn/LNZ+fNsaWkJQPG7Lp8PFRcXhxs3bkAmkykck5+fX+W51fF///d/OH78OL7++mshQJMT+/sil52djaKiIpWfDysrK9ja2qr8fKjz/tHLiQHSC65Vq1b4+++/cevWLZVfVDFUt1qs8h9GVZ78dVVZRUWFwnMjIyOsWbMG586dw6FDh5CUlITo6GgsXboUCxcuxBtvvKF+w9UgxrL3qq6POtdNJpNh1KhRSEtLQ0hICNzc3GBhYQF9fX388ssv+OOPPxQmNaujTp06GpUvKyvD4cOHATz+R/Du3buwtrbWqI7KvL298dNPPyEpKQmnT5+GnZ0dXn31VZSXl6O4uBhnz57FsWPHoK+vDy8vrxqdC6j5Z1QT6kxC1kRtbb1Q3eeguutVVd6T13H+/PnYsGEDAgMDMW7cONjY2MDQ0BAXL15EVFSUxp/ZJy1fvhyxsbEYP348BgwYoNQGsb8vNaULW2dQ7eA7/4KTD+Fs27ZNrfLyIOqff/5RyktNTVUoIxb5L9O8vDylvKp2sfXw8EB4eDjWrFmDPXv2oE6dOli8eHG152ncuDGuXbumFHRVVFTg+vXrtRZA1sTly5eRkpKCsLAwTJs2DYGBgXjttdfg6+ur8g99dcGmthYtWoQLFy5g6tSpMDc3x+TJk7VavfakTp06QSKRIDExEceOHRN6iZycnGBtbY3ExEQcP34cLi4uQg9FbZP3jMg/50+6evUqpFKpwmfEyspKZQ+pql6E2nhfntS4cWNIpVKkpaUp5cnTnuWE4Li4OHTs2BHffPMNBg4ciG7dusHX17fG2wjs3LkTixcvRmBgICZNmqSUX5vfFxsbG5iZman8fOTl5SErK+uF/BtCzw8DpBfc4MGD0bx5c/z4448qx8cB4MKFC4iJiQEAdO7cGaampti4caPCWH1BQQE2btwIU1NTdO7cWdQ2yrv6K89t+uOPP5CZmamQVnkeBQA0bNgQNjY2KgOsJ/n5+SE7O1spWNy6dSuys7NfyE0G5b8+K/dyXLlyReV8GPn8C7GGNg8cOIC1a9di4MCBGDNmDL766itcv35dYeWXNmxsbODo6Ii//voLFy5cEAIk+Yq1+Ph4/PPPP2oPr5mamtb4NderVw9t27bF/v37ceXKFSFdJpMJS9yf7KFs1qwZCgsLce7cOSFNKpUqrOZ6sn2A6h8BYpB/dleuXKnwWbly5Qr27duH9u3ba7xitCb09PSUPrNFRUUqr426zpw5g4iICLRp0wbz589XGdzU5vdFT08P3bt3x6VLl3Dw4EGFvJUrV0Iqlb6Qf0Po+eEQ2wuuTp06WLFiBcLCwhAeHo4uXbrA19cXVlZWyM7OxvHjx3H48GGMGTMGwOP5BFOmTMHs2bMxZMgQDBw4EMDjZf43btzA7NmzFSYci6FFixbw9fXFli1bIJPJ4OLiguTkZCQkJKBp06YKPT7ff/89jhw5gtdffx0ODg6QyWTYv38/rl69KryGqowZMwbx8fGYPXs2Ll26JJzn559/RvPmzZ96/PPw6quvolWrVvjhhx9QUlKC5s2b49q1a9iyZQscHR1x8eJFhfJt2rTBxo0bhb2eDA0N4eHhodUv28zMTERERKBp06aYNWsWAKB79+4ICQnB+vXrhblfcj169MDt27eV9t2pire3N9atWyf8/yfT4+PjldKr4+npicTERKxcuRKvvPIKJBKJQtvUNWPGDAQHB2PEiBEYPnw4bG1tsX//fhw+fBh9+/YVtiUAgCFDhmDNmjUIDw9HSEgIDA0NsXv3bpVDbC1btoSZmRk2bdoEExMTWFpawsbGRqG+mujcuTMCAgKwY8cO5OXloXv37sjKysKmTZtgbGyMmTNninIedfn7+2PLli348MMP4evri/v37+OXX34R5n1p4/3330dFRQV69+4tfD7kzMzM4OfnV+vfl48++ghHjx5FeHg4hg8fjiZNmiApKQk7d+5Ex44dhb+XRAADJJ3QtGlT/Prrr9iyZQt2796N5cuXo6ioCHXr1oWbmxvmz58v7A0CPN7/xs7ODqtXrxY2CXR2dsayZctq7RdSZGQk5syZg99//x2//fYb2rdvj/Xr1+Pzzz9XmBDt5+eHrKwsxMfH4/79+zAxMUHTpk0xd+5cpT1SKrOwsMDmzZsRHR2Nffv2ITY2FvXq1cPQoUMxceLEF2YX4Sfp6+tjxYoV+Prrr7F9+3YUFxejVatW+Prrr5GSkqL0B79v375ITk7Gjh07EB8fD6lUiq+++krjAEkqlWLatGnCXlZProScOnUqkpKS8H//938K/5gUFhbCzs5O7XPIA6TGjRsrTLCXBw2GhoZKE3Cr8tlnn2H27NlYvny5sChBmwDJ3d0dP/30E6Kjo7F582YUFRWhcePGmDJlCkaNGqVQtnHjxli2bBkWLVqEb7/9FlZWVujfvz8GDRqEgIAAhbImJib45ptvsHjxYsybNw9lZWXw8vISLUACgKioKLi6umL79u2YP38+TE1N0bFjR0yaNOmZ3/Nw+vTpMDMzQ3x8PPbu3YtGjRohKCgI7u7uCA0N1apO+YTr+fPnK+XZ29vDz8+v1r8v9vb22Lp1K6Kjo/Hbb7/h4cOHaNCgAcaOHYvx48dz521SIJGJPcORiHROSkoK+vfvj3nz5mHQoEHPuzlERM8d5yAREQ4fPgxnZ2cOMRAR/Ys9SERERESVsAeJiIiIqBIGSERERESVMEAiIiIiqoQBEhERkQ7LzlP/fp2kPk7S1nEDFhzA3dyS590MIrUc+LzX824CkVokAIwNa/cWM2Lq+e4ipGdotxu9QwMr7F3zkcgt0n3cFUvH3c0tQXp2ze6rRfSs8NcYUe1Iz8zHzXta3q5HwsEkVRggERER6TqJ5PFD22NJCQMkIiIiXSeRaN8TxABJJfarEREREVXCHiQiIiJdxyE20TFAIiIi0nUSvRoMsXEwSRUGSERERLqOPUiiY9hIREREVAl7kIiIiHQdh9hExwCJiIhI59VgiA0cYlOFARIREZGu4z5IomO/GhEREVEl7EEiIiLSdVzFJjoGSERERLqOk7RFxwCJiIhI17EHSXQMG4mIiIgqYQ8SERGRruMQm+gYIBEREek6LvMXHQMkIiIiXSeRAHqcgyQm9qsRERERVcIeJCIiIl3HOUiiY4BERESk67jMX3QMkIiIiHQde5BEx6tCREREVAl7kIiIiHSdBDUYYhO1JS8NBkhERES6jkNsouNVISIiIqqEPUhERES6jqvYRMcAiYiISNdxiE10DJCIiIh0Xg16kDhLWyWGjURERESVsAeJiIhI10kkNRhiYw+SKgyQiIiIdB0naYuOARIREZGu4yRt0fGqEBEREVXCHiQiIiJdxx4k0TFAIiIi0nWcgyQ6BkhERES6jqvYRMd+NSIiIqJK2INERESk6zjEJjoGSERERDqvBpO0OZikEgMkIiIiXcceJNExbCQiIiKqhD1IREREOk4ikUCiZU+Qtse97BggERER6TgGSOJjgERERKTrJP8+tD2WlHAOEhEREaktMzMTUVFRCA4ORtu2beHk5ITjx4+rLLt3714MHDgQ7u7ueP3117F06VJUVFQolcvPz8esWbPg7e0NT09PhISEIDk5+ZnVqQoDJCIiIh0nH2LT9qGJa9euYdWqVcjIyICTk1OV5Q4cOIDw8HDUrVsXs2bNgp+fH5YtW4avvvpKoZxUKkVYWBh27NiBkSNHYurUqXjw4AGCg4Nx8+bNWq+zKhxiIyIi0nGPV/lrOwdJs/KtW7fGsWPHYG1tjYSEBISHh6ssFxkZCVdXV6xevRr6+voAADMzM6xcuRLBwcFo1qwZACA+Ph6nT5/GsmXL4OfnBwAICAiAv78/li5disjIyFqtsyrsQSIiItJxz7IHydzcHNbW1tWWSU1NRWpqKoKCgoRABgCGDx8OqVSKP//8U0jbvXs37Ozs0LNnTyHNxsYGAQEBSEhIQHl5ea3VWR0GSERERIS7d+8iPT1d4ZGfn69VXZcuXQIAuLm5KaQ3aNAADRs2FPIBIDk5Ga1bt1YK1Nzd3VFYWCgMidVGndVhgERERKTjJKhBD9K/y9hGjBiBnj17KjzWrVunVXuysrIAALa2tkp5tra2yMzMVChrZ2enVE6eJi9bG3VWh3OQiIiIdJ0Iy/xjYmLw6NEjhSxLS0utqiwpKQEAGBkZKeUZGxujuLhYoayqcvI0eV21UWd1GCARERERGjVqJFpdJiYmAICysjKlvNLSUiFfXlZVOXmavGxt1FkdDrERERHpuGc5SVsd8mEw+bDYkyoPf1UeHpOTp8nL1kad1WGAREREpOtqEhzVQoDk4uICALhw4YJCekZGBu7duyfkA4CzszMuXrwImUymUPbcuXMwNTVFkyZNaq3O6jBAIiIi0nEvWg9Sq1at0KJFC2zZskVhXtPmzZuhp6eHXr16CWm9e/dGZmYm9u7dK6RlZ2cjPj4ePXv2hKGhYa3VWR3OQSIiIiKNfPfddwCAtLQ0AEBcXBxOnjwJS0tLjBw5EgAwbdo0jB8/HqNHj0ZgYCCuXLmCmJgYBAUFoXnz5kJd/v7+8PT0xLRp0zBq1ChYW1tj8+bNkEqlmDhxosJ5a6POqkhklfufSKd0mrEb6dlFz7sZRGpJjR7wvJtApBYJABND3bmLq+dHv+HW/UKtjm1c3wxnFr2p0TFV3WLE3t4e+/btE54nJCRg6dKlSEtLg42NDQYNGoT3338fBgaK/TN5eXmIjIxEQkICSktL4e7ujoiICLRu3VrpHLVRpyoMkHQcAyTSJQyQSFfoXID0cQ0DpIWaBUj/BRxiIyIi0nE1mUtUG3OQXgacpE1ERERUCXuQiIiIdJwE2vcEsf9INQZIREREOo5DbOJjgERERKTrRLgXGyniHCQiIiKiStiDREREpOM4xCY+BkhEREQ6jgGS+BggERER6bqa3FONAZJKnINEREREVAl7kIiIiHQch9jExwCJiIhI13GZv+gYIJFOq29hjCl9XdDTrQHqW5ogK78Eu87cwcI/UpBfXC6UG9uzJfw8GuLVBhawMjVEblE50u49xOr9aYg/e1ep3gn+jnBvbAX3plZoWt8Mtx4Uwnvmn9W2ZZBXYwR3bQ7nVyyhJ5HgVnYRfk9Kx+Jdl0V/3fRyys7OxoKv5+H33+JwOz0dFhYWcG3thlmffYHOXV5DSUkJNsVswK4dO3D+/FlkZmSgYaNG6NixE6bPmAVnFxelOq+mpWHOF59h/74E5ObmwqFxYwQNHY6pn0yHiYnJc3iVVBskqEEPEiMklRggkc6qZ2GEPz7phgZ162DjoWu4fCcfTq9YIqRrC3i3qo/+Cw6ipPwRAMCzmTXSHxRh34UMZBeUwcrMEH3b2WP1OG8s+O2SUhAzfUBr5BSU4fytXFjWMXxqWxYGt8Ng7ybYefo2Yk/cglQqQ+P6ZrCvZ1orr51ePjdv3ID/G91RWFCAd0JHoaWjI/Lz8nD+/HncuX0bAHDj+nVMGD8Wvp274J3QUWj0yiu4dvUqfli5HHG/xiLuj13o9np3oc7LKSno3tUXFRUVGDvufTRt3hwnjh3D/Hlz8fffJxD3+04OrxBVgQES6awPejuhcT0zvL/6b8QlpQvpSVez8d3ojhjr1xLf/hv4jF/9t9LxP+xLQ/z07hjfqxWi4y9DKvtfns+s3bh5vwgAsHdWT5gZ61fZjqG+TTHUtyk+WJOEX07cEunV0X/NqNBgVFRU4PjJs2jUqJHKMvVtbZF44hTaeHoqpA8dNgI+Xu3wacQ0HDn2v8/6rBnTkZeXh71/HYK3jy8AYMx7Y9HK0RGfzZqBnzbFYNiIkbX3ouiZ4Rwk8XEVG+ksX0dbFJdVKARHAPDbyXQUlz3CEJ8m1R7/SCrD3dximBoZwFBf8asgD47UMcHfEedu5gjBkZkxf3eQZg4fOoijRw7jo4+nolGjRigvL0dRkfJnsF69ekrBEQC4uLrCtbUbLl28oJB+8MB+tGrlKARHciNDQgEAG9avFe010PMlkfwvSNL88bxb/2JigEQ6y8hADyXlUqV0mQwoKX+EZrbmsDYzUsizMjWEjbkRWja0wIeBTuju2gBHr2ShtEK5HnW82sAcze3MkZSWjQ8DnHBhQR9cWdwPyYv6Yv4wT5hW0/NEJLd7104AgEPjJhg04E3YWJqivpU5PFydsDlm41OPl0qluHfvLuwaNFBILy0tRR1T5WFe03/Tkv4+AZlMppRPOkhSwwcp4U9d0lmX7+ajT0N7tHaoi4vpeUJ6a4e6QmBkb1MHOYVlQt6hL96AjbkxAKD8kRQ7T9/B9J/OaN2GVxtYAADe7OAAQ30Jvt11GbceFMHPrSGCuzbHqw3MMXjxYa3rp/+GK1euAAAmjA/Dqy1bYdXqtSgrL0P0N4sw+t0QlFeUI+Sdd6s8ftXK5bh39y4iPp2pkO7i2hopyZdw7949NGzYUEg/8Nd+AEBBQQFyclPOi8oAACAASURBVHJgY2NTC6+KSLexB+lfsbGxcHJyQnp6uqhlqfb8sC8Nj6QyLB/jhR6tG+AV6zro3roBvh/TEWX/9gjVMVL8DTBmxXEMiz6CyetP4mByJkyM9GFuov3vBPmx9S2MMXbVCaxISMXO03fw0YZT2Jp4A75OtujeusFTaqH/uoKChwAAcwsLxO/Zh6HDRyDknXexZ/9BWFlZ4bNZMyCVqu7lPJZ4FBFTP4a7RxtMi/hUIW/Shx+hpKQEQwYNwKGDB3Dzxg38sm0rJk18H4aGjxcfqBrKI92j/fBaDXbgfskxQKrG5s2bERsb+7ybQVU4kfoA768+ATMTA2yY4Iu/5/XG2vHeOHr5PhLO3wMAFJSUKxxzPPUBDiZnYmviTYQsS0RBSTl+ndINdU2fvlJNFfkqubs5xTiUkqWQt+3YTQCAT6v6WtVN/x0mJnUAAIOHDIWR0f+Gha2trdGn75vIuHcPVy4rbxdx6tRJvNW/Lxq98gpi4/5QWrYfNGw4Fn7zLf65chn+ft3h3Ko5Rr8bgjHvjYVHm8dzmSwtLWvxldGzwgBJfBxi+1f//v3Rp08fhT9OP/30EywtLfHWW289tSw9H3+cuoOdp+/Axb4uzEwMkJbxEA8eluGPT7qh/JEU1zILqz1+27GbGNCxMQI8X8FPR29ofP67OcUAgMz8EqW8jLzHadoGX/TfYe9gDwBo8MQwmJx8aCw3N0ch/fTpU+gX0AuWdesi/s99sLe3V1n3+PCJGDUmDBcunEdZaSlcXFvDysoKK5d/h4aNGjFAIqoCA6R/6evrQ19fvQm1mpSl2ieVQWEOkq2lMdwaW+HYP/eFHp6qmBg+fh+tzLQLdpNv56O47BEaWtVRynvF+nHag4JSreqm/44OHb3ww8oVuKNi2P72v3sg2draCWmnT59C395vPB6S+3MfmjRtWm39xsbGaN++g/D85MkkZGVl4Z13R4n0Cuh54zJ/8b2wQ2xLliyBk5MTrl27hg8++ABt27aFj48PIiMjUV7+v2GTiooKLF26FD179oSbmxv8/PywbNkyPHqk+A/jkSNHMGzYMHTo0AFt27aFv78/Fi1aJORXnlfUo0cPpKSk4MSJE3BycoKTkxOCg4NVlg0LC4O/v7/K1xEYGIjQ0FDhuVQqxerVqxEQEAA3Nzd06dIFc+bMQWFh9T0dpB6JBJgzxAP6Egmi/90DqY6RvsrVZHoSIPT1FgCAU9eytTpfSfkj7Dx9Gw3qmqB3G8W9a4K7NgcA7LuQoVXd9N/R780BsLCwwObNMSgoKBDS7969i99/+xWtWjni1ZYtAQBnTp9Gv4BeMDM3R/yf+9CseXONzlVSUoJpH0+GsbExPpw8RdTXQc9RTYbXGCCp9ML3IH3wwQdo0qQJpkyZgpMnT2L16tUoKirC559/DgCYOXMmtm/fjj59+qB9+/ZISkpCdHQ07t69i7lz5wIA/vnnH4wdOxbt2rXD5MmToaenhxs3buDkyZNVnvfTTz/Fl19+CRMTE4wbNw4AUL++6rkkAQEBiIiIwKVLl+Dq6iqkX758GWlpaQoB0owZM/D7779j0KBBeOedd3Djxg1s3LgRqampWLt2LSN5DZga62PHJ68j/sxd3HxQCEsTQ/Tv6IA2Ta0xP+4ijl65DwBoYWeOnz96DTtO3UZaRgFyi8rQ0KoOBnRwQMuGFtiaeAMnUh8o1D3IqzEc/t0Fu565EQwN9DApwAkAkP6gSGFDyPlxl/Casx2WjuqINX+lIf1BEXq4NYSfe0NsO3YTSVe1C77ov8Pa2hrzvl6Aie+Pw+tdfBAS+i7KysqwauVylJWVYeHiaACPd9vuF9gLOTk5GB8+EccSj+JY4lGFut4cMBBmZmYAgEsXLyJszLsICOwDewcHZGZkIGbDely9moYVq36Ek7PzM3+tVIv4z4eoXvgAqVmzZliyZAkAYMSIETA2NsZPP/2E0aNHo7CwENu3b8fQoUPxxRdfCGUsLCywZcsWjBw5Es7Ozjhy5AiMjY2xZs0atYfG/Pz8sGTJElhaWqJ///5PLWtoaIhdu3YpBEi7du2CgYEB3njjDQBAUlISYmNjER0drdDj5O7ujsmTJ+PQoUPo2rWrRtfnv6y8QopL6XkY0NEBdnVNUFz2CGdv5GB49BEcSM4Uyt3JKcYvx2+iU8v66O35CsxNDPCwuBwXbuVh8c4UbP9beVhjaOem8HW0VUib9ubj9/bolSyFAOlOTjH6RR7AJ/1dEeTTFBZ1DHEjqxCzfz6PlftSa+nV08tm9Jgw1KtXH98sXIDZn/8f9PT00MnbB2vXx8DHtzMA4Pr1a3jw4HEw/+WcL1TWk9z5qhAg1atfH/b2Dljz4w/IysxE3bp14dv5Nfywdj06dvR6Ni+MSEe98AHS8OHDFZ6PGDECsbGxOHz4MPLz8wEA776ruD9IaGgotmzZgoMHD8LZ2RmWlpYoLi7GoUOH8Prrr4veRgsLC3Tp0gXx8fH4+OOPhfT4+Hj4+PjA2tpaeG5lZYWOHTsiO/t/vQodOnSAvr4+Tpw4wQBJA+WPZAj/Memp5XIKyzBzyzmN6h78jWZ7F6VnF2Himqe3hag6Awa+hQED36oyv2u311FUpv6mpg0aNMCWn7kS97+Ac5DE98IHSM2aNVP5/Pbt28jPz4eBgQGaNFG8pUTTpk1hYGAgTG4MDAzEtm3bMHbsWNja2sLX1xdvvPEG/Pz8RPtgBAYGYurUqTh//jzc3d2RkpKCa9eu4b333hPK3LhxA7m5ufDx8VFZx5NBExERkboYIInvhQ+QxGBiYoKYmBgcP34cBw4cwKFDhxAXF4fOnTtj1apVoqxI69GjB4yNjbFr1y64u7tj165dMDQ0FIbXgMcTtG1tbREZGamyDjs7O5XpRERE1anJXGvGR6q98AHS9evXFe5sff36dQDAK6+8AgsLC1RUVODmzZsKPU03b95ERUWFwr4genp68PHxgY+PDyIiIrBq1SpERUXhxIkTVfboaBJVm5ub47XXXkN8fDymTZuG+Ph4dOnSRWGPkSZNmuD48ePo0KED91AiIiJ6gb2wy/zlNm3apPA8JiYGEokEr732Grp16wYAWLdunUKZ9evXA4CQn5OjuMEaALi4uAB4fDPHqtSpU0eY56SOwMBA3L59Gz/99BOuX7+OgIAAhXx/f3+Ul5dj5cqVSseWlZUpLO8lIiJSF3fSFp9O9CCFh4fD19cXJ0+exI4dOxAUFITGjRsDAAYOHIhNmzYhPz8f7dq1w6lTp/DHH3/g7bffhpPT42XZ3333HZKSktC1a1c4ODggOzsbmzZtQsOGDdG+ffsqz926dWts3LgR3333HZo2bQobG5sqe5sAoHv37jAxMUFkZCSMjY3Rs2dPhXxvb28MHjwYS5YswYULF+Dj4wM9PT1cv34du3btQlRUFHx9fUW4akRE9F/CITbxvfABUnR0NBYtWoSoqCgYGxtj1KhR+Oijj4T8uXPnwsHBAbGxsdi9ezfs7OzwwQcfCHsXAY/nB92+fRuxsbHIycmBtbU1vLy8MHHiRFhYWFR57vHjxyM9PR0//PADCgsL4eXlVW2AZGpqim7dumH37t3w8/ODubm5Upk5c+agdevW2Lp1KxYuXAgjIyM4ODhg8ODBcOaeJEREpIXHAZK2k7RFbsxLQiKTyWTPuxGqLFmyBEuXLsXff//NewVVo9OM3UjP5t24STekRg943k0gUosEgImh7kQOPeYfwO0c5XtCqsPe2gT7IrqJ3CLd98L3IBEREVH1OMQmPgZIREREOk4ikUBPj/sgiYkBEhERkY5jD5L4Xthl/hMnTsTly5c5/4iIiIieOfYgERER6TjeakR8DJCIiIh0HIfYxMcAiYiISMexB0l8L+wcJCIiIqLnhT1IREREOo49SOJjgERERKTjOAdJfAyQiIiIdJ72PUiPb6xClXEOEhEREVEl7EEiIiLScRxiEx8DJCIiIh3HSdri4xAbERERUSUMkIiIiHScfIhN24emrl+/jg8//BBdu3aFp6cnAgMDsXLlSpSVlSmUO3XqFIYNG4Y2bdqgc+fOmDt3LoqLi5XqKysrw4IFC9ClSxd4eHhgyJAhSExMVHludeusKQ6xERER6bhnOcSWkZGBwYMHw8LCAiNHjkTdunWRlJSEhQsX4p9//sGCBQsAAMnJyQgNDUXLli0RERGBe/fu4ccff0R6ejqWL1+uUGdERAT+/PNPhISEoGnTpti+fTvee+89bNiwAW3bthXKaVJnTTFAIiIi0nHPcpJ2XFwc8vPzsWnTJrRq1QoAEBQUhNLSUuzcuRPz5s2DoaEhFi1aBCsrK2zYsAFmZmYAAAcHB8ycOROJiYnw8fEBAJw7dw47duzA9OnTERoaCgAYMGAA+vbti6ioKMTExAjnVrdOMXCIjYiIiNRWWFgIAKhXr55Cev369WFgYAB9fX0UFBTg6NGjGDBggBDIAED//v1hamqKXbt2CWnx8fEwNDTE4MGDhTRjY2O8/fbbOHnyJDIzMwFAozrFwACJiIhIxz3uQZJo+Xhcx927d5Genq7wyM/PVzpXx44dAQAzZsxASkoK7t69i99++00YFtPT08Ply5dRUVEBNzc3hWONjIzg4uKC5ORkIS05ORnNmzdXCHoAwMPDAzKZTCirSZ1i4BAbERGRjhNjiG3EiBG4ffu2Qt6ECRMwceJEhbQuXbpg0qRJWLFiBfbt2yekf/DBBwgPDwcAZGVlAQBsbW2Vzmdra4szZ84Iz7OystCgQQOV5QAIPUia1CkGBkhEREQ6r+a3GomJicGjR48UciwtLVUe4eDgAC8vL7zxxhuwsrLCX3/9hSVLlsDGxgbDhg1DSUkJgMe9O5UZGxsL+QBQUlICQ0NDleUAoLS0VCinbp1iYIBEREREaNSokVrlduzYgc8++wzx8fFCz0+vXr0gk8kQGRmJwMBAmJiYAIDSsn/gccAjzwcAExMTlJeXqywH/C9Q0qROMXAOEhERkY57lvsgbdq0Ca1bt1YaFuvRoweKioqQkpIiDIPJh8WelJWVBTs7O+G5ra2tMIxWuRwAoawmdYqBARIREZGO036CtuZDc/fv31caigMg9AI9evQIjo6OMDAwwIULFxTKlJWVITk5GS4uLkKas7Mzrl27JqyOkzt79qyQD0CjOsXAAImIiEjHPcsepObNm+PChQu4efOmQvqOHTugr68PJycnWFhYwMfHB3FxcQqBT1xcHIqKitC7d28hrXfv3igvL8e2bduEtLKyMsTGxqJdu3ZCT5UmdYqBc5CIiIhIbaNHj8bBgwcxbNgwjBgxAnXr1sVff/2FgwcPYujQocL+SJMnT8bQoUMRHByMwYMH4969e1izZg26du0KX19fob42bdqgd+/eiIqKQlZWFpo0aYLt27fjzp07+OqrrxTOrW6dYpDIZDKZqDXSM9Vpxm6kZxc972YQqSU1esDzbgKRWiQATAx15y73g1f9jXv5pVod29DSGNve66jRMefOncOSJUuQnJyM3Nxc2NvbY9CgQRg9ejT09fWFcklJSYiKisKlS5dgbm6OwMBAfPTRRzA1NVWor7S0FIsXL8bvv/+OvLw8ODk54aOPPlIZ9KhbZ00xQNJxDJBIlzBAIl2hawHSkB+SahQgbR3TQeQW6T4OsREREem4Z3kvtv8KTtImIiIiqoQ9SERERDpOm+X6Tx5LyhggERERvQQY54iLARIREZGOYw+S+DgHiYiIiKgS9iARERHpOK5iE1+VAdLSpUs1rkwikSA8PLxGDSIiIiLN6Ekk0NMy0tH2uJcdAyQiIiIdxx4k8VUZIO3du/dZtoOIiIjohVFlgGRvb/8s20FERERaetyDpO0qNpEb85LQapJ2WVkZcnJyYG1tDSMjI7HbRERERBqQSAA9DrGJSqNl/hcvXkRISAjatWuH119/HSdPngQAPHjwAO+88w6OHj1aK40kIiIiepbUDpCSk5MxYsQI3Lp1C/3791fIq1evHkpLS7F9+3bRG0hERETVk28Uqe2DlKk9xPbtt9/Czs4O27dvR2lpKX755ReFfG9vb+zatUv0BhIREVH1JKjBKjZRW/LyULsH6eTJkxg8eDDMzMxURpuvvPIKMjMzRW0cERERPZ2khv+RMrUDpNLSUlhYWFSZX1BQIEqDiIiIiJ43tYfYmjRpgosXL1aZf+zYMbRs2VKURhEREZH69Gqwik3b4152avcg9e3bF3FxcQor1eRDbT/++CMOHTqkNHmbiIiIah8naYtP7R6kUaNG4ciRIxg9ejRatGgBiUSCr776CtnZ2bh//z58fX0xfPjw2mwrERERqcBbjYhP7R4kIyMjrFmzBp988gmMjY1hbGyM69evw9raGlOnTsWKFSugp6fRtkpERERELySNdtI2MDBAaGgoQkNDa6k5REREpCmJRAI9rW81wi4kVbS61QgRERG9ODjEJj6NAqTS0lKsX78eCQkJuHXrFgCgcePG8PPzQ3BwMExMTGqlkURERFS1mky2Zg+SamoHSNnZ2XjnnXfwzz//wNzcHI0bNwYApKWl4ezZs4iLi8P69ethY2NTa40lIiIiehbUDpAiIyORmpqKiIgIDB8+HEZGRgCAsrIybNq0CV9//TUiIyMxf/78WmssERERKeMQm/jUDpD279+Pt99+W2mCtpGREUJDQ/HPP/8gISFB7PYRERHRU+hB+0naerzViEpqr8svKyuDq6trlflubm4oKysTpVFERESkPkkNH6RM7QDJ3d0dly5dqjL/4sWL8PDwEKVRRERERM+T2gFSREQEdu/ejQ0bNqCiokJIr6iowLp167Bnzx5ERETUSiOJiIioGjW5zQgnIalU5RykkJAQpTQrKyvMmzcP0dHRwiq2W7duoaCgAE2aNMH8+fOxbt262mstERERKeHNasVXZYCUnp6uMr1Ro0YAgNzcXACAhYUFLCwsUF5eLuyNRERERM/O444gbfdBErkxL4kqA6R9+/Y9y3YQERERvTB4qxEiIiIdx32QxMcAiYiISMfxViPi0yhAunnzJtauXYuzZ88iPz8fUqlUIV8ikXCzSCIiomeMk7TFp/Yy/8uXL2PgwIHYtm2bMCHb1NQUpaWluH37NvT19YUJ3ERERES6TO0AKTo6GoaGhoiLi8PatWsBAJ9++ikOHz6M2bNnIz8/H5999llttZOIiIiqwn2QRKd2gHTy5EkEBQWhRYsWSuOVQ4YMQdeuXREVFSV6A4mIiKh6vNWI+NQOkAoLC4XNIQ0NDQEARUVFQn67du1w6tQpkZtHRERE9OypPUm7fv36uH//PgDA3NwcderUwfXr14X8/Px8PHr0SPQGEhERUfX0IIGelkNleuxDUkntAMnZ2RkXLlwQnnt5eWH9+vXw8PCAVCrFxo0b4ezsXCuNJCIioqpxHyTxqT3E1q9fP+Tk5KCkpAQAMGnSJDx8+BAhISEIDQ3Fw4cPMXny5FprKBEREamm7QTtmuyf9LJTuwcpMDAQgYGBwnNXV1fs2LEDe/bsgb6+Prp27SrMUSIiIiLSZTXaSbtRo0YICQkRqy1ERESkBQ6xiY+3GiEiItJxEon2k7Q5xKZalQHS9OnTNa5MIpFg3rx5NWoQERERaYY9SOKrMkDavn27xpUxQCIiIvpvOHfuHJYuXYrTp0+joqICjRs3RmhoKN566y2hzN69e7F06VKkpqaiXr16ePvttzFu3DgYGCiGH/n5+ViwYAH27NmDkpISeHh4YPr06XBxcVE6r7p11lSVtaWkpIh6IqodB7/wh+x5N4JITdYdJzzvJhCppUkjG1zeOft5N0NtNVmNps1xBw4cQHh4OLy8vDBp0iQYGBjg+vXruHv3rlIZb29vzJo1C1euXMGyZcuQk5ODWbNmCeWkUinCwsJw5coVjBo1CtbW1ti0aROCg4MRGxuLJk2aaFynGDgHiYiISMfpQYN9e1Qcq4mHDx9i+vTpGDp0KGbOnFllucjISLi6umL16tXQ19cHAJiZmWHlypUIDg5Gs2bNAADx8fE4ffo0li1bBj8/PwBAQEAA/P39sXTpUkRGRmpcpxi0vZ5ERET0gniW+yD9/vvvyM/Px6RJkwAABQUFkMkUxzJSU1ORmpqKoKAgIZABgOHDh0MqleLPP/8U0nbv3g07Ozv07NlTSLOxsUFAQAASEhJQXl6ucZ1iYIBEREREaktMTESLFi1w4MABdOvWDe3bt4eXlxeioqKEW45dunQJAODm5qZwbIMGDdCwYUMhHwCSk5PRunVrpUDN3d0dhYWFuHnzpsZ1ioFDbERERDpOD4CelqvR5D0ld+/eVbqnqqWlJSwtLRXSbty4gXv37iEiIgJjxoyBq6sr9u/fj1WrVqG0tBQzZsxAVlYWAMDW1lbpfLa2tsjMzBSeZ2VlwdvbW6mcnZ0dACAzMxOvvvqqRnWKgQESERGRjpNItA+Q5B03I0aMwO3btxXyJkyYgIkTJyqkFRUVIS8vDx9//DHCwsIAAL169UJRURE2b96M8ePHC7clMzIyUjqfsbExiouLheclJSUqy8nT5HVpUqcYGCARERHpuMf7IGm7iu3x/8bExKjsQarMxMQEANC3b1+F9H79+iE+Ph7nz58XypSVlSkdX1paKuTL61NVTp4mL6tJnWLgHCQiIiJCo0aN4ODgoPBQFSDJh7jq16+vkC5/npeXJ5SRD4s9KSsrSxg+k9enanhMniYvq0mdYtA4QEpPT8e2bdvw/fffIz09HcDjaO7OnTsqozoiIiKqXXqSmj000bp1awBARkaGQvq9e/cAPF6BJt/g8cKFCwplMjIycO/ePYUNIJ2dnXHx4kWllXDnzp2DqampsA+SJnWKQaMAacGCBfD398esWbMQHR2NW7duAXgcIPXp0webNm0StXFERET0dPJbjWj70ETv3r0BAD///LOQJpPJsG3bNpiamsLT0xOtWrVCixYtsGXLFoVhu82bN0NPTw+9evVSqC8zMxN79+4V0rKzsxEfH4+ePXvC0NAQADSqUwxqz0H66aefsHr1agQHB6N79+4YNWqUkGdubo4ePXpg//79CA0NFbWBREREVD29GtysVtPj3NzcMGDAAKxYsQIPHjyAq6srDhw4gMOHD2Pq1KkwNzcHAEybNg3jx4/H6NGjERgYiCtXriAmJgZBQUFo3ry5UJ+/vz88PT0xbdo0YSftzZs3QyqVKk0QV7dOMUhklfu0qvDmm2+iadOmWLJkCXJycuDj44M1a9bAx8cHALBy5Ups3LgRBw8eFLWBVL3SCvBWI6QzeKsR0hW6dquRr/dfRU5xhVbHWtcxwCfdW2h0TFlZGb777jv8+uuvuH//PhwcHBAaGoqhQ4cqlEtISMDSpUuRlpYGGxsbDBo0CO+//77SfdPy8vIQGRmJhIQElJaWwt3dHREREcJwnjZ11pTatV2/fh3Dhg2rMt/a2ho5OTmiNIqIiIjUJ4H2q6606XcyMjLChx9+iA8//LDacn5+fsLtQ6pTt25dfPnll/jyyy+fWlbdOmtK7QDpaXsM3LlzR+VsdyIiIqpd2swlevJYUqZ2wOnh4YE9e/aozCstLUVcXBzatWsnWsOIiIhIPfI5SNo+SJnaAdLo0aNx5swZTJ06FZcvXwYA3L9/H4cOHUJwcDAyMjIUJm4TERER6Sq1h9h8fX3x+eef48svv8Qff/wB4PFscgAwNDTEnDlz0LZt29ppJREREVVJghoMsYnakpeHRlO+g4KC0KNHD8THx+Pq1auQyWRo1qwZAgIC0KBBg9pqIxEREVVDmw0fnzyWlGm8Js7W1hbBwcG10RYiIiKiFwJvVktERKTjJDWYbK3tTW5fdmoHSCEhIU8tI5FIsG7duho1iIiIiDTDZf7iUztAkt+Y9kmPHj1CVlYWpFIprK2tUadOHVEbR0RERE/HOUjiUztA2rdvn8r0srIyrFmzBrGxsdiwYYNoDSMiIiJ6XrTdmVxgZGSEsWPHwsPDA/PnzxejTURERKQBSQ3/I2U1DpDk2rdvj8OHD4tVHREREalJPsSm7YOUibaKLT09HeXl5WJVR0RERGqS1CDQ4SRt1dQOkO7cuaMyPS8vD0ePHsWGDRvg5eUlWsOIiIiInhe1A6QePXpUuVeCTCZD8+bNMXPmTNEaRkREROqRSCRa72fEfZBUUztACg8PV3kRrays0KxZM/j6+kJPT7QpTURERKQmPdRgmb+oLXl5qB0gTZw4sTbbQURERFriRpHiUytwLCwshJ+fH9auXVvLzSEiIiJ6/tTqQTIzM0Nubi7MzMxquz1ERESkIb0a3ItN2+NedmoPPbZp0wbnz5+vzbYQERGRFiQ12AOJ8ZFqagdIU6ZMQXx8PH755RfIZLLabBMRERFpQD4HSdsHKat2iO3OnTuwsbGBiYkJvvrqK1haWmLmzJlYsGABmjRpAhMTE4XyEokE69atq9UGExEREdW2agOknj17YsGCBejbty/S09MBAI0aNQIA3L9/v/ZbR0RERE+lBwn0tLynmrbHveyqDZBkMpkwnLZv375n0iAiIiLSDJf5i0+0e7ERERHR81GTm87yZrWqcQNNIiIiokqe2oOUlJSER48eqV3hgAEDatQgIiIi0szjZf7a3otN5Ma8JJ4aIG3duhVbt259akUymQwSiYQBEhER0TMmQQ3mIInakpfHUwOkIUOGwNPT81m0hYiIiLTAnbTF99QAqUOHDujXr9+zaAsRERHRC4Gr2IiIiHQcl/mLjwESERGRjtOD9svSuZxdNV4XIiIiokqq7UFKSUl5Vu0gIiIibUkkkHCMTVQcYiMiItJxEmi/XJ/hkWoMkIiIiHQcl/mLj3OQiIiIiCphDxIREZGO4xCb+BggERER6TjugyQ+BkhEREQ6TlKDVWxar357yXEOEhEREVEl7EEiIiLScRJo3+PB/iPV4g0bxgAAIABJREFUGCARERHpOA6xiY8BEhERkY7jKjbxcQ4SERERUSXsQSIiItJxEtRgiI19SCoxQCIiItJxetB+SIhDSaoxQCIiItJxnKQtPgaOREREpLVVq1bByckJ/fv3V8o7deoUhg0bhjZt2qBz586YO3cuiouLlcqVlZVhwYIF6NKlCzw8PDBkyBAkJiaqPJ+6ddYUAyQiIiIdJ6nhQ1tZWVn4/vvvYWpqqpSXnJyM0NBQlJaWIiIiAm+//Ta2bNmCyZMnK5WNiIjAunXr8Oabb2LGjBnQ09PDe++9h9OnT2tdZ01xiI2IiEjX1eBebDWJkBYuXAg3NzfIZDLk5+cr5C1atAhWVlbYsGEDzMzMAAAODg6YOXMmEhMT4ePjAwA4d+4cduzYgenTpyM0NBQAMGDAAPTt2xdRUVGIiYnRuE4xsAeJiIhIxz2epC3R8qGdc+fO4bfffsP06dOV8goKCnD06FEMGDBACGQAoH///jA1NcWuXbuEtPj4eBgaGmLw4MFCmrGxMd5++22cPHkSmZmZGtcpBgZIREREpBGZTIY5c+ZgwIABcHFxUcq/fPkyKioq4ObmppBuZGQEFxcXJCcnC2nJyclo3ry5QtADAB4eHpDJZEJZTeoUA4fYiIiIdJykBkNs8uPu3r2LR48eKeRZWlrC0tJS6Zhff/0VqampWLZsmco6s7KyAAC2trZKeba2tjhz5oxC2QYNGqgsB0DoQdKkTjEwQCIiItJxkn//0/ZYABgxYgRu376tkDdhwgRMnDhRIa2goAALFy5EWFgY7OzsVNZZUlIC4HHvTmXGxsZCvrysoaGhynIAUFpaqnGdYmCAREREpOPE6EGKiYlR2YNU2ffffw9DQ0O8++67VdZpYmIC4PHy/cpKS0uFfHnZ8vJyleWA/wVKmtQpBgZIREREhEaNGj21TGZmJtatW4dJkybh/v37QnppaSnKy8uRnp4OCwsLYRhMPiz2pKysLIWeJ1tbW2EYrXI5AEJZTeoUAydpExER6TjtV7A9fqjrwYMHKC8vR1RUFHr27Ck8zp49i7S0NPTs2ROrVq2Co6MjDAwMcOHCBYXjy8rKkJycrDCx29nZGdeuXUNhYaFC2bNnzwr5ADSqUwzsQSIiItJ1z2gfJAcHB5UTsxcvXoyioiJ8+umnaNasGSwsLODj44O4uDiMHTtWWKEWFxeHoqIi9O7dWzi2d+/e+PHHH7Ft2zZhH6SysjLExsaiXbt2wgRuTeoUAwMkIiIiUouFhQX8/PyU0tetWwd9fX2FvMmTJ2Po0KEIDg7G4MGDce/ePaxZswZdu3b9//buPS7G9P8f+KvziVRbzspxJqF0EKVQalNEcqawDrssclqrPtb3a7Esm2PZZWOdc2jVJ2G0rP06L8sHy6ey61BSrEqJUKa5f3/0a9Y0Ex0GjX09PebxMNf9vq/7uqfB23W97/uGu7u7PM7BwQF9+vRBZGQkcnJyYG1tjYSEBGRnZ2Pp0qUKx6lqn+rAJTYiIiINV16kXdPXm9ChQwds3rwZ+vr6WLp0KeLi4jB06FCsWbNGKXb58uUIDQ1FYmIiFi9eDKlUiu+//x7Ozs417rO2tARBENTeK701xVKAP0BlRnqq/8SbmJggt+BJpft9v/47TJ/2KQAg814OLC0tFbY/efIE30avxd7du5CRkQ4DAwO0bSfC+AkfI2T0GD4V+zXMu0x910N4az4b9yEcbZvDsb01WjW3REZ2Hmz7/q/KWO+utgjy6Qyn9i3QoW1TGBro4cMJa3Dy4p9Ksckx09HDpV2lx/351zT0mxwNANDV1cbKuUPhbGcN6yYWqG9igHs5j3DhWgYiNx/Blet3Ffb1cG6LQb5O8HBqC+sm5ih+IcWNjAdYv+cE9h6+WItPQ/NYN7HA9UML3/UwquzkjYd4/kJWo30N9bTh2dZCzSPSfFxio/dWdw9PjJ/wsUKbrop7bZTLzs7G/HnhqFevHp48UU6iZDIZBvTzx69nzyAkdAwmT5mGp0+fYu+eXfh4wkdIS0vFV0uXqf08SDMtmtYfeQVFuJyWiQb1jV4ZOzzABcP8XfDfG/eQdvs+Otu2qDR22cZkbE44o9Q++EMn9O3ZCYdOXJW36evqwsnOGr9euYVdB8/j8dNitGhsjtH9u+HE9s/Qf8q3OP7bH/L4r8IGoFkjM+z/5Xdc+zMbJkb6GPyhE7Yu/Qg9u4gwZdGuGnwS9DZoa5W9arovKeMMkobjDJJqRnpaCAkdg5gftlR5n6GDByIzIwPt7TpgV+wOpRmkX8+ehVcPd0wNm4FvVqySt5eUlMChoy3yHz7E/dwCdZ7Ge+efNIPUstkHSM/KAwBciPsX6hkbVDqD1NSqAXILilDyQooZob2xdNbASmeQKnM5/gvYNP0ArT+ch/zCp6+MbWxpij8OLcLP59IwcNp38nYP57Y4c+kmZLK//1bR0tJCckwYPJ3bwXnwV0i5ea/KY9JkmjaDdPpm7WaQurfhDFJFrEGi91pJSYnK2aCKEv+dgINJ+7F23Xro6OiojHn8uOxJ1U2aNlVo19fXxwcfWMK4wnOE6J+tPDmqiuycRyh5Ia3xsbo7toG4VWPs/+XKa5MjAHjw8DGel7yAuamxQvupizcUkiOg7JlbCUfLHuHQoa3id5/qDq1a/iJlTJDovZUQ/yMsTI1hZV4f1k0bYub0aXj06JFSXGFhIWZOn4oJEz9BF1fXSvtz6eIKMzMzrIpcjn0/xuHOnTu4npaG+fMicOk/F/HF/AVv8GyIKjcmyA0AVC69AYC2thY+MDNBow/qw9nOGluXjEV9E0McPvXfKvXfrKEZAOCvvEL1DJjUTgu1KNJ+14Ovo1iDRO8lly6uCB48BG3atMXjwkIcPnwI67+NxqkTx/HLyTOoV6+ePPaLiLkQZDIs/GrpK3oEzM3NEZewH59+MgEhI4bK2+vXr49de/eh/4CgN3Y+RJWpb2KIYF9H3L6bi/87/4fKGNtWjXHxx3ny9wWPn2L5pmR888NPr+2/iVUDjBvUHbcyc3Dm8k21jZvUSx3PYiNFTJBqKDQ0FACwffv2dzwSUuXkmXMK70eFjkanTvb43/nzsC5qDeZGlP1jceb0aWyM2YDN23aiQYMGr+23nkk92HXoiL79+qObmzvyHz7Ehu/WYWzoSMTFJ6K3j+8bOR+iygzt4wwTIwNsTaw82UnPykPApCjo6+mgTQsrjAjoAtN6RjDQ08XTUuXnWpUzMtTDnhUTUc/IAENmbIBUWrMaFyJNxCW2V8jJyUFUVBRSU1Pf9VBIDWbOngN9fX1IDh0EUFafNPXTj+Hd2wfDho947f7Xrl6FVw939O7ti6XLvsGAoIEYO248fj5+Co0aN8ankyYqPeiR6E0bG+QGqbQU2xN/rTTm6fMS/HLuOpJPpeDbXcfR5+O16N3NFrsiJ1S6j4G+Lvau/BhOdtb4eMEOnL7E2aO6TEvr7yvZqvvi3UlUY4L0Crm5uYiOjlaZIG3atAmbNm16B6OimtLT00OTpk2Rl1f2gMX1367D9bQ0hM2YhZs3bshfj588BgCk376N27duyfePWrMKz58/R/DgIQr9Ghsbo49/X9zJyEBGevpbOx+iDm2bwqVjS/x0JhXZOcr1dZUpelaCxGOX8WF3O7Rqbqm0vTw58u4qxuSFsdh96Dd1DpveABZpqx+X2GpIX1//XQ+Bqun58+fIunsXrl27AQDu3MmQ39tIFU93V4UbS2ZnZwGAylmiUmnZFUhSac2vRCKqro8GlhVnb6mkOPtVjAzK7glmYWqM2y+1lydHPm62mLJ4F7bvr3xmiuqO2twRmzNIqr3TGaSoqCiIxWJkZmbi888/h7OzM5ydnREREYFnz54pxO7btw8DBw6Evb09unbtirlz5yI3N1chRiaTISoqCh4eHnBwcEBoaChu3LgBb29vhIeHy+MKCgqwbNkyBAYGwtHREU5OTpgwYQLS0tLkMefOnUNQUFnRbUREBMRiMcRiMeLj4wGU1SCV1yHl5uaiffv2+O6771DRlStXIBaLkZiYKG+7d+8ePv/8c7i5uaFjx44IDAzEgQMHavlpUrm8PNWXV3/5v/MhlUoR0C8QADB6zEfYuTtO6dWjZy8AwIaYH/DD1h3y/W3b2wEAtm/botBvQUEBkpISYW5ujjZt26r/hIhU0NfTxfAAV9zPLcShk9dUxlia11N5d/dGH9RHsK8THhc9R8qtv+9rpK+niz0rJ8LHzRbTvtqDLQln39j4ieq6OjGDFBYWhhYtWmD27NlISUlBXFwcLCwsMGfOHABAdHQ01q1bh759+2Lo0KHIycnBtm3bcPXqVcTHx8PQ0BAAsGLFCmzcuBHe3t7w8PBAWloaxo8fj+LiYoXjZWZm4ujRo+jTpw+aN2+O3Nxc7NmzByEhITh48CAaNWqENm3aYObMmVi1ahWGDRsmfx6Mk5OT0vgtLS3h4uICiUSCyZMnK2yTSCQwMDBA7969AQAPHjzA0KFDoaenh9GjR6NBgwb4+eefMXv2bJSUlCA4OFjtn+8/zddLFuP8uV/Rs5cXWrSwxpOiJ0iWHMLx//sFXVy74tMp0wAA9g4OsHdwUNpfcrAsWQ3oF6hwo8ipYTMQu2Mb5v8rHP+9ehVu7t2Rn/8QP2yKwf1797B67bpK76FE/zwj+naBdZOym+9ZmteDvp4u5k7wAwDcufcQuw7+vWzVsV1T9O3ZCQDg1rk1AGBkvy5wdyz7/Xe7j6PwyXOF/vt72eMDMxOs2HwEpaWqi6eH+7tg6igv7D92BenZeSh5UYp2Ng0xql9XmJsaYfLCWDx7/kIev2XJGPh174Cff03D0+clGB7QRaG/a39m4dqf2bX5WOgN0ULNL9fnBJJqdSJB6tSpExYu/PuOpQUFBfjxxx8xZ84c3L17F99++y3mzJmDcePGyWN69OiB4cOHIyEhASNGjEBubi62bNkCPz8/rF27Vh4XHR2NqKgoheOJxWIkJydDW/vvCbQBAwbA398fP/74I6ZMmQJLS0v07NkTq1atQufOnTFgwIBXnkNAQAAWLFiAW7duoXXrsr/UBEFAcnIyevToIb+sfPXq1dDW1sa///1vmJqaAgBGjhyJCRMmYOXKlQgKClIYF1Vfj569kJaagh3bt+JhXh50dHTQtm07fLnoK4TNmCVPqKvLxsYGJ8+cx5LFC/HLLz8jbu9uGBkZwd6hM75evgJBA5nc0t/GBrkrPTNtwZSy2csTF/5USJA627aQb3t5/3K7Dv6mlCCV3/toy78rn+U5fekmnOys4d+jIxpbNoC+ng4e5D3GL+fSsG7X/+HXK7cV4p3srAEAvbvZonc3W6X+Fq8/xASpjtLW0oJ2DdfKarrf+65OJEjDhw9XeO/i4oIjR47gyZMnOHr0KARBgK+vLx4+fCiPsba2hpWVFc6fP48RI0bg7NmzkEqlGDlypEJfISEhSgnSy/VDpaWlKCwshLGxMVq1aoWUlJQanYOfnx8WLVoEiUSCKVOmAAAuX76M7Oxs+UyYIAg4cuQI+vXrB6lUqnA+np6eOHnyJG7fvo02bdrUaAxUJrD/AAT2f3VC+yoxP2yp9BElrdu0wcbNW2vcN/1z+E2s+tPFdySdw46kc68PfEngp+teG3MpNRPjvthW5T4rexQK1X2cQVK/OpEgNWnSROF9+czKo0ePkJ6eDplMBh8fH5X7licZ2dll/6uxsbFR2G5mZqZ0fxuZTIZt27YhNjYWd+/eVSi6NTMzq9E5WFhYoGvXrgoJkkQigZGREXr16iUfa2FhIWJjYxEbG6uyn/z8/Bodn4iIiNSnTiRIldVtCIIAmUwGHR0dxMTEqCw2LE+mqmP9+vVYs2YNBg0ahOnTp6NBgwbQ1tbGkiVLUJtn9/r7+2P+/Pm4ceMG2rRpg+TkZPTs2RPGxmXPO5LJyuoEgoODERgYqLKPdu3aqWwnIiKqFKeQ1K5OJEivYm1tjdLSUtjY2KB58+aVxjX9/w8QzcjIUJiRys/PV3r+VnJyMrp27YolS5YotBcWFsLc3Fz+XlVC9iq+vr748ssvIZFI0L17d9y/fx8BAQHy7RYWFjAxMYEgCHB3d39FT0RERNXD+xmpV52vBvb19YW2tjbWrVNeb5fJZCgoKAAAuLm5QVdXV2npaufOnUr76ejoKM0USSQS/PXXXwptRkZGAMoSp6owNzdHt27dIJFIIJFIYGxsjJ49eyoc19fXF4cOHcKtl25AWO7lmiQiIqKqqvGDankn7UrV+RkkGxsbhIWFYfXq1cjMzISXlxeMjIyQmZmJ5ORkTJ48GUOGDIGlpSVGjx6NH374AZ9++im6d++O69ev48SJEzA3N1eYDerVqxfWrVuHiIgIODo64o8//kBSUhJatGihcOxmzZrBzMwMu3fvhomJCYyNjWFvb68U9zJ/f3/MmzcP9+/fh7e3t9IVU7Nnz8a5c+cwaNAgDBs2DK1bt0Z+fj6uXr2KlJQUHDt2TL0fIBEREVVbnU+QAGDy5MmwsbHBtm3bEBUVBS0tLTRt2hQ+Pj4KS1WfffYZDA0NERcXh9OnT6Nz587YtGkTRo4cqXDl2qRJk/Ds2TMkJSXh0KFDsLOzw4YNG7BixQqF4+rq6mLZsmWIjIzEggULIJVKsXTp0lcmSL6+vliwYAGKiorg7698h+aGDRsiLi4O0dHRkEgkyMvLg5mZGcRiMaZPn66GT4uIiP5pWIKkflpCbaqSNUBhYSG6dOmCGTNmKN3E8X1QLAXe6x8gvVfMu0x910MgqhLrJha4fmjh6wPriCuZhSiR1uxfA31dLTi0qP4FT+87jZhBqqrnz58rLWlt3Vp2zxpXV9d3MSQiIqI3rjaPnGVxt2rvVYKUlJSE/fv3o2fPnjAyMsLFixdx8OBBeHh4yB8VQkRERPQ671WCZGtriwMHDiAmJgZFRUWwtLTEmDFjMGPGjHc9NCIiojemNlej8So21d6rBKlTp07yJTUiIqJ/ChZpq1+dvw8SERER0dv2Xs0gERER/WNxKkitmCARERFpOF7Fpn5MkIiIiDQci7TVjzVIRERERBVwBomIiOg9wIkg9WKCREREpOl4nb/aMUEiIiLScCzSVj/WIBERERFVwBkkIiIiDcer2NSPCRIREZGGYwmS+jFBIiIi0nTMkNSONUhEREREFXAGiYiISMPxKjb1Y4JERESk4VikrX5MkIiIiDQcS5DUjzVIRERERBVwBomIiEjTcQpJ7ZggERERaTgWaasfEyQiIiJNV4sibeZHqrEGiYiIiKgCziARERFpOJYgqR8TJCIiovcBMx21YoJERESk4VikrX6sQSIiIqIq+/333/Hll18iICAAnTt3Rq9evTBz5kxkZGQoxf7nP//BiBEj4ODggO7du2Px4sV49uyZUlxJSQm++eYbeHh4wN7eHkOHDsXZs2dVHr+qfdYWEyQiIiINV/6okZq+qmPjxo04cuQI3N3dMW/ePAwdOhTnz59HUFAQbt68KY9LTU3F2LFjUVxcjPDwcAwePBh79uzBzJkzlfoMDw/H1q1b0b9/f8ybNw/a2tqYOHEiLl26pBBXnT5ri0tsREREGu5tFmmPHTsWkZGR0NfXl7cFBAQgMDAQMTEx+PrrrwEAK1euhJmZGbZv3w4TExMAQPPmzfHFF1/g7NmzcHNzA1A2I3Xw4EFERERg7NixAICgoCD069cPkZGR2Llzp/w4Ve1THTiDRERERFXm5OSkkBwBQMuWLdGuXTv5DNKTJ09w5swZBAUFyRMZABgwYACMjY0hkUjkbYcPH4aenh6GDBkibzMwMMDgwYNx8eJFPHjwoNp9qgNnkIiIiDSdGqaQ7t27h9LSUoVNpqamMDU1fW0XgiAgNzcXtra2AIDr169DKpWiY8eOCnH6+vpo3749UlNT5W2pqalo1aqVQtIDAPb29hAEAampqWjYsGG1+lQHJkhEREQaTh1XsY0aNQpZWVkK26ZOnYpp06a9to/9+/fjr7/+ktcC5eTkAACsrKyUYq2srHD58mX5+5ycHDRq1EhlHAD5DFJ1+lQHJkhEREQaribF1i/vCwA7d+5UOYP0Ojdv3sTChQvh7OyMAQMGAACeP38OAEpLcUDZ8ln59vJYPT09lXEAUFxcXO0+1YEJEhEREaFJkybV3icnJweffPIJGjRogDVr1kBbu6y02dDQEEDZ5fsVFRcXy7eXx7548UJlHPB3olSdPtWBCRIREZGGexePGnn8+DEmTpyIx48fY9euXQpLX+W/L18We1lOTg4aNmyoEFu+jFYxDoA8tjp9qgOvYiMiItJ0WrV8VVNxcTEmTZqE9PR0bNiwAa1bt1bYLhKJoKuri2vXrim0l5SUIDU1Fe3bt5e32dra4vbt2ygqKlKIvXLlinx7dftUByZIRERE7wGtGv6qrtLSUsyYMQOXL1/GmjVr0LlzZ6WY+vXrw83NDYmJiQqJT2JiIp4+fYo+ffrI2/r06YMXL14gLi5O3lZSUoL4+Hg4OTnJC7ir06c6cImNiIiIquzrr7/GsWPH4OXlhYKCAiQmJsq3mZiYwMfHBwAwc+ZMDB8+HKGhoRgyZAju37+PzZs3o0ePHnB3d5fv4+DggD59+iAyMhI5OTmwtrZGQkICsrOzsXTpUoVjV7VPddASBEFQa4/0VhVLAf4ASVOYd5n6rodAVCXWTSxw/dDCdz2MKssqKEaprGb76mgDzcwMqhwfGhqK8+fPq9zWrFkzHDt2TP7+woULiIyMREpKCurVq4eAgADMmjULxsbGCvsVFxdj9erVSEpKwqNHjyAWizFr1iyVSU9V+6wtJkgajgkSaRImSKQpNC1Byq5lgtS0GgnSPwWX2IiIiDScOu6DRIpYpE1ERERUAWeQiIiINB6ngdSNCRIREZGG4xKb+jFBIiIi0nDv4k7a7zvWIBERERFVwBkkIiIiDaeFWiyxqXUk7w8mSERERBquZg8NKd+XVGGCREREpOlqk+UwQ1KJNUhEREREFXAGiYiI6D3AiSD1YoJERESk4bS0anGZPzMrlZggERERaTgWaasfa5CIiIiIKuAMEhERkabjVWxqxwSJiIhIw/FRI+rHJTYiIiKiCjiDREREpOF4FZv6MUEiIiLScLyKTf2YIBEREWk4ziCpH2uQiIiIiCpggkRERERUAZfYiIiINByX2NSPCRIREZHGq3mRNqnGJTYiIiKiCjiDREREpOFqs0zGJTbVmCARERFpOD6KTf2YIBEREb0PmOmoFWuQiIiIiCrgDBIREZGGq801bJx4Uo0JEhERkYarVZG2+obxXmGCREREpOFYpK1+rEEiIiIiqoAzSERERJqO00BqxwSJiIhIw7FIW/2YIL0H+OUmTWHdxOJdD4GoSpo1NHvXQ6iW2jysllTTEgRBeNeDICIiIqpLWKRNREREVAETJCIiIqIKmCARERERVcAEiYiIiKgCJkhEREREFTBBIiIiIqqACRIRERFRBUyQiIiIiCpggkRERERUARMkIgDx8fEQi8W4e/euWmOJ6qLQ0FCEhoa+62EQ1WlMkIgqsWvXLsTHx7/rYRDVSE5ODqKiopCamvquh0KkkfgsNiIApaWlkEql0NfXh5ZW2SMfBwwYAFNTU2zfvv21sUR1TWpqKoKCgrB06VIEBwcrbCspKQEA6Ovrv4uhEWkE3Xc9AKK6QEdHBzo6OmqPJaqLmBgRvR6X2KhOioqKglgsxu3btxEWFgZHR0e4ublh+fLlePHihTxOKpUiOjoavXv3RseOHeHj44N169ahtLRUob/Tp09jxIgRcHFxgaOjI/z8/LBy5Ur59op1Rd7e3khLS8P58+chFoshFovlNRsVYz/++GP4+fmpPI+AgACMHTtW/l4mk2HTpk3w9/dHx44d4eHhgUWLFqGoqEgtnxu9PeXf0czMTHz++edwdnaGs7MzIiIi8OzZM4XYffv2YeDAgbC3t0fXrl0xd+5c5ObmKsTIZDJERUXBw8MDDg4OCA0NxY0bN+Dt7Y3w8HB5XEFBAZYtW4bAwEA4OjrCyckJEyZMQFpamjzm3LlzCAoKAgBERETIv8PlS8Yv1yDl5uaiffv2+O6775TO8cqVKxCLxUhMTJS33bt3D59//jnc3NzQsWNHBAYG4sCBA7X8NInqHs4gUZ0WFhYGa2trfPbZZ7h48SI2bdqEp0+fYsGCBQCAL774AgkJCejbty+cnZ1x4cIFrF27Fvfu3cPixYsBAH/++Sc++eQTODk5YebMmdDW1kZGRgYuXrxY6XH/9a9/4auvvoKhoSEmTZoEALC0tFQZ6+/vj/DwcKSkpMDOzk7efv36ddy8eVMhQZo3bx6SkpIwaNAgjBkzBhkZGdixYwdu3LiBLVu2cMlOA4WFhaFFixaYPXs2UlJSEBcXBwsLC8yZMwcAEB0djXXr1qFv374YOnQocnJysG3bNly9ehXx8fEwNDQEAKxYsQIbN26Et7c3PDw8kJaWhvHjx6O4uFjheJmZmTh69Cj69OmD5s2bIzc3F3v27EFISAgOHjyIRo0aoU2bNpg5cyZWrVqFYcOGwdnZGQDg5OSkNH5LS0u4uLhAIpFg8uTJCtskEgkMDAzQu3dvAMCDBw8wdOhQ6OnpYfTo0WjQoAF+/vlnzJ49GyUlJUpLeUQaTSCqg9auXSuIRCJh6tSpCu3h4eGCWCwW7ty5I6SmpgoikUj4n//5H4WY+fPnCyKRSEhNTRUEQRA2b94sODk5CVKptNLj7du3TxCJREJmZqa8rX///kJISMhrYwsLC4UOHToIkZGRCnGrVq0S7OzshIcPHwqCIAi//fabIBKJhMOHDyvEHTx4UBCJRMLx48df97FQHVL+HZ0/f75C+5QpUwRXV1dBEAQhMzNTaN++vbBp0yaFmEuXLglisViIjY0VBEEQcnJyBDs7O2HatGkKcVFRUYJIJBLmzp36VT5OAAAQtklEQVQrbysuLhZKS0sV4jIzM4WOHTsK0dHR8raUlBRBJBIJ+/btUxp7SEiIwnc7NjZWEIlEws2bN+VtMplM6NWrlzBlyhR5W0REhNCjRw/h0aNHCv2NHz9e6N69u9K4iDQZl9ioThs5cqTC+1GjRkEQBJw6dQrHjx8HAHz00UcKMeUzNidOnAAAmJqa4tmzZzh58uQbGWP9+vXh4eGBw4cPK7QfPnwYbm5uMDc3l783MzNDly5d8PDhQ/nLxcUFOjo6OH/+/BsZH71Zw4cPV3jv4uKCgoICPHnyBEePHoUgCPD19VX4mVtbW8PKykr+Mz979iykUqnS9z0kJETpePr6+tDWLvuru7S0FPn5+TA2NkarVq2QkpJSo3Pw8/ODjo4OJBKJvO3y5cvIzs5GQEAAAEAQBBw5cgTe3t6QSqUK5+Pp6YmcnBzcvn27Rscnqou4xEZ1WsuWLVW+z8rKQmFhIXR1dWFtba0QY2NjA11dXWRlZQEoqwOKi4vDJ598AisrK7i7u8PX1xc+Pj5qW9IKCAjAnDlzcPXqVXTq1AlpaWm4ffs2Jk6cKI/JyMhAQUEB3NzcVPbx8OFDtYyF3q4mTZoovDc1NQUAPHr0COnp6ZDJZPDx8VG5b/nPPDs7G0DZd/dlZmZmaNCggUKbTCbDtm3bEBsbi7t37yrU25mZmdXoHCwsLNC1a1dIJBJMmTIFQNnympGREXr16iUfa2FhIWJjYxEbG6uyn/z8/Bodn6guYoJE7z1DQ0Ps3LkT586dw/Hjx3Hy5EkkJiaie/fuiImJUcsVad7e3jAwMIBEIkGnTp0gkUigp6cHX19feYxMJoOVlRWWL1+uso+GDRvWehz09lX2/REEATKZDDo6OoiJiVGZjJcnU9Wxfv16rFmzBoMGDcL06dPRoEEDaGtrY8mSJRBqcdcWf39/zJ8/Hzdu3ECbNm2QnJyMnj17wtjYGEDZ9xcAgoODERgYqLKPdu3a1fj4RHUNEySq09LT0xX+h56eng4AaNq0KerXrw+pVIo7d+4ozDTduXMHUqkUzZo1k7dpa2vDzc0Nbm5uCA8PR0xMDCIjI3H+/PlKZ3SqM7tUr149eHp64vDhw/j8889x+PBheHh4KPwDaG1tjXPnzsHFxYWXWf9DWFtbo7S0FDY2NmjevHmlcU2bNgVQNsv48vc9Pz8fjx49UohNTk5G165dsWTJEoX2wsJC+XIuUL3vLwD4+vriyy+/hEQiQffu3XH//n358hpQNstkYmICQRDg7u5erb6JNBFrkKhOqziVv3PnTmhpacHT0xM9e/YEAGzdulUhZtu2bQAg365q2r99+/YAoHSF0MuMjIxQWFhY5bEGBAQgKysLu3fvRnp6Ovz9/RW2+/n54cWLF/j++++V9i0pKcGTJ0+qfCzSDL6+vtDW1sa6deuUtslkMhQUFAAA3NzcoKurq/L7XpGOjo7STJFEIsFff/2l0GZkZAQAVf4Om5ubo1u3bpBIJJBIJDA2Npb/GSo/rq+vLw4dOoRbt24p7c8lYnrfcAaJ6rT09HRMmTIF7u7uuHjxIg4ePIhhw4ahRYsWAICBAwciNjYWhYWFcHJywn/+8x8cOHAAgwcPhlgsBgB8++23uHDhAnr06IHmzZvj4cOHiI2NRePGjeWXP6vSoUMH7NixA99++y1sbGxgYWFR6WwTAHh5ecHQ0BDLly9XuDS6XLdu3TBkyBBERUXh2rVrcHNzg7a2NtLT0yGRSBAZGcn/mb9nbGxsEBYWhtWrVyMzMxNeXl4wMjJCZmYmkpOTMXnyZAwZMgSWlpYYPXo0fvjhB3z66afo3r07rl+/jhMnTsDc3FxhNqhXr15Yt24dIiIi4OjoiD/++ANJSUnyPxPlmjVrBjMzM+zevRsmJiYwNjaGvb29UtzL/P39MW/ePNy/fx/e3t7yWxCUmz17Ns6dO4dBgwZh2LBhaN26NfLz83H16lWkpKTg2LFj6v0Aid4hJkhUp61duxYrV65EZGQkDAwMMG7cOMyaNUu+ffHixWjevDni4+ORnJyMhg0bIiwsTH7vIqCsPigrKwvx8fHIz8+Hubk5XF1dMW3aNNSvX7/SY0+ePBl3797Fxo0bUVRUBFdX11cmSOX/405OToaPjw/q1aunFLNo0SJ06NABe/fuxYoVK6Cvr4/mzZtjyJAhsLW1reGnRHXZ5MmTYWNjg23btiEqKgpaWlpo2rQpfHx8FBLizz77DIaGhoiLi8Pp06fRuXNnbNq0CSNHjlRYkp00aRKePXuGpKQkHDp0CHZ2dtiwYQNWrFihcFxdXV0sW7YMkZGRWLBgAaRSKZYuXfrKBMnX1xcLFixAUVGR0gwoUFYnFxcXh+joaEgkEuTl5cHMzAxisRjTp09Xw6dFVHfwWWxUJ0VFRSE6Ohq//fZbjQpZid4HhYWF6NKlC2bMmKF0E0cierNYg0REVAc8f/5cqa28vs7V1fVtD4foH49LbEREdUBSUhL279+Pnj17wsjISF5z5+Hh8cpaOSJ6M5ggERHVAba2tjhw4ABiYmJQVFQES0tLjBkzBjNmzHjXQyP6R2INEhEREVEFrEEiIiIiqoAJEhEREVEFTJCIiIiIKmCCRPQPcvfuXYjFYkRFRb2yrS4JDw+X3xX9dby9vREaGlrjY4WGhsLb27vG+7+KWCxGeHj4G+mbiNSPV7ERvWHnzp3D6NGjFdqMjY3RqlUrDBgwACEhIZU+Eb6uu3v3LhISEuDj4yN/vh0R0fuACRLRW9KvXz/06NEDgiDgwYMHSEhIwJIlS3Djxg0sWrTonY2rWbNm+P3332uUpGVlZSE6OhrNmjVjgkRE7xUmSERviZ2dHQYMGCB/P3LkSPj7+yMuLg7Tp0+HpaWlyv2ePHmi8rlu6qKlpQUDA4M31j8RkSZiDRLRO1KvXj04OjpCEARkZmYC+LuGJiUlBePHj4ezszP69+8v3yc9PR1z5syBh4cHOnbsCG9vbyxbtgxPnz5V6v/ChQsYPnw47O3t4e7ujoULF6qMe1UNUnJyMkJDQ+Hi4gIHBwf4+flh8eLFKCkpQXx8vHzpMCIiAmKxGGKxWKEGSBAExMbGIjg4GA4ODnB0dERoaCh+/fVXpWMVFxdj2bJl8PDwgL29PQYPHoxTp05V/4Ot4NSpU5gxYwZ69+4Ne3t7uLi4YNy4cTh//nyl+2RmZmLy5MlwdnaGk5MTpkyZIv8Zvaw650dEmoUzSETviCAIyMjIAACYm5vL27OzszFmzBj06dMHH374oTypuXbtGsaMGQNTU1MMGzYMjRo1QlpaGrZv345Lly5h+/bt0NPTAwBcuXIFH330EUxMTDBx4kTUr18fhw4dwty5c6s8vlWrVmH9+vVo27Ytxo4dCysrK9y5cwc//fQTwsLC0KVLF0yaNAnr16/HsGHD5I/DeHkmbM6cOTh48CD8/PwQHByMkpISJCUlYdy4cYiKikLv3r3lsbNmzcLRo0fh5eUFT09P3LlzB9OmTUPz5s1r/iEDSEhIwKNHjxAUFITGjRvjr7/+QlxcHMaOHYtt27bBxcVFIf7p06cIDQ2Fvb09Zs2ahYyMDMTGxuLKlStISEiAlZVVjc6PiDSMQERv1K+//iqIRCIhKipKyMvLE/Ly8oTU1FRh3rx5gkgkEoYOHSqP9fLyEkQikbB3716lfgIDAwU/Pz/h8ePHCu0//fSTIBKJhH379snbhg0bJnTo0EG4deuWvK24uFgYNGiQIBKJhLVr18rbMzMzldquXLkiiEQiITQ0VHj+/LnC8WQymSCTyRTO7eVjVxzX7t27FdpfvHghDBw4UPDy8pL3c/LkSUEkEglz585ViD1y5IggEokEkUik1L8qXl5eQkhIiEJbUVGRUlxOTo7g6uoqTJgwQaE9JCREEIlEwuLFi1Wey/z582t0foIgqDw/Iqq7uMRG9JZERUXBzc0Nbm5uGDBgAPbt2wdvb2+sW7dOIc7MzAzBwcEKbdevX8f169fRr18/lJSU4OHDh/KXs7MzjI2Ncfr0aQBAXl4eLl26BG9vb7Rq1Ureh76+PsaOHVulse7fvx8AMHv2bKX6JC0tLWhpaVWpDxMTE/j4+CiMt7CwEN7e3sjKykJ6ejoA4OjRowCA8ePHK/Th4+OjcA41YWxsLP99UVER8vPzoa2tDQcHB/z+++8q9/n4448V3vv6+qJVq1b4+eefa3R+RKR5uMRG9JYMGzYMffr0gZaWFoyMjNCyZUuYmZkpxbVo0ULpirKbN28CKEuyKrtfUW5uLgDIa2Vat26tFNO2bdsqjTUjIwNaWlqwtbWtUrwqN2/eRFFREdzd3SuNycvLQ6tWrZCZmQltbW20bNlSKaZNmza4fft2jcdx584drFq1CqdOnUJhYaHCNlWJnqmpqcIy2svjOHr0KJ4+fQpjY+NqnR8RaR4mSERviY2NzSv/MS1nZGRU6bZx48bB09NT5TZTU9Maj02Vqs4UVUYQBFhYWGDFihWVxrRr167G/VdFUVERRo0ahWfPnmHMmDEQiUQwMTGBtrY2NmzYUKti6rpwfkT05jBBItIANjY2AABtbe3XJlnlRc23bt1S2nbjxo0qHa9ly5Y4ceIE0tLSYG9vX2ncqxIoGxsbpKenw8HBASYmJq88XosWLSCTyZCenq6UVJTPntXE2bNn8eDBAyxZsgSDBg1S2LZ69WqV+xQWFiInJ0dpFunmzZv44IMP5Et21Tk/ItI8rEEi0gB2dnYQiUTYvXu3ysvNpVIpCgoKAJRdRda5c2ccO3ZMYWmqpKQEW7ZsqdLxAgMDAQArV65ESUmJ0nZBEAD8Xd/z6NEjpZigoCDIZDKsXLlS5THKlwQByK/22rRpk0LM0aNHa7W8Vr5UWT7ecqdOncKVK1cq3e/7779XeH/kyBHcvn0bPj4+8rbqnB8RaR7OIBFpAC0tLSxfvhxjxoxB//79MWjQILRt2xbPnz9HRkYGjhw5glmzZsmLu8PDwxEaGooRI0Zg1KhR8sv8S0tLq3Q8e3t7TJw4ETExMQgODoa/vz+srKxw9+5dJCcnIy4uDqampmjbti1MTEwQGxsLQ0NDmJqawsLCAm5ubujTpw+Cg4OxY8cO/Pe//4WXlxfMzc1x//59XL58GRkZGfKiZ09PT3h5eSEhIQEFBQXw9PREZmYm9uzZA5FIhD/++KNGn5uzszOsrKywbNkyZGVloXHjxkhNTUViYmKl/Zqbm+PIkSN48OABXF1d5Zf5W1paYurUqfK46pwfEWkeJkhEGqJ9+/ZISEjAhg0bcOzYMezevRsmJiZo1qwZBg4cCDc3N3mso6MjNm/ejBUrVuD7779H/fr14efnhxEjRshnh17ns88+g62tLXbs2IGNGzdCEAQ0btwYPXr0gKGhIQDA0NAQq1atwurVq7FkyRKUlJTA1dVVPpalS5eia9eu2Lt3LzZs2IAXL17AysoKdnZ2mD17tsLxVq9ejdWrVyMpKQlnzpyBSCRCVFQUDhw4UOMEydTUFBs3bsQ333yDHTt2QCqVomPHjoiJicGPP/6osl9jY2Ns3boVS5YswYoVKyAIAjw9PREeHo6GDRsqxFbn/IhIs2gJFeeeiYiIiP7hWINEREREVAETJCIiIqIKmCARERERVcAEiYiIiKgCJkhEREREFTBBIiIiIqqACRIRERFRBUyQiIiIiCpggkRERERUARMkIiIiogr+H8y1WlCMWjWGAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFXyQR063Z4M"
      },
      "source": [
        "A useful tool to determine an optimal threshold  is the **Receiver Operating Characteristic curve (ROC)**. It is a plot of the false positive rate (x-axis) versus the true positive rate (y-axis) for a number of different candidate threshold values between 0.0 and 1.0.\n",
        "\n",
        "The **true positive rate** is calculated as the number of true positives divided by the sum of the number of true positives and the number of false negatives. It is also called **hit rate** and describes how good the model is at predicting the positive class when the actual outcome is positive. It is also known as **sensitivity** or **recall**.\n",
        "\n",
        "\\begin{equation}\n",
        "    TPR = \\frac{TP}{TP \\ + \\ FN}\n",
        "\\end{equation}\n",
        "\n",
        "The **false positive rate** is calculated as the number of false positives divided by the sum of the number of false positives and the number of true negatives. It is also called the **false alarm rate** as it summarizes how often a positive class is predicted when the actual outcome is negative.\n",
        "\n",
        "\\begin{equation}\n",
        "    FPR = \\frac{FP}{FP \\ + \\ TN}\n",
        "\\end{equation}\n",
        "\n",
        "The complement of the FPR is the **specificity** and it is calculated as:\n",
        "\n",
        "\\begin{equation}\n",
        "  \\text{Specificity} = 1 - FPR\n",
        "\\end{equation}\n",
        "\n",
        "The Geometric Mean or G-Mean is a metric for imbalanced classification that, if optimized, will seek a balance between the sensitivity and the specificity.\n",
        "\n",
        "\\begin{equation}\n",
        "    \\text{G-Mean} = \\sqrt{\\text{Sensitivity} \\cdot \\text{Specificity}}\n",
        "\\end{equation}\n",
        "\n",
        "One approach to determine the optimized threshold would be to test the model with each threshold and select the one with the largest G-Mean value."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VdRopsSbtvBh"
      },
      "source": [
        "from sklearn.metrics import roc_curve, auc\n",
        "\n",
        "def evaluate_roc(probs, y_true):\n",
        "    fpr, tpr, thresholds = roc_curve(y_true, probs[:, 1])\n",
        "    roc_auc = auc(fpr, tpr)\n",
        "\n",
        "    # calculate the g-mean for each threshold\n",
        "    gmeans = np.sqrt(tpr * (1-fpr))\n",
        "    # locate the index of the largest g-mean\n",
        "    ix = np.argmax(gmeans)\n",
        "    \n",
        "    # title\n",
        "    plt.title(f'Receiver Operating Characteristic\\nAUC = {roc_auc:.2f}, Best G-Mean = {gmeans[ix]:.2f}')\n",
        "    # plot ROC curve\n",
        "    plt.plot([0, 1], [0, 1],'--', label='No skill')\n",
        "    plt.plot(fpr, tpr, marker='.', label='Bert Classifier')\n",
        "    plt.plot(fpr[ix], tpr[ix], marker='o', color='black', label=f'Best threshold = {thresholds[ix]:.2f}')\n",
        "    # show legend\n",
        "    plt.legend(loc = 'lower right')\n",
        "    # axis labels\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    # show plot\n",
        "    plt.show()\n",
        "    return tpr, fpr, thresholds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "ojBpEORfuTfI",
        "outputId": "1798b133-b621-497e-f4f4-7623642b42a2"
      },
      "source": [
        "tpr, fpr, thresholds = evaluate_roc(probs, y_true)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGuCAYAAAA3TXjMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gU5frw8e/upkAqCYQWIKFtQgs99I6gEIrSDhCQJmAH8Yj6w+PRw6ugKChHpdhoR0pCDSgdEUIo0iFUSaElQEgvm+zO+0fcJctuQgKBJcn9uS4vzDPPPHPPzG5y7zP3zKoURVEQQgghhBBC2Iza1gEIIYQQQghR1klSLoQQQgghhI1JUi6EEEIIIYSNSVIuhBBCCCGEjUlSLoQQQgghhI1JUi6EEEIIIYSNSVIuhHhqrF27Fj8/Pw4ePGjrUMR9unfvzqhRo2wdRpGV1Lht5erVq/j5+TF//vxiHffdd9/Fz8+vWMcUorSxs3UAQojH5+DBg4wePdqszcnJCV9fXwYMGEBwcDB2dvJroKhSU1NZsmQJO3bsICoqCoPBgLe3N126dGH8+PFUqlTJ1iE+lPnz59OgQQN69uxp61AKlJGRwapVq9i2bRuXLl0iLS0Nd3d3GjVqxHPPPUf//v1L5ev6559/xs3NjRdeeMHWoVi1du1akpOTGTNmjK1DEaJEUsmXBwlRehmT8qCgIDp37oyiKNy+fZsNGzZw4cIFhg4dyn/+8x9bh2mi1+vJycnB3t4etfrpvJB35coVxo8fz/Xr1+nVqxdt2rTBzs6O48ePs2nTJpydnVmwYAHNmze3dahF5ufnx/PPP8+sWbMslul0OgAcHByedFhmoqOjmThxIlFRUbRv354OHTrg4eHBnTt3OHDgAOHh4YwfP5533nkHyJ0p9/b2ZtmyZTaNuzg8iX1RFAWdTodGoynyB5tRo0Zx7do1du3aZbEsOzsbg8GAo6NjcYUqRKlT+qYShBAWGjZsyIABA0w/jxgxgueee441a9YwdepUPD09bRjdPRqNBo1GY7PtZ2ZmYmdnl28ykpGRweTJk4mPj2fBggV07drVtGzYsGGMGDGCsWPH8sorr7Bp0yabzZg/aD8ehq2Tccjdr0mTJnH16lXmz59Pr169zJZPnDiRkydPcurUKZvEp9fr0el0lC9f3ibbfxSpqam4uLigUqkeS+Jsb29f7GMKUdo8nVNRQojHysnJiaZNm6IoCjExMWbL4uPj+fDDD+natSuNGzemY8eOfPDBB9y5c8dinNTUVObOnctzzz1HkyZNaNOmDcOHD2fz5s0PNeb9NeW///47fn5+LF261Op+DBs2jLZt25KdnW1qi4qK4p///CcdO3akcePGdO/endmzZ5Oenm62rrHGNSEhgffee4/27dvTrFkzbt68me9xCwkJISoqitGjR5sl5EZNmjRh6tSpJCQk8MMPP5jaDx48iJ+fH2vXrmXZsmX07t2bJk2a0Lt373xnPYtrP1asWMG4cePo1KmT6di//fbbXL161TSGsY4YYN26dfj5+Zn+M7JWm21su3z5MhMnTqR58+a0bNmSN954g1u3blns07lz5xg3bhzNmjWjTZs2TJ8+nYSEBPz8/Hj33XfzPe5Ga9as4cqVK4wdO9YiITcKCAhg5MiRFu2FiTEuLo5Zs2YxYMAAWrduTZMmTejTpw+LFi1Cr9eb9TW+VsPDw/nmm2/o2bMnAQEB/PrrrwDs27ePKVOm0KNHDwICAmjVqhXjxo3j0KFDVuOOjo7mvffeo3Pnzqbz9PLLL3P69Gkg9yrGtWvXOHTokNn5yXseT506xauvvkqbNm1o3LgxvXv35rvvviMnJ8dsW6NGjaJ79+7ExsbyxhtvEBgYSMuWLYH8a8rXr1/P4MGDadWqFc2aNaNHjx5MmzaNhIQEIPe1cOjQIa5du2YWn/G9nF9N+a1bt5g5cyY9evSgcePGtGvXjrFjx7J//36rx0mI0kxmyoUoo2JjYwFwd3c3tV2/fp1hw4aRnZ3N4MGDqVWrFtHR0fzyyy8cPHiQ0NBQXF1dAUhOTmbEiBFcvHiR3r17M3z4cAwGA2fPnmX37t307du3yGPer2PHjnh5ebF+/XqL2vioqCiOHz/OqFGjTLNwp0+f5sUXX8TNzY1hw4ZRpUoVzp07x7Jlyzh27BjLli2zmLEbO3YslSpV4pVXXiE9PR0nJ6d8j9nWrVuB3A8D+XnhhRf49NNP2bp1K9OnTzdbtnz5cm7dusWwYcNwcXEhLCyMmTNnkpSUxGuvvWbqV5z78eOPP9KsWTNGjRpFhQoVuHDhAiEhIURERLBp0yY8PDzw9PTks88+45133qFVq1YMHTo03/27X1xcHKNHj6Znz5688847nDt3jlWrVpGamsqPP/5o6hcVFcXIkSMxGAyMGjWKKlWq8PvvvzNhwoRCb6swx/9RYjx//jzbtm3jmWeeoVatWmRnZ/PHH3/wxRdfcPXqVT7++GOLsWfPnk1OTg5Dhw7F2dmZ2rVrA7kfbpKSkhg4cCBVq1YlLi6ONWvWMGbMGJYuXUqrVq1MY5w6dYoxY8aQk5PD4MGDqV+/PklJSRw6dIhjx47RuHFjPvvsMz799FM8PDyYPHmyaV3jVa49e/bw2muv4ePjw7hx43B3d+f48eN8/fXXREZG8vXXX5vFnZaWRnBwMC1atGDKlCmm5Nqa9evXM336dFq1asUbb7xBuXLluHHjBr///jt37tzB09OT999/ny+++IK7d+/y3nvvmdatW7duvuNevXqV4cOHc+fOHQYMGEDjxo3JyMjgxIkThIeH06FDh3zXFaJUUoQQpVZERISi1WqV+fPnK3fu3FHu3LmjnDt3Tvn3v/+taLVaZfDgwWb9J0+erLRt21a5ceOGWfvJkyeVBg0aKF9//bWp7cMPP1S0Wq2ycuVKi+3q9fqHGjM0NFTRarVKRESEqW3WrFmKVqtVLl68aLb+3LlzFa1Wq5w+fdrU1q9fP6V3795KSkqKWd9t27YpWq1WCQ0NNbVNnz5d0Wq1yrRp0ywPXD4CAwOV5s2bP7BfUFCQotVqldTUVEVR7p2HZs2amR2HrKwsZdCgQUrDhg3N2otzP9LS0izawsPDFa1WqyxatMisXavVKtOnT7c6Trdu3ZTg4GCLNq1Wq2zevNms3fj6unz5sqntjTfeULRarXLkyBGzvm+++WaB280rMDBQadGixQP7PWyMGRkZisFgsBjj7bffVvz9/ZW4uDhTm/G12qtXLyU9Pd1iHWvH/datW0pgYKAyYcIEU5vBYFD69u2rNG7cWImMjLRYJ+97ydo5UBRFyczMVNq3b6+MGDFCyc7ONlv2008/WbyngoODFa1Wq3z55ZcWY8XGxipardbsffnqq68qzZs3txj7fsHBwUq3bt2sLjO+TvOaMGGCotVqlb1791r0z7vfQpQVUr4iRBkwf/582rVrR7t27ejfvz//+9//6NWrF99++62pT0pKCnv27KF79+44ODiQkJBg+s/b25tatWqZLikbDAa2bNlC3bp1rc5aGm/SLMqY+Xn++eeB3Nk6I0VR2LhxI1qtlkaNGgG5s5znz58nKCgInU5ntq2WLVvi5ORkdVvjx48v9HFMTU3Nd1Y/LxcXF1P/vPr160fVqlVNPzs4OJhmSI03xxX3fhhnzA0GAykpKaZyEVdXV06ePFm4HS9A5cqV6dOnj1lb27ZtgdySDMittd67dy8BAQGmMgmjcePGFXpbqampODs7P5YYAcqVK4dKpQJyb2xNTEwkISGBjh07YjAYTKUkeQ0fPtxqDXneKy5paWncvXsXtVpN06ZNzY57ZGQkFy9e5IUXXsDf399inMLc8Lx//35u377NCy+8QHJystlrpnPnzqY+9yvsa9/V1ZXMzEz27NmDUkzPhkhMTOSPP/6gU6dOdOrUyWL503qjtxCPk5SvCFEGDBs2jGeffZbs7GwuXLjA999/z82bN81u6Lpy5QoGg4GQkBBCQkKsjlOzZk0A7t69S1JSktU/pnkVZcz8GBPvTZs28dZbb6FWqzl8+DDXrl3jn//8p6nf5cuXgdwPIPk9Y/n27dsWbb6+vgVuPy8XFxeLRNsaYx9jcm5k7VJ+vXr1gHvlRMW9HwcOHODbb7/lxIkTZGVlmS1LSkoqYC8Kx9r5q1ChApCbeAEkJCSQnp5uKu3Iy1pbflxcXEhLS3ssMQLk5OSwaNEiNmzYQHR0tEUCmpycbDFOfvHHxMQwd+5c9u3bZ7GeMfGH3LIeyL0Z+2EZXzPvv/9+vn3uf814enri5uZWqPEnTZrE4cOHefXVV6lQoQKBgYF07tyZ5557zuI1XlgxMTEoivJI+y1EaSNJuRBlgI+PD+3btwegS5cutGzZkhEjRvDhhx8yd+5cAFMC0r9/f9Ps9P2K+lSG4hpzwIABfPLJJ0RERNC+fXvWr1+PRqOhf//+Fn2NNzVaYy0JKcqTMurXr8/hw4eJjo7Gx8fHap+MjAyuXLmCt7f3Q83qGhXHfpw8eZLx48dTq1Ytpk2bRo0aNUyzwVOnTi2WWc+CnpZTXLOqRsbjHxsb+8APc3kVNsZZs2axbNky+vTpw+TJk/H09MTe3p4zZ84wZ84cDAaDxfrlypWzaEtLS2PkyJFkZGTw4osvotVqcXZ2Rq1Ws3DhQiIiIgode2EY9+Gdd96hQYMGVvtUrlzZ7OeivO59fX3ZsmULBw4c4MCBAxw6dIgZM2bw9ddfs2LFCmrVqvXwwQshTCQpF6IMatGiBQMGDGD9+vWMGjWKFi1aUKtWLVQqFdnZ2aYEPj8eHh64u7tz7ty5AvsVZcyC9OvXj88//5z169fTokULtm7dSvv27c0SDWOSrFarH2lbBXnmmWc4fPgwa9as4e2337baZ/369WRnZ1t9OohxRjOvS5cuAfdmc4tzP8LCwtDr9SxevNgsiU1PT7c66/u4eHp64uTkxJUrVyyWWWvLT69evUzH/6233irOEAHYsGEDrVu3Nn1QNcpb4lIYBw4cID4+nk8++YRBgwaZLZs3b57Zz8aZ9sjIyIeIOJfxKkn58uUf22vfwcGBLl260KVLFyD3yUgTJ07kp59+4sMPPyzyeMbfDY+y30KUNlK0JUQZ9corr6DRaExPZfDw8KBLly5s376d48ePW/RXFMX0hAa1Wk3fvn25dOkSa9assdq3qGMWxNPTk06dOrF9+3Y2bdpEamqqxcx7w4YN0Wq1rFy50lQKkldOTo5ZqcLDGDJkCD4+Pvz888/s3bvXYvmZM2f48ssv8fT0tFqvu2nTJrNHLup0On7++Wc0Gg3dunUr9v3Ib4Z44cKFVmd9nZycHvkY5RdHp06dOHnyJH/++afZsrxPP3mQIUOGULt2bX788Ud27Nhhtc/p06dZsWLFQ8WpVqstZvfT09P5+eefizSO8bjfP9a+ffs4ceKEWZu/vz/169cnNDSUixcvWoyVdwxnZ2er56djx45UrFiRxYsXW12emZlZqLKr/Fh7jxrLTvKWQDk7O5OUlFSoKyQVKlSgc+fO7N27l/DwcIvlxX2VRYiSQGbKhSijfHx86NOnD5s2beLIkSO0atWKf//734wYMYLg4GAGDBhAw4YNMRgMxMbGsnPnTgYOHMjrr78OwJQpU4iIiGDGjBns37+fli1boigKkZGR5OTk8PnnnwMUacyCPP/88+zatYtZs2bh6upq8VXwKpWKzz77jBdffJH+/fszaNAg6tWrR2ZmJtHR0Wzfvp233nrrkb6i3MnJie+++44JEyYwadIkevXqRWBgIHZ2dpw8eZINGzbg7OzMN998g5eXl8X6tWvXZsiQIfzjH//A2dmZsLAwTp06xSuvvEK1atWKfT969uzJzz//zEsvvcSwYcOwt7dn//79nD9/Hg8PD4v+zZo148CBAyxatIjq1aujUqlMj7Z8VFOmTGHfvn1MmDCB4OBgqlatyp49e0wJX9466/yUL1+ehQsXMnHiRF599VU6duxI+/btqVChAgkJCRw8eNC0jYfRu3dvVq1axZQpU2jfvj23b98mNDTUVH9eWC1btsTLy4vZs2dz7do1qlatSmRkJBs2bECr1XLhwgVTX5VKxSeffMKYMWMYMmSI6ZGIycnJHD58mE6dOpmeD9+0aVNCQkKYN28edevWRa1W061bN5ycnJg9ezavvvoqzz77LIMGDcLHx4fk5GT++usvtm/fzn//+1/atGnzUMdl/PjxuLq60qpVK6pVq0ZycjLr1q1DpVKZfSlZ06ZN2b17Nx9//DHNmzdHo9HQtm1bKlasaHXcDz74gLNnz/LSSy8xcOBAGjVqRFZWFidOnMDb29vsnhEhygJJyoUow15++WU2b97MV199xbJly6hWrRqhoaEsXryYXbt2sXHjRhwdHalWrRrdunXjueeeM63r7u7OqlWrWLBgAdu3b2fHjh04OztTt25dgoODTf2KMmZBunbtSoUKFUhMTGTIkCFWa9EbNGjAunXrWLhwIbt27WLlypU4Ozvj7e3N888/T7t27R75mNWtW5eNGzeyZMkStm/fzt69e9Hr9VSvXp1Ro0Yxbtw4qwk5QHBwMKmpqSxfvpzr169TvXp13n//fV588cXHsh8tW7Zk/vz5fPvtt3z11Vc4OjrSvn17li9fbnaOjD788EM+/vhjFixYYLqhsriS8jp16rBixQpmz57N0qVLcXR0pGvXrvzrX/+iZ8+ehb5fwcfHh/Xr17Nq1Sq2bt3KggULSE9Px93dncaNGzNr1iz69ev3UDG+9957ODs789tvv7Fz506qVavGsGHDaNKkCWPGjCn0OG5ubnz//fd8/vnnLF++nJycHBo3bszixYsJCQkxS8oh9wuPQkJC+Pbbb/n1119ZuXIlFSpUICAggBYtWpj6TZ06laSkJP73v/+RnJyMoijs3LkTJycnOnXqREhICIsWLWLjxo3cvXsXNzc3atWqxZgxY6x+cU9hDR8+nF9//ZVVq1aRlJREhQoVaNCgATNmzDA9xQZgzJgxxMbGsnXrVlauXInBYGDp0qX5JuU1a9YkNDSUb775hr1797Jhwwbc3Nzw9/cv8rPohSgNVIpcIxJCiMfq4MGDjB49mk8//fSRZupLo9OnTzNo0CCmTZvGxIkTbR2OEELYjNSUCyGEeCIyMzPNflYUhe+//x7gsd2gKIQQJYWUrwghhHgiBgwYQNu2bdFqtWRkZLB7926OHDlCnz59aNy4sa3DE0IIm5KkXAghxBPRo0cPdu/ezcaNG8nJyaFGjRq8+eabvPTSS7YOTQghbE5qyoUQQgghhLAxqSkXQgghhBDCxiQpF0IIIYQQwsYkKRdCCCGEEMLG5EZPIcqopKQkOnXqRFZWFrNnz2bgwIFW+/n5+REYGMiyZcusLh81ahSHDh3i/PnzFsuuXLnCkiVLiIiI4ObNmxgMBqpVq0ZgYCBDhgwhICCgWPepsOLi4vjiiy/Yu3cv6enp1KtXj5deeqnQX2QEcODAARYuXMipU6fIycmhbt26jB492upxzM7OZvXq1YSGhhITEwPkfnHKwIEDGT58OA4ODg+1H8bnn+fl4OBA5cqVCQwMZMKECdStW/ehxi6sHTt2EBkZWahvZb3fuXPnWL58OYcPHyYuLg69Xk+lSpVo1KgRPXv2pG/fvtjb2z9wnLVr1/Lee+8B8M477zB+/HiLPmfPnuX5558Hcr8ddtasWUWOtzT566+/mDNnDocPHyY7O5uGDRvy+uuvF+kLtn799VeWLFnCuXPnUKlUNGjQgEmTJtGlSxeLvvl9eZGTkxPHjh176P0QojSRpFyIMmrTpk3odDpq1KhBaGhovkn5w1qzZg0fffQRDg4OBAUF4e/vj52dHVeuXGHbtm2sXr2azZs3U69evWLd7oMkJiYyYsQIEhISGDNmDFWrViUsLIwpU6aQnp7OoEGDHjhGWFgYb7/9NjVq1GDSpEmUL1+ebdu2MX36dG7evMnkyZPN+r/77ruEhYXRu3dvBg8ejMFgYPfu3XzyySccO3aMefPmPdI+BQUF0blzZwCysrI4f/48a9asYevWrWzatAlvb+9HGr8gO3bsYN26dUVOyhcsWMBXX32Fm5sbffr0oX79+tjb2xMXF0d4eDjTp0/n6NGjfPzxx4Ue09HRkbVr11pNykNDQ3F0dCQrK6tIcZZGMTExDB8+HI1Gw4QJE3BxcWHNmjVMmDCBxYsXF+qZ8YsWLeKLL76gYcOGvPnmmwBs3LiRSZMm8dlnn9G/f3+LdVq1asXQoUPN2grzoUuIMkMRQpRJAwYMUEaPHq0sWbJE8fPzU2JiYqz202q1SnBwcL7jBAcHK1qt1qxt//79ir+/vxIUFKTcvHnTYp3s7Gzlp59+Ui5evPhoO/EQZs+erWi1WmXnzp2mtpycHGXQoEFKYGCgkpqaWuD6Op1OadOmjdK+fXslKSnJ1G4wGJTx48crjRo1MjuWN2/eVLRarfLKK6+YjWMwGJThw4crfn5+SmJi4kPtS0REhKLVapXvv//eYtmSJUsUrVar/PTTTw81dmFNnz7d4vw/SGhoqKLVapUXX3xRSU5Ottrn7NmzyvLly4s03ltvvaVotVrlxIkTZsuzsrKUwMBA0/Lp06cXKd7S5o033lD8/f2Vs2fPmtpSU1OVrl27Kr169VIMBkOB69+6dUtp1KiREhQUpOh0OlO7TqdTgoKClNatWyspKSlm68hxF+LBpKZciDLozJkzREZG8vzzzxMUFISdnR0hISHFNv6cOXNQFIW5c+dSpUoVi+V2dnaMGTPmic+SQ+4sd61atejevbupTaPREBwcTGJiIr///nuB61+8eJG7d+/So0cP3NzcTO0qlYqBAweSnZ3Nxo0bTe1paWkAVK5c2WwclUqFl5cXarUaR0fH4tg1M8btWZuJ3LJlC8OHD6d58+Y0bdqUIUOG8Ntvv1n027NnD8HBwbRp04aAgAC6du3Ka6+9xpUrV4Dc0qV169YBueUJxv/Wrl2bb1w6nY4vv/wSZ2dn5s2bh6urq9V+DRo0YOTIkUXa527duuHh4UFoaKhZ+44dO0hMTCzwKkh4eDjjxo2jVatWNGnShH79+vHLL79Y9Nu3bx9TpkyhR48eBAQE0KpVK8aNG8ehQ4cs+o4aNYru3bsTFxfHW2+9RevWrWnatCnjx483HcMnLT09nV27dhEYGEiDBg1M7c7OzgwePJioqChOnTpV4BjHjh0jOzubfv36mb2+7O3tCQoKIikpiR07dlhdV6fTmd4TQghzUr4iRBkUEhKCk5MTvXr1wsnJia5du7J+/XrefPNN1OpH+6weGxvLmTNnaNWq1SMn3dnZ2aSkpBS6v6enZ4HL4+PjiYuLo1+/fhbLmjVrBsCpU6fo06dPvmPodDoAypcvb7GsXLlyAJw4ccLUVqtWLWrVqkVoaCgNGzakXbt2GAwGdu7cyfbt25k4caJpvYeVkZFBQkICkFu+cuHCBebOnYuHhwe9evUy6zt37lwWLFhAp06dTOd7+/btvPnmm/zrX/8yJcKHDh3i5Zdfpn79+kyaNAlXV1fi4+M5cOAAMTEx1K5dm8mTJ2MwGDhy5AifffaZaRstWrTIN9ajR49y69YtBg4cSIUKFR5pv+9nZ2dH//79WbduHe+//77pw47x2Pv7+1tdb9WqVXz44Yc0a9aMyZMnU758ecLDw/n3v/9NTEwM06dPN/Vdt24dSUlJDBw4kKpVqxIXF8eaNWsYM2YMS5cupVWrVmZjp6enExwcTNOmTZk6dSpXr15l6dKlvPLKK4SFhaHRaArcJ51OR2pqaqH2X6PR4O7uXmCf8+fPo9PpTK/3vPK+Bwq638P4HrD2us37Hri/JG7r1q1s3LgRvV6Pp6cnffr0YcqUKfl+MBOirJGkXIgyJisry1Tf7OTkBMDAgQPZvn07f/zxh9WbtIri4sWLAGazcA/r6NGjFjcyFsTazaZ5xcfHA1idvTe2Gfvkp3bt2mg0Gg4dOoSiKKhUKtOygwcPAnDjxg1Tm52dHd999x3Tp09nxowZpnZ7e3tmzJjBiBEjHrBXDzZ//nzmz59v1lavXj1WrFiBl5eXqe3MmTMsWLCASZMm8dZbb5naR48ezSuvvMIXX3zBgAEDcHFxYefOnRgMBn766ScqVqxo6vvqq6+a/r9Dhw5s2rSJI0eOMGDAgELFWtDrIzU11ZTwQe7VBA8Pj0KNazRo0CCWLFnCtm3b6NevHzdv3iQ8PJz/+7//s9o/Pj6emTNn0rdvX7744gtT+8iRI5k5cyY///wzI0aMoGbNmgD85z//Mb1vjP7xj3/Qt29fFi5caJGU3717l/Hjx5t9a6mnpyeff/454eHhdOrUqcD9CQsLM93E+iDe3t7s2rWrwD6FeQ/ExcUVOIbxw3ZERITF+9PaewAgICCAZ599Fh8fH1JTU/n9999Zvnw5hw4dYuXKlTg7Oxe4TSHKAknKhShjtm3bRnJystksVpcuXfD09CQ0NPSRk3LjrF5x/JH19/fnp59+euRxjDIzMwGsPu3EOKuakZFR4Bju7u4MGjSI1atX8+677zJ27FjTjZ5r1qwx207esX19fWnSpAlt27YlMzOTdevWmRK8R73JdtiwYTz77LNA7oeuS5cu8dNPPzFx4kSWLl1qutFz06ZNpjIb48y6Uffu3dm5cyfHjx+nY8eOptnLrVu3MnToUOzsiufPRUGvj/fff5+tW7eafn6YJ3P4+fnRuHFj1q5dS79+/Vi3bh12dnb069cPvV5v0X/r1q3odDoGDx5s9ZgsW7aM8PBwhg0bZorJKC0tDZ1Oh1qtpmnTpmZXSIzUarVF4tq2bVsAoqOjH5iUd+zYsdDvgcKUQRlf34/yHvDz86NDhw7s3LmTzz77zFQWtHbtWvbu3QtYvgeM7w2jgQMH4ufnx9y5c1m6dCkvv/zyA2MXorSTpFyIMiYkJARPT0+qVq1KdHS0qb1Dhw789ttvJCQkPLAMpCAuLi4AxVI36u7uXqgnQRSW8dJ63tlYI+NTOayVpdxvxowZqFQqQkNDWb9+PZA7+zlz5kymTZtmOgYAt27dYvDgwQwZMoS3337b1N6/f9nrQ6QAACAASURBVH+GDx/Oxx9/TLdu3R5YdlAQHx8fs+PUrVs3AgMDGTp0KHPmzGHu3LkAXL58GUVRCnz04+3bt4HcmeKdO3fy0UcfMWfOHFq2bEmnTp0ICgp6bK+PV199lX/84x8AzJo1i9jYWNOyxMREsrOzzfrnvQqQ1wsvvMDMmTO5du0a69ato0ePHri7u1sk3ZB7TADGjBmTb8zGYwK5Ty6ZO3cu+/btIzk52axf3qsmRpUrV7ZIlo1lO4mJifluM+/699+P8CiMr+9HfQ/MnTuXGTNm8OOPP/LDDz8AuTP1H374ITNmzDB7D+Rn/Pjx/Pe//+X333+XpFwIJCkXokyJjY3l4MGDKIpC7969rfbZuHGjWYLi6OhY4MxZenq6WW1p/fr1AYiMjHzkeHU6HUlJSYXun1+SZmRMbqxdnje2FSYBcnR05OOPP2batGlcunQJe3t7/P39Tc8gr1OnjqnvqlWrSExMNM1kG6nVanr37s3x48c5e/ZskZ4PXRhNmzbF1dWViIgIU5ux3Gbx4sX51jIbSxM8PDwICQnhyJEjhIeHc/jwYT799FPmz5/PokWLaN68+UPFVdDrI++zrN3d3c2S8tdff93iZsr8ypX69evH7Nmz+eCDD4iOjuaDDz7INx5FUQCYPXt2vufeWLqSlpbGyJEjycjI4MUXX0Sr1eLs7IxarWbhwoVmx9qooJpx47YLkpmZWej7KjQazQM/MBXmPWCttOV+7u7uzJ8/n9u3bxMVFYWTkxP+/v788ccfgPl7ID/29vZUrlyZu3fvPrCvEGWBJOVClCFr165FURRmzpxp9eaqefPmERoaapaU16hRg+joaPR6vUWCkZOTQ3R0NDVq1DC11axZk4YNG3L06FEuX778SF9ec+zYsWKtKa9cuTJVqlSxWmZw/PhxAJo0aVLo7bm7u9OyZUvTz8YntxifGQ73angNBoPF+jk5OWb/Fje9Xm82I+rr68sff/xB9erVC3VeNBoNbdq0oU2bNkDul/0MGjSI7777jkWLFgHWZ4cL0qJFC7y8vNi+fTvvvvtuoWvGp0+fbjEznR83NzeeeeYZwsLCqFatGh06dMi3r6+vL5D7IeRBV2UOHDhAfHw8n3zyicWTXB71WfP52bJlS7HWlGu1WhwcHEyv97yMbY0bNy50fJUqVaJSpUqmn629B/KTlZVFXFwcTZs2LfT2hCjNJCkXoowwGAysW7cOrVbLkCFDrPa5dOkS8+fP5+TJk6anL/Ts2ZOFCxcSEhJiqqs1CgkJISUlxeLRdW+//TYTJkzgrbfe4vvvv7eYwdbr9SxbtoyOHTsW+ISW4q4pB+jbty8//vgju3btMj0WUa/Xs3z5ctzc3MySiYyMDK5fv46rq+sDZ9BjY2NZvHgxvr6+ZuUhxuR37dq1Zk+0yM7OJiwsDDs7Oxo2bFicuwjA/v37SU9PN3sSSv/+/Vm2bBlffvklX3/9tcWHrNu3b5sSLGtlTHXq1MHR0dHs6oWxxjoxMbFQT1NxcHBg6tSpvP/++0ydOpX58+db/YB4/yxyURJFgJdeeglfX18aNmxY4BOFnnvuOb788kvmz59PmzZtLJ4okpKSgqOjIw4ODqbjdX9s+/bts/pBrzgUd025s7Mz3bp1Y/v27Zw7d870RJq0tDRCQkLw9fU1e52mpKQQHx+Ph4fHA2fhT506xZo1awgMDDS74fXu3btWP3zNmzePnJwcunXrVqj9E6K0k6RciDJi37593Lhxg8GDB+fbp1evXsyfP5+QkBDTH+aXXnqJHTt28K9//YuIiAjTY9OOHz/Oli1bqFu3rtmTJSC3Pv3jjz/mo48+4tlnn6Vv3740aNAAOzs7oqOj2bZtGzExMYSFhRUYc3HXlANMnDiRrVu3Mm3aNMaOHUuVKlUICwvj1KlTzJw506wW9uTJk4wePdria9lXrlzJnj17aNmyJR4eHvz111+sWbMGjUbDV199ZXYT3aBBg1i6dCm//PILN2/epFOnTmRkZLBx40bOnz/P+PHjzZ5uYvzK+Ndee63Q35J59uxZNmzYAOSW/Fy6dInVq1djb2/PlClTTP0CAgJ4/fXXmT9/PgMHDqR3795UqVKF+Ph4zpw5w969ezl9+jQAH3zwATdv3qRjx45Ur16dzMxMfv31V9LS0syetNK0aVOWL1/ORx99RJcuXbC3tycgIMBU8mHNoEGDiI+P56uvvqJnz55m3+gZHx/P/v37+fPPP9FqtYXaf2v8/f3zfQRiXlWrVuXf//43M2bMoE+fPvTv3x9vb28SEhK4cOECO3bsYPPmzdSoUYOWLVvi5eXF7NmzuXbtGlWrViUyMpINGzag1Wq5cOHCQ8ebn+KuKQeYNm0aERERjBs3jjFjxuDs7MyaNWuIi4tj4cKFZlc/tm/fbvX1OG/ePKKjowkICMDFxYWzZ8+ydu1aqlSpYvZ4TIDvvvuOEydO0KZNG6pVq0Z6ejq///47Bw8epGnTpowaNapY90+IkkqSciHKCOOXAz3zzDP59tFqtfj6+rJlyxbef/99ypUrh6urK6tWrWLRokXs2LGDnTt3ArmXyidOnMjEiROt3tQ1ZMgQWrZsyZIlS4iIiGDDhg0YDAaqV69O27ZtmTdvnk2+PMjDw4NffvmFOXPmsGLFCtLT06lXrx5z584t8PnkedWrV4/Nmzfzww8/kJqaipeXF3379uXll1+2qMd1cXFh9erVfPPNN+zZs4d9+/ZhZ2dHvXr1+M9//mNx1cJ4A2Rh6nqNwsLCTB9w1Go1FSpUoEOHDkycONHiedOvvfYajRs3ZtmyZSxdupT09HQqVqxI/fr1zR4bOGDAANauXcu6detISEjAxcWFevXq8fXXX5vdjxAUFERkZCSbN2/mt99+w2Aw8OmnnxaYlAO8/PLLdO3aleXLl7N//37Wrl2LwWCgYsWKNGrUiM8++6zQ5+NRDRo0CF9fX3788UdWrVpFSkoKFSpUoHbt2rz55pumKz1ubm58//33fP755yxfvpycnBwaN27M4sWLCQkJeSxJ+ePg4+Njeg8sWrSI7OxsGjZsyPfff1/oD8GNGjUiIiKC/fv3k5GRQfXq1QkODmbSpElmX6oFEBgYyOXLl1m3bh2JiYloNBp8fHyYOnUqY8eOfSxfniVESaRSCnOniRBCiCfitdde48KFC2zevNnqt3EKIYQonR7tq/uEEEIUm5ycHCIiIpg2bZok5EIIUcbITLkQQgghhBA2JjPlQgghhBBC2Jgk5UIIIYQQQtiYJOVCCCGEEELYmDwS8W9376ZhMDzZ8vqKFV24cyf1iW5TPHlynssGOc9lg5zn0k/Ocdlgq/OsVqvw8HC2ukyS8r8ZDMoTT8qN2xWln5znskHOc9kg57n0k3NcNjxt51nKV4QQQgghhLAxScqFEEIIIYSwMUnKhRBCCCGEsDFJyoUQQgghhLAxScqFEEIIIYSwMUnKhRBCCCGEsDFJyoUQQgghhLAxmybl8fHxzJkzh1GjRtG8eXP8/Pw4ePBgode/fPky48ePp3nz5gQGBjJ9+nQSEhIeY8RCCCGEEEIUP5sm5VeuXGHx4sXExcXh5+dXpHVv3rzJyJEjiY2NZerUqYwbN47du3czfvx4srOzH1PEQgghhBBCFD+bfqNno0aNiIiIwMPDgx07dvDqq68Wet0FCxaQlZXFsmXLqFKlCgABAQGMHTuWDRs2MHjw4McVthBCCCGEEMXKpkm5i4vLQ6+7bds2unfvbkrIAdq3b4+vry+//vqrJOVCCCHEE6SPu0TO9XPYVfcn68xO9LGn0NRsglP3SaY+usg95Px1BLs6rXBo0BVd5B50J7eBChya9MKhQVdTv6zD60CXCo4uaLwbQkaKaT2AzIOrybnyJypXL1Rgtux+ebcLmMVwf5/kgI5Qs22h9vn+/TG25d2n/LZXVOm7Flo9pta2aW07BR0D4zJ1pVoougwMd6+BPgd7/85my/M7fpkHV5N98QBo7FGXc8Hev7PVftaOV37HsjB9DVnpKCnxVo9JQfRxl7h74Qp699poqtQr9HqPm0pRFMXWQQCmmfKlS5fSpk2bAvvGxcXRuXNn3n33XcaOHWu27J///Cf79+8nPDy8SNu/cycVg+HJHgovL1du3Up5otsUT56c57KhrJ7nvImYPuGq6Y9ozo3z6K/8CQ7lTQkV5V3N/s0vSSnsH2djIqLo0gBQOTqbJUHG7egTb0L6XVQVfXB54V8WCYYxmVFX8jHFZRzj/m16eblybe8ms4RA5VoZtaOT1eQj7z4bbkWj6NJQOTqbbetByam1+Oz9u4CioIvcjT7qKBrfFjj4dUJ3/nf0V46iqdUUe20HMP6JVxSyL+4nJ+YEdrWaYl+vnakdcvtk7l+O/sZ5NNX8KNduONmXDpJz9SR2NZpgXzfw724KussH0V89DfblQZcODk4oaXdQ7t4wjZWXunpDHJsHkR19jJzT2++1V9ViuHnBrK+dtiMoCjkX9+f7mtPUaY2SkYrhRqTlMt9W2FWpa4pDUUAfdxl99J/Wx/JpjsarNvpbf6GPPp6nvRmaSrX//uneMbxHQX8rCn3syXvr1GwCCuivnso3dnX1Bmg8a2JxnO4bO+8/oJAdewZS4u51ca2MfY2GAOjvXrc4juqq9dG4V7sXa+INDHGXrAflWglSbucbM86VIM36crVnTZTsLJSU+PzXB1Qe3rmR3L12r61CddQunqAoGFITUJJu3FuhnCtk3vt9qnKritrZHQBDWiJKcp5jYeTgjNq9imX7ffRZ6ZB8ExWAxgGnoHeeaGKuVquoWNH6pHSJTMpPnTrF4MGD+eKLLwgKCjJb9tlnn/HDDz9w9uxZNBrN4wxZCFGC3Nm5jLTzEdhVqAooGDLT0d2KRu3ohGfnf+DW4hmr68Wt/4qMy0cpX7cFVQa+CUDy0e0kHtqEITM3GdSUd8GhSh306UlonNzJunkZQ2YamvIuuLcOMhs7+eh2Us8dwMU/NynKO47R/etZW0elUuHeOoiMmLNkXD6KnUc1cpLiUbIzcdK2ITvhBrq4v7CvUpsaYz4BxYBiMJB8bCcJe1ZAThb2VWrj0rADyce2o2TlxqB2dMahUg2ybl1FhYJLky64NGgPioKiGHIDVAygKMRvWUD2zb/+jlqFtYTsQZwbdqJcTf/cMVHIiD1HeuS9SRV7Lx+yb0Wbfi5fryWOVeuQdf0yGX8dLfL2sHOEnKyirVLRG3u3iiiKQk7yHXISrufbV+NWCcWgx5B6t/Ab0NiBPse8Tf333y+DvkixiqecSo3KoVzu/5rajP+nytPP/H8MGZYf+DXO7oAKfUYqGO57/Wjs0JR3M42jT08BfX732z3gvatS3fehIc8ie0cUvd5y+/f3cygPgKLLyNPmhEMlb1Cp0N2+ipKVnmcF9d+/E3KpHZ1xqOIDqNDFR1n8zsztpKG8b0ABUSiAisw71zEkxuUedpUajy7D8ejwQoHxPyklMik/cuQII0eOZP78+fTq1cts2VdffcW3337L0aNHcXZ2LvT2ZaZcPC6l6TwbLxfb1W6J2q2yaSZRF7kX5U40OHmgsnOwmLk0zZoqBlCrUVWoAdmZVi+1mi5b52SA2j53Jg4V2Nmh8W2JITEO5XYUaOxBo0Hl4IS6an30V8/A39tFpQFH59zky84eMlJBKTi50dRvj101v9w/Pn/PHurO7c3d1t9UlXxQu1ZGf+VwkY6b2qsOavcqGBJvYrh9pdDrqdyq5CbDD5iFEo+ZfXnUnt6oUKEkXseQZSUhMHL8ewYsK7Xw46vtLJIalUtFQIWSan2GUuXqlbs872tD4wB63b0+7lVxaNQTVKA7sxMl8d5MpKpCddNVBVQqssJXQM69de9PxNQe3jg064vu+BYMd68Wft/4e6a85QCyo4+Tc/LXe+01AzDkmWkGcGg1KDfew6H5jufQdjiGlNvknNluuazdCBz+Lp0wZqS6C3vR7V9hfawOo3Dw74Lu3O/o9i/L0z469/fSfcmxSnUvcdZF7iHrj59NPzt2GgNg1nY/x05jHqqEJX3XQvSXDph+1tRrZyrXuD8Oa9ux1ifvWHnHLspyx05jMCTHk31iS4HxWzs2eWO8P777t1lQ37zrWCthuZmQzubwKDKz9bz6fBP0cZdID5ud+4FXbfdUzZTbtKb8YTk6OgKg0+kslmVl5c6AlCtX7onGJERx0cddIuOPJbl/QNV2qFw8TX88dSe3oaQlAAZUFWqgdnTKvTSfdjt3BqaiL2Rnmi6RGxPe9F0L0V8+9HdSrPl79u3vP7hqTe7soS49d3aivDvkZKGuqkVTyYfsi/tBl5mbNGRnApj9AtZfO30v+LTbZvMtSkaS5S9PAyi3/zL9mPXHz2Sf3Y3atRKG5HgMCbF5Ohvf4wrk6Mz/MOizQA+KLh39pfsTFz2k68yHeAD9xXD0Fwsue1NuR6O/HV1gH2sMCbEomSko6UlFWk/JLEJiVyAVDq1fIPv8PuuXfR9A7VkDhxYD/h5KDSoVmbsWFnnG2RqHNkOx13YElQoVKnQX9qOL+MW0XFOnDfq/7j0q16Hjizg27FZgklGgAi7F58ex7TBTQuAYG8HtXxfk3zcw936mosSmqdPaIulxaN6vwHEcmvW1WK6p3dJsHIeAZ+8lZmo7s773fxjOuX7ObF1Vxdpm71P7xs9gX789So6ucPvmXh0yk81qfe2q+aFzr1KomnJVOdcH15Tb2Reqptyx0TOo1PYF1kQ7NuqBSq0h568jeAR0JKsQNeXGda2WVhVzTblT90mkg9WaclOyWkBN+f2x3h+TrppfwTXlfy8vqLSrMDXl+bVZO5Z5t5lf34Jqyq/eSiUsPIrD5+Kx16jp2twbg6KgqVIPp6DpOCZdIUtqyq2TmnJRUmQeXE326Z25SWFBHJz/vlyYe6kfRxdUzhVRbl/B7FKhSo3auxHo0lC5VSlwxuKh2JeH7IwH97MlOwfUrpUxpN6C7EdP9B6GQ+vB2Ndvl5t0Qm7iuW8Z+qh7taia2q3RVK2P7sD/ijS2cZanqIlkYWbeCkNVqY6plvphxrI2u3f/zB2osG/6HIbbMVJTnmcfHldNeX43zhXlRrr73X8jYUHH/f59IyMFBVBSbmFXuyXl2gy1GL+kkL/NJd+BMzdZvOksjg4aurfwpnfrWrg5O5j1sdV5LnU15QDt2rWjffv2fPHFF2btvXv3pkaNGvzwww9F2r4k5WWDLnIPWUfDLGfKNA65CbRajcrTByX1DmTmndVUgVot9Z2PycMmrQ9L5VIJyrmh3L0KDuVxbPV8vjNY1p54YC0ZdPKuR2ZigkXyZa085/6E0DiOKb771ssviTSVBsWeQuVaOfd1m5Nxr8znTrQpEc27/awDK0GfhapibRwadLbYF3UlHwy3ogt8koPp2EQfR+XqRflOo5+qGafHRX5vl35yjkumy9eTUBSo5+1OSrqOnX9epWermriUt7fa/2lMyktE+UpMTAwAtWrVMrX16tWLjRs3EhcXZ3os4oEDB4iKimLChAk2iVM8Gfq4S2Qd35J797/+vtoElTq3PtPUrga1KretoEvtxv4Gvdkl23uUEpuQqyrVyWefHo3au7HpknFx1ZQb/324mvKzuZe6jR+gHN0gJxPsyuX+W84NlaIv8iyetRpFhwZdLRLVwv6Cv3/dwlzOLmidol4Otxb7o1xSF0IIWzofc5dN4VGcjbpLo9qeTBvWDFcnBwZ2qmPr0IrM5jPl3377LQCXL18mLCyMQYMGUaNGDdzc3AgODgage/fuAOzatcu03o0bNxg4cCAVKlQgODiY9PR0fvjhB6pVq8aaNWtwcHCw3FgBZKbcdtI2z8GQty65rMt717nGAZVrpSdQU14u9yZJlRqcPCEnE03NJrk1oH/PoqocnVE5e5aKy9OPg7yfywY5z6WfnOOS4UJsImv3/sWF2ETcnOzp3aYW3Zp7U86hcPPNT+NMuc2Tcj8/P6vt3t7epiTcWlIOcPHiRWbNmsWff/6Jvb09Xbt25b333sPT07PIcUhSXvz0cZdI3/4NpBfh0WClSRFqyjV125hqRTWeNUzPfS4L5QClRWl/P4tccp5LPznHTy9FUVCU3MR299GrhB2I5tk2tejctDqO9kV7DLYk5U8xScofjj7uEukbZto6jEdXqJpyDaDkPm7P3hF7v06Fmi0uDedZPJic57JBznPpJ+f46WNQFI5duE1YeBSdm1ajW4sa5OgNKArY26kfasynMSkvETXl4umR8v2EB35JgE3lW1Nu//czeHPLQtTejXHu+7bNwhRCCCFEwQwGhcPn4gk7EMW1W2lU9iiPq1NuebKd5uGS8aeZJOWiQClL37xvxvhpoELlWbPMPO1BCCGEKIsWbTrDoch4qldyZmK/hrRuUBmNuvQl40aSlAsLKT9Ozn1axRMjSbYQQghR1uXoDYSfvknz+pVwdXKgW3NvWvlVpoWfF+o836ZaWklSLoDHXJZSzh3X0V89nrGFEEIIUaLpsvX8cfIGWyKiuZuShS67Pj1b1cSvloetQ3uiJCkvY0JDV/P//t9HXLt2leqVPJjWpQ4DmtV68IoP4l4d12GfPPo4QgghhCgTFEVh2+FYfjsYQ1Kajvo13Bnbx59GvkV/il5pIEl5GRIaupq33nqdjIzcr1y/diuB/9uQWy9e1MTcacAMKTURQgghRJFl5xiwt1OjUqm4EJtI9UrOTOrfCL9aFVCVgTKV/EhSXobM/Nc7poTcKCNbz5ztZx6clMtMuBBCCCEeQVpmNtsPx7Lr6DXeH9WSqp5OTB7QCHu7oj1jvLSSpLwMMD5L/PqtBKvLbyRlWG2X2XAhhBBCPKrkdB3bDsWy6+hVMnV6mtevhHE+XBLyeyQpL+XyfrlPNffyXLeSgFdzL3/vB7tyuI5b8KTCE0IIIUQppsvW83+LIkjPzKF1g8r0bedLzcrWvzynrJOkvBRL2zwHw7XTpp/ffqYR/7fhGBnZelNbeXsNbz/bDNeJP9sgQiGEEEKUNgnJmRw5F0+vwFo42GsY0VOLbzVXqlV0tnVoTzVJykuplEVjLNqMdeNztp/hRlIG1Txc+eCTuQwa9OCvihdCCCGEKEh8YgZbDkSx/9RNAJrWr0QVDyfaNa5q28BKCEnKS5nMg6vJPrEl3+UDmtViQLNaMjMuhBBCiGKRnKZj9e5LRJyJQ62Gzs2q06eNDxXdy9k6tBJFkvJSJGXF25B2u8A+cvOmEEIIIYpDlk6Po4MGR3sN52Pu0rNVDXoH1sLD1dHWoZVIkpSXEmmb50hCLoQQQojHLvpmCpvCo7h+O42ZE9rg6KDh00ntsNOobR1aiSZJeSmR94ZOa6RcRQghhBCP4vK1JDaFR3Hy8h3KO9rxTKsa5OgNOKg1kpAXA0nKSwFrN3Uaqb0b49z37ScXjBBCCCFKnbNRCcxZeRyX8va80LkO3VvUwKmcpJHFSY5mCZeyIv+EW2bHhRBCCPEwFEXhbNRdUtJ1tG1UFf9aHozq7Ue7RlUo5yDp4+MgR7UE08ddyr+OXC2nVgghhBBFoygKJy7fISw8ir+uJ1OrsgttGlZBrVbRrbm3rcMr1SRzK8GM39RpjeuE759gJEIIIYQo6S7EJvK/HReIiUulkns5Rj/rR4fG1VCpVLYOrUyQpLyEKqiOXMpWhBBCCFEYBoNCVrae8o65KWGWTs+4Pg1o26iK3Lz5hElSXgKlrHo/32WSkAshhBDiQXL0BiLOxLH5QBQNa3syqpcf2poV+H8vtUWtlplxW5CkvCRKum693b36k41DCCGEECVKdo6B/adusCUimttJmdSq7EJjX0/TcknIbUeS8hJGH3cpnyUqXId98kRjEUIIIUTJErLnMtuPxFK7mhsjntHStG5FqRl/SkhSXsKkh8222u468acnHIkQQgghnnaZuhz2HLtOAx8PfKq60qNVDQLqVqShr4ck408ZScpLGn22ZVs59ycfhxBCCCGeWumZOew6epVth2NJzcimfwdffKq6UrlCeSpXKG/r8IQVkpSXAq6jv7J1CEIIIYR4Svx2MIaw8CjSs3IIqFuRoPa+1POWCbynnSTlJUj+9eRCCCGEKMtS0nU4l7dHrVKRkZWDv48HQe198K3qZuvQRCFJUl6CpG//xkqr1IMJIYQQZVViaha/HYxhz7FrTBrQiOb1vRjYqbbUi5dAkpSXJOl3LZrU3o1sEIgQQgghbOlOUiZbDkbzx4kbGAwKbRtVoXpFZwBJyEsoScpLiMyDq622O/d9+wlHIoQQQghbUhSFOSuPcTspkw5NqtKnrQ+VPZxsHZZ4RJKUlxDZJ7bYOgQhhBBC2MiNO2ns+vMaQ7vXxd5Ow9g+DajoVo6K7uVsHZooJpKUl2R28kYUQgghSrOr8amEHYjicGQ89vZq2jSsQr0a7mhrVrB1aKKYSVJeAqRtnmO13XXcgicciRBCCCGehIysHL4PO8uxi7dxdNDwXFsfegXWxM3JwdahicdEkvISwHD9jK1DEEIIIcQTkJCciadbOco5aMjOMdC/gy89W9XEpby9rUMTj5kk5SWBolg0aeq1s0EgQgghhChuiqJwPiaRTeFR/HU9mc9eboerkwNThzaVJ6mUIZKUl1BO3SfZOgQhhBBCPAJFUTgTlcCm/VFcvJqEm7MDAzrWxsFeA8ijDcsaScqfcik/TrZ1CEIIIYR4DG4mpPPlqhN4uDoy8hktnQKqmRJyUfZIUv60y8m0dQRCCCGEKAYGReHo+VtcvZXKwE51qFbRmalDm+JfywN7O7WtwxM2Jkl5CeTYaYytQxBCCCFEIekNBg5HxhN2IJrrt9OoVtGJvu18sLfT0KRORVuHJ54SkpQ/xfJ7FKJDg65PNhAhhBBCPJTL15NYvOks8Xcz8K7kzMT+DQn0r4JaLfXiwpwk5U8xw7XTtg5BCCGEEEWU5ojJWQAAIABJREFUnWMgJV2Hp1s5KrqVw7W8PYO71KWFnxdquXlT5EOS8qeUPu6S1Xb7pn2ecCRCCCGEKAxdtp7fT1znt4MxeFUoz7sjW1DBxZH/G93K1qGJEkCS8qdUethsq+3l2gx9wpEIIYQQoiCZuhz2HLvOb4diSE7Toa3hTr/2viiKIo81FIUmSfnTSp9t2eZc6cnHIYQQQogC/XHiBqt3X6Khrwf9BjTCr5aHrUMSJZAk5SWI60jrN34KIYQQ4slJzchmx5FYqldyJrBBFTo3rU7t6m7U83a3dWiiBLNpUq7T6fjqq6/YsGEDycnJ+Pv7M3XqVNq1e/BXyIeHh/Pdd99x4cIFDAYDderU4cUXX6RPH6m5FkIIIUTxS07TsfVwDLuOXiNLp6dX65oENqiCo4NGEnLxyGyalL/77rts27aN0aNH4+Pjw7p163jppZdYtmwZzZs3z3e93bt38/LLL9O8eXNef/11ADZv3szUqVNJS0tjyJAhT2oXHiMVoNz7US0XNYQQQghb2XY4lrW/XyY7x0DrBpUJau9LDS8XW4clShGbZXonT55k8+bNvPfee4wZMwaAgQMHEhQUxJw5c1ixYkW+665YsQIvLy+WLFmCg4MDAEOHDqVHjx5s2LChxCflmQdXY5aQAxhybBKLEEIIUVbdScrEqZwd5R3t8HB1pLV/Zfq086FaRWdbhyZKIZt9p+tvv/2Gvb29WQLt6OjI4MGD+fPPP4mPj8933dTUVNzd3U0JOYCDgwPu7u44Ojo+1rifhOwTv9o6BCGEEKLMir+bzk9bInl34QF2/HkVgNb+lRkf1FAScvHY2Cwpj4yMpHbt2jg7m7+4AwICUBSFyMjIfNcNDAzk4sWLzJs3j5iYGGJiYpg3bx5RUVGMGzfucYf+BCiWTe7Vn3wYQgghRBly/XYaX/zvT95bFMGBM3F0beZNh8ZVbR2WKCNsVr5y69YtqlSpYtHu5eUFUOBM+eTJk4mJiWHBggV89913ADg5OfHtt9/SoUOHh4qnYkXb1IV5eblatKVY6VfntfmPPxjx2Fg7z6L0kfNcNsh5Lr3+u+40Z67cYUDnujzftR6ebuVsHZJ4jJ6297LNkvLMzEzs7e0t2o3lJ1lZWfmu6+DggK+vL88++yzPPPMMer2e1atXM2XKFH7++WcCAgKKHM+dO6kYDFZmqB8jLy9Xbt2yloJbKmw/8fQpynkWJZec57JBznPpEnUzmc3h0QzrUY9K7uUZ0rUOb41ogS5Dhz4rm1u3rHxniCgVbPVeVqtV+U4E2ywpL1euHNnZli92YzJeUG34f/7zH06dOkVISAhqdW4FznPPPUdQUBCffPIJK1eufDxBPwH6uEu2DkEIIYQo1S5dTWJTeBSn/rqDk6Md12+nUcm9PFU8nHB3ceRWhs7WIYoyyGZJuZeXl9USlVu3bgFQuXJlq+vpdDpCQkKYNGmSKSEH/j97dx4QVbm/Afw5Mwww7Pu+u4AbCK6AK+5bLrmUlfrTtDK93Va1vdtNy7xqZWmalZpZaqKhZprL1QQ1VzTFHRQHZJF9GWY5vz+8TiGCw3qAeT5/ed6zPTNHmC9n3vc9UCgU6NmzJzZs2ACtVgszs6Y5hWDxzsUVG+VNf/AqERGR1PSiiKUbz+Dc9TuwUSrwaO8gxET4QGnRNGsGal4k+18YEhKCdevWoaioqNxgzzNnzhjWP0hubi60Wi10Ol2FdVqtFlqtFqLYsN1Q6pSmuEKTon0/CYIQERE1faIoIjm9AIGedpAJAnzdbdAu0Al9OnrDwlwudTwiA8lmXxk8eDA0Gg02bdpkaCsrK8OWLVsQERFhGASqUqlw9epVwzbOzs6ws7PDnj17ynV/KSoqwv79+9G6desH9lVvyiy7jZc6AhERUZMiiiJOXc7Ev9cex/trjiM5PR8AMK5PSwzq6seCnBodye6Uh4WFYfDgwVi0aBEyMzPh5+eH2NhYqFQqLFiwwLDdnDlzcOzYMVy8eBEAIJfLMXXqVCxduhQTJkzAI488Ar1ej82bNyM9PR1z5syR6iURERGRxPSiiBMXMxF3OBmpmYVwsbfE5MHBfPomNXqSdqJauHAhli5dim3btiEvLw/BwcFYuXIlOnXqVOV+zz33HHx8fLB27Vp8/vnnKCsrQ3BwMJYtW4YBAwY0UHoiIiJqbErVOnz7ywXYWVtg2rA26NbWHWZyyToGEBlNEJt0B+y601imRCxYOaXCdrYzvm2YQFQvOIWaaeB1Ng28zo2PVqdHwp/pSLySjedGt4dMEJCaWQgvZ2vIZEK1j8drbBo4JSIRERFRHdBo9Th8Ng07j6QgK68Ufu42KCgqg72NBbuqUJPEopyIiIialFtZRVj842nkFKgR5GWHJwa0RmgLZwhC9e+MEzUWLMobPf6CISIiKi3TIv1OMQI87ODuqEQrH3v0DPNCW39HFuPULFRr5ENaWhrmzZuHXr16oX379khISAAA3LlzB/PmzUNiYmK9hDQVpUc3PqCVXf6JiMh0FZdqERefjNeWJ+CTzYnQ6vQwk8vw7Mj2aBfgxIKcmg2j75TfvHkTEyZMgFqtRseOHREfH29Y5+TkhHPnzmHz5s0IDQ2tl6CmQHPml4qNcvOGD0JERCSxwhIN9vxxE7+dSEWJWouwFs4YHh3AmVSo2TK6KF+6dClkMhm2b98OCwsLREVFlVvfu3dv7N+/v84DmpaKd8UtoiZKkIOIiEhayWn5iItPRqfWrhgeFQB/D1upIxHVK6OL8vj4eDz55JPw9PRETk5OhfVeXl5IT0+v03AEmLfpI3UEIiKiepdToMYvR1OgNDfD6F5BaBfohPkzusPDyUrqaEQNwuiivLCwEG5ubpWu12g00Ol0dRKKiIiITENWXgl+OXIDhxJV0OuBPuFeAABBEFiQk0kxuij39PTE5cuXK11/5swZ+Pn51UkoIiIiav4OnL6F9bsvAQB6hHpiaHd/uDooJU5FJA2ji/IBAwbghx9+wNixY+Hq6goAhhHPv/76K3bt2oXZs2fXT0oTULzvS6kjEBER1TtVVhHMzGRwc1CipZc9+oR7Y0g3PzjZWUodjUhSRhflzz33HA4cOIDx48ejc+fOEAQBq1atwpIlS5CYmIg2bdpg6tSp9Zm1WdNdSZA6AhERUb25cbsA2xNScCIpA93beWD6iLbwcbPBEwNaSx2NqFEwuii3sbHBjz/+iKVLl2L79u0QRRGHDx+GnZ0dJk6ciBdffBEWFhb1mdXkCC5BUkcgIiKqletp+Yg7nIzTV7JgaS7H0Eh/DOziK3UsokanWk/0tLGxwZtvvok333wTd+7cgSiKcHLixP31xWbM21JHICIiqpWj52/j0s1cjOwRiP6dfWBtqZA6ElGjZPQM/MuWLcOlS5cMy05OTnB2djYU5JcvX8ayZcvqPiERERE1CaIo4kJKDhZ+fxJ/Xr8DABgRHYCPZ0ZhZI9AFuREVahWUX7x4sVK11++fBmff/55nYQiIiKipkMURZy9lo0F60/i4w2nkJZdjBK1FgBgbamA0qJaX8wTmaQ6+ylRq9WQy+V1dTgiIiJqIj776SxOX8mCk50FnhjQGr3CPKEwY01AVB1VFuWFhYXIz883LOfm5kKlUlXYLi8vD3FxcfD09Kz7hERERNSo6EURpy9nIbSFM8zkMnQKdkXHVi6Iau8BM7nRX8IT0d9UWZR/++23hi4pgiBg/vz5mD9//gO3FUURr776at0nJCIiokZBp9fj2IUMbI9PRlp2MZ4d2Q5d27gjugNvyhHVVpVFedeuXQHcLbg///xzDBgwAMHBwRW2s7a2RlhYGCIiIuonJREREUlGrxdx+GwadhxJQUZOCbxdrfHsyHboHOwmdTSiZuOhRfm9wlylUuGxxx5DWFhYgwQjIiIiaYmieHeWNQHYffwmlOZmmDWmAzq2coGM0yET1SmjB3ouWLCgPnMQERFRI6HW6PDf0yocOqPCvCc7wcrSDK88Fg47KwWfTUJUT6o9+4pOp8O1a9eQl5cHURQrrO/SpUudBCMiIqKGVaLW4sCpW/j12A3kF2sQ7OuAwpIyWFmawd7aXOp4RM1atYrylStXYtWqVSgsLKx0mwsXLtQ6FBERETWs/KIyvLHqCIpKtWgX6IQRUQFo7esgdSwik2F0Ub5p0yYsXrwYXbp0QY8ePbBkyRJMmTIFZmZm2Lx5M3x9fTFx4sT6zNpsFax9QeoIRERkggpLNLh4Ixedgl1hZ22Ofp18ENrCBUFedlJHIzI5RhflGzZsQMeOHbFu3Trk5ORgyZIl6N27NyIjIzFp0iSMGjUKOp2uPrM2X6V5UicgIiITkldUhl+P3cD+k7eg0+vxn+ejYWtljlE9g6SORmSyjC7Kr127hn/+858AYBjkodfrAQBubm4YP3481q5di7Fjx9ZDTNMj824vdQQiImpm8orKsCM+Gf89o4JWp0e3Nu4YFukPWyv2FyeSmtFFuUwmg1KpBABYWVkBuPuEz3u8vb2RkpJSx/FMl/WwV6SOQEREzYReFCETBGg0Ohw8o0LXNm4YFhkADycrqaMR0f8YXZR7eXkhNTUVAGBubg5PT08cP34cw4YNAwCcPXsW9vb29ZOSiIiIqu12TjF2xKegsESDf4wNhYuDEouej4aNUiF1NCK6j9FFeefOnXHgwAG8/PLLAIDBgwdjzZo1KC0thSiK+Pnnn/Hoo4/WW1AiIiIyjiqrCNsTknH0/G2YyWXoFeYFvV6ETCawICdqpIwuyidNmoSQkBCUlpbC0tISs2fPxvXr17F161YAQHR0tKFgJyIiImkcT8rA8q3nYK6QY1AXPwzq6gt7GwupYxHRQxhdlAcFBSEo6K9R2VZWVlixYgUKCgogk8lgbW1dLwGJiIioatfT8qHR6tHa1wFtAxwxIjoA/Tr5cAAnURMiq+0BbG1tYW1tDVEUDXfNiYiIqP5dTs3F4h9P4/01xxF78BoAwMpSgVE9g1iQEzUx1Xqi54OIoojt27fjiy++QHJyMkaNGlUXuYiIiKgSV1LzsOXgVSTdyIWtlQJj+7RA33BvqWMRUS08tCg/fvw4Vq9ejZSUFNjb22PkyJF47LHHAACHDh3Chx9+iGvXrsHKygrTp0+v98BERESmSBRF6EURcpkMadlFSLtTjMf6tULvMC9YmMuljkdEtVRlUX7ixAlMmTIFWq3W0Hb69GmUlJRArVZj6dKlsLOzw8yZMzFp0iROiVgDZRcOSB2BiIgaMb0o4szlLMTFJ6N7Ow8M7OKLyPYe6N7OHQozFuNEzUWVRfmqVatgbm6OTz/9FJGRkUhJScGcOXOwfPlyFBUVYcKECXj55ZdhZ2fXUHmbHfWhb6WOQEREjZBeL+L4xQxsj09BamYhXB0s4WBzt5+4mbzWQ8KIqJGpsihPTEzEhAkTEBMTAwAICQnBnDlzMHXqVIwePRrvvfdeg4Q0OeacyYaIyNSt3nEBCX+mw8PJCk8Pb4Nubd0hl7EYJ2quqizKc3Nz0apVq3JtLVu2BAD069ev/lKZONspn0sdgYiIGphWp0f8uXSEtXCGvY0F+oR7IaylMzoHu0EmE6SOR0T1rMqiXK/XQ6Eo/+Sve8ucl5yIiKj2NFodDiWm4ZcjKcjOV2NCTEsM6uqHVj4OUkcjogb00NlXSkpKkJuba1jOy8sDABQVFZVrv8fBgb9EiIiIjPHb8ZvYcSQFeYVlaOFth6cGhaBDkJPUsYhIAg8tyt955x288847Fdpnz55doU0QBJw/f75ukhERETVDGq0eCrO7fcOv3MqDp5MVZgxvixB/RwgCu6kQmaoqi/LRo0c3VA4iIqJmrbhUg9+Op+K3E6l4bWI4fFxtMG1YG05rSEQAHlKUL1iwoKFyEBERNUsFxWXYc/wm9p5IRYlah44tXSD73x1xFuREdM9Du68QERFRzWi0erz11VEUFGvQKdgVw6MC4OduK3UsImqEWJQTERHVoZwCNY6ev41BXX2hMJPhsX6t4OtuC28XzlpGRJWTtCgvKyvDJ598gm3btiE/Px8hISF48cUXERkZadT+cXFxWLNmDa5cuQJzc3O0bt0ar732GkJDQ+s5ORERUXlZuSXYeSQFv59NgygCHVo4w9vFGt3beUgdjYiaAEmL8rlz52L37t2YNGkS/P39ERsbi+nTp2PdunUIDw+vct8lS5bgq6++wiOPPIIJEyaguLgYSUlJyMzMbKD0REREd/uMb9p/FQl/pkMQgB4dPDG0uz9cHJRSRyOiJkSyojwxMRE7duzAvHnzMGXKFADAqFGjMHz4cCxatAjr16+vdN+TJ0/iyy+/xGeffYYBAwY0UGIiIqK/qMt0sDCXw0Ihx8WbOegb7o3B3fzgZGcpdTQiaoJkUp14165dUCgUGDdunKHNwsICY8eOxYkTJ5CRkVHpvmvXrkWHDh0wYMAA6PV6FBUVNURkIiIi3LhdgM9jz+Ltr49Cq9PDXCHHB9O7Y+KA1izIiajGJCvKL1y4gMDAQFhblx/4EhoaClEUceHChUr3TUhIQIcOHbB48WJ06tQJERERiImJwc8//1zfsYmIyERdU+Xj/dVH8e43f+B88h10a+sBnU4EAJjJJfs4JaJmolrdVwoLC/Htt9/i8OHDyM7OxkcffYTw8HDcuXMH33//PYYMGYIWLVoYdazMzEy4u7tXaHd1dQWASu+U5+XlITc3Fzt27IBcLscrr7wCBwcHrF+/Hq+++iqUSiW7tBARUZ26dDMXH64/CVsrBUb1DET/Tj6wslRIHYuImhGji/I7d+7g8ccfR2pqKvz8/HDz5k2UlpYCAJycnLB161YUFBRg3rx5Rh2vtLQUCkXFX2gWFhYAALVa/cD9iouLAQC5ubnYuHEjwsLCAAADBgzAgAED8Pnnn9eoKHd2tqn2PvXF1ZVz2DY3vKamgde5+RBFEYmXs5CdX4qYzr5wdrbBTLUOvcO9WYybAP4sm4bGdp2NLsqXLl2KrKwsbNy4EZ6enoiKiiq3vl+/fkhISDD6xJaWltBoNBXa7xXj94rz+91r9/HxMRTkAGBubo5BgwZh7dq1KCoqqtAt5mGyswuh14vV2qe2KvvPkJlZ0KA5qH65utrympoAXufmQRRFnL2Wjbj4ZFy9lQ8vF2u087OHTBDQuaUzrCwVvM7NHH+WTYNU11kmEyq9EWx0Ub5//35MnDgR7dq1Q05OToX1vr6+iI2NNTqUq6vrA7uo3JvS0M3N7YH7OTg4wNzcHC4uLhXWubi4QBRFFBYWVrsoJyIi03blVh7W77mElPQCONtZ4KmBrdEj1BMyQZA6GhGZAKOL8pycHPj5+VW6XhCESrucPEhISAjWrVtX4a72mTNnDOsfRCaToU2bNrh9+3aFdenp6ZDL5bC3tzc6BxERmS69XoRao4PSwgwyQUCJWov/GxKCyPYeHLxJRA3K6N84rq6uuHnzZqXrL1y4AE9PT6NPPHjwYGg0GmzatMnQVlZWhi1btiAiIsIwCFSlUuHq1asV9k1LS8Phw4cNbYWFhfjll18QHh4OS0tOSUVERJXT6fU4fDYNb351FD/svQwACPKyw/wZ3dEzzIsFORE1OKPvlPfq1QubN2/Gk08+WWGA5pkzZ7B161ZMnjzZ6BOHhYVh8ODBWLRoETIzM+Hn54fY2FioVCosWLDAsN2cOXNw7NgxXLx40dD2+OOPY9OmTZg9ezamTJkCOzs7/PTTTygoKMBLL71kdAYiIjItWt3dYnznkRRk5pbCx9UGoS2cDevZVYWIpGJ0UT5r1izs27cPo0ePRkxMDARBwNatW7Fp0ybs3r0bbm5umD59erVOvnDhQixduhTbtm1DXl4egoODsXLlSnTq1KnK/ZRKJdauXYuFCxfiu+++Q2lpKdq1a4dvvvnmofsSEZHpij14Db8cvYEAD1s89mgrhLV0YSFORI2CIIqi0VOOpKWl4V//+hf++9//Qq/X3z2AIKB3795499134eHhUW9B65tUs69c++DRCu22M75t0BxUvziS3zTwOjdOao0O/z2tQktvewR52SErrwRp2cVoH+gEoQbFOK9z88drbBqa9OwrAODp6Ynly5ejsLAQ165dAwD4+fnBwcGh9ilNUMpnz0odgYioWSpRa7H/1C38euwGCoo1GBbpjyAvO7jYK+Fir5Q6HhFRBdWafcXR0REAYGNjg9DQ0HoLZSp0+ZlSRyAianb2/HETPx++jqJSLdoHOmFEdABa+fDmERE1bkYX5T179kSfPn0watQo9OnTB2Zm1brJTsYy48wxRETVVViigZWFGWQyAaVlWrTyccCI6AAEetpJHY2IyChGV9YDBw7Evn37sHfvXtjb22P48OEYOXIkOnToUJ/5TI7t1BVSRyAiajLyCtXYdewG9p+6hWnD2qJLiBuGRwXUqL84EZGUjC7KFy9ebJgLfNu2bVi/fj3Wr1+PoKAgjB49GiNGjDDMLU5ERFSf7uSX4pejN3DwjApanR7d27rDx/Xug+hYkBNRU1St2Vf+7tatW9i6dSt+/vlnpKSkQC6Xo3v37li9enVdZ2wQUsy+UrBySoU2zrzS/HAkv2ngdW44oiji7dXHkH6nGJHtPTAs0h/ujlYNcm5e5+aP19g0NMbZV2r8yDJvb288//zz+PXXX7Fo0SIolUrEx8fXOCQREVFlbt8pxne7L0JdpoMgCJg0OBgLZnTH1KFtGqwgJyKqTzUerVlUVGToynLixAno9Xq0atWqLrMREZGJu5VZiO0JKTh24TbM5DJ0DnZDiL8jZ1MhomanWkW5KIo4dOgQtm3bhr1796K0tBSOjo544oknMHr0aLRt27a+chIRkQlRa3T4Ku48TlzKhIVCjsFd/TCwqx/src2ljkZEVC+MLso/+ugjxMXFITs7G2ZmZujbty9GjhyJ3r17c3pEIiKqE3fyS+FkZwlzMxk0Oj1GRAVgQBdf2CgVUkcjIqpXRlfT33zzDTp06IDnnnsOw4cPh729fX3mIiIiE3LpZi7iDl/HpdQ8LHw2EvY2FnhhbChnUiEik2F0Ub5jxw60aNGiPrMQEZEJEUUR51NyEHc4GZdu5sLOSoFRPQNhaX73o4kFORGZEqOLchbkRERUl7LySrH4h9OwtzHH4/1aoVdHL1go5FLHIiKSRKVF+datWwEAI0eOhCAIhuWHGTVqVN0kIyKiZkUvijh1KQvJ6fl4tHcLuDoo8eL4MAT7OUBhxmKciExbpUX53LlzIQgChg4dCnNzc8NyVc8aEgSBRTkREZWj14v4IykD2xOScSuzCO6OSgyPDICFuRztg5yljkdE1ChUWpSvXbsWAGBubl5umYiIyFgp6QX48uc/kX6nGJ7OVpg+oi26tnGDXFbjZ9cRETVLlRblXbt2rXKZiIjoQbQ6PfIKy+BsbwlHOwtYK83w3Kj26BTsChkHbxIRPZDRtyrmzZuHM2fOVLo+MTER8+bNq5NQRETU9Gi0Ouw9kYq5XyZg+bZzEEURdlbmeOOpzugS4saCnIioCkYX5bGxsbhx40al61NTU40eDEpERM2HukyHX4/dwGvLE7B+zyU42VliVI9AqWMRETUpdfYozuLiYj7Zk4jIBMX/mY4f911BG39HzHikHUL8HDjHOBFRNVVZRatUKty6dcuwfO3aNfzxxx8VtsvLy8OGDRvg7+9f9wmJiKhRKSrV4LfjqXBzUCKyvQei23vA19UGLX34pGciopqqsijfsmULli1bBkEQIAgCVqxYgRUrVlTYThRFyGQyzJ8/v96CEhGRtPKLy7Dnj5vYeyIVpWU69IvwQWR7D5gr5CzIiYhqqcqivH///vD29oYoinj99dcxfvx4hIeHl9tGEARYWVmhQ4cO8PT0rNewREQkjb0nUrHpwBVoNHp0DnHD8KgA+LrZSB2LiKjZqLIoDwkJQUhICIC7XVkGDhyI1q1bN0gwIiKS1p38Uliay2FlqYCTnQU6tXbFsMgAeLlYSx2NiKjZMXpk5qxZs+ozBxERNRIZuSXYmZCCw2fTMCzSH6N6BiG8lSvCW7lKHY2IqNmqtCi/N6CzS5cu5ZYf5t72RETUtKRlF2FnQgoS/rwNmQzo1dELPUO9pI5FRGQSKi3Kn3rqKQiCgDNnzsDc3NywXBlRFCEIAi5cuFAvQZubwi3/kjoCEVE5m/ZfxfnkO+jXyQeDu/nB0dZC6khERCaj0qJ8/vz5EAQBCoUCALBgwYIGC2UKxKxrUkcgIhOXkl6A7QnJGNunBdwdrfBY/1awUMhhb20udTQiIpNTaVE+ZsyYcsujR4+u9zAmz5yDp4io/l1V5SHucDISr2ZDaWEGVWYR3B2t4OaglDoaEZHJ4iM4GxHbKZ9LHYGImjFRFPHp5kScuZoNa0szjO4VhH4R3rCyVEgdjYjI5BldlCcmJiIpKQnjx483tP3222/45JNPkJubi9GjR+Oll16ql5BERFQzoigiOb0AgZ52EAQBvu42aO3ngL7h3rA0530ZIqLGwujfyMuWLYNMJjMU5SqVCi+//DKUSiWcnJywatUq+Pv749FHH623sEREZBxRFJF4NRvb45NxVZWP15/shJY+9hjTq4XU0YiI6AFkxm6YlJSEiIgIw/KOHTsgiiK2bduGnTt3Ijo6Ghs3bqyXkEREZBy9KOLExQy89+0f+GRzInILyzBpUDD8PWyljkZERFUw+k55bm4uXFxcDMu///47unTpAnd3dwBATEwMPvnkk7pPSERERlOX6fDtL0mwVirwf0NDENnOA2Zyo++/EBGRRIwuyu3s7JCVlQUAKCsrw5kzZ/DMM88Y1guCALVaXfcJiYioUlqdHkfP38aJi5mY9WgHKC3MMPeJCHg4W0EuYzFORNRUGF2Uh4SEYPPmzYiKisKePXugVqvRo0cPw/rU1FQ4OzvXS0giIipPo9Xj8Lk07ExIQVZeKXzdbJBXWAZ27wUgAAAgAElEQVRHWwt4u9pIHY+IiKrJ6KJ85syZmDZtGsaNGwdRFBEdHY0OHToY1h84cABhYWH1EpKIiP5y+04xFm44hZwCNQI9bTGxf2uEtXSu8qnLRETUuBldlEdERGDLli34/fffYWtri6FDhxrW5eTkIDo6GgMGDKiXkEREpk5dpoMquwiBnnZwcbBEsJ8Dotp7oF2AE4txIqJmoFqT1AYGBiIwMLBCu6OjI15//fU6C0VERHeVqLXYdzIVvx67CUEAFs2MgsJMjhkj2kkdjYiI6lC1nxxRWFiI+Ph43Lx5EwDg6+uLqKgo2NiwDyMRUV0pKtVgzx838dvxVBSrtQht4YzhUQFQmMmljkZERPWgWkX5pk2b8OGHH6K4uBiiKAK4O+uKlZUV5s6di3HjxtVLSCIiU3PzdiF+PpyM8FYuGBEdgAAPO6kjERFRPTK6KN+7dy/eeust+Pr64oUXXkCrVq0AAJcvX8Z3332Ht99+G87OzoiJiam3sEREzVVuoRq7jt6AXC5gXJ+WCPZzwIIZ3eHuZCV1NCIiagBGF+VfffUVWrRogY0bN8La2trQHhkZiTFjxmDChAlYtWoVi3IiomrIzivFL0dTcPBMGvR6ET3DPCGKIgRBYEFORGRCjC7Kk5KS8Pzzz5cryO+xsbHBqFGj8MUXX9RpOCKi5uxQogprd10EAES198CwSH+4ObIQJyIyRdUe6FkZTslFRPRwadlFkMkEuDtaoaW3PXqFeWFIdz+42CuljkZERBIy+hnMwcHBiI2NRXFxcYV1RUVFiI2NRUhISJ2GIyJqLlIzC7Fi2zm8ueooYg9eAwB4OlvjqUHBLMiJiMj4ovzpp5/G1atXMXr0aKxfvx5HjhzBkSNH8N1332HMmDG4du0apk2bVq2Tl5WV4eOPP0aPHj0QGhqK8ePHIyEhodovYvr06QgODsYHH3xQ7X2JiOpTSnoBlm05i7dXH8OZq9kY3N0PE/u3ljoWERE1MkZ3X+nfvz/eeustLFq0CO+//76hu4ooilAqlXjrrbfQv3//ap187ty52L17NyZNmgR/f3/ExsZi+vTpWLduHcLDw406xoEDB3D8+PFqnZeIqKH8kZSBCyk5eCQ6AP07+8JGqZA6EhERNULV6lP+xBNPYMSIETh8+DBSU1MB3H14UHR0NGxtbat14sTEROzYsQPz5s3DlClTAACjRo3C8OHDsWjRIqxfv/6hxygrK8OCBQswbdo0fPbZZ9U6PxFRfbh4Iwdx8ckY0NkXYS1dMLS7P4Z294eVZZ0N4SEiombooZ8SWq0We/fuRUpKChwdHdGvXz8MGTKk1ifetWsXFApFuQcOWVhYYOzYsViyZAkyMjLg5uZW5THWrl2L0tJSFuVEJClRFHHqYga+23kel1LzYGdtjtIyHQCwGCciIqNU+WmRl5eHp556CpcvXzbMm7to0SKsXr0a7du3r9WJL1y4gMDAwApTLIaGhkIURVy4cKHKojwzMxNffPEF3n77bSiVHCRFRNJZvvUcjl/MhKOtBSb2b4VeYV4wV8iljkVERE1IlUX58uXLcenSJfTp0wc9e/bE9evX8cMPP+Dtt9/Gli1banXizMxMuLu7V2h3dXUFAGRkZFS5/+LFixEYGIiRI0fWKsc9zs42dXIcYxU8oM3VtXpdgKjp4LVtXvR6EcfOp6NTiBsUZnL07uyHbh280K+LLxRmLMabO/48N3+8xqahsV3nKovy/fv3o2fPnlixYoWhzcfHBx999BHS09Ph4eFR4xOXlpZCoag44MnCwgIAoFarK903MTERW7duxbp16+psfvTs7ELo9WKdHKumMjMfVKpTU+fqastr20zo9SKOJd3GjvgU3MoqwrRhbRDdwRPtfO15nU0Er3Pzx2tsGqS6zjKZUOmN4CqnRExLS0Pv3r3LtfXt2xeiKOLWrVu1CmVpaQmNRlOh/V4xfq84v58oivjggw8wcOBAdO7cuVYZiIiMoRdF/J6YhjdWHcHKn89DBDDjkbaIbFfzGxNERER/V+Wd8rKyMtjb25drs7OzM6yrDVdX1wd2UcnMzASASvuT79mzB4mJiXjxxRcNM8DcU1hYiNTUVLi4uMDS0rJW+YiI7o2lEQDsPZEKC4UcM0e1R0SwK2R8ijEREdWhGk8LUNtuIyEhIVi3bh2KiorKDfY8c+aMYf2DqFQq6PV6TJ48ucK6LVu2YMuWLVi1ahV69epVq3xEZLrKNDocPKPC/lO3MO/JTrBRKvDi+DDYWinqrMscERHR3z20KP/mm2+wY8cOw7JWq4UgCFi6dCkcHBzKbSsIApYvX27UiQcPHoyvv/4amzZtMsxTXlZWhi1btiAiIsIwCFSlUqGkpAQtWrQAAMTExMDHx6fC8Z5//nn07dsXY8eORbt27YzKQET0d6VlWhw4pcKuYzeQX1SG1j72KCzRwEapgJ21udTxiIioGXtoUX7+/HmcP3++Qvvp06crtFXnDlJYWBgGDx6MRYsWITMzE35+foiNjYVKpcKCBQsM282ZMwfHjh3DxYsXAQB+fn7w8/N74DF9fX2r/VRRIiIAKCzR4PWVR1BYokEbf0c8N7Idgv0cpY5FREQmosqiPCkpqV5PvnDhQixduhTbtm1DXl4egoODsXLlSnTq1Klez0tEBNwtxJNSctA5xA02SgUGdPZBmwAntPS2f/jOREREdUgQRVHaeQAbiYaeErFg5ZQKbbYzvm2w81PD4fRajU9+URl+/eMG9p28Ba1Wj0Uzo2Bv8+AZn4zF62waeJ2bP15j09AYp0Tk85+JyGQUFJdhR0IKDpy6BY1Wjy5t3DAsMqDWBTkREVFtsSgnomZPL4qQCQI0Wj0OnL6FziFuGBbpD09n64fvTERE1ABYlBNRs5WRW4KdCcm4k6/GSxM6wsnOEv95PhrWlhWfJkxERCQlFuVE1OykZRdhR0IKjvx5GzKZgF5hntDq9DCTy1iQExFRo8SinIialVOXMrFsy1koFDL07+yDQV394GjLPuNERNS4sSgnoiYvOT0fpWodQvwd0SbAEcOjAtCvsw/srPjAHyIiahqqXZSnpqYiISEBWVlZGDFiBHx8fFBWVoasrCy4uLjA3JwfgkTUMK7cykPc4WScvZaNFt52eOOpzrA0N8PoXkFSRyMiIqqWahXlH3/8Mb799lvodDoIgoCOHTsaivJhw4bhhRdewJQpU+opKhHRXVdVedjy32u4kJIDG6UCj/YOQkyEj9SxiIiIakxm7IY//PADVq9ejYkTJ+Lrr7/G3585ZGNjg5iYGOzfv79eQhIRiaIInV4PAMjIKcGtrCKM79sSHz8XhWGRAVBasDceERE1XUZ/in3//fcYMGAA3njjDeTk5FRYHxwcjD/++KNOwxERiaKIM1eyERefjE7Brhja3R/d2rijU2tXmCvkUscjIiKqE0YX5cnJyXj88ccrXe/o6PjAYp2IqCb0ooiTFzMRF5+MmxmFcLG3hJPd3VlUZDIB5jIW5ERE1HwYXZRbWFigpKSk0vUqlQp2dnZ1EoqIaO2uJBw8kwZ3JytMG9YG3dq6w0xudI87IiKiJsXoojw0NBR79uzB1KlTK6xTq9XYtm0bIiIi6jQcEZkOrU6PhD/T0S7ACU52lugZ5oUQf0d0DXGHTCZIHY+IiKheGX3badq0aTh9+jReffVVXLx4EQCQlZWFQ4cO4amnnsLt27cfWLATEVVFo9XjwKlbeH3lEXyzMwnx59IBAC287NG9rQcLciIiMglG3ymPiorCu+++iw8++ADbt28HALz22msAAIVCgffffx/h4eH1k5KImqV9J1OxIyEFOQVqBHraYeKA1ghr4Sx1LCIiogZXrTnEJkyYgJiYGOzatQvXrl2DKIoICAjAkCFD4O7uXl8ZiagZ0Wj1UJjd/ZLu6q18uNpbYurQNmgb4AhB4F1xIiIyTdWe2NfV1RVPPfVUfWQhomasuFSLfSdTsfuPm3h5Qkf4e9hiypBgKMw4iwoRERGftiEZAYB43zJR81NYosFvx2/it+OpKFZrEdrCGWbyu//fWZATERHdZXRRPmnSpIduIwgC1qxZU6tApkN8yDJR06fV6fHO18eQU6BGRGtXjIgKgL+HrdSxiIiIGh2ji/LU1NQKbTqdDpmZmdDr9XB0dIRSqazTcETU9OQUqHHkz3QM7uYHM7kME2JawsvFGj6uNlJHIyIiarSMLsr37dv3wPaysjJ888032LJlC9atW1dnwYioacnKK8EvR27gUKIKej3QLtAJfu626NqGg8CJiIgeptZ9ys3NzfHMM8/gypUr+PDDD7F48eK6yNX8CTJA1JdfJmqCiko12LjvimF+8egOnhga6Q83B35zRkREZKw6G+jZqVMnFuTVwS7l1MSpy3SwMJfDQiHHpZu56NPRG0O6+8HJzlLqaERERE1OnRXlqamp0Gg0dXU4E6B/yDJR43QzoxDb45NxTZWH+TMioTCT4f2nu8FMzm97iIiIasroolylUj2wPS8vD/Hx8Vi3bh26du1aZ8GIqHG5npaP7fHJOHU5C5bmcvTr5AOdXg8FZCzIiYiIasnoojwmJqbSp+2JoojAwEC8+eabdRaMiBqPq6o8fLD2BKwszDCyRyD6dfKBjVIhdSwiIqJmw+ii/Pnnn39gUe7g4ICAgABERUVBJuPdMqNxoCc1YqIo4uKNXGTmlaBnqBeCPO0waXAwurVxh9KCzxwjIiKqa0Z/us6ePbs+cxBRIyCKIv68fgdx8cm4nJoHd0clott7QiYT0Kejt9TxiIiImi2jivKioiKMHDkSTz75JKZMmVLPkUyEKFa9TNTArqnysX7PJVxPy4ejrQWeGNAaPUPvFuRERERUv4wqyq2trZGbmwtra+v6zmNCBJSbB7GS/vpE9UkvilCX6aC0MIOZXEBhSRkmDw5GVHtPKMzYpYqIiKihGP2pGxYWhrNnz9ZnFtPCGpwkpNPrceTPdLy9+hi+230JAODnbosFz0Sid0dvFuREREQNzOg+5a+88gomT56MsLAwjBkzptKZWMhI7K1CEtDq9Ej4Mx07ElKQkVMCbxdrhLV0NqyX8eeaiIhIElUW5SqVCk5OTrC0tMSCBQtgZ2eHN998Ex9//DH8/PxgaVn+yX2CIGDNmjX1GrjZkAnlnxfE2VeoAcQdTkZcfDL83G3w/OgOCG/twkKcqBHSajUoKsqHWl0CvV4ndRyTkpEhg17PB/o1d3V9neVyBWxs7KFU1ryrd5VFeb9+/fDxxx9j+PDhSE1NBQB4enoCALKysmp8UgKgv3+gJ38BUN1Ta3Q4eEYFf3dbtPZ1QJ9wbwR52SG0hTO/7SJqpLRaDe7cuQ0rK1s4OXlALpfz57UBmZnJoNXyM7m5q8vrLIoiNBo1cnOzYGamgEJhXrNMDzuJ+L9ZQfbt21ejE1BlOPsK1Z/SMi32n7qFX4/eQH6xBgO7+KK1rwMcbS3gaGshdTwiqkJRUT6srGxhY2MvdRQiMoIgCDA3t4S1tT0KC3Ph6OhWo+PwKSCS4ewrVD/2nkjF1kPXUFSqRbsARwyPCkCwn6PUsYjISGp1CZycPKSOQUTVZGmpRFFRXo33Z1EuFbkc0P3taxMZLwXVXGGJBkoLOeQyGcq0OrT0tsfw6AC08OKdNqKmRq/XQS6XSx2DiKpJJpPXagzIQyvB48ePQ6cz/gSjRo2qcRiTUqG7CruvUPXlF5Xh12M3sO/ULUwaGIzI9h4Y3NUPQ7r5Sx2NiGqBfciJmp7a/tw+tCjfuHEjNm7c+NADiaIIQRBYlBurwhM9OaiEjJdToMYvR1Nw8LQKGp0eXdu4w9/DFgA/zImIiJqihxbl48ePR8eOHRsii2kRhPI3xzklIlXDp5sTcTOjEJHt3TEsMgAeTlZSRyIiIqJaeGhR3rlzZ4wYMaIhspgWuTmg15ZfJqrE7Zxi7P7jJsb2bgGlhRmeHNQadlbmcHVQSh2NiKhJOnnyOP7xj2cxf/4i9OrVp9LtPvjgXZw6dQKbN8cBANLSVBg37hG8/vo7GDp0xAO3IaoJ3p6VCnsYkBFUWUVYGfcnXl95BL8npuFaWj4AoIWXPQtyImpSdu6MQ48endGvXzSysys+62TKlImYNWuGBMmIGgdO+SGVCk+R4kBP+otGq8equD9x4mImFAoZBnXxw6CuvrC34RzjRNS0qdVqfP/9Osye/aLUUYwyZ86bfMInNQgW5VIxUwDa0r8ts9gi4E5+KZzsLKEwk0EvAkMj/TGwiy9srdi9iYiah1atWmPbtp/w5JOT4ejoJHWchzIzY6lEDaPK7itJSUnsT15ftJr7ltXS5KBG4XJqLhb/eBpzv0zAnfy7f6w9P7o9Hu3dggU5ETUrTz01FRqNBhs2rHvotsXFxfj00/9g1Kgh6Ns3Ek8+OQ6xsZuNOk9S0nm89NIsDBvWDzEx0Rg37hHMn/9elfuUlpZi1qxnMWxYP1y6lATgbn/xsWNZC1H9k/TPv7KyMnzyySfYtm0b8vPzERISghdffBGRkZFV7rd7927s3LkTiYmJyM7OhqenJ/r27YuZM2fC1ta2gdLXUoVp69jJ3NSIooikG7mIO3wdSTdyYaNUYGSPQCgt7v5YcmpDImqOfH190b//QMTGbsbEiZPh4ODwwO1EUcTcuS/h1KkTeOSR0QgKaoHDh3/Hf/7zIfLz8zB58rRKz5GTcwcvvjgLnp5emDx5GiwtlUhLU+Hgwf2V7lNcXIzXXvsnUlKS8emnX6JFi5a1fq1E1SFpUT537lzs3r0bkyZNgr+/P2JjYzF9+nSsW7cO4eHhle731ltvwc3NDSNHjoSXlxcuXryIdevW4dChQ/jpp59gYdEEuoLI7ntam5xfj5manAI1/vPDadhaK/BYTEv07ugNC3M+xY+IKvfR+pMV2rq0cUNMhA/UGh2WbjxTYX10B0/0CPVEQXEZvog9V2F93whvdG3jjjv5pVgVd77C+kFd/dCxlQvSsouwdtdFzHkiotavY/Lkadiz51f88MN3ePbZWQ/c5vff/4uTJ4/j2Wdn4cknpwAAxowZj1dffQFr1qzGyJGPVlrQnz2biIKCfHz//U9wdHQ0tD/zzPMP3L6oqBCvvPIPqFS38MUXK+HrG1Cr10dUE5JVgomJidixYwfmzZuHKVOmALj7NNDhw4dj0aJFWL9+faX7fvrpp+jWrVu5tvbt22POnDnYsWMHxowZU5/RiWpEFEWcvpKFK6l5GNe3JZzsLPHi+DC09rWHwozFOBGZDj+/APTrNxA//bQREyc+BTs7+wrbJCQchpmZGR59dIKhTRAEjBv3OI4cicfx40fRv/+gBx7fxsYGAHDw4H6MGDEKMlnlvXULCwvwz38+j6ysTHz22UoEBgZCq+XATmp4khXlu3btgkKhwLhx4wxtFhYWGDt2LJYsWYKMjAy4ubk9cN/7C3IA6N+/PwDg6tWr9ROYqIb0ehF/JGUg7nAyUjML4eagxPCoACgtzNAusPEPciKixqOqu9QWCnmV622tzKtc72RnWeV6T2frOrlLfs/kydOwd+9u/PDDesyYMbPC+vT0dLi6ukOpLD/9q79/wP/Wp1V67PDwTujTJwYffzwfX375OSIiOqNHj17o128gFApFuW2XLPkYWq0Ga9b8AD8//9q/MKIakmye8gsXLiAwMBDW1tbl2kNDQyGKIi5cuFCt42Vl3Z3z9O9fUzVqel35ZZ32wdtRk3bjdgFmLdqH5VvPQavTY9qwNvhgRjdDv3EiIlMVEBCIvn3746effkR+fn6dHlsQBPz73wvx5ZffYtSoR5GenoZ///sdPP30JBQXF5fbtlev3tDpdPj++7V1moGouiSrDDIzM+Hu7l6h3dXVFQCQkZFRreOtWrUKcrkcAwcOrFEeZ2ebGu1XUwVC+XnJBUGEq2sTGaRKVdLq9LiTVwo3JysorS1gozTHa092RlSYF+QyDt5szvgzbBrq+zpnZMhgZtb8nu0n+9/vP7n8r9f39NMzsH//b/jppw0QhLvF9L11Xl6eOHnyD2g06nJ3y2/dugEA8Pb2fuj7FBYWirCwUMycOQu//bYbb745FwcO/IZHHhkFufzuvjEx/dGpUxfMn/8v2Nra4oUXXip33HuD7u+13dtPJvsr6/3bUNNQH9dLJpPV+HeEZEV5aWlpha+QABgGaarVxk8RGBcXh82bN+OZZ56Bn59fjfJkZxdCr2/AB/iYWQFlf81TLprbIDOzoOHOT3VOo9Xh98Q07DySAmulAu9M6QJBELBwdk9kZhbgTnah1BGpHrm62vJn2AQ0xHXW6/XNsk/zvc9Yne6v1+frG4A+ffrhxx83wNbWDtbWNoZ13bpFYevWLdi06UdMnDgJwN2xOT/+uAHm5uYID+9S6fuUn58PW1vbcrNYBQW1AgCUlJRCq9VDp9P/L4+IoUMfQUFBAT77bAlsbW0xadJfM7uI4t3c9851bz+9XjS03b8NNX5mZrJ6uV56vb7K3xEymVDpjWDJinJLS0toNJoK7feKcWNnUDl+/DjeeOMN9OnTBy+88EKdZqxX8vJ/nQmcfaXJUmt0+O9pFXYdTUFuYRlaeNthRFSg1LGIiJqEKVOmYf/+31BYWAh3dw9De3R0L0REdMaKFcugUqkQGBiEhITfceRIPJ5++tlKZ14BgF27tmPLls3o1as3vLx8UFpagu3bt8Ha2hqRkdEP3GfChCdQWFiIlSuXw9raptwAU6KGIFkl6Orq+sAuKpmZmQBQ6SDPv0tKSsJzzz2H4OBgLFmyBHJ5E5rBQrjvrb9/ikRqMo5duI0f9l5GsK8Dnh7eFm38HTnHOBGRkYKCWqJ37xgcOLC3XLtMJsOHHy7GqlXLsW/fHmzfvhXe3j546aU5GDNmXCVHu6tjxwicP/8n9u7dg5ycO7C2tkGbNu3w5pv/gpeXd6X7TZv2DIqLC7F06SJYW9tg8OBhdfIaiYwhiPe+c2lgH330EdatW4ejR4+WG+y5YsUKLFmyBAcPHnxgn/N7bty4gYkTJ8La2hobNmyAk1PtZrFo6O4rhZvegJhzy7AscwmC9Zi3G+z8VHPFpRr8diIVjrYW6BnqBa1Oj+tp+Wjl8+C7NuzWYBp4nU1DQ1zn9PQUeHhwFhCp1Fe3Bmpc6us6P+znt6ruK5KNSBg8eDA0Gg02bdpkaCsrK8OWLVsQERFhKMhVKlWFaQ4zMzMxdepUCIKA1atX17ogl4JgXT6z4FD5HyDUOBQUl2HLwat4dXk8th66jmuqu7MFmMlllRbkRERERMaQrPtKWFgYBg8ejEWLFiEzMxN+fn6IjY2FSqXCggULDNvNmTMHx44dw8WLFw1tTz/9NG7evImnn34aJ06cwIkTJwzr/Pz8qnwaaGMhFt0pv5x3W6IkZIwDp27hx31XoNbo0CnYFSOiAuDnzpk2iIiIqG5IOrpw4cKFWLp0KbZt24a8vDwEBwdj5cqV6NSpU5X7JSUlAQC++uqrCutGjx7dNIpyne6+5YqDXklaOQVqKMxksFEq4GRniY6tXDA80h/erg07fSYRERE1f5L1KW9sGrpPecF3/wSKc/9qsHGB7cRFDXZ+qlxWbgl2Hr2B3xNVGNDZF+P6tqzV8djX2DTwOpsG9ilv/tin3DQ0xj7lnIdPKvc/wZN3yiV3O6cYO+JTkPBnOgCgR6gn+oRXPkqfiIiIqK6wKCf6n58OXMWZq9noE+6NId384GRnKXUkIiIiMhEsyslk3bhdgO0JKRjVIxBeLtYYH9MSTwxoDXsb4x5cRURERFRXWJRL5f6HBckV0uQwQdfT8hF3OBmnr2RBaSFH1xA3eLlYw8VeKXU0IiIiMlEsyqWiUAIleYZFwZwFYX0TRRGfx57DyUuZsLY0w6gegejf2QdWlvyDiIiIiKTFolwq+vsGeup1D96OakUURVxPK0Cgpy0EQYCfmw2CvOzQN9wbSgv+9yciIqLGgVWJVGT3vfX3d2ehWhFFEWev3cH2+GRcuZWHVx8PRxt/RzzSI1DqaEREREQVyKQOYKoEefm3XpCxC0Vd0IsiTl3KxPtrjmPppjO4U1CKJwe2RktvO6mjERFRM9KjR2esXv2lJOdOS1OhR4/O2Lkzrlx7fPzvmDz5MfTtG4kePToDAGbNmoFZs2ZIEZOqiXfKJSJYO0PMUf217OAuYZrmQ6vVY82uJFiYyzFlSAii2nvATM6/PYmIpLZzZxzmz3+vXJujoxNatmyFSZOmIjy86qd5V1d2dha2bv0JvXr1QatWwUbvd/PmDaxduwbHjx9FVlYmzM3N0bJla/TvPwjDh4+EQtE4b6Ll5ubinXfmoVWrYLz88txGm5Mqx6JcImJeWrllfVaKREmaNp1ej2PnM3D0wm3MfrQDzBVyvDoxAh5OSshlLMaJiBqbGTNmwt3dA6IoIjs7C9u3b8NLL83C8uWrERLSts7Oc+dONr75ZhU8Pb2MLsp///0g3nlnHiwsLDF48FAEBraAWl2K06dPYenSj5GWdgszZ75QZxlrysPDE3v3HoaZ2V9lXFLSeZSUlGD69OfK/YGzZMnnUkSkGmBRLhGxrKT8srpIoiRNk1anR/y5dOxMSEFGbgl8XK2RU6CGi70S3i7WUscjIqJKREZGlyuSBw0ailGjhmD//r11UpRrNBoIglDt/W7dSsV7770BLy9vfPrpCjg6OhnWjR37GK5fv4YzZ07WOl9dEAQBFhbln6mRk3MHAGBjU/4R7nV5x/zee/v3Pwao7vBdlcr9Az05T7nRMnNLsPD7k8jOV8Pfwxazx3RAWCsXyGrwS5iIiKRlZ2cPuVwOrbb8rGRqdSnWrPkae/b8iqysDDg5OWPIkOGYMuVpQ1GYlqbCuHGPYPbsF6HXi9iyZSNu307H3LlvGbrKzJ//nuHfr7/+DoYOHfHAHN9/vxYlJSV44423yxXk9wQGBiEwMKjS11ts6mwAACAASURBVJGenobvvluDEyeO4fbt27C0tERERGc8//wL8PT0Mmyn1Wqxdu3X2L37F2Rk3IalpRL+/gGYOnU6unTpDuBuF5oVKz7D2bOJKCwsgL29A0JDw/Dqq2/AxsbG8LrvvZ5Zs2bg9Om7fzD83/89AQAYMmQ43njjXUN/8mXLVtbJe/vjj1vLvR6qOyzKpaKw5Dzl1aDW6JCaWYgWXvZwtrdEa19HdGvrjg5BTjW6I0JEZIp0t69Aq0qCmVcI5O4tJclQUFCA3NxciKKIO3eysWHDOgiCgJiYAYZt9Ho9XnvtJZw/fw4jR46Br68fLl68gLVrv0ZGxm28/vo75Y4ZF7cNWq0Wo0ePhSDI0L17FGbMmImVK7/AI4+MRlhYOACgffvQSnMdPnwI3t4+aN8+FFqtvtqv68KFP3HuXCL69x8EV1c3pKWpsHXrT5g9+xl8990mWFpaAgBWr/4SGzasw5gx4xAU1AIFBYVISvoTFy8moUuX7tBoNHjppdmQy2WYMGEi7O3tcfv2bcTH/47CwoIKd8IBYPLkqfDz88fPP8caugd5e/s8MGdt31ul0qra7w0Zh0W5VO6fl5zzlD9QiVqL/adu4ddjN6DXi1g0MxoW5nJMH1F3/Q6JiJoCzaXD0Fw8WOP9xbIS6LNvAhBRBgEyZ98a3RBSBPeConV0jXP84x/PlltWKpV4770FaNeuvaFt9+5fcPr0CXzxxepy7V5e3lixYhmeeGIy/P0DDO1ZWZn48cdY2Ns7GNoiI6OxcuUXaN8+FIMGDa0yU1FRIbKyMtGzZ+8av66oqB7o27d/ubbo6F549tn/w4EDezF48DAAQELCYYwYMRr/+MfLDzxOcvI1pKXdwqpVa9CmTTtD+7Rpz1R67i5duiMzMxM//xxboXvQ/erivaX6waJcIoK5FcS/L1tW/MvXlBWXavDbiVTs+eMmikq1aBfohBFRAbAw53zuREQ1IZYVA4ZPHhFiWbEk39K+8so8w13crKxMbNu2Bf/615tYvHiZ4Y72gQN7ERjYAt7ePsjNzTXs27lzVwDAqVPHyxWOffv2r1XRWFR0d1yXlVXN7wJbWFga/q3ValFUVAgfH1/Y2Nji0qUkQ1FuY2OD8+fPISPjNtzcKs68Zm19tx44fPgQWrZsXeezqDT0e0vGY1EuEcFcWb4ot2BR/neq7GJsPXQdHVu6YHhUAIK8OM84EZk2RevoWt2h1t2+guLtC+8+UVpmBmXMs5J0YWnXrn25O7l9+/bHhAkj8ckni/D11+sBAKmpN5GcfB3Dh/d/4DH+XkwCgJdX7fo4W1vfnSCguLi4xsdQq0uxbt232LkzDpmZGRDFvz7lCwsLDf9++ulnMXfuy3j00eFo1SoY3bpFYuDAIYb+6l5e3pgw4Ql8++1X+PHH7xEeHoGoqJ4YOHAwrKxqP5FBQ7+3ZDwW5dQo5BWq8esfNyGKIibEtEJLb3ssmNEd7k7su0ZEVBfk7i1hNfw1yfuU38/S0hJt23bAoUMHUFJSAqVSCb1ej9atg/Hcc/944D5eXt7llu+fiaS6rK1t4OzsgmvXrtb4GEuWfIydO+MwbtzjaN++w//6fgt4993XyxXoHTtGYOPGrfj994M4duwItm79Cd9/vxavvfYGhg17BAAwe/aLGDZsBA4d+i+OHTuCxYs/wtq1X+PLL7+Bq6tbrV5rQ7+3ZDwW5RIRNfdPiVhYyZbNW06BGr8cScF/z6ig1ekR3d4ToihCEAQW5EREdUzu3rLRFON/p9PdnXmlpKQYSqUS3t4+uH79Grp06VaLo1ZvEoCoqJ6Ii4vFuXNnERLS7uE73Odev/HZs180tKnV6nJ3ye+xs7PH0KEjMHToCJSUlGD27GewevWXhqIcAIKCWiIoqCUmT56GP/88h2eemYKtW3/C9OnPVTvb3/1/e3ceF1XVP3D8MzOsiigobiCiwoyi4oK5hYnigmnuSypqoaVWZvlUamY+/R6zNFPLsnKplEpLAbcyxbVyQ01FXKhQcktADGXf5v7+4GEeR0C2Gcbk+369fL2cc8+Z8733DPCde8891zTHVpiDPF3FQpQs40tkSmbVS8oPn73BzE8Pse/kNTq1qMeCZzoT3L+FrKYihBBVSEpKCtHRZ3B2rm1YitDfP4AbN/5ix47theqnp6eTlZVV4vva2+fPl09NTSlVHGPHjsfOzo533vkPf//9d6HtcXGX2Lw5tNj2anXhe55CQ78lL894IYfbt42nh9jb2+Pm1ojs7Px9SktLLbQ8ZNOmzdBoNGRnZ5dqX+7HFMdWmIecKbeUe1ZbUfJyi6n4cIm/lY5eUWhQuzrNXGvi59OQxzu5U6eWLAkphBBVweHDBw3TRAqe6Hnnzm3+9a9ZhpMygYH92bMnggUL3uLYsaO0bNma3Nwc4uIusXdvBGvWfIWbW6P79lO/fgMcHWuyeXMo1apVw87OHm/vVoWmZxRwc2vEm2/O59//fp2goOEEBvanSZOmZGVlERV1iv379zJq1Nhi++va1Y+dO3+genUHPDyacPbsGY4fj6RmzZpG9YKCRtK2bXuaN2+Bo2NNYmLOs3dvBEOHjgDgxInjLF26CH//ANzdG6PX57Fz5w5UKhXdu/cs9XEujimOrTAPScotJfeeb6L3TGd52Fy7mcb3h+I4ej6etp51mDbMh7q17Bnft3SPPhZCCPFwWLlyheH/tra2NG3qybx58+ndO9BQrtFoWLhwCevXh7Br1w727duNvX01XF3dCAp6ChcXlxL7sbKy4o033uKTTz7kvffeIS8vj9dfn1dsUg7w2GP+hIRsICRkLQcO7CMsbCM2NjZ4eemYMWOm0fSSe02f/gpqtZqIiB1kZWXTunUbli37mBkzphnVGz58FL/88hPHjh0lJyeb+vUbMGnSFMaMGQ+Ap6cXHTt25tChn9myJQw7Ozs8Pb1YvPhDWrVqXeJ+l8QUx1aYh0q5++6DKiwpKRW9vvIORcpXL0P6XZfHatSlxuhFldZ/Zbkcn8K2Q3H8GpOIjbWGHu1d6dvRnZrVbSwdWqVxcalBYmLpLp+Kfy4Z56qhMsb5xo0/qV+/sVn7EMWzslKX6+FB4p/FXONc0s+vWq2idu2iV9yTM+WWcs/cM5Xm4RyKX39L5FzcLfp39aB3BzdqVKs6ybgQQgghRGk9nJngP4Hqnntsi7hB5J/otyvJbDsUh3/bhvjq6tK3ozt9HmlENTvTPvxACCGEEOJhIkm5pdwzp1zJ/ufOKVcUhfN//s22g3HEXEnGsZo12Tn5l4TsbeUjJoQQQghREsmYRIWt2naOI+fiqeVgw+gALx5r2xBb64fjzL8QQgghRGWQpNxSrIyfkKWy+ecsCahXFE7/fpOWTZyxsdbQTuuCl1tN/HwaYG0lybgQQgghRFlJUm4p/8DpK3q9wvGYBLYdiuNaYhpP9WvOY20a8kjzij3yVwghhBCiqpOk3FLufVhQXo5l4igFvaJwOPoG3x/+kxu30mlQuxrPDPCmo7ck40IIIYQQpiBJuSiWoiioVCpUwP5T17DSqJk6uBW+WhfUapWlwxNCCCGEeGhIUi4KycnN4+eov9hz4iozx7THsboN04b64FDNGrVKknEhhBBCCFOTpNxS7n1YkMby63hn5eRx4OQ1dkRe5nZqNp6uNUnJyMGxug2OVegJnEIIIYQQlU2Sckt5wFZfycjKZfbKI9xJy6a5ey2eHeBN88ZOqOTMuBBCCCGE2alLriLM4gFYfSU9M4djFxKA/If89O7gxuyg9rw2pj0tPJwlIRdCCCH+69dfj+Pn14Gfftpv6VAAWLPmM/z8OpCSkmKy9/Tz68CaNZ+Vum9hWnKm3FLumb6i0lTe9JCU9Gx2HbvC3l+vkpmdR7OGXXF2tKN/F49Ki0EIIUTV8sMP21iw4C2jMicnZzw9vRg/Pph27XzN0m909BmOHj3EyJFjqFGjRon1IyJ2kpiYyMiRY8wSjyi/lJQUVqz4kJ9/3kdmZibe3q2YNu1lvLx0Jba935eIDh06smzZCqOyy5f/ZPXqT/n112Okp2fQoEEDHn/8CcaOnVDh/SiOJOUWoqnnRV5qkuG1lUdbs/eZmpHDD4f/ZN/Ja2Tn5OHbvC4DujTG2dHO7H0LIYQQAM8++xz16tVHURSSkm6yffsWZsx4gU8+WUPz5t4m7+/cuTN88cUqHn/8iVIn5b/9FiNJ+QNGr9fz2mvTiY2NZfToIBwdaxIevolp0yazZs1XuLq63bf93Ln/Z/RarVZx7tw5Nm5cT8eOnY22xcRc4MUXJ+Pu7kFQ0FNUq1ad69evkZAQb/L9upsk5RaipN0yeq2/fcNsfekVBbVKRZ5eYd+pa7TT1qF/Fw9c61Q3W59CCCFEUbp0edTozGbfvo8zeHA/9u3bY5ak/EGRmZmJnZ2cBCuvffv2cOZMFAsWLOaxx/wB6NmzN6NHD+Xzz1cWSrrv1bfv40avrazUnDhxHJVKRa9efQ3leXl5/Oc/b+Lr25H58xeiVlfeTG9Jyi1Ef+uq0eu8+FiT95GYnMEPR/4k/lY6r45uR83qNix+rivV7Sy/0osQQggB4OhYE41GQ26u8UP1srIyWbv2cyIidnLzZgLOzrXp128ATz01CSur/6Uvu3fv5JtvQrhy5TIqlYr69eszYMBgRo4czZo1n/HFF6sAGDFioKHNxo1badCgYaFYXnjhWU6d+hX433SH+vUbsGnTNkMdRdHz5Zer2bw5lDt3btO6dRteffV13NwaGb1Pamoqr732OsuXLyUm5gJjx45n4sTJJtmvu6Wk3GHZsvf45ZcDAHTv3pMZM2YafQHIzc1l3brP2bHje27eTMDFpS6PP/4E48Y9jUajue/4nD59iuXLl3Dx4h/UqePCmDHj71vfXPbv30OdOi5069bdUObk5ETPnr3YvXsXubm5RsevJNnZ2ezfv5e2bdtTt249Q3lk5BHi4i4aEvL09HTs7OwqJTmXpNxS7n2CZ06myd76xq10vj8cx+HoeNRq6ObTkNw8BWsrlSTkQghRRYWGfsfbb7/FtWtXcXV1Y86ceQwbNrLS40hJSSE5ORlFUbh1K4n160NQqVT07NnbUCd/qsIMzp2LZtCgoTRq5E5MzHnWrfuchIR4Xn99HgDHjh3h3/+eQ/fuPRg4cAh5eXnExV3izJnTjBw5mu7de3L9+lV27tzBiy/OoGbNWgDUquVUZGwTJgSTmZnBjRt/MW3aDADs7asZ1Vm7dg1qtYYxY8aTknKH9etDeOutN1i1aq1RveTkv3nttZfp0yeQwMD+1KtX32T7dbc33niNhg3dmDJlGr/9doFt2zZTq5YTzz33oqHOwoXz2bFjOwEBffDxGUtU1ElWr/6U+PgbzJz5RrFjFRv7BzNmPI+TkzPBwc+Sm5vL55+vxMnJ+b5jXCAzM5PMzJLzG7VajaOj433r/P57DDpd80KLUHh7t2Tr1nCuXr2Ch0eTUsUFcOjQL6SmptCnTz+j8uPHI6levTqJiQnMnv0vrly5jJ2dHb17BzJ9+itmvdohSbmlqO75xmVtmkGOik3ig02nsdaoCfB1I7CTO041bEtuKIQQ4qEVGvodM2ZMIyMjf6Wvq1evMGPGNIBKT8xffHGK0Wt7e3veeusdWrZsZSjbtWsHp06dYMWKNUblDRu68umnHzF27AQaN/bg0KGDNGnSlLfffq/Ivjw9vdDpWrBz5w66dfMv8uz43R55pDPh4ZtITk4uNN2hQH5iutZwVtbRsSYffLCYixf/oGlTT0O9mzcTmTVrLgMGDDKU/fjj9ybZr7s1b+7Na6/NMby+ffs233+/xZCU//77b+zYsZ3Bg4fxyiuzgfwxd3CowZYtYQwbNgpPT68i33v16k9RqVR88skaXFzqAuDvH8CECU+WGBfA11+vNVypuJ97r0YUJSnpJu3bF75Zs3btOkD+8S5LUr5z5w5sbGzw9w8wKr927Qp5eXnMnv0v+vcfyJQpL3DmTBTffvs1ycl/884775e6j7KSpNwC8uL/KLQkorpmvWJql+zPGymkZebg7eGMzr0WT3T1oEd7N2rKA3+EEOKh8e2337B+/VflanvixDGysoz/7mRkZPDSS88TEvJlmd5r9OggRo0q/02Qr7wy23BT3s2biWzZEsb//d8bLFnyEW3atAPypyo0adIMV1c3kpOTDW07dOgIwMmTx2nc2AMHBwcSEuI5ezbaKMk1p/79BxpNk2jTJn+hhuvXrxkl5XZ2dgQG9jdqa479Gjx4mNHrNm3a8tNP+0hLS6V6dQeOHDkIwKhRY43qjRo1hi1bwjh8+GCRSXleXh6RkYfx9+9pSMgBPDya0LFjZw4fPnjfuAACA/vj41PyQha2tiWfPMzKysLGpnBeY2Nja9heWmlpqRw69AudOz9a6ObfjIwMMjMzGTx4GC+//BqQPyVIrVbxzTch/P77b3h5aUvdV1lIUm4BWad/KFSmdnIt8/vEXr/N9oNxnI5NonG9Gsx72hlbaw2DuzU1RZhCCCEeEsUlLGVJZEylZctWRjd69ujRi1GjBvHBB4v5/POvgfwz+XFxlxgwoFeR71GQ0A4dOoJ9+3YzefJTNGjgyiOPdKRHj1488kgns8Vfr159o9c1auRPu7h3vXAXl7qF5jibY7/uF0/16g7cuPEXGo2m0Ookrq6N0Gg0xMf/VUwsf5OVlYWbm3uhbe7ujUuVlLu6upW4Kkpp2drakp2dXag8OzvLsL209u/fS1ZWFn36BBbZD2B08ydA7979+OabEM6cOS1J+cOkqJs6bbSPlrr9pb/uEHYglrNxf1Pdzooh3ZoQ4GuaD70QQogH06hRY8p9hrp9+5ZcvXqlULmbWyM2by58oqgy2dnZ4e3dmp9/3k9GRgb29vbo9Xq0Wh1Tp75YZJuGDfNPZDk5OfPFF98QGXmEI0cOceTIIbZuDad//4HMnv2mWeJVq4u+MVJRFKPXtraFp6WaY79KG48lpKenk5GRXmI9tVqDk1PR8/wL1K5dh6Skm4XKC8rq1HEpdVy7dv2Ig4MDXbt2K7IfAGdn43nzBa9TUu6Uup+ykqTcErLv+YCqNWjqeRZd978URSFPr2ClUZN0O5MrCamM6NEM/7au2NvKMAohhCjenDnzjOaUQ/5c7jlz5lkwqv/Jy8tfeSUjIx17e3tcXd24dOliqc54W1tb8+ij3Xj00W4oisLSpYsIC9vI+PHB/z1LW7anU5vzadam3a/SqV+/AXl5eVy7dpVGjf531vvatavk5eVRr16DItvVquWEra0tV69eLrTt8uU/S9X3+vUhJptT7umpJTo6CkVRjMbo7Nmz2NtXM1r95n5u3rzJyZPH6d//iSKnw+h0zdm2DRITE3F39zCUJybmPwG9uJuETUGyOUu49yZPdfHDoCgKUbFJbD8UR+umtRno14T2OhdaN6uNrfX9lzESQggh4H83cz4Iq6/cKyUlhejoMzg71zas6uHvH8DhwwfZsWM7/foNMKqfnp6ORqPB1taW27eTDSuqQH5C3axZ/vzogqk59vb2AKSmlu5x9HZ2dqSmplZ4v4piyv0qrc6dH+Wzzz7mu+/W869/zTSUb9y4AYCuXf2KbKfRaOjYsQsHDuxjypQEw7zyuLhLREYeKVXfppxT3qNHAPv37+Hnnw8Y1ilPTk5m377ddOvW3Wiq0LVr+ctOF/XlZc+enej1evr27VdoG4CfX3c++OB9tm/fgq/vI4byrVvDUalURmWmZtGkPDs7mw8++IAtW7Zw584dmjdvzssvv0yXLl1KbBsfH8+CBQs4ePAger2ezp07M3v2bBo1Kt03JYuyrwkpCcav76FXFE7+dpPth+L4Mz6F2o621K6ZfylMrVJJQi6EEKJMhg0b+UAk4YcPH+TixfxpnAVP9Lxz5zb/+tcswxnQwMD+7NkTwYIFb3Hs2FFatmxNbm4OcXGX2Ls3gjVrvsLNrRHvvjuflJQ7tG/fgbp16xIfH8+mTd/i5aU1rMSh0zUHYOXKFQQE9MHKyopHH33MkKzfq3nz/NVali9fQvPm3tjbV8PP7zGT7Lsp96u0vLy09Os3gPDwjaSk3MHHpy1RUafYvXsnAwYMolmz4q/UT5w4maNHDzN16kQGDx5GXl4eoaHf4eHRlNjY30vs25Rzyv39A2jZsjXz589j9OggatasRXj4RhRFT3Dws0Z1p0+fClDk2fddu36kTh0X2rfvgF5fuJ86dVwICnqKL75YRU5ODr6+jxAdfZqdO3cwZMiIUp+RLw+LJuWzZs1i165djB8/nsaNGxMeHs4zzzxDSEgI7dq1K7ZdWloa48ePJy0tjSlTpmBlZcWXX37J+PHj2bx5MzVrFk5yHyhZafd/DXwT8Rt7f71G3Vr2PN2vOV1a1cdKU3lPlRJCCCHMYeXKFYb/29ra0rSpJ/Pmzad37//ddKfRaFi4cAnr14ewa9cO9u3bjb19NVxd3QgKegoXl/z5w3379mPr1nDCwzeRmpqCs3NtevbsRXDws4aHvWi1zZk8+XnCwjZy9Ohh9Ho9GzduLTYpHzJkGBcunOeHH7bz7bffUL9+A5Ml5abcr7KYOfMNGjRoyI4d29m/fw8uLnWZNGkK48Y9fd92np5eLFmynOXLl7JmzWe4uNQlOPhZkpJuliopNyWNRsN7733AihUfsGnTBrKysmjRoiVvvPF/pU6UL1+OIybmPKNGjUWtVqMvKisHgoOfxcHBgfDwTfzyywFcXOryzDNTSzxeFaVSLHQnQFRUFCNGjGD27Nk89dRTQP4lmQEDBlC3bl2+/vrrYtuuWrWK999/n7CwMLy98x/JGxsbyxNPPMHkyZOZPn16meNJSkpFr6+cQ5Gy8qlCZdUmfc6Rs/HoGtWiTi174m7c4a+kdDq2qIumEh/xKkzPxaUGiYmlu2wq/rlknKuGyhjnGzf+pH79xmbtQxTPykpNbm7RyZp4eJhrnEv6+VWrVdSu7VD0NpNHU0o//vgj1tbWjBgxwlBma2vL8OHDOXHiBAkJCcW23blzJ23btjUk5ADNmjWjS5cu7Nixw6xxm4MeeH3lEdZ8f56D0TcA8KjvSJeW9SUhF0IIIYSoAiyW8Z0/f54mTZpQvXp1o3IfHx8UReH8+fNFttPr9cTExNCqVeHF9Fu3bk1cXJzR3eUPOkUB9FDdzpppw1oz8FEPS4ckhBBCCCEqmcXmlCcmJlKvXuGnWBbMpyruTHlycjLZ2dmGeve2VRTlv8vYFF7s/n6Ku5RgDkYXPlX5/z58pYdZl2ESluXiUqPkSuIfT8a5ajD3OCckqLGykqukliTHv2owxzir1epy/46wWFKemZmJtbV1ofKCZXFKevpYUWtLFrTNzMwsczyVOaccjQ3k5T+VSgWoNDbcvGme5ZeE5clc46pBxrlqqIxx1uv1MqfZgmROedVgrnHW6/X3/R3xQM4pt7OzIycnp1B5QdJd3JqVBeVFPWq1oK2dXeGnaD1IakxcmZ+YA2hs8l8LIYQQQogqy2Jnyl1cXIqcopKYmAhA3bp1i2xXq1YtbGxsDPXubatSqYqc2vKgqTFxpZxZE0IIIYQQgAXPlDdv3pxLly6Rlma8Rvfp06cN24uiVqvRarVER0cX2hYVFUXjxo2LXXtUCCGE+Cew0GrFQogKqOjPrcWS8sDAQHJycti4caOhLDs7m7CwMNq3b2+4CfT69evExsYate3bty+nTp3i3LlzhrKLFy9y5MgRAgMDEUIIIf6pNBprcnLK9ih1IYTl5eRko9GUfxKKxaavtGnThsDAQBYvXmxYLSU8PJzr16/zzjvvGOrNnDmTyMhIYmJiDGVjxoxh48aNPPvsszz99NNoNBq+/PJLXFxcDA8iEkIIIf6JHBxqkpx8k+rVa2JnZ49arZHVuYR4gCmKQk5ONsnJidSo4VTu97FYUg6waNEili1bxpYtW7h9+zY6nY6VK1fi6+t733YODg6EhISwYMECVqxYgV6vp1OnTsyZMwcnp/IfDCGEEMLS7O2rY2VlTWpqMmlpt9Hr8ywdUpVyv8evi4eHqcdZo7GiRg0n7O2rl1y5GCpFJq4Blbwk4n/JjZ5Vg4xz1SDjXDXIOD/8ZIyrBkuN8wO5JKIQQgghhBAinyTlQgghhBBCWJgk5UIIIYQQQliYJOVCCCGEEEJYmCTlQgghhBBCWJhFl0R8kKjVllkD1lL9isol41w1yDhXDTLODz8Z46rBEuN8vz5lSUQhhBBCCCEsTKavCCGEEEIIYWGSlAshhBBCCGFhkpQLIYQQQghhYZKUCyGEEEIIYWGSlAshhBBCCGFhkpQLIYQQQghhYZKUCyGEEEIIYWGSlAshhBBCCGFhkpQLIYQQQghhYZKUCyGEEEIIYWGSlAshhBBCCGFhkpSbWHZ2Nu+99x5+fn74+PgwcuRIDh8+XKq28fHxTJ8+nQ4dOtC+fXuee+45rly5YuaIRXmUd5x37drFSy+9RM+ePWnTpg2BgYEsXLiQlJSUSohalFVFfp7v9swzz6DT6Xj77bfNEKWoqIqO87Zt2xg+fDht27alY8eOBAUFERUVZcaIRVlVZIwPHTrEuHHj6NSpE4888gijRo3ihx9+MHPEojwSEhJYvHgx48aNo127duh0Oo4ePVrq9rGxsUycOJF27drRsWNHZs6cya1bt8wYsTFJyk1s1qxZrF27loEDBzJnzhzUajXPPPMMJ0+evG+7tLQ0xo8fz4kTJ5gyZQovvvgi586dY/z48dy+fbuSohelVd5xnjt3LrGxsQwaNIg33ngDPz8/QkJCGD16NFlZWZUUvSit8o7z3fbv38/x48fNGKWoqIqM89KlS5k1axZeXl7MmTOH559/nkaNGpGYmFgJkYvSKu8Y79u3j+DgYHJzc5k2bRrTp09HrVbz8ssvs3HjxkqKXpTWpUuXWLVqFfHx8eh0ujK1vXHjxw4QZgAAFQ9JREFUBmPHjuXKlSu8/PLLBAcHs2/fPiZOnEhOTo6ZIr6HIkzm9OnTilarVb744gtDWWZmptKrVy9lzJgx9227cuVKRafTKWfPnjWU/fHHH0qLFi2UZcuWmStkUQ4VGecjR44UKgsPD1e0Wq0SGhpq6lBFBVRknAtkZWUpffr0UZYvX65otVpl/vz5ZopWlFdFxvnEiROKTqdTdu3aZeYoRUVUZIwnTpyo+Pn5KVlZWYayrKwsxc/PTxk7dqy5QhbllJKSoty6dUtRFEWJiIhQtFptkX93izJv3jylbdu2yo0bNwxlBw8eVLRarbJx40azxHsvOVNuQj/++CPW1taMGDHCUGZra8vw4cM5ceIECQkJxbbduXMnbdu2xdvb21DWrFkzunTpwo4dO8watyibioxzp06dCpX16tULyL9sJh4cFRnnAuvWrSMzM5OJEyeaM1RRARUZ53Xr1tG6dWt69+6NXq8nLS2tMkIWZVSRMU5NTaVmzZrY2NgYymxsbKhZsya2trZmjVuUnYODA05OTuVqu2vXLnr27Em9evUMZV27dsXDw6PS8jBJyk3o/PnzNGnShOrVqxuV+/j4oCgK58+fL7KdXq8nJiaGVq1aFdrWunVr4uLiyMjIMEvMouzKO87FuXnzJkC5f5EI86joOCcmJrJixQpefvll7O3tzRmqqICKjPPhw4dp3bo1S5YswdfXl/bt29OzZ0+2bt1q7rBFGVRkjDt27Mjvv//OsmXLuHz5MpcvX2bZsmXExcURHBxs7tBFJYmPjycpKanIPMzHx6fMf9fLy6pSeqkiEhMTjb5hFXBxcQEo9tt4cnIy2dnZhnr3tlUUhcTERNzd3U0bsCiX8o5zcVatWoVGo6FPnz4miU+YRkXHecmSJTRp0oRBgwaZJT5hGuUd59u3b5OcnMz333+PRqPhlVdeoVatWnz99de8+uqr2Nvb07t3b7PGLkqnIj/LU6ZM4fLly3z66ad88sknAFSrVo0VK1bw6KOPmidgUekKPgPF5WFJSUnk5eWh0WjMGock5SaUmZmJtbV1ofKCS1zF3chXUH735bF722ZmZpoqTFFB5R3nomzbto1NmzYxefJk+dL1gKnIOEdFRbF582ZCQkJQqVRmi1FUXHnHOT09Hcg/qfLdd9/Rpk0bAHr37k3v3r35+OOPJSl/QFTkZ9nGxgYPDw8CAwPp3bs3eXl5fPfdd7z00kt8+eWX+Pj4mC1uUXlKm4fde7XF1CQpNyE7O7si79AtGOzi5p8VlGdnZxfb1s7OzlRhigoq7zjf6/jx48yZMwd/f3+mT59u0hhFxZV3nBVF4e2336ZPnz506NDBrDGKiqvo7203NzdDQg75f9T79u3LunXrSEtLM/sfcVGyivzO/s9//sOZM2fYtGkTanX+jN9+/foxYMAAFixYwIYNG8wTtKhUD0oeJnPKTcjFxaXIy2AFS2PVrVu3yHa1atXCxsamyCW0EhMTUalURV5SEZZR3nG+24ULF5g6dSo6nY6lS5ea/ZKYKLvyjnNERARRUVGMHj2aq1evGv5B/k1jV69elStfD5CK/t6uU6dOoW116tRBURRSU1NNG6wol/KOcXZ2Nps2bcLf39+QkANYW1vTrVs3zpw5Q25urnmCFpWq4DNQXB5Wu3btSvk7LUm5CTVv3pxLly4VugP/9OnThu1FUavVaLVaoqOjC22LioqicePGcqPYA6S841zg8uXLTJo0CWdnZz777DOqVatmtlhF+ZV3nK9fv45er2fChAkEBAQY/gGEhYUREBBAZGSkeYMXpVaR39stWrQgPj6+0LYbN26g0WioWbOm6QMWZVbeMU5OTiY3N5e8vLxC23Jzc8nNzUVRFNMHLCpdvXr1cHZ2LjYPa9GiRaXEIUm5CQUGBpKTk2P0QIHs7GzCwsJo37694UaT69evF1r+rm/fvpw6dYpz584Zyi5evMiRI0cIDAysnB0QpVKRcU5MTCQ4OBiVSsWaNWtwdnau1NhF6ZV3nHv27MnHH39c6B9Ajx49+Pjjj2nZsmXl7owoVkV+ngMDA/nrr784ePCgoSw1NZUdO3bQrl07mXb4gCjvGNeuXRtHR0ciIiKMpr+kpaWxb98+tFptkXPVxYOvYCWdu/Xp04e9e/cafdE+fPgwcXFxlZaHqRT5mmdS06dPZ8+ePUyYMAF3d3fCw8OJjo5m7dq1+Pr6AjBu3DgiIyOJiYkxtEtNTWXIkCFkZGTw9NNPo9Fo+PLLL1EUhc2bN8tyeQ+Y8o7zoEGDuHDhApMmTUKr1Rq9p7u7O+3atavU/RD3V95xLopOp2P8+PHMmTOnMkIXZVDecc7IyGDo0KHEx8fz1FNP4ejoSGhoKJcuXTJqKyyvvGP8ySefsGzZMlq2bMnAgQPR6/Vs2rSJ2NhYli5dyuOPP26pXRLFWLFiBZD/7I/t27czbNgw3NzccHR0JCgoCMg/eQKwd+9eQ7u//vqLwYMHU6tWLYKCgkhPT2fNmjU0aNCAjRs3FnkTqKnJjZ4mtmjRIpYtW8aWLVu4ffs2Op2OlStXlvjL2cHBgZCQEBYsWMCKFSvQ6/V06tSJOXPmSEL+ACrvOF+4cAGA1atXF9o2ZMgQScofMOUdZ/HPUt5xtre3Z926dSxatIivvvqKzMxMWrZsyRdffCGfkQdMecd46tSpuLm5sW7dOj7++GOys7PR6XR89NFHsrrOA+qDDz4weh0aGgqAq6urISkvSoMGDfjqq6949913ef/997G2tsbf35/Zs2dXSkIOcqZcCCGEEEIIi5M55UIIIYQQQliYJOVCCCGEEEJYmCTlQgghhBBCWJgk5UIIIYQQQliYJOVCCCGEEEJYmCTlQgghhBBCWJgk5UIIIYQQQliYJOVCCFEBy5cvR6fTcfXqVUuHUqnKut9hYWHodDqOHj1q5siEEOKfSZ7oKYSoUo4ePcr48eOL3f7tt9/Stm3bSoyo/K5evUpAQIBRmZ2dHY0aNSIwMJBJkyZhZ2dXafEcPXqUyMhIJkyYgKOjY6X1W1oFj1EvYGVlhZOTEx06dOC5555Dq9WW+713797N+fPnmTZtmilCFUJUQZKUCyGqpAEDBvDYY48VKnd3d7dANBXz6KOPMmjQIAD+/vtvfvjhB5YvX87JkydZs2aNWfqcOnUqzz77rNHjpyMjI/noo48YMmRIoaR80KBB9O/fH2tra7PEU1o2NjbMnz8fgKysLKKjowkLC+PAgQOEhobStGnTcr3v7t27CQ8Pl6RcCFFukpQLIaokb29vQyL7T+fh4WG0L0FBQQwfPpxffvmFqKgofHx8TN6nlZUVVlal/xOi0WjQaDQmj6OsrKysjI7VyJEj8fT05O233+brr79m7ty5FoxOCFGVyZxyIYS4R1RUFLNmzaJv3760adOGdu3a8eSTTxIREVGq9snJySxYsIBevXrRunVrOnXqxNChQ1m9enWhuj/88AOjR4+mXbt2tGnThhEjRvDjjz9WKH4rKyu6dOkCwOXLlw3lGzduZMiQIfj4+ODr60twcDDHjx8v1H7//v0EBQXRqVMnfHx88Pf354UXXuDSpUuGOvfOKZ81axYfffQRAAEBAeh0OnQ6HcuXLwcKzyk/cOAAOp2OdevWFbkPo0aNonPnzuTk5BjK4uLiePXVV/Hz86NVq1b07NmThQsXkp6eXpHDZThWcXFxRuWl/RyMGzeO8PBwAMN+63Q6wsLCDHUSEhKYN28e/v7+tGrVCj8/P+bOnUtSUlKFYhdCPDzkTLkQokrKyMjg1q1bRmU2NjY4ODgQERHBxYsXCQwMxNXVleTkZMLDw3nhhRdYvHgxTzzxxH3fe/r06Rw/fpwnn3wSnU5HZmYmsbGxREZGMmnSJEO9pUuX8umnn9KtWzemT5+OWq0mIiKC6dOn8+abbzJ27Nhy719Bgunk5ATAe++9x+rVq/Hx8WHGjBmkpqby3XffMWHCBFasWEH37t2B/CkoU6dOxcvLi8mTJ1OjRg0SEhI4fPgwly9fpkmTJkX2N2rUKFJTU4mIiGD27NmGfnU6XZH1/fz8cHFxYfPmzYXm+MfFxXHq1CnGjRtnmO4SHR1tmKs+atQo6tWrx4ULFwgJCeHkyZOEhISUe2pMwReXWrVqGZWX9nMwZcoU9Ho9x48fZ9GiRYb27du3B+D69euMGjWKnJwchg8fjru7O3/++Sfr16/n6NGjhIaGUqNGjXLFLoR4iChCCFGFHDlyRNFqtUX+e+mllxRFUZS0tLRC7dLT05U+ffoo/fr1Myr/8MMPFa1Wq1y5ckVRFEW5c+eOotVqlXnz5t03jujoaEWr1Srvv/9+oW1Tp05V2rVrp6SkpNz3Pa5cuaJotVrl9ddfV5KSkpSkpCTljz/+UJYsWaJotVqlR48eSlZWlhIbG6vodDrlySefVLKysgztb9y4ofj6+io9evRQcnNzFUVRlAULFiharVa5efPmffu+d7+LKysQGhqqaLVa5ciRI4ayd999V9Fqtcrvv/9uVHfp0qWKVqtVoqOjDWVPPPGE0rdv30LHZNeuXYpWq1VCQ0PvG6+iKEpQUJDStm1bw7G6fv26EhERofTo0UPRarXK/v37jeqX5XMwc+ZMRavVFtnvlClTlM6dOyt//fWXUXlUVJTSokUL5cMPPywxdiHEw0/OlAshqqRRo0YRGBhoVFanTh0AqlWrZijLyMggMzMTRVHo3LkzGzZsIDU1FQcHhyLf19bWFhsbG6Kiorh69Spubm5F1tu2bRsqlYrBgwcXOmPfs2dP9uzZw6lTp/Dz8ytxXzZt2sSmTZuMyh555BHmz5+PjY0Ne/bsQVEUJk2aZHRjZr169Rg6dChr167l3LlztG7d2nDGdufOnYwcObJM88bLasiQIXz++eds3ryZV155BQBFUdi6dStarZaWLVsCEBMTQ0xMDNOmTSM7O9voePn6+lKtWjUOHjzI0KFDS+wzPT3dMF2lgIuLCwsXLjRcLShQkc9BgZSUFPbv38/QoUOxsbExit3V1RV3d3cOHjwoN4gKIWT6ihCiamrcuDFdu3YtcltSUhLLli1jz549Rc75vXPnTrHJmI2NDa+//jpvv/02AQEBeHp60rlzZ3r16mWUDMbGxqIoCv369Ss2xps3b5ZqXwICAggKCkKlUmFjY0Pjxo0NXzAAw7xvLy+vQm0Lyq5cuULr1q0ZO3Yse/bs4a233mLx4sX4+vrSrVs3BgwYgLOzc6niKa2CxHvbtm3MmDEDtVrNsWPHuHbtGq+++qqhXmxsLJA/j71gjvq9SnusbG1t+fTTT4H8uf9btmzh4MGD6PX6QnUr8jkocOnSJfR6fZFfnAo0atSoVLELIR5ukpQLIcRdFEUhODiY2NhYxo8fT6tWrahRowYajYbQ0FC2b99eZAJ3t9GjRxMQEMCBAweIjIxk586dfPXVVzz++OMsXbrU0I9KpWLVqlXFrkri6elZqpjr169f7BeMsnJycmLTpk0cP36cQ4cOcezYMd555x2WL1/OypUradeunUn6KTBo0CAWLFjAkSNH6Nq1K5s3b0aj0TBw4MBCdYODg+nWrVuR71PaddE1Go3RsQoMDGTy5Mm8+eabeHt707x5c8A0n4OC9wEYOHAgQ4YMKbKOra1tqWIXQjzcJCkXQoi7xMTEcOHCBZ5//nlefPFFo20bN24s9fvUrVuXESNGMGLECPLy8njttdfYvn07Tz/9ND4+Pnh4ePDzzz/TsGFDmjVrZurdMFJwJvb3338vtA77H3/8YVQH8hPXTp060alTJwAuXLjAsGHD+OSTT1i5cmWx/ahUqjLH9sQTT/Dee++xefNm2rdvz86dO+natSt169Y11GncuDEAarXaZF8+CqjVaubMmUP//v1ZtGgRn3/+OVD2z0Fx++7u7o5KpSInJ8fksQshHi6yJKIQQtxFrc7/tVhwhrPAb7/9VqolETMyMsjIyDAq02g0hlVIbt++DWA4E7xkyRLy8vIKvU9pp2OURs+ePVGpVKxZs8ZoicGEhATCwsJwdXXF29sboND8doCmTZtia2triL04BXOwS6p3N2dnZ7p160ZERATbtm0jNTW10Bllb29vtFotGzZs4MqVK4XeIzc3l+Tk5FL3eS8PDw8GDBjAwYMHDUtElvVzULDv98bh5ORE9+7diYiI4NSpU4XaKYpS5DEXQlQ9cqZcCCHu0qxZM7y8vFi9ejWZmZk0adKES5cu8e2336LVajl79ux928fFxREUFETv3r3x8vLC0dGRixcvsn79etzc3OjQoQMAPj4+TJs2jeXLlzN48GD69u1LvXr1SEhI4OzZs/z0009ER0ebZJ+aNm3KxIkTWb16NUFBQfTr14+0tDS+++470tPTWbx4sWEKzdy5c7lx4wZ+fn40bNiQzMxMduzYQVpaWokPW2rTpg2AYblAW1tbvLy8Snx8/ZAhQ9i7dy/vvvsuNWrUoFevXkbbVSoVixYtYsKECQwcOJBhw4bh6elJZmYmf/75JxEREcyYMaNUN3oWZ/LkyWzdupXly5ezdu3aMn8O2rRpw1dffcVbb71F9+7dsba2xsfHh0aNGvHvf/+bMWPGEBQUxKBBg/D29kav13PlyhX27NnD4MGD5UZPIYQk5UIIcTeNRsNnn33GwoULCQ8PJyMjAy8vLxYuXMiFCxdKTMrr16/PsGHDOHr0KLt37yY7O5t69eoxYsQInnnmGezt7Q11X3jhBVq1akVISAjr1q0jPT2d2rVr4+XlxZw5c0y6X6+++iqNGzfmm2++4f3338fa2po2bdrw/vvvG74oQP4c77CwMMLDw7l16xYODg54enry4Ycf0rdv3/v24evryyuvvMKGDRuYO3cuubm5vPDCCyUm5f7+/tSqVYvk5GRGjBhR5BzrFi1aEB4ezmeffcbevXvZsGED1atXx9XVlSFDhhRaUaWsmjZtSr9+/fj++++JjIykY8eOZfocDBgwgPPnz/P999/z448/otfreeedd2jUqBENGjQgNDSUVatWsXfvXrZu3YqtrS0NGjSgR48e973ZVwhRdaiUe6/NCSGEEEIIISqVzCkXQgghhBDCwiQpF0IIIYQQwsIkKRdCCCGEEMLCJCkXQgghhBDCwiQpF0IIIYQQwsIkKRdCCCGEEMLCJCkXQgghhBDCwiQpF0IIIYQQwsIkKRdCCCGEEMLC/h85epFv6VD/aAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fXWo7a4bZpKs"
      },
      "source": [
        "An excellent model has AUC near to 1 which means it has a good measure of separability. A poor model has AUC near to 0 which means it has the worst measure of separability. In fact, it means it is reciprocating the result, predicting 0s as 1s and 1s as 0s. And when AUC is 0.5, it means the model has no class separation capacity whatsoever."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SDdnlu47DubS"
      },
      "source": [
        "To confirm the result we can check it with the [Youden's J statistic](https://en.wikipedia.org/wiki/Youden%27s_J_statistic) which is defined as:\n",
        "\n",
        "\\begin{equation}\n",
        "    \\begin{split}\n",
        "        &J = \\text{Sensitivity} + \\text{Specificity} - 1 = \\\\\n",
        "        &TPR + (1 - FPR) - 1 = TPR - FPR\n",
        "    \\end{split}\n",
        "\\end{equation}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ILv2twvdv7OE",
        "outputId": "991afb0f-708f-40a3-d6b3-624bb7771929"
      },
      "source": [
        "# get the best threshold\n",
        "J = tpr - fpr\n",
        "ix = np.argmax(J)\n",
        "best_thres = thresholds[ix]\n",
        "print('Best threshold = %.2f' % (best_thres))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best threshold = 0.76\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WPu5KLWpl7B5"
      },
      "source": [
        "Another way to evaluate the skill of a prediction model is with the **Precision-Recall** curve.\n",
        "\n",
        "Precision is a ratio of the number of true positives divided by the sum of the true positives and false positives. It describes how good a model is at predicting the positive class.\n",
        "\n",
        "\\begin{equation}\n",
        "    \\text{Precision} = \\frac{TP}{TP \\ + \\ FP}\n",
        "\\end{equation}\n",
        "\n",
        "As it has been mentioned, **recall** is the same as TPR.\n",
        "\n",
        "F-Measure or **F1 score** is defined as the harmonic mean of precision (P) and recall (R).\n",
        "\n",
        "\\begin{equation}\n",
        "    F1 = \\frac{2PR}{P \\ + \\ R}\n",
        "\\end{equation}\n",
        "\n",
        "As in the ROC curve, the approach to finding the optimal threshold would be to calculate the F-measure for each threshold and select the largest one."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5pEb0H4pmE4"
      },
      "source": [
        "from sklearn.metrics import precision_recall_curve\n",
        "\n",
        "def evaluate_prec_recall(probs, preds, y_true):\n",
        "    precision, recall, thresholds = precision_recall_curve(y_true, probs[:, 1])\n",
        "\n",
        "    # convert to f score\n",
        "    fscore = (2 * precision * recall) / (precision + recall)\n",
        "    # locate the index of the largest f score\n",
        "    ix = np.argmax(fscore)\n",
        "\n",
        "    # title\n",
        "    plt.title(f'Precision-Recall\\nBest F1 score = {fscore[ix]:.2f}')\n",
        "    # plot the precision-recall curves\n",
        "    no_skill = len(y_true[y_true==1]) / len(y_true)\n",
        "    plt.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No skill')\n",
        "    plt.plot(recall, precision, marker='.', label='Bert Classifier')\n",
        "    plt.plot(recall[ix], precision[ix], marker='o', color='black', label=f'Best threshold = {thresholds[ix]:.2f}')\n",
        "    # show legend\n",
        "    plt.legend()\n",
        "    # axis labels\n",
        "    plt.xlabel('Recall')\n",
        "    plt.ylabel('Precision')\n",
        "    # show the plot\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "AixYhUNZmycW",
        "outputId": "394d9f70-787d-4a21-a421-b21915925101"
      },
      "source": [
        "evaluate_prec_recall(probs, preds, y_true)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGuCAYAAAA3TXjMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1QUZxsF8LsLLHXBBgiIgsGADewoorErROyKBRU7iRpLjJril2iMKWKMwViwdxMLIsYa1FijMbGABWMvKEUE6Qvsfn8QNq67lIWFodzfOZ4j77RnZiyX4Zl3RQqFQgEiIiIiIhKMWOgCiIiIiIiqOoZyIiIiIiKBMZQTEREREQmMoZyIiIiISGAM5UREREREAmMoJyIiIiISGEM5EVEFNnfuXDg7O2u93ZMnT+Ds7IygoKBSqKryunDhApydnbF3717lGK8lEemCvtAFEBFVBBcuXMCoUaNUxkxMTODo6Ii+ffvCz88Penp6AlVXcT158gRdu3ZVGTM0NIS9vT169uyJCRMmwNjYWKDqiIjKDkM5EZEWevfujY4dO0KhUCA2NhYhISFYtGgR7ty5gy+//LLM6/nyyy8xf/58rbezs7PDtWvXys03Eu3bt0ffvn0BAC9fvsSRI0fw008/4cqVK1i/fr3A1RERlT6GciIiLTRq1EgZHgFg+PDh8PLywq5duzBt2jTUqlVLbZuUlBSYmZmVSj0GBgbF2k4kEsHQ0FDH1RSfg4ODynUdOXIkhgwZgrNnzyIyMhJNmjQRsDoiotLHnnIiohIwMzND8+bNoVAo8PjxY3Tp0gUjR47EjRs3MG7cOLRs2RJ9+vRRrv/gwQN89NFH8PT0RJMmTdClSxd8++23SEtLU9t3XFwcFi5ciK5du6JJkyZo164dxowZg7NnzyrX0dRT/uzZM3z88cfo3LmzcruhQ4ciJCREuU5+fdDZ2dkIDg6Gt7c3mjZtCnd3d0yePBlRUVEq672+/YkTJzBw4EA0bdoUnp6e+Pbbb5GdnV2i66qnp4c2bdoAAB4+fKiyLDk5GYsXL0b37t3RpEkTtG3bFjNnzsTjx4/V9iOTybBmzRr07dsXbm5uaNmyJQYMGICtW7cq14mJicE333yDvn37onXr1mjatCm8vb0RHByMnJycEp0HEVFR8Uk5EVEJKBQKZWisXr06ACA6OhqjR49Gr1690KNHD2XgjoyMxOjRo2Fubg5fX19YW1vj1q1b2LJlCy5fvowtW7Yon3w/efIEw4YNw4sXL9C3b180adIE6enpuHr1Ks6dO4f27dtrrCc7OxtjxoxBTEwMhg8fDgcHB6SkpCAqKgqXLl1C//79CzyfWbNm4dChQ2jfvj2GDRuG+Ph4bNu2DUOHDsW2bdvQqFEjlfV///13bN++HUOHDsXAgQMRHh6O9evXw8LCAgEBASW6tnkh28LCQjmWnJyMoUOHIjo6GgMHDkSDBg0QFxeH7du3Y/DgwdizZw/s7OwA5AbycePG4eLFi/D09ESfPn1gaGiI27dv4+jRo/Dz8wMAREVF4ejRo+jevTvq1q2LrKwsnD59GkuWLMGTJ0+wYMGCEp0HEVFRMJQTEWkhPT0dCQkJAIDY2Fhs3boVt27dQrNmzeDg4AAgN1AvXLgQgwcPVtn2k08+gaWlJXbv3q3SztKuXTtMmTIFYWFhGDBgAABg/vz5iI2Nxdq1a9GhQweV/cjl8nzru3PnDu7fv49Zs2ZhwoQJWp3b2bNncejQIXh5eWHp0qUQiUQAAC8vLwwYMAALFy7E9u3b1Y534MAB1KlTBwAwbNgw+Pj4YOvWrVqF8szMTOV1ffnyJQ4dOoRjx46hdu3ayifmALBs2TI8fvwYv/zyC1xcXJTj/fv3h4+PD4KCgvDNN98AADZt2oSLFy9i0qRJmDlzpsrxXr+Gbdq0QXh4uPJ8AcDf3x8fffQRdu3ahSlTpsDKyqrI50JEVBxsXyEi0kJQUBDatWuHdu3aoW/fvtizZw+6dOmCn376SblOtWrVlOE6T1RUFKKiotC7d2/IZDIkJCQof7Vs2RImJibKtpTExEScPn0aHTp0UAvkACAW5/9Pt1QqBZA7W8yLFy+0Ordjx44BAAICAlQCqouLCzp37oy//vpLGZzzdO3aVRnIgdxedXd3d8TFxSE1NbXIx969e7fyunp7eyMoKAju7u7YuHEjJBIJgNyfSoSFhaF169awsrJSuYbGxsZo1qwZzpw5o9xnWFgYLCwsMHnyZLXjvX4NjYyMlOcrk8mQmJiIhIQEeHp6Qi6XIzIyssjnQURUXHxSTkSkBV9fX/Tq1QsikQjGxsZwcHBAtWrVVNaxt7dXm9Xk7t27AHJDfX7zWcfHxwMAHj16BIVCodYqUhR2dnYICAhAcHAwPD090bBhQ7Rt2xa9evWCq6trgds+efIEYrEYb731ltoyJycn/Pbbb3jy5Alq1KihHLe3t1dbN+96JCYmwtTUFKmpqWo98xYWFsqwDeSGez8/P+Tk5ODhw4dYu3Ytnj9/rrJOQkICEhMTcebMGbRr107jObweth8+fIiGDRsW+kJrXh99aGgoHj58CIVCobL81atXBW5PRKQLDOVERFqoV68ePDw8ClynoHm1x44dq/HpNwCYm5uXqLY8M2bMwKBBg3Dy5ElcunQJu3fvxrp16zB+/Hh89NFHOjlGnoKmVMwLt+vXr8fy5ctVlm3evBnu7u7Kr2vXrq28rh06dEDHjh3Rp08fzJw5Ezt37oRIJFLuz8PDQ+vWnIJ888032LJlC7y9vREQEIAaNWrAwMAA169fR2BgYIHtQkREusJQTkRUBurVqwcg90luYaG+bt26EIlEuHnzZrGPZ29vj5EjR2LkyJHIzMzEuHHjsHbtWowdOxY1a9bMdxu5XI67d++q9GsD/z3pf71Vpaj69euHli1bqoy9uf831a1bF2PHjsVPP/2EAwcOwMfHBzVq1IC5uTlSUlIKvYZA7jSL9+7dg0wmU3ni/qbQ0FC0bt0aS5cuVRl/c9YXIqLSxJ5yIqIy0KhRI7z99tvYuXOnxqn7srOzkZiYCCC3/aNjx444deoUzp07p7bum+0Vr0tOTkZWVpbKmKGhIerXrw8ASEpKynfbbt26AQCCg4NVjnH79m0cP34cLVu2VGldKSp7e3t4eHio/Hp9RpX8+Pv7w8zMDMuXL0dOTg7EYjF8fHxw7do1HD58WOM2r/fR+/j4ICkpCStWrFBb7/XzE4vFatc0LS0NGzduLOIZEhGVHJ+UExGVAZFIhO+++w6jR49Gnz59MHDgQDg5OSEjIwMPHz7EsWPHMHPmTOULovPmzcONGzcwYcIE9OvXD40bN0ZmZiauXr0KOzu7fNtQLly4gHnz5qFHjx5wdHSEqakpIiMjsXv3bri5uSnDuSbt27eHl5cXfv31VyQlJaFz587K6QYNDQ3x2Weflcq1yY+5uTn8/PywatUqhIWFoV+/fpgxYwb+/vtvTJ8+HV5eXnBzc4OBgQGio6Nx6tQpNG7cWDn7yqhRo3DixAmsXLkSERER8PT0hEQiUc5Qkxe6e/bsiZ9//hnTp0+Hh4cH4uPjsWfPHrV3BYiIShNDORFRGWnYsCFCQkKwevVqHD9+HDt37oSpqSns7OzQv39/lZcX7e3tsWfPHvz00084deoUQkNDYW5uDhcXF/j6+uZ7DGdnZ3Tv3h0XL15EWFgY5HI5bGxsMGnSJIwdO7bQGgMDA9GoUSOEhITgm2++gYmJCVq3bo1p06apfUhRWfD398fmzZuxYsUK+Pj4QCqVYseOHVi/fj0OHz6M8PBw6OnpoXbt2mjZsqXKNJQSiQTr16/H+vXrceDAAXz//fcwNDREvXr1VGbH+fjjj2Fqaqrcn42NDXx9fdG0aVP4+/uX+TkTUdUkUhT0c1AiIiIiIip17CknIiIiIhIYQzkRERERkcAYyomIiIiIBMZQTkREREQkMIZyIiIiIiKBMZQTEREREQmMoZyIiIiISGD88CAiqhQuXLiAUaNGqYxJJBJYWVmhTZs2GD9+PN56661SreG3337DzZs3MXXq1CJvM3LkSFy8eFHjMk9PT6xbtw4AEBsbi23btiEyMhLXr1/Hy5cv0b9/f+WnV5LuyWQyrFy5EqGhoYiNjUXt2rUxYMAATJgwAQYGBkXax6NHj7Bs2TKcP38er169go2NDXx8fDBp0iQYGhqqrDt37lyEhIRo3M+yZcvQq1evEp8TEZVfDOVEVKn07t0bHTt2BABkZmYiKioKu3btwpEjRxAWFgY7O7tSO/Zvv/2GkJAQrUI5kPvNw8KFC9XGrayslL+/f/8+Vq1aBRsbGzRt2hSnTp0qcb1UsOnTpyM8PBwDBw5E8+bNcfnyZSxbtgyPHj0q0jdDd+/exdChQ5GdnY0RI0agTp06uHLlClasWIGrV69i7dq1EIlEatt99913amOurq46OSciKr8YyomoUmnUqBH69u2rMlavXj189dVXOHbsWLn82HR9fX21mt/UuHFjnD9/HjVq1EBCQgLatWtXRtWVDoVCgbS0NJiamgpdika///47wsPDMWbMGMydOxcAMHjwYJibm2PDhg0YMmQIWrRoUeA+lixZguTkZGzfvl257tChQ+Ho6Ijvv/8e+/fv13jfC/uzQESVE3vKiajSy3virKnl4ODBgxg2bBiaN28ONzc3DB48GIcPH1Zb7+TJk/Dz84O7uztcXV3RqVMnTJkyBffv3weQ24aS13rg7Oys/LV3716dnIOZmRlq1KhRon0kJiZi0aJF6NatG5o2bQp3d3cMGDAAa9euVVv3yJEjGDlyJFq1agU3Nzf07NkTCxcuhEwmU66TlpaGJUuWoFu3bmjSpAnat2+P2bNn4+nTpyr7unDhgvJabNu2Dd7e3mjatCnWr1+vXKeo96GshIWFAQBGjx6tMp739f79+wvdx4ULF+Dg4KAW3vv37w8A+f7ZUCgUSElJgVwu17puIqq4+KSciCqV9PR0JCQkAMhtX7l9+zaWLl2K6tWro0ePHirrLl26FKtWrUKHDh0wbdo0iMViHDt2DNOmTcP//vc/jBgxAgBw8eJFvPfee2jQoAEmTZoEqVSK2NhYnD9/Ho8ePYKjoyMCAgIgl8tx6dIllfaDwp6m5smr+XUWFhbQ09Mr7qVQM23aNFy6dAlDhw6Fs7MzMjIycPfuXVy8eBHjx49Xrpd3XZycnODv7w9LS0s8evQIR48exQcffACJRIKsrCyMGzcOf//9N3r27IkxY8bg4cOH2LFjB86ePYs9e/agdu3aKsfftGkTEhMTMXjwYFhaWiqXF/U+FETT9cuPVCottCc8IiIC1tbWsLGxURm3sbGBlZUVIiIiCj2OTCaDsbGx2nje2LVr16BQKNRaWFq2bInU1FQYGBigdevWmD59Otzc3Ao9HhFVbAzlRFSpBAUFISgoSGXMyckJ27Ztg6WlpXLs+vXrWLVqFSZNmoSZM2cqx0eNGoX3338fS5YsQd++fWFmZobw8HDI5XJs2LABNWvWVK47efJk5e/bt2+PsLAwXLp0Sev2g7S0NI3tKAcPHtTZy6nJycn4448/MGzYMMybNy/f9a5du4ZVq1bB3d0da9asUXkZcdasWcrfh4SE4O+//8a4ceMwe/Zs5biHhwcmTZqEJUuWYPHixSr7fvbsGQ4dOqRyDbW5DwXRpp1n8+bNcHd3L3Cd2NhYODk5aVxmbW2N58+fF3qcBg0a4M6dO4iLi1P5s3fhwgUAufc9KSkJ1apVAwDUqlUL/v7+aNy4MUxMTHDr1i1s2rQJI0aMQHBwMDw8PIp6ikRUATGUE1Gl4uvrq5ylIjMzE3fu3MGGDRswceJEbN68WfmiZ1hYGEQiEfr166f2lLVLly4IDw/HlStX4OnpCalUCiC3pWPIkCHQ19ftP52GhoZYtWqV2ritra1OjyGRSHDt2jU8efIEderU0bheXlvGhx9+qDY7yOtPdI8dOwaxWIxJkyaprNOpUyc0bNhQ+Y2MWPxfl2Tfvn1VAjmg3X0oyIYNGwpc/joXF5dC18nIyIBEItG4zNDQEBkZGYXuY8yYMZg1axbef/99fPTRR7Czs8PVq1exaNEiGBgYICsrS2U/r3/TAwDdunVD79690a9fP3zxxRc4evRoocckooqLoZyIKpV69eqpPFHs3Lkz2rRpgyFDhiAwMBBLly4FkDszhkKhgJeXV777io+PBwCMGDEC4eHhmD9/PgIDA9GyZUt06NABvXv3LnGfNwDo6emV+lNQiUSCTz75BF999RW6du0KJycntG3bFt26dVN5yvzw4UOIRKJCg+uTJ09gZWUFCwsLtWVOTk64efMmXr58qRLCHRwc1NbV5j4URNfXz8jISKV//nWZmZkwMjIqdB8+Pj5ITEzEsmXLMHLkSAC57zUEBATg5MmTiIiIKPQnAA4ODvDy8sLevXtx//59ODo6an8yRFQhMJQTUaXn5uYGqVSKP/74QzmW18u7Zs2afPu289oXqlevjt27d+PSpUs4d+4c/vzzT3z99dcICgpCcHAwmjdvXibnUVLDhg1D165d8fvvv+PixYs4cuQItm7dCm9vb+U3K0DuE3FNU/WVlKb+am3uQ0Hi4uKKXIeFhUW+T8HzWFlZISYmRuOymJgYWFtbF+lYI0eOhK+vL6KioiCTydCgQQOYm5sr26kKC+UAlD/defnyJUM5USXGUE5EVUJOTo7Kk08HBwecPn0atra2Rerb1tPTg7u7u7IX+datWxg4cCBWrlyJ4OBgACiVIKtrVlZWGDx4MAYPHoycnBzMnj0bBw4cwJgxY+Dq6goHBwecOnUKt27dKnBubHt7e5w+fRqvXr2Cubm5yrK7d+/CzMwM1atXL7Qebe9Dfgprb3ldUXrKmzZtirCwMDx79kzlZc9nz54hNjYWXbp0KfLxJBIJmjZtqvw6IiICCQkJGDRoUJG2f/DgAYDcnnMiqrw4JSIRVXpnz55FWloaGjdurBzr06cPAOD7779HTk6O2javt0xomtmjfv36MDQ0RFJSknLMxMQEQO7Ug+VNeno60tPTVcb09PTg7OwMAMrz8PHxAZB7XTS1bygUCgC5/c5yuVz5DUme33//HTdu3ECXLl1U+snzo819KMiGDRuK/KsoPeW9e/cGkDtjzOvyvs67Tnnu3r2LR48eFbrfzMxMLFq0CBKJBGPHjlWOp6WlITMzU239Gzdu4PDhw3jrrbdQt27dQvdPRBUXn5QTUaVy48YNhIaGAsidku7OnTv45ZdfYGBggOnTpyvXc3V1xdSpUxEUFIR+/fqhZ8+esLa2RmxsLK5fv45Tp04hMjISADBv3jw8f/4cnp6esLW1RUZGBg4dOoTU1FSVmVbc3NywdetWzJ8/H++88w4MDAzg6uoKe3t7nZzbihUrAED5cmBUVJRyrHXr1mjdunW+2z548AB+fn7o3r27soXi3r172LFjB+rUqYNWrVopr8uECROwZs0aDBgwAF5eXrC0tMSTJ09w5MgR7Nq1C+bm5ujfvz9CQkKwZs0aPH36FK1atcKjR4+wfft21KpVS2UmlYJocx8Kouue8k6dOqFz587YsGEDkpOT0axZM1y5cgW7d+9Gnz59lNcrj7e3N+zs7HD8+HHl2D///IO5c+eic+fOsLa2xosXLxASEoLHjx9j0aJFKj8ZePjwISZMmICuXbvCwcEBxsbGuHXrFvbs2QM9PT0sWLBAp+dHROWPSJH32IOIqAK7cOECRo0apTImFotRrVo1tGzZEhMnTtTYjnHy5Els2bIFERERSEtLQ82aNdGgQQN07doVw4YNAwAcPXoUe/fuxY0bN5CQkAAzMzM4OTlh5MiR6Nmzp3Jfcrkcixcvxq+//oq4uDjI5XJ8/fXXGDBgQL51jxw5EpGRkbh8+XKh55j3VFuTKVOmYOrUqfkuf/nyJVauXIkLFy7g6dOnkMlksLa2RqdOnTBhwgTlByzlOXDgALZu3YqoqCgoFArUrl0bHTp0wEcffaTsx05LS8PKlStx8OBBxMTEQCqVwtPTE9OnT1f2QQP/3ZuCrkVR7kNZy8zMxIoVKxAWFobY2FhYW1tjwIABmDhxoto8587OzmqhPD4+Hp9//rmyXcXMzAytWrXS+GcxLi4O3333HSIiIhAbG4vMzExYWlrC3d0dEydO1NnUmERUfjGUExEREREJjD3lREREREQCYygnIiIiIhIYQzkRERERkcAYyomIiIiIBMZQTkREREQkMM5T/q+XL1Mhl5ftRDQ1a5rhxYuUMj0mlT3e56qB97lq4H2u/HiPqwah7rNYLEL16qYalzGU/0suV5R5KM87LlV+vM9VA+9z1cD7XPnxHlcN5e0+s32FiIiIiEhgDOVERERERAJjKCciIiIiEhhDORERERGRwBjKiYiIiIgExlBORERERCQwhnIiIiIiIoEJGspjY2MRGBiIkSNHonnz5nB2dsaFCxeKvP3du3cxbtw4NG/eHG3atMGcOXOQkJBQihUTEREREemeoKH8/v37WLNmDWJiYuDs7KzVts+fP8eIESPw+PFjzJgxA2PHjsWJEycwbtw4ZGVllVLFRERERES6J+gnejZu3Bh//PEHqlevjt9++w2TJ08u8rarVq1CZmYmtmzZAmtrawCAq6srxowZg9DQUAwaNKi0yiYiIiIi0ilBQ7mZmVmxtz169Ci6dOmiDOQA4OHhAQcHBxw6dKjch3LZzZO4t2YzoJD/N2hkAcPW/SFp2Emwukj3Mp5EIePiMcjTkiAysYBIYgx5/COIa9WFSGICfVsX6Fk7CV0mERERCUjQUF5cMTExePHiBZo0aaK2zNXVFWfPnhWgqqKT3TyJzNMb1RdkJCnHy1Mwz4m5g+zoWxAZmUEe/xDytCTlMpGJBfRq1VMZ1zRWlHUVgMbtNG2f/7qK3HWNzSGuWRfyF4+gSE/6b5FCAZGxFOIa9pAnPIYi/RVgJIXIwBDymDuAkQX0bV2QE30T8vQk6Fm9BUVyHOSvYiEyNIXI3Ari6raQv4yGIiMZIkNTiC1skBPzDxQZyRDXqgdkZUKRkZx7SFk6khMea76uTyMBADKIIKphB7FZrX/rzD0HiMUARP+dWWYKFBkpEBmZQWSY+w2tPDkOirREiEyqQSQxVlmuyEyBPCX3HQuRxBgKWTqQkwXoGUBsViN3nxkpgFgvd5mG9VS+VjuBbECeDYj1AT191a/zLri+ISBLy/3S2ALIzszdV964PDv3fBUKQCQCxHq555yTlbu9SJw7nreOyh8I0b/HUuR+c6uyzmvrSq1g0mUiv/EhIqJyrUKG8tjYWACApaWl2jJLS0u8ePECOTk50NPTK/I+a9Ys/lN7bUUfu1zgctGTy7Ds6KNxWcaTKCRHnIQs7jFy0l9BUsMO+jVsIIu5DzMXD5i36J7vNjkpiUWqT2xsBoOadki/dxWy+MeQp7wscP3sIo4VZd38tivOusUlexrx3zFeC9SK5Dgg/gFy3lj/9a9zXj4txhEVUCQ8QU7CkyKurWEsIxka4mi+28hT4ou0b037UpeZ/9d5gRwAkmM1jysPpgBy5G+MyfMvQqEAcmSFl5cci7TQhYWvBxGKesYaD1PsLSsuI8dmsB0+T+gyypylpVToEqiU8R5XDeXtPlfIUJ6ZmfufvkQiUVtmaGgIAMjIyICpqWmR9/niRQrk8uL/h6wNRZ3mwP2rBS6Pi8v9L1528yRkEUehyJblZoZk1TCVHf9fCMy4fw0vLvwKsfTfb1bk2ZCnJ0ER/1Dn50BU+ZTN3//KJOP+Fdz7aqDQZQhIDOhLgOwMtSUGbt4wch8iQE1UUpaWUuX/wVR5CXWfxWJRvg+CK2QozwveMpn6U7K8wG5kZFSmNWkjrzUl88wbPeUAJG2HQtKwU26Ly6UQIF29jaMgioTHyMmnXYKEJIK4dgOIq9spe8oVAOT/trAQUUUk1xjIASDr6kFkXT1YxvUUkUgP0gnrhK6CiN5QIUO5lZUVACAuLk5tWVxcHGrWrKlV64oQJA07wa6jj8oT8czTGyGWWiFl9zwoyl2wFsHAzQuQpZfznvL8j5XfsryXL3Oib0FkUg36dV2R/fgaFCmJ0LNzgTzpOeSJzyEyMoO4up3atnq16qms/+Y1MqlRCzl122jsaZbdPImsW6cAPX1ln3h+2FNezJ5yIlKlyEFysL/u9mdhCyQ9Q97fO8MO/uXqvSiiiqJChnJra2vUqFEDkZHqTxmvXbuGhg0bClBVyYj0c1txMo79WLQNpFYQ6elDbFEbMDBEzp0/UHAQyX1SW1jwA6ASNOWJzyG2qA3DZt5V6kU5bf9DKWj9gn5EJmnYif95laK046uRc+8iIH/zTQBNStZTTlRlJUWrfJl5eqPmyQy0wGBPVVGFCOWPHj0CANStW1c51qNHD+zfvx8xMTHKaRHPnz+PBw8eYPz48YLUWSL6hgUvl5hCZFYDIrEBDBp2VPvHKqdxV2TdPpvvU2PJ2+21DtX8B5EqOpMuk4Auk8rkWFWpDzV58zQgQ7vWOiJt6CLYq9GTQDouWLf7JNIhkULx5s+Ey9aKFSsAAHfv3sWBAwcwcOBA1KlTB+bm5vDz8wMAdOnSBQBw/Phx5XbPnj1Dv379UK1aNfj5+SEtLQ3r1q2DjY0Ndu3apfEl0IKU5YueeV7/Tzz97FZkX/9NfSXOXV7hVaWwVpXxPlcNlpZSPI+8rDajj55TO5h0maTbthAqF/jUvnIqjy96Ch7KnZ2dNY7b2dkpQ7imUA4A//zzD7755hv89ddfMDAwQKdOnfDxxx+jRo0aWtchdChP3j4T+Lf/N0/eP/JUsTGsVQ28z1VDRb7PGRd+Kb8vn1ZS0okbhS6B8sFQXo4JHcpTfvkYisRn/y00toB05LIyrYdKR0X+T5yKjve5auB9/k/y2vG5L2uTVkz6flal3tEqr8pjKK8QPeVVgaRpT5X+OcNW/YUrhoiIqBDS8Wt1ur/UXwOrxDSxBX6YmWktSEcEll0xVK4wlJcTef1q2fcuQb9+K/avERFRlWL67qxS2W/yxsmALLVU9q1zqfGFv5fA4F5psX3lX0K3r1DlxftcNfA+Vw28z5Vf3sSoWy8AACAASURBVD2uaC/tsn9dO2xfISIiIqoAihtyhXqhVqtvIixsIfVdVGq1UPEwlBMRERHpiJH7EBi5D1EbL1dP3pOi1erhk3bhMZQTERERlbKCQm95COwqNUhMIfX/SbBaqiqGciIiIiIBFeUpdXLwGABl9O6bLFUlpBu4eWt8+k+6xVBOREREVM5JJ25QG0vZuwCK+Hulfuysqwf/65MX6+t8OkzKxVBOREREVAGZDfifVuvr5Gm7PFvlKTp70XWHoZyIiIioCnjzaXvy+gAgO6NE+8wL6AznJcdQTkRERFQFSceuUvm6JC+cMpyXHEM5EREREakF6uKE9ORgf0BPAum4YJ3UVJUwlBMRERGRmtdDulYBPUeWu75pLUhHBOq6rEqLoZyIiIiICvR6QE87vho5d84XvlFqPNtatCAWugAiIiIiqjhMukzKDdkWtkXeJjnYH8nbZpVeUZUAQzkRERERaU3quwjSiRth4OZdtA1S45G8dnzpFlWBMZQTERERUbEZuQ8penvKv/OcJ6+bWKo1VUQM5URERERUYtKJG4sezvNeBiUlhnIiIiIi0pm8cC62a1Lougzm/2EoJyIiIiKdM313VpGenPMF0FwM5URERERUagoN5qnxSNm7oExqKc8YyomIiIioVOXXbx565RE6LD6EtwIC0cylDvbs+aXsiysnGMqJiIiIqEy8HsxDrzzCp6GXEZ2UDgWA6IRXmDljSpUN5gzlRERERFRm8uY1Dzx2HelZOSrL0jMy8NWCeUKUJTh9oQsgIiIioqrDyH0IAOBZ0l6Ny58+e1aW5ZQbfFJORERERGXKyH0IbKpLNS6zsTCuklMlMpQTERERUZmbt2gpjA1UmzaMDfQwq3tjAFVvDnOGciIiIiIqcwMHDsH3P66CrYUxRABsLYzxVd/m6NusrnKdqjSHOXvKiYiIiEgQAwcOwcCBQ/J/Kp4aX6b1CIlPyomIiIhIUAV9wFBVaWNhKCciIiIiwRUYzDdOLrtCBMJQTkRERETlmyxV6ApKHUM5EREREZULBT0tr+wYyomIiIio3MgvmFf23nKGciIiIiIigTGUExEREVG5YuDmrXG8Mj8tZygnIiIionLFyH2I0CWUOYZyIiIiIip3TPp+JnQJZYqhnIiIiIjKHT1rJ43jlbWFhaGciIiIiEhgDOVEREREVC6JatUXuoQyw1BOREREROWS2YD/aRyX3TxZtoWUAYZyIiIiIqpQMk9vFLoEnWMoJyIiIiISGEM5EREREZVb0okbNY6n7F1QtoWUMoZyIiIiIqpwFPH3hC5BpxjKiYiIiIgExlBOREREROVafi0slQlDORERERFVSJXp0z0ZyomIiIiIBMZQTkRERETln2ktoSsoVQzlRERERFTuSUcECl1CqWIoJyIiIiISmKChXCaTYfHixfD09ISrqyuGDBmC8+fPF2nbffv2wcfHB02bNoWnpycWLlyI1NTUUq6YiIiIiMqT5J8/EboEnRA0lM+dOxebNm1Cnz598Omnn0IsFmPChAm4fPlygdtt2rQJc+bMgaWlJebOnYsBAwZg9+7deP/996FQKMqoeiIiIiISXFK00BXohL5QB7527Rp+/fVXfPzxx/D39wcA9OvXD71790ZgYCC2bdumcTuZTIagoCC0bdsW69atg0gkAgA0b94cAQEBCA8PR7du3crqNIiIiIiISkywJ+WHDx+GgYEBBg8erBwzNDTEoEGD8NdffyE2Nlbjdv/88w+Sk5Ph7e2tDOQA0LlzZ5iYmODgwYOlXjsRERERlb3K/CFCgoXymzdvwtHREaampirjrq6uUCgUuHnzpsbtZDIZgNwA/yYjIyNcv35d98USEREREZUiwdpX4uLiYG1trTZuaWkJAPk+Ka9Xrx5EIhH+/vtv9OvXTzl+7949JCQkICMjo1j11KxpVqztSsrSUirIcals8T5XDbzPVQPvc+XHe1y+JWsaCx6L+p/u0mo/5e0+CxbKMzIyYGBgoDae9wQ8MzNT43Y1atSAl5cX9uzZg/r166Nr166IiYnBl19+CQMDg3y3K8yLFymQy8v2JVFLSyni4jT90aLKhPe5auB9rhp4nys/3uOKSq7VfRPqPovFonwfBAsWyo2MjJCVlaU2nheqNbWn5FmwYAEyMjLw9ddf4+uvvwYA9OnTB3Xr1i3ylIpEREREVAFJTAFZ5ZsGW7BQbmlpqbFFJS4uDgBgZWWV77ZSqRQrV65EdHQ0nj59CltbW9jZ2WHo0KGoV69eqdVMRERERMKS+v+E5GB/ocvQOcFe9HRxccH9+/fVPvDn6tWryuWFsbW1RevWrWFnZ4dXr14hMjIS7dq1K5V6iYiIiIhKi2ChvFevXsjKysKuXf815ctkMuzduxctWrRQvgQaHR2Nu3fvFrq/JUuWQCwWw9fXt9RqJiIiIiIqDYK1r7i5uaFXr14IDAxEXFwc6tati5CQEERHRyv7xAFgzpw5uHjxIqKiopRjK1euxN27d+Hm5gY9PT2Eh4fjzJkzWLBgAezt7YU4HSIiIiISUHKwf4Wex1ywUA4A3333HX744QeEhoYiKSkJzs7OCA4ORsuWLQvcztnZGeHh4QgPDwcANG7cGGvWrEHHjh3LomwiIiIiIp0SKRSKsp0HsJzilIhUWnifqwbe56qB97ny4z2uGFL2LoAi/p7aeFGflJfHKREF6yknIiIiIioOswH/E7oEnWMoJyIiIiISGEM5EREREVUKFXn+coZyIiIiIiKBMZQTERERUYUjtmsidAk6xVBORERERBWO6buzhC5BpxjKiYiIiIgExlBORERERJVG2vHVQpdQLAzlRERERFRp5Nw5L3QJxcJQTkREREQkMIZyIiIiIqqQpBM3Cl2CzjCUExEREREJjKGciIiIiEhgDOVEREREVKkkB/sLXYLWGMqJiIiIiATGUE5EREREFZbYronQJegEQzkRERERVVim784SugSdYCgnIiIiIhIYQzkRERERkcAYyomIiIiIBMZQTkRERESVjuzmSaFL0ApDORERERFVOpmnNwpdglYYyomIiIiIBMZQTkREREQVmnTiRqFLKDGGciIiIiIigTGUExEREREJjKGciIiIiEhgDOVERERERAJjKCciIiIiEhhDORERERGRwPSFLqCiyM7OQmrqK2RmpkMuz9HJPmNjxZDL5TrZF5VfvM/C0NMzgJmZBYyNTYUuhYiIqFAM5UWQnZ2FhIQYmJhIUaNGbejp6UEkEpV4v/r6YmRnM6xVdrzPZU+hUCArKxOJifHQ1zeAgYFE6JKIiIgKxPaVIkhNfQUTEynMzCygr6+vk0BORKVHJBJBIjGCqakFUlIShS6HiIioUAzlRZCZmQ4jI/4InKiiMTIyRlaWTOgyiIiICsVQXgRyeQ709PSELoOItCQW6+nsHRAiIqp4koP9hS6hyBjKi4gtK0QVD//eEhFRRVGiFz3T09ORmJgIhUKhtszW1rYkuyYiIiIiKjp9IyA7Q+gqik3rUC6Xy7F27Vps2bIF8fHx+a538+bNEhVGRERERFRU0rGrKlS7ypu0DuWBgYFYv349GjRogJ49e6JatWqlURdVQX//fQkffBCARYsC0bFjp3zX++qrL3D58l/YvTsMAPDsWTQGD+6DTz75HN7ePhrXISIiIirPtA7l+/fvR4cOHbBmzZrSqIfK0MGDYVi0aD4MDQ3xyy+hqFmzlspyf//hMDMzw/LlwQJVSERERFQ1aP2i56tXr9C1a9fSqIUEkpmZie3btwhdRpHNmfMZtm/fI3QZRERERDqjdSh/++23ERcXVxq1kEAaNHgboaF78PJlgtClFIm+vj4kEn5CIxEREVUeWofyKVOmYOfOnXj27Flp1EMCGDlyLLKysrBjR+FPy9PS0vDjj0vQr58XOnduBz+/wQgJ2V2k49y6dQMzZ07Bu+92RZcu7TF4cB8sWjS/wG0yMjIwbdr7ePfdrrh9+xaA3H7xQYN8inRMIiIioopA657yyMhI2NrawtvbG927d0edOnUgFqtme5FIhMmTJ+usSCpd9vb26NatB0JCdmP48NH5vryrUCgwd+5MXL78F/r06Y/69d/C2bNnsGTJN3j1KgmjR4/L9xgvXyZgxowpsLGxxejR42BkZIxnz6Jx6tSJfLdJS0vD7NnT8fDhA/z442q89ZZTic+ViIiIqpbkYH9IJ24UuoxCaR3Kly9frvz9/v37Na5TlUL5t9v+Vhtr3dAKXVrUQWZWDn745ara8vZNbeDpaoPkNBl+3HVNbXnnFnZo09AaCa8ysCbshtrynm3qolmDWnj2IhWbD0dhzogWJT6P0aPH4dixI9i5cysCAqZoXOfMmd/x99+XEBAwBX5+/gCAAQOG4KOPpmHTpnXo23dgvoE+IuIakpNfYfv2PahevbpyfNIkzX9OUlNTMGvWB4iOfoqgoNVwcHAs2QkSERERlWNah/Lw8PDSqIMEVreuA7p27YE9e37B8OEjYW5uobbO+fNnoa+vj4EDfZVjIpEIgwcPwx9/nMOlSxfQrVtPjfs3MzMDAJw6dQI+Pv3UfrryupSUZEyfPhnx8XEICgpG3br1Snh2REREVCWI9QF5ttBVFIvWodzOzq406qiwCnpKbWigV+ByqYmkwOU1zI0KXG5T01QnT8nzjB49DuHhR7Fz5zZMnPi+2vLnz5/D0tIaxsbGKuP16jn8uzz/9wyaN2+JTp26YPHiRVi9+ie0aNEKnp4d0bVrDxgYGKisu3TpYmRnZ2HTpp0M5ERERFRk0vFrK+wHCGn9oufrXr58iYiICERERODly5e6qokE4uDgiM6du2HPnp/x6tUrne5bJBJh4cLvsHr1RvTrNxDPnz/DwoWfY/z4UUhLS1NZt2PHd5CTk4Pt2zfrtAYiIiKi8qpYofzWrVvw8/ODh4cHhgwZgiFDhsDDwwMjR47ErVu3dF0jlSF///FIT0/Hrl071JbVrl0bcXExSE9PVxl/9Ojhv8ttCt1/48ZNMHHi+1i7djPmz/8ad+/+g+PHj6qs8847XTF79ic4cCAUQUFLS3A2RERERBWD1u0rt2/fxrBhwyCTydC1a1c4OeXOiHHnzh2cOHECI0aMwM6dO9GgQQOdF0ulz9GxPjp16opdu3ZAKjVX9oIDQLt27bF/fwhCQnZh+PBRAHJnZNm9eyckEglatXLPd7+vXr2CVCqFSCRSjjVo8DYAIDNTprZ+7979kJqaiqCgpZBKpfD3H6+rUyQiIiIqd7QO5T/++CMMDAywY8cOuLi4qCy7ffs2/Pz88OOPPyIoKEhnRVLZ8vcfhxMnfkNKSgqsrWsrx9u374gWLVph1arliI6OhqNjfZw/fwZ//HEO48cH5DvzCgAcPnwAe/fuRseO78DWtg4yMtJx4EAoTE1N0a5de43b+PqOQEpKCtauXQWpVKrygikRERFRZaJ1KP/zzz8xfPhwtUAO5H7a57Bhw7Bz506dFEfCqF/fCe+80wUnT6rOtCMWi/HNN99jzZqVOH78GA4c2Ac7uzqYOXMOBgwYXOA+mzVrgRs3riM8/BhevkyAqakZGjZsjM8+WwBb2/xfHh43bhJSUlLwww+BMDU1Q69e7+rkHImIiIjKE5FCoVBos4GrqyvmzJmDESNGaFy+bds2fPvtt7h2TX3+7TfJZDIsW7YMoaGhePXqFVxcXDBjxgy0a9eu0G3PnTuHlStX4vbt25DL5ahfvz5Gjx4Nb29vbU5H6cWLFMjlmi/F8+cPUbu27mcB0dcXIztbrvP9UvnC+yys0vr7+yZLSyni4pJL/TgkLN7nyo/3uOLTNPvKmx8eJNR9FotFqFnTTPMybXdmb2+PEyfy/xTGEydOwN7evkj7mjt3LjZt2oQ+ffrg008/hVgsxoQJE3D58uUCtztx4gTGjh2L7OxsTJ06FdOmTYNYLMaMGTOwa9curc6HiIiIiEhoWofyvn374syZM/jwww/xzz//ICcnBzk5Obh9+zY+/PBDnD17Fv379y90P9euXcOvv/6KWbNmYfbs2fD19cWmTZtgY2ODwMDAArfdtm0bLC0tsWnTJvj5+cHPzw+bNm2ClZUVQkNDtT0lIiIiIiJBad1TPm7cONy4cQO//vorDh48qPxkRrlcDoVCAS8vL4wdO7bQ/Rw+fBgGBgYYPPi/XmRDQ0MMGjQIS5cuRWxsLKysrDRum5KSAgsLC0gkEuWYRCKBhYUFDA0NtT0lIiIiIiJBaR3K9fT08MMPP+Ds2bP47bff8OTJEwC5bS3dunWDh4dHkfZz8+ZNODo6wtTUVGXc1dUVCoUCN2/ezDeUt2nTBqtXr8YPP/yAAQMGAAD27t2LBw8e4OOPP9b2lIiIiIiIBKV1KM/Tvn17tG+veSq7ooiLi4O1tbXauKWlJQAgNjY2320DAgLw6NEjrFq1CitXrgQAmJiYYMWKFSWqiYiIiIgqn+SNkyH1/0noMgpU7FBeUhkZGTAwMFAbz2s/yczMzHdbiUQCBwcH9OrVC927d0dOTg5++eUXTJ8+HRs3boSrq6vW9eT3JiwAxMaKoa9frA8/LVRp7ZfKF95n4YjFYlhaSsvkWGV1HBIW73Plx3tcsWmcU0WWqnZfy9t9LjSUL1++HCKRCO+99x7EYjGWL19e6E5FIhEmT55c4DpGRkbIyspSG88L4wX1hn/55ZeIiIjA7t27lT3tXl5e6N27NxYtWlSsedILmhJRLpeXypR2nCqvauB9FpZcLi+Taa84jVrVwPtc+fEeV16v39fyOCVikUP5hAkTIJFIdBbKLS0tNbaoxMXFAUC+/eQymQy7d+/GpEmTlIEcAAwMDNChQwfs2LED2dnZ0NcX7IcARERERCQQ6cSNGucqL+8KTa7h4bmf6pg300ne1yXl4uKCLVu2IDU1VeVlz6tXryqXa5KYmIjs7Gzk5OSoLcvOzkZ2dja0/DwkIiIiIiJBFdroamdnBzs7O7WvC/tVmF69eiErK0vlw35kMhn27t2LFi1aKF8CjY6Oxt27d5Xr1KxZE+bm5jh27JhK+0tqaipOnDiBt99+W2OvOhERERFReaWzHo+EhAS8evUKDg4ORVrfzc0NvXr1QmBgIOLi4lC3bl2EhIQgOjoaX3/9tXK9OXPm4OLFi4iKigKQOyXj2LFj8cMPP8DX1xd9+vSBXC7H7t278fz5c8yZM0dXp0REREREVCa0nhJi3759mDdvnsrYkiVL0L59e3h5eWHo0KFISUkp0r6+++47jBw5EqGhoVi4cCGys7MRHByMli1bFrjde++9h8DAQOjp6eGnn37CsmXLYGZmhuXLl8Pb21vbU6JKwtOzFdatWy3IsZ89i4anZyscPBimMn7u3BmMGDEEnTu3g6dnKwDAlCkTMWXKRCHKJCIionJK6yflO3fuhKOjo/LriIgIrFmzBq1bt4ajoyP27NmDjRs3YsqUKYXuy9DQEHPmzCnw6faWLVs0jvv4+MDHx0fb8uk1Bw+GYdGi+Spj1avXgJNTA4waNRbNmxf8zZG2XryIx759e9CxYyc0aOBc5O2ePHmMbds249KlC4iPj4NEIoGT09vo1q0nevfuW27blRITE/H55x/j7bed8eGHc8ttnURERCQ8rUP5o0eP0KtXL+XXhw8fhoWFBdatWweJRAKRSIRDhw4VKZRT+TBx4vuwtq4NhUKBFy/iceBAKGbOnIKVK9fBxaWRzo6TkPACGzasgY2NbZFD+Zkzp/C//30MIyMj9OrlDUfHt5CZmYErVy7jhx8W49mzp3j//Wk6q7G4ate2QXj4WZVZf27duoH09HRMmjQZrq7NleNLl5bvDy8gIiKisqd1KE9OToZU+t9k6+fPn4eHh4dydpYmTZpg//79uquQSl27du1VQnLPnt7o188LJ06E6ySUZ2VlQSQSab3d06dPMH/+p7C1tUNQ0CpUr15DuWzQoKG4f/8erl79u8T16YJIJFKbW//lywQAgJmZ6nykunxinndtOQUoERFRxab1/+SWlpZ4+PAhgNyXO2/duoWBAwcql6elpUFPT093FVKZMze3gJ6eHrKzs1XGMzMzsGnTehw7dgTx8bGoUaMmvLx6w99/vDIUPnsWjcGD+2Dq1BmQyxXYu/cXxMQ8x9y585StMosWzVf+/pNPPoe3t+Y2pO3bNyM9PR1z585TCeR5HB3rw9Gxfr7n8fz5M2zdugl//XURMTExMDIyQosWrTB58jTY2Ngq18vOzsbmzetx9OghxMbGwMjIGPXqOWDs2Alo3botAODx40dYtSoIERHXkJKSDAuLanB1dcNHH30KMzMz5Xnnnc+UKRNx5UruNwyjRg0DAHh59cann36h7CdfvjxYJ9f255/3qZwPERERVTxah3J3d3ds27YNFhYWuHDhAkQiEd555x3l8vv37yunM6SCZT//B5mPb0Lf1gV61k6C1ZGcnIzExEQoFAokJLzAjh1bIBKJ0KVLd+U6crkcs2fPxI0bkejbdwDs7esiKuomNm9ej9jYGHzyyecq+wwLC0V2djb69x8EkUiMtm09MHHi+wgOXoE+ffrDzS23naNJE9d86zp79jTs7OqgSZOmxTqvmzevIzLyGrp16wlLSys8exaNffv2YOrUSdi6dReMjIwAAOvWrcaOHVswYMBg1K//FpKTU3Dr1nVERd1C69ZtkZWVhZkzp0JPTwxf3+GwsLBATEwMzp07g5SUZLUn4QAwevRY1K1bD/v3hyAgYDIsLa1hZ1dHY50lvbbGxibFuj5ERERUfmgdyqdNm4bLly9j8eLFAHJnQqlTJzdsZGdn4+jRo+jRo4duqyynsm6fRVbUqWJtq5ClQ/7iMQAFZBBBXNMeIolxsfZl4NwRBm+3L9a2APDBBwEqXxsbG2P+/K/RuHET5djRo4dw5cpfWLFincq4ra0dVq1ajhEjRqNePQfleHx8HH7+OQQWFtWUY+3atUdw8Ao0aeKKnj0LniUnNTUF8fFx6NDhnQLXK4iHhyc6d+6mMta+fUcEBIzByZPh6NXrXQDA+fNn4ePTHx988KHG/Tx4cA/Pnj3FmjWb0LBhY+X4uHGT8j1269ZtERcXh/37Q+Dh4Yn69Rvku64uri0RERFVbFqH8tq1a+PXX3/FnTt3IJVKYWv734/NMzIysGDBgnw/jZP+o5ClAcj75FEFFLK0Yofykpo162PlU9z4+DiEhu7FggWf4fvvlyufaJ88GQ5Hx7dgZ1cHiYmJym1btWoDALh8+ZJKcOzcuVuJQmNqaioAwMSk+E+BDQ2NlL/Pzs5GamoK6tSxh5mZFLdv31KGcjMzM9y4EYnY2BhYWan/lMfUNPdJ+Nmzp+HkpPsPpyrra0tERFQVJQf7Qzpxo9Bl5KtYb4fp6enB2Vl99gwzMzN069ZNwxaVk8Hb7Yv9hDon5g7Sfv0OyMkGxPow7hIgWAtL48ZNVF707Ny5G3x9+2LZskCsX78NQO60hA8e3Efv3prv7+thEoDKN2vFYWpqCiD3HYXiyszMwJYtG3HwYBji4mKhUCiUy16fS3/8+ADMnfshBg7sjQYNnOHu3g49engp+9Vtbe3g6zsCGzeuxc8/b0fz5i3g4dEBPXr0gomJabHry1PW15aIiIjKH07ZIBA9aydI+8wpFz3lbzIyMkKjRk1x+vRJpKenw9jYGHK5HG+/7Yz33vtA4za2tnYqX785E4m2TE3NULNmLdy7d7fY+1i6dDEOHgzD4MHD0KRJ0397v0X44otPVAJ6s2Yt8Msv+3DmzClcvPgH9u3bg+3bN2P27E/x7rt9AABTp87Au+/64PTp33Hx4h/4/vtvsXnzeqxevQGWllYlOteyvrZERESVndiuCeRPI4UuQyuFhnIXFxeIxWJcuXIFEokELi4uhU5vJxKJcOPGDZ0VWVnp124A1HpL6DI0ysnJnXklPT0NxsbGsLOrg/v376F1a/cS7FW7aRE9PDogLCwE169HqvRaF1Ve3/jUqTOUY5mZmRo/cdbc3ALe3j7w9vZBeno6pk6dhHXrVitDOQDUr++E+vWdMHr0OFy/HolJk/yxb98eTJjwnta1vU4315aIiIjymL47C8nB/kKXoZVCQ3m/fv0gEomU0xzmfU2VV3JyMiIjI1CjRk3lVISdOnXF+fNncejQAXh59VZZP28azMKe4Bob5/bMp6QkF6mOESNG4dixQ/j22y+xbNkqVK9eXWX5gwf3ceXK3+jXb6DG7cVi9ak59+z5GTk5OSpjSUmJKj3axsbGqFPHHs+fRwPIfenU0NBIZS7w+vXfgp6eHmQyWZHOpSC6uLZERERUsRUayr/55psCv6aK7/z5s8o2kbxP9Hz1KgkffjhX+Q1Yr17vIjz8GBYtmo8//7yAxo2bIjs7Cw8e3Mfx48ewbt1W1KljX+Bxate2gbm5Bfbt2wMTExMYGRmjUaMmau0ZeerUscf//rcQX3zxCfz8BqFXr3fh6FgfmZmZuHbtCk6ePA5f3xH5Hs/DwxNHjhyEqakZHBwccf16BC5duggLCwuV9fz8hqBZsxZwcWkIc3MLREXdxPHjxzBgwGAAwF9/XcLSpd+hU6euqFu3HuTyHBw5cujf6UC7FPk650cX15aIiIgqNvaUE4KDVyh/b2hoiPr1nfD55wvRvXsv5bienh6+/fZ77NixBUePHsKJE7/B2NgEdnZ14OfnD0tLy0KPo6+vj88+m4+VK3/E4sVfIycnB5988nm+oRwAOnbshI0bt2P79i34/fcT2Lt3FyQSCRo0cMbMmXNU2kveNG3aLIjFYhw7dgiZmTI0beqGH374CTNnTlVZb9AgX5w5cwp//nkBWVky1K5tg/HjAzB8+CgAgJNTA7Rp0xbnzp1GaOheGBkZwcmpAQIDfyz2HOqv08W1JSIioopNpHj9jbciOHfuHM6fP48PP9Q8p/OSJUvQvn17tG3bVicFlpUXL1Igo8oyMAAAIABJREFUl2u+FM+fP0Tt2vV0fkx9fTGys+U63y+VL7zPwiqtv79vsrSUIi6uaK1ZVHHxPld+vMeVh6ae8rwpEYW6z2KxCDVrqn/oIACItd3Z2rVr8fDhw3yXP3nyBGvWrNF2t0REREREVZbWofzWrVto1qxZvsvd3NwQFRVVoqKIiIiIiKoSrUN5cnKychYNTQwNDZGUlFSiooiIiIiIqhKtQ7m1tTWuX7+e7/Lr16/zxTQiIiIiIi1oHco7deqEffv24dy5c2rLzp8/j3379qFjx446KY6IiIiIqCrQekrEgIAAHDlyBOPGjUPHjh3h4uICILfX/NSpU6hVqxbef/99nRdKRERERFRZaR3Ka9WqhZ07d+KLL77AqVOn8PvvvwMARCIROnbsiHnz5sHKykrnhRIRERERVVbF+vAgOzs7rFmzBklJScrpEevVq6f2SYlEREREROVFcrC/cq7y8qZEn+hpYWEBV1dXXdVCRERERFQlaf2iJwDk5ORg3759mDVrFsaMGYMbN24AAJKSkrBv3z7ExMTotEgiIiIiospM61Cenp4OPz8/zJ07F+Hh4fjjjz+U85KbmZkhMDAQ27dv13mhRERERERFVV7bVPKjdSgPCgpCZGQkli9fjvDwcCgUCuUyPT099OjRA2fOnNFpkUSa/P33JXh6tsKpUyeFLgUAsG7danh6tkJycrLO9unp2Qrr1q0u8rGJiIioYtK6p/zw4cPw9fVFt27d8PLlS7XldevWxcGDB3VSHJWugwfDsGjRfJWx6tVrwMmpAUaNGovmzVuWynEjIyNw4cI5DBkyHFKptND1f/vtCBISXmDIkOGlUg8VX3JyMlas+BGnT59ARkYGGjVqgqlTZ6BBA+cCt5PL5Th06AB+//0E7ty5jVevkmBjY4vu3Xth6FA/SCQS5brPnkVj8OA+GvcTGPgj2rb10Ok5ERERCUHrUB4bGwtn5/z/wzU2NkZqamqJiqKyNXHi+7C2rg2FQoEXL+Jx4EAoZs6cgpUr18HFpZHOj3fjRgQ2bFgDb2+fIoXy8PCj+Oef2wzl5YxcLsfs2dNw9+7d/7d35+E13vn/x1/JkURsEcRSaVCVWEqISqelLSpEGVHrj8ZS2qp2tJi2qsx0b6cYS9VSS0swSiSxlgbFNaOW0tZSSzVlLCGCJpLIfs7vD9+ccZyEbCd3JM/Hdbku+dz355z37X0kr9znc99HgwaFqlo1D0VFrdGYMaO0ePFy1a/vnefctLQ0ffLJ+2rRoqVCQvrI07OGjh49rEWL5uvgwQOaNWuu3Zxu3borMPBRm7EHH/Qt9uMCAMAIBQ7l1atXv+OFnKdOneI+5feYRx9tb3Nms1u3p9W7d3ft2LHdIaG8tEhLS1PFihWNLuOetWPHdh05clgffzxNTzzRUZLUuXOQBg3qoy+/XKC//e39POe6uLho3rzFatnS3zrWq9czqlfvPi1e/IV+/PGAAgJsl+P4+TVTt25PO+RYAAAwWoFD+aOPPqrIyEiNHDnSbtu5c+cUERGhkJCQYikOxqhWzUMmk0lZWVk24+npaVq69Ett3fqtrly5rBo1aqp7954aPvx5Vajwv5fStm3f6l//WqZz587KyclJdevWVc+evTVgwCAtXvyFvvpqoSTZLEkID1+vevXus6vlL395UT///KMkWddM161bT2vWbLDuY7GYtWTJIq1dG6Hr1xPVsqW/3njjbXl732/zOMnJyXrzzbc1e/YMnTx5Qs8+O1QjR44qluO6VVLSdc2cOVX/+c/ND9bq2LGzxo2bYPMLQFZWlsLCvtTmzZt05cpleXnV1tNP/1lDhjwnk8l0x/4cOvSzZs+ert9//021anlp8OChd9zfUXbu3K5atbz0+ONPWsc8PT3VuXMXbdsWraysLJt/v1u5uLjYBPIcTzzRSYsXf6H//veMXSiXbl5oXqFCBbm4uBTfgQAAUAoUOJT/5S9/Ud++fdWvXz/16NFDTk5O+ve//63vv/9eX3/9tVxdXTVq1ChH1FqmRESs1kcfvacLF86rfn1vTZr0jvr2HWBILUlJSUpISJDFYtG1a1e1cuUyOTk5qXPnIOs+N5cqjNexY0cVEtJH99/vo5Mnjyss7Etdvhynt99+R5L0ww979e67k/Tkk53Uq9czys7O1pkzp3XkyCENGDBITz7ZWbGx5/Xtt5v16qvj5eFRXZJUvbpnrrUNGzZCqampiou7qDFjxkuS3N0r2eyzdOliOTubNHjwUCUlXdfKlcv03nuTtXDhUpv9EhL+0JtvjlPXrsEKDu6hOnXqFttx3Wry5Dd1333eeumlMfr11xPasGGtPDw89fLLr1r3+fTTD7V580Y99VRXtWr1rA4f/kmLFs1XXNwlTZgwOc9excT8pvHjX5GnZw2NGPGisrKy9OWXC+TpWeOOPc6RlpamtLS0u+7n7OysatWq3XGfU6dOys+vqZycnGzGmzdvofXro3T+/Dk1bNgoX3XluHbtiiRZXxe3Wrhwvj77bLqcnJzUvPlDGj16jFq3DijQ4wMAUFoVOJQ3aNBAS5Ys0dtvv63PPvtMkvTll19Kkpo0aaKpU6eqXr16xVtlGRMRsVrjx49RamqqJOn8+XMaP36MJBkSzF999SWbr93d3fXee5+oRYuHrGPR0Zv1888HNXfuYpvx++6rr/nzP9ezzw5TgwYN9f33u9Wo0QP66KOpuT7Xgw82kZ9fM3377WY9/njHXM+O36pduz8pMjJciYkJeS5duBlMl1rPylar5qFZs6bp999/0wMPPGjd78qVeL311t/Us+f/3snZsmVTsRzXrZo2ba4335xk/fr69URt2rTOGspPnfpVmzdvVO/effX66xMl3ex7lSpVtW5dpPr2HagHH2yS62MvWjRfTk5Omjdvsby8bi4T69jxKQ0b9v/uWpckrVix1PpOxZ3c/m5Ebq5evZLr2eyaNWtJuvnvXdBQvmJFmKpUqaLAwD9Zx5ycnBUY+KieeKKjatXy0vnzZ7Vy5XKNHfuyZs2aJ3//NgV6DgAASqNCfaLnQw89pPXr1+vXX39VTEyMLBaLGjZsqObNy+7649ysWvUvrVy5vMDzDh78Qenp6TZjqampGjv2FS1btqTAjzdoUKgGDiz8RZCvvz7RelHelSvxWrcuUu+/P1nTp39uDTw7d25Xo0aNVb++txISEqxzH344UJL0008H1KBBQ1WpUkWXL8fpl1+O2oRcR+rRo5fNMgl//9aSpNjYCzahvGLFigoO7mEz1xHH1bt3X5uvW7duo127diglJVmVK1fR3r27JUkDBz5rs9/AgYO1bl2k9uzZnWsoz87O1v79e9SxY2drIJekhg0bKTDwT9qzZ/cd65Kk4OAeatWq9V33c3Nzu+s+6enpNndJyeHq6mbdXhBhYV/qwIH9euONt1WlShXreN26dTV9+mybfbt06abQ0P6aP3+25s37skDPAwBAaVSgUJ6SkqKQkBCFhoZq+PDh8vX1la8vdz8oqLzCSkFDTHFp0eIhmws9O3XqooEDQzRr1jR9+eUKSTfP5p85c1o9e3bJ9TFyAm2fPv21Y8c2jRo1XPXq1Ve7doHq1KmL2rV7xGH116lT1+brqlVvLru4/X7hXl617dY4O+K47lRP5cpVdOnSRZlMJru7k9Svf79MJpPi4i7mUcsfSk9Pl7e3j902H58G+Qrl9et73/GuKAXh5uamjIwMu/GMjHTr9vzavj1aCxfOU0hIH4WE9Lnr/rVqealLl25avz6KC3YBAGVCgUJ55cqVlZCQoMqVKzuqnnvKwIGDC3WGOiCghc6fP2c37u19v9auNf4e7xUrVlTz5i3173/vVGpqqtzd3WU2m+Xr66fRo1/Ndc5999WXdPM+51999S/t379Xe/d+r717v9f69VHq0aOXJk78u0PqdXbO/cLIWz/YSpLc3OyDmyOOK7/1GOHGjRtKTb1x1/2cnU3y9Mx9nX+OmjVr6erVK3bjOWO1annlq6YfftirDz98R+3bP67x4yfka44k1a5dR2azWcnJSYRyAMA9r8DLV/z9/XXkyBH179/fEfWUC5MmvWOzply6uY570qR3DKzKVnb2zTuvpKbekLu7u+rX99bp07/n64y3i4uL2rd/XO3bPy6LxaIZM6YoMjJcQ4eO+L+ztE53fQxbBd0//4r3uPKnbt16ys7O1oUL53X//f87633hwnllZ2erTp3cr8moXt1Tbm5uOn/+rN22s2f/m6/nXrlyWbGtKX/wQV8dPXpYFovF5mLPX375Re7ulWzufpOXX345qrfffkNNmzbXe+99fNc7z9wqNvaCTCZTvu51DwBAaVfgUP76669r2LBh8vf3V58+fezuvIC7y7mYs7TcfeV2SUlJOnr0iGrUqGm9q0fHjk9pz57d2rx5o7p372mz/40bN2QymeTm5qbExASbO2c4OTmpceOb66Nzlue4u7tLkpKT8/dx9O7u7kpOTi7yceWmOI8rv/70p/b64os5Wr16pf761/+dGQ4P/1qS9NhjHXKdZzKZFBj4qHbt2qGXXrpsXVd+5sxp7d+/N1/PXZxryjt1eko7d27Xv/+9y3qf8oSEBO3YsU2PP/6kzVKhCxfOS5LNLy9nzpzWm2++prp16+nTT2fk+k6GJP3xxx92Z+3Pnz+nbdu+lb9/mzznAQBwLylwKP/kk09UrVo1TZ48WVOnTpWPj4/dW8dOTk5aunRpHo8A6WYwHzjw/ykry2x0KdqzZ7d+/z1Gkqyf6Hn9eqL++te3rL90BQf30PbtW/Xxx+/phx/2qUWLlsrKytSZM6f13XdbtXjxcnl7369//ONDJSVdV0DAw6pdu7bi4uK0Zs0qNWnia70Th59fU0nSggVz9dRTXVWhQgW1b/+ENazfzs+vqaKjN2v27Olq2rS53N0rqUOHJ4rl2IvzuPKrSRNfde/eU1FR4UpKuq5WrVrr8OGftW3bt+rZM0SNGz+Y59yRI0dp3749Gj16pHr37qvs7GxFRKxWw4YPKCbm1F2fuzjXlHfs+JRatGipDz98R4MGhcrDo7qiosJlsZg1YsSLNvu+9tpoSbKefb9xI0Xjx/9FSUlJGjRoiPbs+Y/N/o0bN7Fe7Dpv3meKjb2gtm3bqWbNWrpw4bzWrYuQJL3yythiORYAAIxW4FB+/vzNM145tz28csV+TSnuLQsW/O8jzd3c3PTAAw/qnXc+VFBQsHXcZDLp00+na+XKZYqO3qwdO7bJ3b2S6tf3VmjocHl53Vw/3K1bd61fH6WoqDVKTk5SjRo11blzF40Y8aKcnZ0lSb6+TTVq1CuKjAzXvn17ZDabFR6+Ps9QHhLSV7/+ekLffLNRq1b9S3Xr1iu2UF6cx1UQEyZMVr1692nz5o3auXO7vLxq6/nnX9KQIc/dcd6DDzbR9OmzNXv2DC1e/IW8vGprxIgXdfXqlXyF8uJkMpk0deoszZ07S2vWfK309HQ1a9ZCkye/f9elK4mJibp8+eYnA8+f/7nd9ueee8Eaytu1e0Tr1kUqImKVkpKSVLVqNbVr9yc999wLeuCBxsV/YAAAGMDJUoCrz65du6Zz587J09NTPj72d4C4l129miyzOfd/ikuX/qu6dRsU+3NWqOBcKs6Uw7Hos7Ec9f/3dl5eVRUfn78lWbh30eeyjx6XLUkLhtuNVX1xiWF9dnZ2Us2aVXLdlq8z5WazWe+++67WrFljvYNE69atNWfOHNWokb9PEgQAAACQu3y97758+XKtXr1atWrVUlBQkHx9ffXTTz/p7393zC3uAAAAgPIkX2fK165dq8aNG2vVqlXWT9qbPHmyoqKidP36dVWrVs2hRQIAAADFIWnBcHlNijC6DDv5OlN++vRpPfPMMzYffR0aGqrs7GydOXPGUbUBAAAA5UK+Qnlqaqpq165tM5bz9Y0bd/90QAAAAKCkOdV6wOgS8i3f93K7/UOCcr4uDR8dDgAAANyuSp975/rHfN+nfNeuXTb3JE9NTZWTk5O2bNmiEydO2Ozr5OSk4cOHF1uRpcHtHyUOoPTjpAEA4F6R71C+ceNGbdy40W581apVdmNlLZSbTC7KzEyXqysf5w3cSzIzM2QyFfgz0gAAKHH5+mkVFhbm6DpKtSpVPJSQcEWVK3uoYkV3OTubOGsOlGIWi0WZmRlKSIhX1aqeRpcDAMBd5SuUBwYGOrqOUs3dvbIqVHBRcnKCUlISZTZnF8vjOjs7y2zmkx7LOvpsDJOpgqpW9ZS7e2WjSwEA4K54XzefXFxc5elZ++47FgAf5Vs+0GcAAHA3+b77CgAAAADHIJQDAAAABiOUAwAAAAYzdE15RkaGZs2apXXr1un69etq2rSpxo0bp0cfffSO8zp37qwLFy7kuq1BgwaKjo52RLkAAACAQxgayt966y1FR0dr6NChatCggaKiovTCCy9o2bJlatOmTZ7z3n77baWkpNiMxcbGaubMmWrfvr2jywYAAACKlWGh/PDhw9q0aZMmTpxo/aCh3r17q2fPnpo2bZpWrFiR59wuXbrYjc2dO1eS9Oc//9kh9QIAAACOYtia8i1btsjFxUX9+/e3jrm5ualfv346ePCgLl++XKDH27hxo7y9vRUQEFDcpQIAAAAOZVgoP378uBo1aqTKlW0/2KNVq1ayWCw6fvx4vh/r2LFjiomJUc+ePYu7TAAAAMDhDFu+Eh8frzp16tiNe3l5SVKBzpRv2LBBktSrV69C11OzZpVCzy0KL6+qhjwvShZ9Lh/oc/lAn8s+ely25PXxfaWtz4aF8rS0NLm4uNiNu7m5SZLS09Pz9Thms1mbNm1S8+bN1bhx40LXc/VqssxmS6HnFwaf9Fg+0OfygT6XD/S57KPH5YcRfXZ2dsrzRLBhy1cqVqyozMxMu/GcMJ4Tzu9m//79iouL4wJPAAAA3LMMC+VeXl65LlGJj4+XJNWuXTtfj7NhwwY5OzurR48exVofAAAAUFIMC+VNmzbV6dOn7e43fujQIev2u8nIyFB0dLQCAwNzXZ8OAAAA3AsMC+XBwcHKzMxUeHi4dSwjI0ORkZEKCAiwhuzY2FjFxMTk+hi7du3S9evXWboCAACAe5phF3r6+/srODhY06ZNU3x8vHx8fBQVFaXY2Fh98skn1v0mTJig/fv36+TJk3aPsWHDBrm6uqpbt24lWToAAADuYb9/1FdVX1xidBk2DAvlkjRlyhTNnDlT69atU2Jiovz8/LRgwQK1bdv2rnOTk5O1c+dOdezYUVWrlq5b2gAAAAAF4WSxWEr2PoClFLdEhKPQ5/KBPpcP9Lnso8dlT9KSV6SMFLtxI86Ul8pbIgIAAACOVnX4HKNLyBdCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAMqdpCWvGF2CDUI5AAAAyp+MFKMrsEEoBwAAAAxGKAcAAECZVvXFJUaXcFeEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAJRLSQuGG12CFaEcAAAAMBihHAAAADCYoaE8IyNDU6dOVYcOHdSqVSsNGDBAe/bsyff8DRs2qF+/fmrdurUCAwMVGhqqw4cPO7BiAAAA3IuqvrjE6BLuqIKRT/7WW28pOjpaQ4cOVYMGDRQVFaUXXnhBy5YtU5s2be44d8aMGVq0aJF69eqlgQMH6saNGzpx4oTi4+NLqHoAAACgeBgWyg8fPqxNmzZp4sSJGj58uCSpd+/e6tmzp6ZNm6YVK1bkOffHH3/UF198odmzZysoKKiEKgYAAAAcw7DlK1u2bJGLi4v69+9vHXNzc1O/fv108OBBXb58Oc+5YWFhatmypYKCgmQ2m5WSklISJQMAAAAOYVgoP378uBo1aqTKlSvbjLdq1UoWi0XHjx/Pc+6ePXvUsmVLTZ8+XW3btlVAQIA6d+6s9evXO7psAAAAoNgZtnwlPj5ederUsRv38vKSpDzPlCcmJiohIUGbNm2SyWTS66+/rurVq2vFihV644035O7uXqglLTVrVinwnOLg5VXVkOdFyaLP5QN9Lh/oc9lHj8uupFzGSku/DQvlaWlpcnFxsRt3c3OTJKWnp+c678aNG5KkhIQErV69Wv7+/pKkoKAgBQUFac6cOYUK5VevJststhR4XlF4eVVVfHxuLw+UJfS5fKDP5QN9LvvocflTkv12dnbK80SwYctXKlasqMzMTLvxnDCeE85vlzPu7e1tDeSS5Orqqm7duunEiROsMQcAAMA9xbBQ7uXllesSlZxbGtauXTvXedWrV5erq6tq1aplt61WrVqyWCxKTk4u3mIBAAAABzIslDdt2lSnT5+2O6t96NAh6/bcODs7q1mzZoqLi7PbdunSJZlMJnl4eBR/wQAAAICDGBbKg4ODlZmZqfDwcOtYRkaGIiMjFRAQYL0INDY2VjExMXZzL168qN27d1vHkpOTtXnzZrVp00YVK1YsmYMAAAAAioFhF3r6+/srODhY06ZNU3x8vHx8fBQVFaXY2Fh98skn1v0mTJig/fv36+TJk9axQYMGKTw8XGPGjNHw4cNVrVo1RUREKCkpSePHjzficAAAAIBCMyyUS9KUKVM0c+ZMrVu3TomJifLz89OCBQvUtm3bO85zd3dXWFiYpkyZouXLlystLU0tWrTQV199dde5AAAAQGnjZLFYSvY+gKUUt0SEo9Dn8oE+lw/0ueyjx2Vb0oLhdmNVX1xSYs9fKm+JCAAAAOAmQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwQjkAAABgMEI5AAAAYDBCOQAAAGAwJ4vFYjG6iNLg6tVkmc0l+08xPfyQMjOybcbaNautzgHeSs/M1szVh+zmtG9ZTx1a1VPSjQzNjTpqt71TQH0FNquja9fTtHDDMbvt3QJ91LpJLV28mqKwLSfttvds31AtGtbQ2bgkrdx2ym573ycb60FvD/12PlERu2Lstg/q0kQ+darqlzPXtHH3GbvtQ4P9VK9mZf186oq+3X/WbvsLf26uGtUqav/xOO348YLd9pefeUhVK7nqP4cvaveRi3bbxw7wl5uLSd/9eF4/HL9st33CswGSpC37zurQb1dstrm4OGv8gNaSpPW7T+v4mT9stldxd9ErfVpKktbsjFHMhUSb7Z7V3PTin1tIkv617Vedi0u++biuJmVmZKtOjUoa3r2pJGnJ5hOKu3bDZv79dapocBdfSdKCDb/oj+vpNtsb1/dQv46NJUlzIo8oOTXTZnuzhp7q1b6RJGn66p+VmWm22e7/YC0FP+IjSfp0xY92/za89or22vvw5fZKSkwtVa+9HLz2iu+1t37PGbvv20a/9krj970c9+JrL+d7tlS6Xnul8fvevfraezHlM1X4v79XfXGJXd2O5OzspJo1q+S6rUKuowAAAEAZtKDyq5o29knFxycZXYoNzpT/HyPOlHt5VS11LwgUP/pcPtDn8oE+l330uHwwqs93OlPOmnIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBghHIAAADAYIRyAAAAwGCEcgAAAMBgFYwuoLRwdnYqV8+LkkWfywf6XD7Q57KPHpcPRvT5Ts/pZLFYLCVYCwAAAIDbsHwFAAAAMBihHAAAADAYoRwAAAAwGKEcAAAAMBihHAAAADAYoRwAAAAwGKEcAAAAMBihHAAAADAYoRwAAAAwGKEcAAAAMBihHAAAADAYobyYZWRkaOrUqerQoYNatWqlAQMGaM+ePfmaGxcXp9dee00PP/ywAgIC9PLLL+vcuXMOrhiFUdg+R0dHa+zYsercubP8/f0VHBysTz/9VElJSSVQNQqqKP+fb/XCCy/Iz89PH330kQOqRFEVtc8bNmxQv3791Lp1awUGBio0NFSHDx92YMUoqKL0+Pvvv9eQIUP0yCOPqF27dho4cKC++eYbB1eMwrh8+bKmTZumIUOGqE2bNvLz89O+ffvyPT8mJkYjR45UmzZtFBgYqAkTJujatWsOrNgWobyYvfXWW1q6dKl69eqlSZMmydnZWS+88IJ++umnO85LSUnR0KFDdfDgQb300kt69dVXdezYMQ0dOlSJiYklVD3yq7B9/tvf/qaYmBiFhIRo8uTJ6tChg5YtW6ZBgwYpPT29hKpHfhW2z7fauXOnDhw44MAqUVRF6fOMGTP01ltvqUmTJpo0aZJeeeUV3X///YqPjy+BypFfhe3xjh07NGLECGVlZWnMmDF67bXX5OzsrHHjxik8PLyEqkd+nT59WgsXLlRcXJz8/PwKNPfSpUt69tlnde7cOY0bN04jRozQjh07NHLkSGVmZjqo4ttYUGwOHTpk8fX1tXz11VfWsbS0NEuXLl0sgwcPvuPcBQsWWPz8/Cy//PKLdey3336zNGvWzDJz5kxHlYxCKEqf9+7dazcWFRVl8fX1tURERBR3qSiCovQ5R3p6uqVr167qiTPLAAANB0lEQVSW2bNnW3x9fS0ffvihg6pFYRWlzwcPHrT4+flZoqOjHVwliqIoPR45cqSlQ4cOlvT0dOtYenq6pUOHDpZnn33WUSWjkJKSkizXrl2zWCwWy9atWy2+vr65/tzNzTvvvGNp3bq15dKlS9ax3bt3W3x9fS3h4eEOqfd2nCkvRlu2bJGLi4v69+9vHXNzc1O/fv108OBBXb58Oc+53377rVq3bq3mzZtbxxo3bqxHH31UmzdvdmjdKJii9PmRRx6xG+vSpYukm2+bofQoSp9zhIWFKS0tTSNHjnRkqSiCovQ5LCxMLVu2VFBQkMxms1JSUkqiZBRQUXqcnJwsDw8Pubq6WsdcXV3l4eEhNzc3h9aNgqtSpYo8PT0LNTc6OlqdO3dWnTp1rGOPPfaYGjZsWGI5jFBejI4fP65GjRqpcuXKNuOtWrWSxWLR8ePHc51nNpt18uRJPfTQQ3bbWrZsqTNnzig1NdUhNaPgCtvnvFy5ckWSCv2NBI5R1D7Hx8dr7ty5GjdunNzd3R1ZKoqgKH3es2ePWrZsqenTp6tt27YKCAhQ586dtX79ekeXjQIoSo8DAwN16tQpzZw5U2fPntXZs2c1c+ZMnTlzRiNGjHB06SghcXFxunr1aq45rFWrVgX+uV5YFUrkWcqJ+Ph4m9+wcnh5eUlSnr+NJyQkKCMjw7rf7XMtFovi4+Pl4+NTvAWjUArb57wsXLhQJpNJXbt2LZb6UDyK2ufp06erUaNGCgkJcUh9KB6F7XNiYqISEhK0adMmmUwmvf7666pevbpWrFihN954Q+7u7goKCnJo7cifovxffumll3T27FnNnz9f8+bNkyRVqlRJc+fOVfv27R1TMEpczmsgrxx29epVZWdny2QyObQOQnkxSktLk4uLi914zltceV3IlzN+69tjt89NS0srrjJRRIXtc242bNigNWvWaNSoUfzSVcoUpc+HDx/W2rVrtWzZMjk5OTmsRhRdYft848YNSTdPqqxevVr+/v6SpKCgIAUFBWnOnDmE8lKiKP+XXV1d1bBhQwUHBysoKEjZ2dlavXq1xo4dqyVLlqhVq1YOqxslJ7857PZ3W4obobwYVaxYMdcrdHOandf6s5zxjIyMPOdWrFixuMpEERW2z7c7cOCAJk2apI4dO+q1114r1hpRdIXts8Vi0UcffaSuXbvq4YcfdmiNKLqift/29va2BnLp5g/1bt26KSwsTCkpKQ7/IY67K8r37A8++EBHjhzRmjVr5Ox8c8Vv9+7d1bNnT3388cf6+uuvHVM0SlRpyWGsKS9GXl5eub4NlnNrrNq1a+c6r3r16nJ1dc31Flrx8fFycnLK9S0VGKOwfb7ViRMnNHr0aPn5+WnGjBkOf0sMBVfYPm/dulWHDx/WoEGDdP78eesf6eZFY+fPn+edr1KkqN+3a9WqZbetVq1aslgsSk5OLt5iUSiF7XFGRobWrFmjjh07WgO5JLm4uOjxxx/XkSNHlJWV5ZiiUaJyXgN55bCaNWuWyM9pQnkxatq0qU6fPm13Bf6hQ4es23Pj7OwsX19fHT161G7b4cOH1aBBAy4UK0UK2+ccZ8+e1fPPP68aNWroiy++UKVKlRxWKwqvsH2OjY2V2WzWsGHD9NRTT1n/SFJkZKSeeuop7d+/37HFI9+K8n27WbNmiouLs9t26dIlmUwmeXh4FH/BKLDC9jghIUFZWVnKzs6225aVlaWsrCxZLJbiLxglrk6dOqpRo0aeOaxZs2YlUgehvBgFBwcrMzPT5gMFMjIyFBkZqYCAAOuFJrGxsXa3v+vWrZt+/vlnHTt2zDr2+++/a+/evQoODi6ZA0C+FKXP8fHxGjFihJycnLR48WLVqFGjRGtH/hW2z507d9acOXPs/khSp06dNGfOHLVo0aJkDwZ5Ksr/5+DgYF28eFG7d++2jiUnJ2vz5s1q06YNyw5LicL2uGbNmqpWrZq2bt1qs/wlJSVFO3bskK+vb65r1VH65dxJ51Zdu3bVd999Z/OL9p49e3TmzJkSy2FOFn7NK1avvfaatm/frmHDhsnHx0dRUVE6evSoli5dqrZt20qShgwZov379+vkyZPWecnJyXrmmWeUmpqq5557TiaTSUuWLJHFYtHatWu5XV4pU9g+h4SE6MSJE3r++efl6+tr85g+Pj5q06ZNiR4H7qywfc6Nn5+fhg4dqkmTJpVE6SiAwvY5NTVVffr0UVxcnIYPH65q1aopIiJCp0+ftpkL4xW2x/PmzdPMmTPVokUL9erVS2azWWvWrFFMTIxmzJihp59+2qhDQh7mzp0r6eZnf2zcuFF9+/aVt7e3qlWrptDQUEk3T55I0nfffWedd/HiRfXu3VvVq1dXaGiobty4ocWLF6tevXoKDw/P9SLQ4saFnsVsypQpmjlzptatW6fExET5+flpwYIFd/3mXKVKFS1btkwff/yx5s6dK7PZrEceeUSTJk0ikJdChe3ziRMnJEmLFi2y2/bMM88QykuZwvYZ95bC9tnd3V1hYWGaMmWKli9frrS0NLVo0UJfffUVr5FSprA9Hj16tLy9vRUWFqY5c+YoIyNDfn5++vzzz7m7Tik1a9Ysm68jIiIkSfXr17eG8tzUq1dPy5cv1z/+8Q/985//lIuLizp27KiJEyeWSCCXOFMOAAAAGI415QAAAIDBCOUAAACAwQjlAAAAgMEI5QAAAIDBCOUAAACAwQjlAAAAgMEI5QAAAIDBCOUAAMNERkbKz89P+/bts47t27dPfn5+ioyMNLAyAChZfKInAJQx+/bt09ChQ23GKlWqpIYNGyokJEShoaGqUIFv/wBQmvBdGQDKqJ49e+qJJ56QxWLRlStXtG7dOn3yySeKiYnRBx98YHR5AIBbEMoBoIxq3ry5QkJCrF8PHjxY3bt3V3h4uMaNG6caNWoYWB0A4FaEcgAoJypVqiR/f399++23Onv2rDWUX758WXPmzNGuXbt05coVVa9eXZ06ddLYsWNVs2ZNm8dITk7WwoULFR0drfPnz6tSpUp64IEHFBoaqh49ekiSYmJitGzZMv3www+KjY2V2WxW48aNNWjQIPXv37/EjxsA7gWEcgAoR86dOydJ8vDwkCTFxsZq4MCByszMVL9+/eTj46P//ve/Wrlypfbt26eIiAhVrVpVknT9+nUNHjxYp06dUrdu3TRo0CCZzWYdO3ZMO3bssIby/fv368CBA+rYsaO8vb2VmpqqLVu2aPLkybp27ZpGjRplzMEDQClGKAeAMio1NVXXrl2TJMXHx+vrr7/WsWPH1KpVKzVq1EiS9MEHHygrK0tr165V3bp1rXODg4M1cOBALVmyRGPGjJEkTZ8+XadOndL777+vgQMH2jyX2Wy2/j0kJESDBg2y2T58+HANGzZMCxYs0IgRI+Ti4uKQYwaAexWhHADKqNmzZ2v27Nk2Y127dtXf//53SVJSUpJ27typPn36yNXV1RrgJal+/fry8fHR7t27NWbMGJnNZn3zzTdq3LixXSCXJGfn/91ht1KlSta/p6en68aNG7JYLGrfvr3279+v33//XX5+fsV9uABwTyOUA0AZNXDgQAUHByszM1O//vqrFi1apEuXLsnNzU2SdPr0aZnNZq1Zs0Zr1qzJ9THuv/9+SdIff/yhxMREPf7443d93pSUFH3++efavHmzLl68aLf9+vXrRTgqACibCOUAUEY1aNBAjz32mCTpySefVNu2bTV48GC98847mjFjhiwWiySpV69eeuaZZ3J9jJwAXxB//etftXPnTg0YMEDt2rVT9erVZTKZtGvXLi1ZssRmqQsA4CZCOQCUEwEBAQoJCdHatWs1ZMgQNWrUSE5OTsrMzLSG97x4enrKw8NDJ06cuON+169f186dOxUSEqL333/fZtv3339f5GMAgLLK+e67AADKipdfflkmk0mfffaZPD099eSTT2rr1q36+eef7fa1WCzWdebOzs7q0aOHfvvtN4WHh+e6b85+t36d4/Lly7nOAwDcxJlyAChHGjRooKefflobNmzQgQMH9O6772rw4MEKDQ1VSEiImjdvLrPZrHPnzmn79u3q3bu39e4rY8eO1d69ezV58mTt3r1bbdu2lcVi0fHjx5WVlaWpU6eqSpUqat++vdavX6+KFSuqZcuWunDhglatWiVvb28lJCQY/C8AAKUToRwAypnRo0dr06ZNmjVrlpYtW6aIiAgtXLhQ3333ndavXy83NzfVq1dPnTp1Uvfu3a3zPDw8tGrVKs2fP19bt27Vtm3bVLlyZTVu3FihoaHW/aZOnap//vOf+u677xQVFaWGDRtq3LhxqlChgiZOnGjEIQNAqedkuf09RgAAAAAlijXlAAAAgMEI5QAAAIDBCOUAAACAwQjlAAAAgMEI5QAAAIDBCOUAAACAwQjlAAAAgMEI5QAAAIDBCOUAAACAwf4/mzLWzT3RhhAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9bXiLFZnBvmk"
      },
      "source": [
        "A model with perfect skill is depicted as a point at (1,1). A skilful model is represented by a curve that bows towards (1,1) above the flat line of no skill. Mention thresholds do not have to match, since in each curve the optimization goal is different."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PwA726ILA6kv"
      },
      "source": [
        "ROC curves should be used when there are roughly equal numbers of observations for each class. Precision-Recall curves should be used when there is a moderate to large class imbalance. The reason for this recommendation is that ROC curves present an optimistic picture of the model on datasets with a class imbalance."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SkcwMyOGJX-M"
      },
      "source": [
        "Finally, we can predict test samples with the optimized threshold."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "OUPJwQ5K2cS1",
        "outputId": "0c8ca5db-2bd9-43ee-abe4-e131480a73f6"
      },
      "source": [
        "# Get new predictions respect best threshold\n",
        "preds_thres = np.where(probs[:, 1] > best_thres, 1, 0)\n",
        "\n",
        "num_examples = 5 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "df = test_df.sample(num_examples)\n",
        "df['real'] = np.where(df.label==0, 'negative', 'positive')\n",
        "df['predicted'] = np.where(preds_thres[df.index]==0, 'negative', 'positive')\n",
        "df['positive probability'] = probs[df.index, 1]\n",
        "df['negative probability'] = probs[df.index, 0]\n",
        "pd.set_option(\"max_colwidth\", 150)\n",
        "print(f'Threshold: {best_thres:.2f}')\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Threshold: 0.76\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "      <th>real</th>\n",
              "      <th>predicted</th>\n",
              "      <th>positive probability</th>\n",
              "      <th>negative probability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>17964</th>\n",
              "      <td>option to slap her creators because they 're clueless and inept</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.002045</td>\n",
              "      <td>0.997955</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5055</th>\n",
              "      <td>the fetid underbelly of fame has never looked uglier .</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.010490</td>\n",
              "      <td>0.989510</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13665</th>\n",
              "      <td>profanity and violence</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.002618</td>\n",
              "      <td>0.997382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7671</th>\n",
              "      <td>feel cheated</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.002861</td>\n",
              "      <td>0.997139</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16903</th>\n",
              "      <td>familiar and predictable , and 4/5ths of it</td>\n",
              "      <td>0</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.001858</td>\n",
              "      <td>0.998142</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                               sentence  ...  negative probability\n",
              "17964  option to slap her creators because they 're clueless and inept   ...              0.997955\n",
              "5055            the fetid underbelly of fame has never looked uglier .   ...              0.989510\n",
              "13665                                           profanity and violence   ...              0.997382\n",
              "7671                                                      feel cheated   ...              0.997139\n",
              "16903                      familiar and predictable , and 4/5ths of it   ...              0.998142\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mA2b-qLbJb3v"
      },
      "source": [
        "And check the new results in the confusion matrix."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 446
        },
        "id": "imFdzQGPtlKv",
        "outputId": "b3e5dae3-f418-4a6c-9c6b-342b42620ae3"
      },
      "source": [
        "plot_confusion_matrix(confusion_matrix(preds_thres, y_true), ('positive', 'negative'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGtCAYAAADtf4sDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5f4H8M8Mq2wCCmosLimLsrkhoGkiiaKl5oIbRGqooXktNUy7lfozQzRTLJdrSopLJkSlYKJm7uaKCy6oqOACiILsIPP7w+bkMAPOwEEZ+7x7zeveec5znnnmMINfvs9yJDKZTAYiIiIiEkhfdAeIiIiI6hsGSERERESVMEAiIiIiqoQBEhEREVElDJCIiIiIKmGARERERFQJAyTSSEpKCt555x107twZjo6OWLZsWZ28TmxsLBwdHXH06NE6af9l5OjoiPDw8BfdjWfStJ/a8r7+rY4ePQpHR0fExsYKZenp6XX6+0Edy5Ytg6OjI9LT019YH0i76b7oDpB6ioqKsGXLFvz+++9ITU1FQUEBGjZsiHbt2qFv37546623oKtbtz/O8vJyTJ48GeXl5ZgyZQpMTU3h6OhYp6/5b5OUlISUlBRMnjz5RXfluVq2bBmcnZ3h5+f3oruiUnp6OuLi4uDn5wdnZ+cX3R3627/1+0LPBwMkLXDjxg2EhoYiLS0NPj4+CA0NhYWFBe7fv4/Dhw9j5syZSE1NxYwZM+q0H7du3cKtW7cQHh6O0aNH1+lrDRgwAP369YOenl6dvk59k5SUhLi4uBr9wk9OToZUWv+Twqr6GRUVhUGDBtXbACkjIwNRUVGwsbFhgKQGGxsbJCcnQ0dHp05fp7rvy8SJExEaGgp9ff067QO9vBgg1XPFxcUYP3480tPTsWzZMvTu3VvheGhoKJKTk3H27Nk670t2djYAoGHDhnX+Wjo6OnX+y/VlUFxcDF1dXejq6sLAwOBFd0ct2tJPbZOfnw8TE5MX3Q0AgEQieeE/Z/n3gqim6v+fm/9yW7duxfXr1/Huu+8qBUdybm5uGDVqlEJZUlIShg8fDg8PD7Rv3x7Dhw9HUlKS0rm+vr4ICgrC1atXERoaivbt26Njx4744IMPkJWVJdQLCgoSskYzZ86Eo6OjML5f3XyhoKAg+Pr6KpSdPHkS48aNQ9euXeHq6orXXnsN7733Hk6fPi3UqarNnJwcfPHFF+jRowdcXFzQo0cPfPHFF3jw4IFCPfn5hw8fxpo1a+Dn5wcXFxf4+/sjLi5O5XWs7Om5FTExMfD394erqyvefPNN7N27FwBw6dIljB07Fh06dECXLl0wb948lJWVKbSTnJyM8PBw+Pv7w93dXfh57Nq1S+layfsmv75Pz+0IDw+Ho6MjcnJyMHPmTPj4+MDDwwN3794Vznl6rk5MTAwcHR2xfPlyhde5d+8evLy80LdvXxQWFqp1LZ42c+ZMuLq6oqSkRCg7deoUHB0d4enpiYqKCqF83759cHR0xI4dO4Syp/spn6sCAHFxcQrvu7JTp05h9OjR8PDwQJcuXTBr1iwUFBQo1bt48SLCwsLQpUsXuLq6IiAgAKtXr8bjx48V6qn6bD7dJ/n8mdjYWAQHBwvvXd6/oKCgaq+TfA7MtWvXsHjxYnTv3h0uLi546623sG/fPqX65eXlWLVqFQICAuDq6oouXbogLCwMly5dqrJ/O3bswNtvvw03NzfMmzdP4foePnwYgYGBcHd3R/fu3bFq1SoAQG5uLj755BN4e3vD3d0d48ePx7179xRe4969e1iwYAEGDBiAzp07C9dx1apVStdRFVVzkIKCghR+vk8/nv45iPV9qWoOUnp6OqZPnw4fHx+4uLjAz88PixcvRlFRUa1+fvTyYXhdz+3cuRMAEBgYqPY5MTExmDNnDlq1aoX3338fwJN/fMLCwjBnzhyltu7du4fg4GD4+flhxowZuHjxIrZs2YL8/Hx8//33AIAJEyagQ4cOWLFiBQIDA9GxY0cAgKWlpUbv59q1axgzZgwaN26M4OBgNGrUCPfv38eJEydw8eJFeHh4VHnuo0ePMGLECNy4cQODBw9G27ZtkZKSgk2bNuHIkSPYunWr0l/QX3/9NYqLixEYGAh9fX1s2rQJ4eHhsLe3F97Ds8TExCAvLw9Dhw6Fvr4+1q9fj0mTJuGbb77B7Nmz0b9/f/j5+eHgwYNYv349LC0thesOALt27cK1a9fQp08f2NjY4OHDh4iLi8OkSZMQGRmJN998U7jGFRUVOH78OCIiIoTzO3TooNCfd999F40bN8b777+PwsJCGBkZqez3qFGjcOTIESxfvhxdunRBp06dUFFRgWnTpqGgoADr1q2r8tzqdOnSBbGxsTh58iS8vb0BAIcPH4ZUKkVubi4uXLgAFxcXAMCRI0cgkUjQpUsXlW1ZWloiIiICM2bMQKdOnTBs2DCV9VJSUjBhwgS8/fbb6N+/P44dO4affvoJUqkUc+fOFeqdPXsWQUFB0NXVxahRo9C4cWPs3bsXkZGRuHjxIhYtWqTx++3cuTMmTJig9Nlv3LixWueHh4dDV1cXY8aMQVlZGaKjoxEWFobExETY2toK9aZNm4aEhAR07doVI0aMQHZ2NmJiYjB8+HDExMSgbdu2Cu0mJSVh/fr1GDFiBIYPH67w2b9w4QL27t2LYcOGYcCAAUhISMCiRYtgYGCAn3/+GTY2Npg0aRJu3ryJ9evX4+OPP8a6deuE8y9duoTff/8db7zxBuzt7VFWVob9+/dj0aJFSE9Px5w5czS+jhMmTMCQIUMUym7duoVly5ahUaNGQpnY35enZWRkYOjQoXj06BFGjhyJ5s2b49ixY1i5ciVOnjyJdevWKWWd1P350UtIRvWap6enrEOHDmrXf/jwoczDw0Pm5+cne/TokVD+6NEjWa9evWQeHh6y3Nxcobxnz54yBwcH2fbt2xXa+fzzz2UODg6yq1evCmVHjhyROTg4yLZt26ZQd9u2bTIHBwfZkSNHlPozevRoWc+ePYXn0dHRMgcHB9mZM2eqfR+q2ly8eLHMwcFBtmHDBoW6GzZskDk4OMi+/vprpfMHDBggKykpEcrv3r0ra9eunWzq1KnVvv7T77dbt26yvLw8oTwlJUXm4OAgc3R0lO3cuVPhnEGDBsm6du2qUFZQUKDUdmFhoax3796yvn37KpR//PHHMgcHB5X9kR/76KOPVB53cHCQffzxxwplDx8+lPXs2VPWo0cP2cOHD2VRUVEyBwcH2fr166t+489w9+5dmYODg2zx4sVCWVBQkGzChAmy9u3by1atWiWUDxo0SNa/f/9n9lNV2dPHHB0dZadPn1Yof++992Rt27aV5efnC2WBgYEyZ2dnWUpKilBWUVEh++CDD2QODg6yQ4cOCeWVP5tyt27dkjk4OMiWLl0qlFX12a/O0qVLZQ4ODrLQ0FBZRUWFUH7mzBmZg4ODLDIyUig7cOCAzMHBQTZlyhSFuikpKTJnZ2fZiBEjlPrXtm1bWWpqqtLrqrpeJSUlsq5du8ocHR1lc+fOVag/f/58pe96UVGRQj/kpk2bJnNycpLdu3dPKFN1bVRdw8oePnwo8/f3l3l6espu3LghlIv1fZFf/1u3bgllH374oczBwUH2xx9/KNRdsGCBzMHBQfbjjz8qna/Oz49eThxiq+fy8/NhbGysdv2DBw+isLAQQUFBCn9RmpiYICgoCIWFhTh06JDCOdbW1ggICFAo8/LyAvBkgriYTE1NAQC7d+9WGKJRx65du2BpaamUAQsMDISlpaXKIcSRI0cqTNJs0qQJWrZsibS0NLVf9+233xb6DQBOTk4wMTGBtbW10rBnhw4dkJWVpTD083SWpqioCA8ePEBRURG8vLxw9epV5Ofnq90XABg7dqzadRs2bIjIyEhkZWXhvffew/Lly+Hr61urSfZNmjRBixYtcOTIEQBASUkJTp8+jW7dusHT01Moz8vLQ0pKSpXZI014eHjA3d1doczLywvl5eXIyMgAANy/fx+nTp2Cr68vnJychHoSiQQTJ04EAKVhmuchODgYEolEeO7m5gYjIyOF75a8XxMmTFCo6+TkhJ49e+LEiRPIyclRaLdHjx549dVXVb5m5eulr68PV1dXyGQypaHBTp06AVD8rhsaGgr9KC0txcOHD5GTk4Nu3bqhoqIC586d0+gaVFZWVobJkycjPT0dy5cvh729vXBM7O+LXEVFBfbs2YO2bduiR48eCsfGjx8PqVSq8neIOj8/ejlxiK2eMzExUTnPoiry8fY2bdooHZOX3bp1S6Hczs5Oqa65uTkA4OHDh2q/tjr69euHX375BStWrMC6devg7u6Obt26oV+/frCxsan23PT0dLi4uCilwHV1ddGiRQtcuHBB6Zyq3pv8H1V1qEqjN2zYEE2bNlVZDjy5bvLA9v79+1iyZAl2796N+/fvK52Tl5en0eTaFi1aqF0XeBK0jRs3DitWrICVlRXmz5+v0fmqeHl54aeffkJ+fj7Onj2LkpISeHl5oaysDEuWLEFpaSmOHTuGiooKIdiuDXU+o/LPfuvWrZXqtmrVClKpVOmz/zyo6ruFhYXCvLn09HRIpVKVAU/r1q2RlJSE9PR0hSHt6j4Hql5T/tms/Hk2MzMDoPhdl8+Hio+Px40bNyCTyRTOycvLq/K11fHf//4XR48exVdffSUEaHJif1/kcnJyUFhYqPLzYW5uDisrK5WfD3V+fvRyYoBUz7Vp0wZ//fUXbt26pfKLKobqVotV/sWoytN/XVVWXl6u8FxfXx9r165FcnIy9u/fj+PHj2Pp0qWIiorCokWL8MYbb6jfcTWIsey9quujznWTyWQYM2YMrl69iuDgYLi4uMDU1BQ6OjrYtm0bfvvtN4VJzepo0KCBRvVLS0tx4MABAE/+Ebxz5w4sLCw0aqMyLy8vbN68GcePH8epU6dgbW2NV199FWVlZSgqKsKZM2dw5MgR6OjowNPTs1avBdT+M6oJdSYha6Kutl6o7nNQ3fWq6tjT13HBggVYv349AgICMGHCBFhaWkJPTw/nz59HZGSkxp/Zp61YsQKxsbGYOHEiBg4cqNQHsb8vtaUNW2dQ3eBPvp6TD+Fs3bpVrfryIOrKlStKx1JTUxXqiEX+l2lubq7Ssap2sXVzc0NYWBjWrl2LXbt2oUGDBliyZEm1r2NnZ4fr168rBV3l5eVIS0urswCyNi5duoSLFy8iNDQUM2bMQEBAAF577TX4+Pio/EVfXbBZU4sXL8a5c+cwffp0mJiYYOrUqTVavfa0Ll26QCKR4PDhwzhy5IiQJXJ0dISFhQUOHz6Mo0ePwtnZWchQ1DV5ZkT+OX/atWvXUFFRofAZMTc3V5khVZVFqIufy9Ps7OxQUVGBq1evKh2Tlz3PCcHx8fHo3Lkzvv76awwaNAg9evSAj49PrbcR2LFjB5YsWYKAgABMmTJF6Xhdfl8sLS1hbGys8vORm5uLrKysevk7hF4cBkj13NChQ9GyZUt8//33KsfHAeDcuXOIiYkBAHTt2hVGRkbYsGGDwlh9fn4+NmzYACMjI3Tt2lXUPspT/ZXnNv3222/IzMxUKKs8jwIAmjZtCktLS5UB1tP8/PyQk5OjFCz++OOPyMnJqZebDMr/+qyc5bh8+bLK+TDy+RdiDW3u27cP69atw6BBgzBu3Dh8+eWXSEtLU1j5VROWlpZwcHDAH3/8gXPnzgkBknzFWmJiIq5cuaL28JqRkVGt33OjRo3Qvn177N27F5cvXxbKZTKZsMT96QxlixYtUFBQgOTkZKGsoqJCYTXX0/0DVP8RIAb5Z3fVqlUKn5XLly9jz5496Nixo8YrRmtDKpUqfWYLCwtVXht1nT59GuHh4XB3d8eCBQtUBjd1+X2RSqXo2bMnLly4gD///FPh2KpVq1BRUVEvf4fQi8MhtnquQYMGWLlyJUJDQxEWFoZu3brBx8cH5ubmyMnJwdGjR3HgwAGMGzcOwJP5BNOmTcOcOXMwbNgwDBo0CMCTZf43btzAnDlzFCYci6FVq1bw8fHBli1bIJPJ4OzsjJSUFCQlJaF58+YKGZ/vvvsOBw8exOuvvw5bW1vIZDLs3bsX165dE95DVcaNG4fExETMmTMHFy5cEF7np59+QsuWLZ95/ovw6quvok2bNvjf//6H4uJitGzZEtevX8eWLVvg4OCA8+fPK9R3d3fHhg0bhL2e9PT04ObmVqO/bDMzMxEeHo7mzZvj008/BQD07NkTwcHB+OGHH4S5X3K+vr7IyMhQ2nenKl5eXoiOjhb+/9PliYmJSuXV8fDwwOHDh7Fq1Sq88sorkEgkCn1T16xZsxAUFIRRo0Zh5MiRsLKywt69e3HgwAH0799f2JYAAIYNG4a1a9ciLCwMwcHB0NPTw86dO1UOsbVu3RrGxsbYuHEjDA0NYWZmBktLS4X2aqNr167o27cvtm/fjtzcXPTs2RNZWVnYuHEjDAwMMHv2bFFeR13+/v7YsmUL/vOf/8DHxwfZ2dnYtm2bMO+rJt5//32Ul5ejT58+wudDztjYGH5+fnX+ffnwww9x6NAhhIWFYeTIkbC3t8fx48exY8cOdO7cWfh9SQQwQNIKzZs3x88//4wtW7Zg586dWLFiBQoLC9GwYUO4uLhgwYIFwt4gwJP9b6ytrbFmzRphk0AnJycsX768zv5CioiIwNy5c/Hrr7/il19+QceOHfHDDz/g888/V5gQ7efnh6ysLCQmJiI7OxuGhoZo3rw55s2bp7RHSmWmpqbYtGkTli5dij179iA2NhaNGjXC8OHDMXny5Hqzi/DTdHR0sHLlSnz11VeIi4tDUVER2rRpg6+++goXL15U+oXfv39/pKSkYPv27UhMTERFRQW+/PJLjQOkiooKzJgxQ9jL6umVkNOnT8fx48fx3//+V+Efk4KCAlhbW6v9GvIAyc7OTmGCvTxo0NPTU5qAW5XPPvsMc+bMwYoVK4RFCTUJkFxdXbF582YsXboUmzZtQmFhIezs7DBt2jSMGTNGoa6dnR2WL1+OxYsX45tvvoG5uTkGDBiAwYMHo2/fvgp1DQ0N8fXXX2PJkiWYP38+SktL4enpKVqABACRkZFo27Yt4uLisGDBAhgZGaFz586YMmXKc7/n4cyZM2FsbIzExETs3r0bzZo1Q2BgIFxdXRESElKjNuUTrhcsWKB0zMbGBn5+fnX+fbGxscGPP/6IpUuX4pdffsGjR4/QpEkTjB8/HhMnTuTO26RAIhN7hiMRaZ2LFy9iwIABmD9/PgYPHvyiu0NE9MJxDhIR4cCBA3BycuIQAxHR35hBIiIiIqqEGSQiIiKiShggEREREVXCAImIiIioEgZIREREWiwnV/37dZL6OElby70V8QfuPCx60d0gUssfn/m/6C4QqUUCwMhAe3IIvd5djPR7NduN3raJOXav/VDkHmk/7oql5e48LEL6/drdV4voeeFfY0R1Iz0zDzfv1vB2PRLtCQSfJwZIRERE2k4iefKo6bmkhAESERGRtpNIap4JYoCkEvNqRERERJUwg0RERKTtOMQmOgZIRERE2k4ircUQGweTVGGAREREpO2YQRIdw0YiIiKiSphBIiIi0nYcYhMdAyQiIiKtV4shNnCITRUGSERERNqO+yCJjnk1IiIiokqYQSIiItJ2XMUmOgZIRERE2o6TtEXHAImIiEjbMYMkOoaNRERERJUwg0RERKTtOMQmOgZIRERE2o7L/EXHAImIiEjbSSSAlHOQxMS8GhEREVElzCARERFpO85BEh0DJCIiIm3HZf6iY4BERESk7ZhBEh2vChEREVElzCARERFpOwlqMcQmak9eGgyQiIiItB2H2ETHq0JERERUCTNIRERE2o6r2ETHAImIiEjbcYhNdAyQiIiItF4tMkicpa0Sw0YiIiKiSphBIiIi0nYSSS2G2JhBUoUBEhERkbbjJG3RMUAiIiLSdpykLTpeFSIiIqJKmEEiIiLSdswgiY4BEhERkbbjHCTRMUAiIiLSdlzFJjrm1YiIiIgqYQaJiIhI23GITXQMkIiIiLReLSZpczBJJQZIRERE2o4ZJNExbCQiIiKqhBkkIiIiLSeRSCCpYSaopue97JhBIiIi0nLyAKmmD01kZmYiMjISQUFBaN++PRwdHXH06FGVdXfv3o1BgwbB1dUVr7/+OqKiolBeXq5ULy8vD59++im8vLzg4eGB4OBgpKSkPLc2VWGAREREpO0ktXxo4Pr161i9ejXu3bsHR0fHKuvt27cPYWFhaNiwIT799FP4+flh+fLl+PLLLxXqVVRUIDQ0FNu3b8fo0aMxffp03L9/H0FBQbh582adt1kVDrERERGR2tq1a4cjR47AwsICSUlJCAsLU1kvIiICbdu2xZo1a6CjowMAMDY2xqpVqxAUFIQWLVoAABITE3Hq1CksX74cfn5+AIC+ffvC398fUVFRiIiIqNM2q8IMEhERkZZ7nkNsJiYmsLCwqLZOamoqUlNTERgYKAQyADBy5EhUVFTg999/F8p27twJa2tr9OrVSyiztLRE3759kZSUhLKysjprszoMkIiIiLTck1X+NQ2QxO/PhQsXAAAuLi4K5U2aNEHTpk2F4wCQkpKCdu3aKQVqrq6uKCgoEIbE6qLN6jBAIiIi0nJiZJDu3LmD9PR0hUdeXl6N+pOVlQUAsLKyUjpmZWWFzMxMhbrW1tZK9eRl8rp10WZ1OAeJiIiIMGrUKGRkZCiUTZo0CZMnT9a4reLiYgCAvr6+0jEDAwMUFRUp1FVVT14mb6su2qwOAyQiIiItJ0Et9kH6exlbTEwMHj9+rHDMzMysRm0aGhoCAEpLS5WOlZSUCMfldVXVk5fJ69ZFm9VhgERERKTtarBcX+FcAM2aNROrN8IwmKqhrqysLLRv316hrqohL3mZ/Py6aLM6nINEREREonJ2dgYAnDt3TqH83r17uHv3rnAcAJycnHD+/HnIZDKFusnJyTAyMoK9vX2dtVkdBkhERERa7nku81dHmzZt0KpVK2zZskVh2G7Tpk2QSqXo3bu3UNanTx9kZmZi9+7dQllOTg4SExPRq1cv6Onp1Vmb1eEQGxERkbarTaBTg/O+/fZbAMDVq1cBAPHx8Thx4gTMzMwwevRoAMCMGTMwceJEjB07FgEBAbh8+TJiYmIQGBiIli1bCm35+/vDw8MDM2bMwJgxY2BhYYFNmzahoqJCaYJ4XbRZ5WWRVc4/kVbp/EkC0u8XvuhuEKnlytJBL7oLRGqRADA20J5BFvep8biVXVCjc+0aG+PM1wM0OqeqW4zY2Nhgz549wvOkpCRERUXh6tWrsLS0xODBg/H+++9DV1cxP5Obm4uIiAgkJSWhpKQErq6uCA8PR7t27ZReoy7aVIUBkpZjgETahAESaQsGSMQhNiIiIi1Xm7lEdTEH6WXAAImIiEjbibDMnxQxQCIiItJyzCCJT3sGWImIiIieE2aQiIiItJwENc8EMX+kGgMkIiIiLcchNvExQCIiItJ2nKQtOs5BIiIiIqqEGSQiIiItxyE28TFAIiIi0nIMkMTHAImIiEjbPeeb1f4bcA4SERERUSXMIBEREWk5DrGJjwESERGRtuMyf9ExQCKt1tjUANPfbIterk1hZWaIrLxiJJy6jYW/XkBeUZlQb7xfG/R2a4ZXm5rA3EgfDwtLkXr3EdbsuYqE07dVtm1upIcP+jqhj8craGbRAAXF5bh4OxcLf7mAo6n3q+zTrLddMMnfEQXF5Wg9JV7090wvr5ycHERGfIntv8QjIyMdJqamaNvWBbP++zm6dnsNN9LS4OL0arVt/G/tDwgcMUqhLCM9HQvmz0XS7zuRmXkP5hYWcHf3wPyvIuHk3LYu3xI9JxLUIoPECEklBkiktRqZGmBHeE80MW+A9fuv4VJGHhxtzBDcoxW6tGmMARF/oKjsMQCgfUsL3LpfgN3n7iInvwTmxvp4s6Mtvp/ojYj48/h6x0WFtm0tjbDto+4wNtDFxoNpuHbvEcwa6MHZtiGamjeosk/tbBtivF8b5BeX8ZcOaeTmjRsI6O2L/IJ8BIeMQevWbZCXl4dzZ5Nx53YGAKCxlRVWfx+t8vxpUz9AUVERer3hr1B+5vQpvBnQG6Ympgh6513Y2tnhwYMHOHXyOLKzsgDnOn9rRFqJARJprSl9HWHX2BgT/3cUP/+VLpQfv3of343rgvFvtMGSvwOfCauPKZ2/encqdn7ii/f9HfBNwkVUyP45FjWmM3SkEvjOSUJmXrFa/ZFKgMigDthz7i5MDfXg3tyidm+Q/lXGvRuM8vJyHPnrNJo2a6ayjrGxMYaPHK1UfvTIYeTm5mLg24PRuHFjoby4uBjvjBoOOzt7JOzaCzMzszrrP71YnIMkPq5iI63l42CFotJyheAIAOKPp6Oo9DECfZpXe/7jChnuPiyGkb4u9HT++Sp4tWmMLm0a49vfLyMzrxi6Ugka6Ok8sz/jfFvDoZkZZm8+U7M3RP9aB/b/icOHDuA/H01H02bNUFZWhsLCQrXPj167BgDwzrtjFcpjf/oRV6+mYtZ/P4eZmRlKSkpQUlIiat+pfpBI/gmSNH+86N7XTwyQSGsZ6OmguKxCqVwmA4rLHqOFlQksjfUVjpkb6aGRiT7aNDXF1H5O6NmuCQ5ezkJJ+T/t+Lo0BQBk5BQiOswH16MG4lrUQByY0xuDu9ip7IutpRFmvNUOi35LQXqO+v+wEQHA7zsTAAC2dnYY+vZbsDI3RhNLU3i4OGHzxg3Vnpufn4+4bVthb98cvr3eUNluw4bm8O/1OqzMjdG4oRG6dumIpF076+bN0IshqeWDlDBAIq116XYeLIz10c62oUJ5O9uGsPg7MLKxNFI4dnCuP84tehN/ftEbU/s5Y/upDExcfVShTusmJgCAyNEdYG6khynrjuM/0cdRVl6BqDGeKjNTC0a2x43sAqxMuiLmW6R/iSuXLwEAJr8/Hg9yHmDl/9bi25X/g76+Pt4b8w7WR6+t8txtW7cgPz8fo4NDIJUq/kq/cvkyAGD0iKEwa2iGdes3Ysmy5bifnY3BA/pj7+6kuntTRFqOAdLfYvl4vdIAACAASURBVGNj4ejoiPT0dFHrUt1ZvfsKHlfIsDK0C3xdmsLGogF82zXBive6oPTvjFADfcWhsbErjmD4kv34T/Rx/HkhE4Z6OjA21FOoI3+eX1KOIYv/ROyxW9hy6AYGRu7Dw4JSzBzYTiElPbCzLXq2a4KPY07i8dMTmYjUlP8oHwBgamKKHb/vRuCIUQh6513s3L0P5ubm+OKz2aioUM6WAkD0uu8hlUoxOjhERbuPAAAODk74cVs83h4yDGPfm4DfdiZBKpVizuef1tl7ouer5sNrtdiB+yXHAKkamzZtQmxs7IvuBlXhaOp9TFh9FCaGuoiZ3BXHFwQgOswHhy5lIensHQDAo+IyhXOOXMnGvpRMbDl0A6OjDiK/uBy/zOiBhkb/BEnFf698+/mvWyh7/E/Ak1tYht+T76BJwwZo3cQUwJMhuznD3LHpYBqOX8up67dMLynDBoYAgCHDhkNf/59hYQsLC/Tt9ybu3b2Ly39nmZ52MeUC/jp6BL69/GBnb6+i3ScrLkeMGq3wj2Dr1m3QxcsHJ47/hYKCArHfDr0ADJDEx1VsfxswYAD69eun8Mtp8+bNMDMzw9tvv/3MuvRi/HYyAztOZcDZpiFMDHWRei8f9x+VYEd4T5Q9rkBaZvW//LceuYFBnnYIaG+DTQfTAAB3HhQBADJzlSez3st9sqJNHlB91L8tjPR1seHAdbSwMhbqGerrQCIBWlgZo7S8Arf/bpNIFRsbWwBAk6ZNlY7JV7Q9fPBA6Vj0uu8BKE/O/qddG1w4f66KdptCJpMhLzcXxsbGKs4m+ndjgPQ3HR0d6Og8e6WSpnWp7lXIgPPpucJzKzMDuNib4/DlbGEfpKoY/r06zfypydyn0nLwTo9WaGahvN/RK3+XZT96EjzZNjKCsaEuEmb6qmz/8Lw+uJiRi55zONeDqtaxU2esWb0SGRnKw/a3/x7Kt7K2VigvLS3F5o0b0NjKCv3eHFBFu57Y9ftO3FbRbkZ6BnR1dWFhaSnCO6AXjcv8xVdvh9iWLVsGR0dHXL9+HR988AHat28Pb29vREREoKzsn2GT8vJyREVFoVevXnBxcYGfnx+WL1+Ox48V/2E8ePAgRowYgU6dOqF9+/bw9/fH4sWLheOV5xX5+vri4sWLOHbsGBwdHeHo6IigoCCVdUNDQ+Hvr7g5m1xAQABCQkKE5xUVFVizZg369u0LFxcXdOvWDXPnzmWaWyQSCTAv0AM6Egm++XsPpAb6OjAyUA5opRIg5PUnuxKfvPbPztiJp2/jUVEZBnexUzjP2swQfdxfQerdR0jLevLzitp5CeNWHlF6XLqdh6LSxxi38gg+25pcl2+ZXgL93xoIU1NTbNkUg/z8fKH87p07+O3XeLRu44BXX22tcM6O335BdlYWho8cDT09vcpNAgCGDh8BHR0dRK/9HuXl5UL52eQzOHb0MLr3eB2GhoZ186bo+arN8BoDJJXqfQbpgw8+gL29PaZNm4YTJ05gzZo1KCwsxOeffw4AmD17NuLi4tCvXz907NgRx48fx9KlS3Hnzh3MmzcPAHDlyhWMHz8eHTp0wNSpUyGVSnHjxg2cOHGiytf95JNP8H//938wNDTEhAkTAEBhA7an9e3bF+Hh4bhw4QLatv1n2/5Lly7h6tWrCgHSrFmz8Ouvv2Lw4MF45513cOPGDWzYsAGpqalYt24dI3kNGBnoICHcFwmnb+NmdgFMG+hhkKcd3Jtb4Mufz+HQ5SwAQCtrE8RO64HfTqTj6r18PCwoRVOLBhjU2Q6tm5piy6E0hVuH5BaWYc62s1g4ugO2f9wTmw/dgJ6OBO/0aAU9XSlmbz4t1D1RxbyjMa+/CltLI2w/mVG3F4FeChYWFpj3ZQSmTJoI3+4+CHrnXZSWlmLNqhUoLS1F5OJvlM754RnDawDg4OCI/3w4HYsWLkAfv54YMiwQD3Jy8N23y2BkZIT/W7Cwzt4TvQD850NU9T5AatGiBZYtWwYAGDVqFAwMDLB582aMHTsWBQUFiIuLw/Dhw/HFF18IdUxNTbFlyxaMHj0aTk5OOHjwIAwMDLB27Vq1h8b8/PywbNkymJmZYcAA1enrp+vq6ekhISFBIUBKSEiArq4u3njjyd4kx48fR2xsLJYuXaqQcXJ1dcXUqVOxf/9+dO/eXaPr829WVl6B8+m5GORpB+uGhigqfYwzaQ8w4psD+OPCPaHenQdF+OnITXRp3Qh929vAxFAXj4rKcPbmQ3y9PQWxx24ptb1h/3Xk5Jfg/d4OmPFWW1TIZDhxLQfvr/kLf12t+j5sRDU1ZlwoGjVujCWLIzHvi/9CKpXCs4sX1kRvgLdPV4W66bduYXfSLnTx8oGTU/X3Cvl87v/BvnlzrF75HWbPnIEGDRrgtR6v49PP5sC5bbu6fEtEWq3eB0gjR45UeD5q1CjExsbiwIEDyMvLAwC8++67CnVCQkKwZcsW/Pnnn3BycoKZmRmKioqwf/9+vP7666L30dTUFN26dUNiYiI++ugjoTwxMRHe3t6wsLAQnpubm6Nz587Iyfkn89CpUyfo6Ojg2LFjDJA0UPZYhvfXKN9CpLKcglLMeirro64dp25jxynVN7J9lsGL/6zRefTvNmDg2xgw8O1n1rO1s0NuYdkz68mNGReKMeNCa9M1quc4B0l89T5AatGihcrnGRkZyMvLg66uLuwrLW9t3rw5dHV1kZHxZHgjICAAW7duxfjx42FlZQUfHx+88cYb8PPzE+2DERAQgOnTp+Ps2bNwdXXFxYsXcf36dbz33ntCnRs3buDhw4fw9vZW2cbTQRMREZG6GCCJr94HSGIwNDRETEwMjh49in379mH//v2Ij49H165dsXr1alFWpPn6+sLAwAAJCQlwdXVFQkIC9PT0hOE14MkEbSsrK0RERKhsw7rSKhUiIiJ11GauNeMj1ep9gJSWloZmT93ZOi0tDQDwyiuvwNTUFOXl5bh586ZCpunmzZsoLy+HjY2NUCaVSuHt7Q1vb2+Eh4dj9erViIyMxLFjx6rM6GgSVZuYmOC1115DYmIiZsyYgcTERHTr1k3h7tn29vY4evQoOnXqxD2UiIiI6rF6u8xfbuPGjQrPY2JiIJFI8Nprr6FHjx4AgOjoaIU6P/zwAwAIxx+o2GDN2fnJxMbq7mzdoEEDYZ6TOgICApCRkYHNmzcjLS0Nffv2VTju7++PsrIyrFq1Sunc0tJSheW9RERE6uJO2uLTigxSWFgYfHx8cOLECWzfvh2BgYGws3tyV/VBgwZh48aNyMvLQ4cOHXDy5En89ttvGDJkCBwdHQEA3377LY4fP47u3bvD1tYWOTk52LhxI5o2bYqOHTtW+drt2rXDhg0b8O2336J58+awtLSsMtsEAD179oShoSEiIiJgYGCAXr16KRz38vLC0KFDsWzZMpw7dw7e3t6QSqVIS0tDQkICIiMj4ePjI8JVIyKifxMOsYmv3gdIS5cuxeLFixEZGQkDAwOMGTMGH374oXB83rx5sLW1RWxsLHbu3Alra2t88MEHwt5FwJP5QRkZGYiNjcWDBw9gYWEBT09PTJ48GaamplW+9sSJE5Geno7//e9/KCgogKenZ7UBkpGREXr06IGdO3fCz88PJiYmSnXmzp2Ldu3a4ccff8SiRYugr68PW1tbDB06FE5OTjW8SkRE9G/2JECq6SRtkTvzkpDIZLJ6efvxZcuWISoqCn/99ZfCPB5S1PmTBKTfL3zR3SBSy5Wlg150F4jUIgFgbFDvZ6EIfBfsQ8aD4hqda2NhiD3hPUTukfar9xkkIiIiqh6H2MTHAImIiEjLSSQSSKXcB0lMDJCIiIi0HDNI4qu3A6yTJ0/GpUuXOP+IiIiInjtmkIiIiLQcbzUiPgZIREREWo5DbOJjgERERKTlmEESX72dg0RERET0ojCDREREpOWYQRIfAyQiIiItxzlI4mOAREREpPVqnkF6cmMVqoxzkIiIiIgqYQaJiIhIy3GITXzMIBEREWk5+STtmj40lZaWhv/85z/o3r07PDw8EBAQgFWrVqG0tFSh3smTJzFixAi4u7uja9eumDdvHoqKipTaKy0txcKFC9GtWze4ublh2LBhOHz4sMrXVrfN2mIGiYiIiNR27949DB06FKamphg9ejQaNmyI48ePY9GiRbhy5QoWLlwIAEhJSUFISAhat26N8PBw3L17F99//z3S09OxYsUKhTbDw8Px+++/Izg4GM2bN0dcXBzee+89rF+/Hu3btxfqadJmbTFAIiIi0nLPc4gtPj4eeXl52LhxI9q0aQMACAwMRElJCXbs2IH58+dDT08Pixcvhrm5OdavXw9jY2MAgK2tLWbPno3Dhw/D29sbAJCcnIzt27dj5syZCAkJAQAMHDgQ/fv3R2RkJGJiYoTXVrdNMXCIjYiISMs9zyG2goICAECjRo0Uyhs3bgxdXV3o6OggPz8fhw4dwsCBA4VABgAGDBgAIyMjJCQkCGWJiYnQ09PD0KFDhTIDAwMMGTIEJ06cQGZmJgBo1KYYGCARERFpOXkGqaYPTXTu3BkAMGvWLFy8eBF37tzBL7/8IgyLSaVSXLp0CeXl5XBxcVE4V19fH87OzkhJSRHKUlJS0LJlS4WgBwDc3Nwgk8mEupq0KQYOsRERERHu3LmDx48fK5SZmZnBzMxMoaxbt26YMmUKVq5ciT179gjlH3zwAcLCwgAAWVlZAAArKyul17GyssLp06eF51lZWWjSpInKegCEDJImbYqBARIREZGWe5IJqumtRp7876hRo5CRkaFwbNKkSZg8ebLSOba2tvD09MQbb7wBc3Nz/PHHH1i2bBksLS0xYsQIFBcXA3iS3anMwMBAOA4AxcXF0NPTU1kPAEpKSoR66rYpBgZIREREWk6MSdoxMTEqM0iVbd++HZ999hkSExOFzE/v3r0hk8kQERGBgIAAGBoaAoDSsn/gScAjPw4AhoaGKCsrU1kP+CdQ0qRNMTBAIiIi0nq1v9VIs2bN1Kq9ceNGtGvXTmlYzNfXF7Gxsbh48aIwDCYfFntaVlYWrK2thedWVlbCMFrlegCEupq0KQZO0iYiIiK1ZWdnK2WaAAhZoMePH8PBwQG6uro4d+6cQp3S0lKkpKTA2dlZKHNycsL169eF1XFyZ86cEY4D0KhNMTBAIiIi0nLPcxVby5Ytce7cOdy8eVOhfPv27dDR0YGjoyNMTU3h7e2N+Ph4hcAnPj4ehYWF6NOnj1DWp08flJWVYevWrUJZaWkpYmNj0aFDByFTpUmbYuAQGxERkZar6S1D5OdqYuzYsfjzzz8xYsQIjBo1Cg0bNsQff/yBP//8E8OHDxf2R5o6dSqGDx+OoKAgDB06FHfv3sXatWvRvXt3+Pj4CO25u7ujT58+iIyMRFZWFuzt7REXF4fbt2/jyy+/VHhtddsUg0Qmk8lEbZGeq86fJCD9fuGL7gaRWq4sHfSiu0CkFgkAYwPtGWQZsuoY7uaV1OjcpmYG+CnUU6NzkpOTsWzZMqSkpODhw4ewsbHB4MGDMXbsWOjo6Aj1jh8/jsjISFy4cAEmJiYICAjAhx9+CCMjI4X2SkpKsGTJEvz666/Izc2Fo6MjPvzwQ5VBj7pt1hYDJC3HAIm0CQMk0hYMkIhDbERERFrueQ6x/VswQCIiItJyDJDExwCJiIhIy4mxUSQp0p4BViIiIqLnhBkkIiIiLcchNvExQCIiInoJMM4RFwMkIiIiLccMkvg4B4mIiIioEmaQiIiItBxXsYmvygApKipK48YkEgnCwsJq1SEiIiLSjFQigbSGkU5Nz3vZMUAiIiLScswgia/KAGn37t3Psx9ERERE9UaVAZKNjc3z7AcRERHV0JMMUk1XsYncmZdEjSZpl5aW4sGDB7CwsIC+vr7YfSIiIiINSCSAlENsotJomf/58+cRHByMDh064PXXX8eJEycAAPfv38c777yDQ4cO1UkniYiIiJ4ntQOklJQUjBo1Crdu3cKAAQMUjjVq1AglJSWIi4sTvYNERERUPflGkTV9kDK1h9i++eYbWFtbIy4uDiUlJdi2bZvCcS8vLyQkJIjeQSIiIqqeBLVYxSZqT14eameQTpw4gaFDh8LY2FhltPnKK68gMzNT1M4RERHRs0lq+R8pUztAKikpgampaZXH8/PzRekQERER0Yum9hCbvb09zp8/X+XxI0eOoHXr1qJ0ioiIiNQnrcUqtpqe97JTO4PUv39/xMfHK6xUkw+1ff/999i/f7/S5G0iIiKqe5ykLT61M0hjxozBwYMHMXbsWLRq1QoSiQRffvklcnJykJ2dDR8fH4wcObIu+0pEREQq8FYj4lM7g6Svr4+1a9fi448/hoGBAQwMDJCWlgYLCwtMnz4dK1euhFSq0bZKRERERPWSRjtp6+rqIiQkBCEhIXXUHSIiItKURCKBtMa3GmEKSZUa3WqEiIiI6g8OsYlPowCppKQEP/zwA5KSknDr1i0AgJ2dHfz8/BAUFARDQ8M66SQRERFVrTaTrZlBUk3tACknJwfvvPMOrly5AhMTE9jZ2QEArl69ijNnziA+Ph4//PADLC0t66yzRERERM+D2gFSREQEUlNTER4ejpEjR0JfXx8AUFpaio0bN+Krr75CREQEFixYUGedJSIiImUcYhOf2gHS3r17MWTIEKUJ2vr6+ggJCcGVK1eQlJQkdv+IiIjoGaSo+SRtKW81opLa6/JLS0vRtm3bKo+7uLigtLRUlE4RERGR+iS1fJAytQMkV1dXXLhwocrj58+fh5ubmyidIiIiInqR1A6QwsPDsXPnTqxfvx7l5eVCeXl5OaKjo7Fr1y6Eh4fXSSeJiIioGrW5zQgnIalU5Ryk4OBgpTJzc3PMnz8fS5cuFVax3bp1C/n5+bC3t8eCBQsQHR1dd70lIiIiJbxZrfiqDJDS09NVljdr1gwA8PDhQwCAqakpTE1NUVZWJuyNRERERM/Pk0RQTfdBErkzL4kqA6Q9e/Y8z34QERER1Ru81QgREZGW4z5I4mOAREREpOV4qxHxaRQg3bx5E+vWrcOZM2eQl5eHiooKheMSiYSbRRIRET1nnKQtPrWX+V+6dAmDBg3C1q1bhQnZRkZGKCkpQUZGBnR0dIQJ3ERERETaTO0AaenSpdDT00N8fDzWrVsHAPjkk09w4MABzJkzB3l5efjss8/qqp9ERERUFe6DJDq1A6QTJ04gMDAQrVq1UhqvHDZsGLp3747IyEjRO0hERETV461GxKd2gFRQUCBsDqmnpwcAKCwsFI536NABJ0+eFLl7RERERM+f2pO0GzdujOzsbACAiYkJGjRogLS0NOF4Xl4eHj9+LHoHiYiIqHpSSCCt4VCZlDkkldQOkJycnHDu3DnhuaenJ3744Qe4ubmhoqICGzZsgJOTU510koiIiKrGfZDEp/YQ25tvvokHDx6guLgYADBlyhQ8evQIwcHBCAkJwaNHjzB16tQ66ygRERGpVtMJ2rXZP+llp3YGKSAgAAEBAcLztm3bYvv27di1axd0dHTQvXt3YY4SERERkTar1U7azZo1Q3BwsFh9ISIiohrgEJv4eKsRIiIiLSeR1HySNofYVKsyQJo5c6bGjUkkEsyfP79WHSIiIiLNvIgMUnJyMqKionDq1CmUl5fDzs4OISEhePvtt4U6u3fvRlRUFFJTU9GoUSMMGTIEEyZMgK6uYviRl5eHhQsXYteuXSguLoabmxtmzpwJZ2dnpddVt83aqrK1uLg4jRtjgERERPTy27dvH8LCwuDp6YkpU6ZAV1cXaWlpuHPnjlIdLy8vfPrpp7h8+TKWL1+OBw8e4NNPPxXqVVRUIDQ0FJcvX8aYMWNgYWGBjRs3IigoCLGxsbC3t9e4TTFIZDKZTNQW6bkqKQf4AyRtYdF50ovuApFa7JtZ4tKOOS+6G2r7dGcqcgrLanSupZEe5vq3Vrv+o0eP4O/vj4CAAMyePbvKev369YOBgQG2bt0KHR0dAMDXX3+NVatWISEhAS1atAAA7NixA1OnTsXy5cvh5+cHAMjJyYG/vz969uyJiIgIjdsUg9rL/ImIiKh+ktbyoYlff/0VeXl5mDJlCgAgPz8flXMtqampSE1NRWBgoBDIAMDIkSNRUVGB33//XSjbuXMnrK2t0atXL6HM0tISffv2RVJSEsrKyjRuUwwMkIiIiLScGPsg3blzB+np6QqPvLw8pdc6fPgwWrVqhX379qFHjx7o2LEjPD09ERkZKdxR48KFCwAAFxcXhXObNGmCpk2bCscBICUlBe3atVOaLO7q6oqCggLcvHlT4zbFwFVsREREhFGjRiEjI0OhbNKkSZg8ebJC2Y0bN3D37l2Eh4dj3LhxaNu2Lfbu3YvVq1ejpKQEs2bNQlZWFgDAyspK6XWsrKyQmZkpPM/KyoKXl5dSPWtrawBAZmYmXn31VY3aFAMDJCIiIi0nBSCt4Wo0+VBSTEyM0j1VzczMlOoXFhYiNzcXH330EUJDQwEAvXv3RmFhITZt2oSJEycKd93Q19dXOt/AwABFRUXC8+LiYpX15GXytjRpUwwMkIiIiLScRFLzAEk+stWsWTO16hsaGgIA+vfvr1D+5ptvIjExEWfPnhXqlJaWKp1fUlIiHJe3p6qevExeV5M2xcA5SERERFruyT5INZ2DpNlryYe4GjdurFAuf56bmyvUkQ+LPS0rK0sYPpO3p2p4TF4mr6tJm2JggERERERqa9euHQDg3r17CuV3794F8GQFmnyDx3PnzinUuXfvHu7evauwAaSTkxPOnz+vtBIuOTkZRkZGwj5ImrQpBo0DpPT0dGzduhXfffcd0tPTATxJd92+fVtl2ouIiIjqllRSu4cm+vTpAwD46aefhDKZTIatW7fCyMgIHh4eaNOmDVq1aoUtW7YozGvatGkTpFIpevfurdBeZmYmdu/eLZTl5OQgMTERvXr1gp6eHgBo1KYYNJqDtHDhQqxbtw6PHz+GRCKBh4cHbG1tUVpain79+mHKlCkICQkRtYNERERUved5qxEXFxcMHDgQK1euxP3799G2bVvs27cPBw4cwPTp02FiYgIAmDFjBiZOnIixY8ciICAAly9fRkxMDAIDA9GyZUuhPX9/f3h4eGDGjBnCTtqbNm1CRUWF0go6ddsUg9o7aW/evBmff/45goKC0LNnT4wZMwZr166Ft7c3AOCjjz5CdnY2oqOjRe0gVY87aZM24U7apC20bSftr/Zew4Oi8hqda9FAFx/3bKXROaWlpfj222/x888/Izs7G7a2tggJCcHw4cMV6iUlJSEqKgpXr16FpaUlBg8ejPfff1/pvmm5ubmIiIhAUlISSkpK4OrqivDwcGE4ryZt1pbaAdJbb72F5s2bY9myZXjw4AG8vb0VAqRVq1Zhw4YN+PPPP0XtIFWPARJpEwZIpC0YIJHac5DS0tLg4+NT5XELCws8ePBAlE4RERGR+iSo+W1Gajgy99JTOx/1rE2Ybt++rXJDKSIiIqpbz3MO0r+F2hkkNzc37Nq1S+WxkpISxMfHo0OHDqJ1jIiIiNQjlUhq9SBlagdIY8eOxenTpzF9+nRcunQJAJCdnY39+/cjKCgI9+7dw5gxY+qso0RERETPi9pDbD4+Pvj888/xf//3f/jtt98APFluBwB6enqYO3cu2rdvXze9JCIioipJUIshNlF78vLQaE1cYGAgfH19kZiYiGvXrkEmk6FFixbo27cvmjRpUld9JCIiomrUZMPHp88lZRpvGmBlZYWgoKC66AsRERFRvSDurkpERET03ElqMdlawknaKqkdIAUHBz+zjkQi4U7aREREzxmX+YtP7QBJfmPapz1+/BhZWVmoqKiAhYUFGjRoIGrniIiI6Nk4B0l8agdIe/bsUVleWlqKtWvXIjY2FuvXrxetY0REREQvitr7IFVFX18f48ePh5ubGxYsWCBGn4iIiEgDklr+R8pqHSDJdezYEQcOHBCrOSIiIlKTfIitpg9SJtoqtvT0dJSVlYnVHBEREalJUotAh5O0VVM7QLp9+7bK8tzcXBw6dAjr16+Hp6enaB0jIiIielHUDpB8fX2r3CtBJpOhZcuWmD17tmgdIyIiIvVIJJIa72fEfZBUUztACgsLU3kRzc3N0aJFC/j4+EAqFW1KExEREalJilos8xe1Jy8PtQOkyZMn12U/iIiIqIa4UaT41AocCwoK4Ofnh3Xr1tVxd4iIiIhePLUySMbGxnj48CGMjY3ruj9ERESkIWkt7sVW0/NedmoPPbq7u+Ps2bN12RciIiKqAUkt9kBifKSa2gHStGnTkJiYiG3btkEmk9Vln4iIiEgD8jlINX2QsmqH2G7fvg1LS0sYGhriyy+/hJmZGWbPno2FCxfC3t4ehoaGCvUlEgmio6PrtMNEREREda3aAKlXr15YuHAh+vfvj/T0dABAs2bNAADZ2dl13zsiIiJ6JikkkNbwnmo1Pe9lV22AJJPJhOG0PXv2PJcOERERkWa4zF98ot2LjYiIiF6M2tx0ljerVY0baBIRERFV8swM0vHjx/H48WO1Gxw4cGCtOkRERESaebLMv6b3YhO5My+JZwZIP/74I3788cdnNiSTySCRSBggERERPWcS1GIOkqg9eXk8M0AaNmwYPDw8nkdfiIiIqAa4k7b4nhkgderUCW+++ebz6AsRERFRvcBVbERERFqOy/zFxwCJiIhIy0lR82XpXM6uGq8LERERUSXVZpAuXrz4vPpBRERENSWRQMIxNlFxiI2IiEjLSVDz5foMj1RjgERERKTluMxffJyDRERERFQJM0hERERajkNs4mOAREREpOW4D5L4GCARERFpOUktVrHVePXbS45zkIiIiIgqYQaJiIhIy0lQ84wH80eqMUAiIiLSchxiEx8DJCIiIi3HVWzi4xwkIiIiokqYLtq7gQAAIABJREFUQSIiItJyEtRiiI05JJUYIBEREWk5KWo+JMShJNV4XYiIiLScfJJ2TR+1sXr1ajg6OmLAgAFKx06ePIkRI0bA3d0dXbt2xbx581BUVKRUr7S0FAsXLkS3bt3g5uaGYcOG4fDhwypfT902a4sBEhEREdVIVlYWvvvuOxgZGSkdS0lJQUhICEpKShAeHo4hQ4Zgy5YtmDp1qlLd8PBwREdH46233sKsWbMglUrx3nvv4dSpUzVus7Y4xEZERKTlXtQqtkWLFsHFxQUymQx5eXkKxxYvXgxzc3OsX78exsbGAABbW1vMnj0bhw8fhre3NwAgOTkZ27dvx8yZMxESEgIAGDhwIPr374/IyEjExMRo3KYYmEEiIiLSdpJ/7sem6aOmEVJycjJ++eUXzJw5U+lYfn4+Dh06hIEDBwqBDAAMGDAARkZGSEhIEMoSExOhp6eHoUOHCmUGBgYYMmQITpw4gczMTI3bFAMzSERERFruySTtmkU68kzJnTt38PjxY4VjZmZmMDMzUzpHJpNh7ty5GDhwIJydnZWOX7p0CeXl5XBxcVEo19fXh7OzM1JSUoSylJQUtGzZUiHoAQA3NzfIZDKkpKTA2tpaozbFwACJiIiIMGrUKGRkZCiUTZo0CZMnT1aq+/PPPyM1NRXLly9X2VZWVhYAwMrKSumYlZUVTp8+rVC3SZMmKusBEDJImrQpBgZIREREWk4YLqvhuQAQExOjMoNUWX5+PhYtWoTQ0FBYW1urbLO4uBjAk+xOZQYGBsJxeV09PT2V9QCgpKRE4zbFwACJiIhIy0n+/q+m5wJAs2bN1Kr/3XffQU9PD++++26VdQwNDQE8Wb5fWUlJiXBcXresrExlPeCfQEmTNsXAAImIiEjLiZFBUkdmZiaio6MxZcoUZGdnC+UlJSUoKytDeno6TE1NhWEw+bDY07KyshQyT1ZWVsIwWuV6AIS6mrQpBq5iIyIiIrXcv38fZWVliIyMRK9evYTHmTNncPXqVfTq1QurV6+Gg4MDdHV1ce7cOYXzS0tLkZKSojCx28nJCdevX0dBQYFC3TNnzgjHAWjUphiYQSIiItJyUkhqsYpN/fNsbW1VTsxesmQJCgsL8cknn6BFixYwNTWFt7c34uPjMX78eGGFWnx8PAoLC9GnTx/h3D59+uD777/H1q1bhX2QSktLERsbiw4dOggTuDVpUwwMkIiIiLRdLYbYNImrTE1N4efnp1QeHR0NHR0dhWNTp07F8OHDERQUhKFDh+Lu3btYu3YtunfvDh8fH6Geu7s7+vTpg8jISGRlZcHe3h5xcXG4ffv/27vzuKjK/Q/gHxbZReCCpiK4DuICsqihoIIQQuKuuICaS2nuO2Tea2WahoqClSFXcyGNhKuII0nen0u5pKlZoOUC4g7IkojAMOf3B5ejhxkNcFSGPu9evHKe8z3Pec4wwtfn+Z5zbmHFihWS41S3T03gEhsRERFpXMeOHbF582YYGBhgxYoViI+Px4gRI7Bu3TqV2FWrViE0NBR79uzBsmXLoFAo8OWXX8LNza3WfT4vHUEQBI33Si9NiQLgN5C0hWXX6a96CETVYtfUCpf2f/iqh1FtRy7n4lGZslb7GjXQRa+2/9DwiLQfl9io3jJuoH7e2NTUFDn5DyRtv1+6hMXvLcKxI4dRWlqKLi6uWPKvD9DH20dl/4KCAiz95/vY858E3M/NRes2bTBl6nRMfmfKcz8Vm+qX+RPegEt7W7g42qGVrTUyb+Wi/Zv/Uhvr0709Bvl2gatjC3Rs2wxGhg3wxqR1OHrmj6f23771awib1A+93NvBqpEJcvIe4Mxv1zHj4524d/9PMU5fXxcTBvfEuEEeaN3CGgBw7UYudiSdxJfxR1GmeHzvm8XvBOL9KYFqjxe+JhGR276vzVtBL5gmLvMnKSZIVK/19PTCxElvS9r0q9yQ7OqVK/Du1QP6+vqYO38hzM0bYXNsDIIC/bFnnxw+fR+vqZeWluLNfn44f+4spk6bgfbtHZGSIsesGe/i3r27eP+fS1/GaZGW+GjGAOTmF+HcxSw0amj8zNiRge4IDnDHb5dv4+K1O+jSvsUz4309HPHNmsm4eiMHn+08jHu5hWhs1RDdnFqhoZmRJEGK+SAUIwO7IjH1LLb850fo6eoisFdnRCwchtedWyE0bLNK/ws+/RY5+dKris6mX6/B2dPLpKtT8VXbfUkVEySq11q1ao1RY0KeGbPk/XDk5+fjx5Nn4NylCwBgTOhYuDp3xOyZ03D+14vizNDm2E04c/onrF67Hu9Or7j9/oRJkzFyxFCs+mQ5Qse9BXt7+xd7UqQ1HPv/Cxk3cwEAp+Pfg5mJ4VNjl0YnYfqynSgtU2B2aN9nJkg2lmbYsnw8jpz5A8Nmb4RC8fSllaY2jTAysCv2HjqP0QtixfYvdh1BauxsDPFzwYyPdyL/z2LJfnv/+wuu375f3VMlqndYpE31XmlpKR48eKB2W1FREZKT9qJX7z5icgQAZmZmeGvCJPzx++84/dNPYvuunXEwMTHBhEmTJf1MnzkbZWVl+DZ+14s5CdJKlclRddzKLkBpmaJasZOGe+IfFqZYHLkHCoUSxkYNoK+v/sd5ZVJ2K7tAZdudnEIolQIelao/bkNTI+jp8deENtB5zv9IFT/5VK8lJnwLK3MT2Fg2hF2zxpgzawYKCh7/orjwyy8oKSlB99c9VPbt1v11AMCZ0xUJklKpxLmzP8O5i4vKLe27du0GHR0dMZboRerXsyMK/iyGRUNjnNgZhvvH1yL/RCRSY2fDrYOdJPbqjRxcuZ6NcQNfx/jBHrBraoVWttaYGeKDgT7O+HTzd3hUovqYh5++Cce9YxHIP7EW/90yF2/07PCyTo9qQQeP76Zd469XPfg6iktsVG+5d+2GIcOGo02btvizsBAHDuzHF59F49iRw/jv0R9hZmaG27dvAQCaNWuusn9l261bFU+3zsvLQ3FxMZo1V401NDSEtbU1blV5EjbRi9CuZRPo6+tiz4Z3kXDwLD6JOQD7ZlZYNKkfUjbNglfIp0i/egcAUF6uxLA5G7Hpw1B8/s8xYh+lZQrMXRWPmPhjkr4L/nyITd8ew4lfriG/8CFk9o0xfYw3EtdPwTtLd2B70smXeq5UPSzS1jwmSLUUGhoKANi2bdsrHgk9zdEfpT/Ix4SORefOTvjXksXYELUOi8IXo/jhQwCPH4b4pMpZoof/i6n8v6GB+joSQyMjsT+iF6mhiSH09fXwdfIpvP2v7WL7z2lZ+G7TLLz3doCk8Lr4URkuX8/G6d8ycfin32FsZICQ/t2xdtEIFBWXIm7fKTE2Ou7/JMdKBvDVnhM4Hf8eVs0fisTUsygqVn1YKFF9wyW2Z8jOzkZUVBTS09Nf9VBIQ+bMWwADAwPI9ycDAIxNTAA8fmr0kx49egQAMPlfTOX/S0pVYwGg5NEjsT+iF6n4f0ti26rM5hw98weu374PL/d2YluTfzTEse0LkHXnPmav+AaJqecQt+8U3pwajdO/ZWLtouGw+Isr7O4XFGHTt8dgaW6C151ba/6E6Lnp6Dy+kq2mX7w7iXpMkJ4hJycH0dHRahOk2NhYxMbGqtmL6rIGDRqgabNmyM2teAp106bNADxeRntSZVvlUpulpSWMjY3VLqOVlJQgJydH7fIbkabdvJsPALibU6iy7U52ASzNHyfqE4b2hLWlGRIOnpXECYKAxNSzMDczRhfHZ99SAAAy/3dFm7WF2fMMnV4QFmlrHhOkWjIwMICBgcGrHgbV0KNHj3Dzxg00blzx8MNOnTvD0NAQJ08cV4k9dfIEAMDVzR0AoKuriy4urjh/7qzKjNNPP52CIAhiLNGLdPq3TABA8yYWKtuaN7FA9hP3QGpmUxGjp6v6417/f1eo6evp/eUx27awAQDcva+alNGrV+sCbc4gPdUrTZCioqLg4OCArKwsLFy4EG5ubnBzc0N4eDiKi6X35Ni9ezcGDx4MJycndO/eHYsWLUJOTo4kRqlUIioqCp6ennB2dkZoaCguX74MHx8fhIWFiXH5+flYuXIlgoKC4OLiAldXV0yaNAkXL14UY06ePIlBgwYBAMLDw+Hg4AAHBwckJCQAqKhBqqxDysnJgaOjIz7//HOVczx//jwcHBywZ88ese327dtYuHAhPDw80KlTJwQFBWHfvn3P+W7Sk3Jz1V9e/cG/lkChUCCwfxCAisv5A/sH4cjh/8Mv58+LcQ8ePMDmf29C23bt0LVbN7F9RPAoPHz4ELExX0r6jV4fCX19fQwfHvwCzoZIKi65omZo8jBPSXtgr05o3sQSB46liW0X/1esHTKguyRWX18XIwLcUVZWjvMXswAAenq6MDeTXqEJALZNLDB5uBdy8h7gxPlrGj0XorqqThRpz5w5Ey1atMC8efOQlpaG+Ph4WFlZYcGCBQCA6OhobNiwAW+++SZGjBiB7OxsbN26FRcuXEBCQoJYTLt69Wps2rQJPj4+8PT0xMWLFzFx4kSVf+1nZWUhNTUV/fr1g62tLXJycrBr1y6EhIQgOTkZTZo0QZs2bTBnzhysXbsWwcHB4gPzXF1dVcZvbW0Nd3d3yOVyTJ06VbJNLpfD0NAQffv2BQDcu3cPI0aMQIMGDTB27Fg0atQI33//PebNm4fS0lIMGTJE4+/v39Eny5fh1MkT6N3HGy1a2OFB0QOkyPfj8P/9F127dce702aIsR8tW4H/O/Q9ggLfwPRZc2De0BybY2Nw6+ZNJO5Nljw+ZMKkydj21WYsWjAXmZkZaN/eEQcO7Mfe/yQi7L33Yd+y5Ss4W6qrRr3ZFXZNrQAA1pZmMGigj0WT/AEA12/fx9fJj28L0aldM7zZuzMAwKNLRZ3P6P5d0cOl4s+f7zyMwgcVdXH/PXkJu+SnERzgjsSoqZAf+RV2Ta0wdWRv3M4uwMcb94v9bt17AtNG98E7I3qheRNLpP6YDhOjBhj5Zlc4yWyxZstBZOdV3CfMzNgQ6ckfIOm/v+DStTvIK3wIWcsmGD/YA2bGhhgXvkXtLQHo1dNB7S/X5wSSenUiQercuTM+/PDxQwHz8/Px7bffYsGCBbhx4wY+++wzLFiwABMmTBBjevXqhZEjRyIxMRGjRo1CTk4OtmzZAn9/f6xfv16Mi46ORlRUlOR4Dg4OSElJge4TU84DBw5EQEAAvv32W0ybNg3W1tbo3bs31q5diy5dumDgwIHPPIfAwEAsXboUV69eRevWFT/QBEFASkoKevXqBTOzinX7yMhI6Orq4j//+Q/Mzc0BAKNHj8akSZOwZs0aDBo0SDIuqp1evfvgYnoatm/7Cvdzc6Gnp4e2bdvhg48+xszZcyX3MWrTti0OHf4BSxaHYfWqT8Rnse1NPiB5zAhQsbSanJKKpf98H9/s+lp8FtuayChMeXfayz5NquPGD+qBXk8UTAPA0mkVs5dHTv8hSZC6tG8hbnty/0pfJ/8kJkgAMHHJVlz4/QbGDvTAqvlDUPDgERK/P4ul0Um4/cRNIf8seoTeYyPw3tsB6OfVEW/0cESZohzpV27j3Q/jsDnxRzG2uKQM//n+HLp2aokgbyeYGRsiJ/8B/nvyEtZsSRWX9qju0dXRgW4t18pqu199VycSpJEjR0peu7u74+DBg3jw4AFSU1MhCAL8/Pxw//7j297b2dnBxsYGp06dwqhRo3D8+HEoFAqMHj1a0ldISIhKgvRk7VB5eTkKCwthYmKCVq1aIS0tDbXh7++Pjz76CHK5HNOmVfyiPHfuHG7duiXOhAmCgIMHD6J///5QKBSS8/Hy8sLRo0dx7do1tGnTplZjoMeCBgxE0IBnJ7VPau/oiPiEPX8dCMDCwgKR66MRuT66tsOjvwn/yeuqHbs96WSN7jFUXq7E6i2pWL0l9S9js/MeYM7KeMxZGf/MuNIyBd79MK7aY6C6gzNImlcnEqSmTZtKXlfOrBQUFCAjIwNKpRK+vr7qdhWTjFu3Km74V/U5WBYWFmjUqJGkTalUYuvWrYiLi8ONGzdQXl4uia8NKysrdO/eXZIgyeVyGBsbo0+fPuJYCwsLERcXh7g49T+E8vLyanV8IiIi0pw6kSDpPeUKCkEQoFQqoaenh5iYGEktSKXKZKomvvjiC6xbtw5Dhw7FrFmz0KhRI+jq6mL58uUQBKHG/VUKCAjAkiVLcPnyZbRp0wYpKSno3bu3eP8cpbLigZJDhgxBUFCQ2j7atWuntp2IiOipOIWkcXUiQXoWOzs7lJeXw97eHra2tk+Na9as4n42mZmZkhmpvLw8ybO3ACAlJQXdu3fH8uXLJe2FhYWwtLQUX6tLyJ7Fz88PH3zwAeRyOXr27Ik7d+4gMDBQ3G5lZQVTU1MIgoAePXo8oyciIqKa4f2MNKvOVwP7+flBV1cXGzZsUNmmVCqRn19xwzQPDw/o6+urLF3t2LFDZT89PT2VmSK5XI67d+9K2oyNK+4uW1hYvft+WFpa4vXXX4dcLodcLoeJiQl69+4tOa6fnx/279+Pq1evquz/ZE0SERFRdfE+SJpX52eQ7O3tMXPmTERGRiIrKwve3t4wNjZGVlYWUlJSMHXqVAwfPhzW1tYYO3Ys/v3vf+Pdd99Fz549cenSJRw5cgSWlpaS2aA+ffpgw4YNCA8Ph4uLC37//XckJSWhRQvp3WSbN28OCwsL7Ny5E6ampjAxMYGTk5NK3JMCAgKwePFi3LlzBz4+PipPfZ83bx5OnjyJoUOHIjg4GK1bt0ZeXh4uXLiAtLQ0HDp0SLNvIBEREdVYnU+QAGDq1Kmwt7fH1q1bERUVBR0dHTRr1gy+vr6Spar58+fDyMgI8fHx+OGHH9ClSxfExsZi9OjRkivXpkyZguLiYiQlJWH//v3o0KEDNm7ciNWrV0uOq6+vj5UrVyIiIgJLly6FQqHAihUrnpkg+fn5YenSpSgqKkJAQIDK9saNGyM+Ph7R0dGQy+XIzc2FhYUFHBwcMGvWLA28W0RE9HfDEiTN0xGepypZCxQWFqJr166YPXu2yk0c64MSBVCvv4FUr1h2nf6qh0BULXZNrXBp/4d/HVhHnM8qRKmidr8NDPR14Nyi5hc81XdaMYNUXY8ePVJZ0vrqq68AAN2eeFwEERFRffI8j5xlcbd69SpBSkpKwt69e9G7d28YGxvjzJkzSE5Ohqenp/ioECIiIqK/Uq8SpPbt22Pfvn2IiYlBUVERrK2tMW7cOMyePftVD42IiOiFeZ6r0XgVm3r1KkHq3LmzuKRGRET0d8Eibc2r8/dBIiIiInrZ6tUMEhER0d8Wp4I0igkSERGRluNVbJrHBImIiEjLsUhb81iDRERERFQFZ5CIiIjqAU4EaRYTJCIiIm3H6/w1jgkSERGRlmORtuaxBomIiIioCs4gERERaTlexaZ5TJCIiIi0HEuQNI8JEhERkbZjhqRxrEEiIiIiqoIzSERERFqOV7FpHhMkIiIiLccibc1jgkRERKTlWIKkeaxBIiIiIqqCM0hERETajlNIGscEiYiISMuxSFvzmCARERFpu+co0mZ+pB5rkIiIiIiq4AwSERGRlmMJkuZxBomIiKg+0KnlVw398ssv+OCDDxAYGIguXbqgT58+mDNnDjIzM1Vif/75Z4waNQrOzs7o2bMnli1bhuLiYpW40tJSfPrpp/D09ISTkxNGjBiB48ePqz1+dft8XkyQiIiItJzOc/5XE5s2bcLBgwfRo0cPLF68GCNGjMCpU6cwaNAgXLlyRYxLT0/H+PHjUVJSgrCwMAwbNgy7du3CnDlzVPoMCwvDV199hQEDBmDx4sXQ1dXF5MmTcfbsWUlcTfp8XlxiIyIiomobP348IiIiYGBgILYFBgYiKCgIMTEx+OSTTwAAa9asgYWFBbZt2wZTU1MAgK2tLd5//30cP34cHh4eACpmpJKTkxEeHo7x48cDAAYNGoT+/fsjIiICO3bsEI9T3T41gTNIREREWq7yUSO1/aoJV1dXSXIEAC1btkS7du3EGaQHDx7gxx9/xKBBg8REBgAGDhwIExMTyOVyse3AgQNo0KABhg8fLrYZGhpi2LBhOHPmDO7du1fjPjWBCRIREZGWq2350fMUdz9JEATk5OTA0tISAHDp0iUoFAp06tRJEmdgYABHR0ekp6eLbenp6WjVqpUk6QEAJycnCIIgxtakT03gEhsRERHh9u3bKC8vl7SZm5vD3Nz8L/fdu3cv7t69K9YCZWdnAwBsbGxUYm1sbHDu3DnxdXZ2Npo0aaI2DoA4g1STPjWBCRIREZG208B1/mPGjMHNmzclm6ZPn44ZM2Y8c/crV67gww8/hJubGwYOHAgAePToEQCoLMUBFctnldsrYxs0aKA2DgBKSkpq3KcmMEEiIiLScpp41MiOHTvUziA9S3Z2Nt555x00atQI69atg65uReWOkZERgIrL96sqKSkRt1fGlpWVqY0DHidKNelTE5ggERERabnaFFs/uS8ANG3atEb7/fnnn5g8eTL+/PNPfP3115Klr8o/Vy6LPSk7OxuNGzeWxFYuo1WNAyDG1qRPTWCRNhEREdVISUkJpkyZgoyMDGzcuBGtW7eWbJfJZNDX18evv/4qaS8tLUV6ejocHR3Ftvbt2+PatWsoKiqSxJ4/f17cXtM+NYEJEhERkZZ7mVexlZeXY/bs2Th37hzWrVuHLl26qMQ0bNgQHh4e2LNnjyTx2bNnDx4+fIh+/fqJbf369UNZWRni4+PFttLSUiQkJMDV1VUs4K5Jn5rAJTYiIiJt9xIfxvbJJ5/g0KFD8Pb2Rn5+Pvbs2SNuMzU1ha+vLwBgzpw5GDlyJEJDQzF8+HDcuXMHmzdvRq9evdCjRw9xH2dnZ/Tr1w8RERHIzs6GnZ0dEhMTcevWLaxYsUJy7Or2qQk6giAIGu2RXqoSBcBvIGkLy67TX/UQiKrFrqkVLu3/8FUPo9pu5JWgXFm7ffV0AVtLw2rHh4aG4tSpU2q3NW/eHIcOHRJfnz59GhEREUhLS4OZmRkCAwMxd+5cmJiYSPYrKSlBZGQkkpKSUFBQAAcHB8ydO1dt0lPdPp8XEyQtxwSJtAkTJNIWTJCIS2xERERaThNXsZEUEyQiIiIt9xJLkP42mCARERFpOc4gaR4v8yciIiKqgjNIREREWo/TQJrGBImIiEjLcYlN85ggERERaTkWaWsea5CIiIiIquAMEhERkZbTwXMssWl0JPUHEyQiIiItpwMdLrFpGBMkIiIibfc8WQ4zJLVYg0RERERUBWeQiIiI6gFOBGkWEyQiIiItp6PzHJf5M7NSiwkSERGRlmORtuaxBomIiIioCs4gERERaTtexaZxTJCIiIi0HB81onlcYiMiIiKqgjNIREREWo5XsWkeEyQiIiItx6vYNI8JEhERkZbjDJLmsQaJiIiIqAomSERERERVcImNiIhIy3GJTfOYIBEREWm92hdpk3pcYiMiIiKqgjNIREREWu55lsm4xKYeEyQiIiItx0exaR4TJCIiovqAmY5GsQaJiIiIqArOIBEREWm557mGjRNP6jFBIiIi0nLPVaStuWHUK0yQiIiItByLtDWPNUhEREREVXAGiYiISNtxGkjjmCARERFpORZpax4TpHqAH27SFnZNrV71EIiqpXlji1c9hBp5nofVkno6giAIr3oQRERERHUJi7SJiIiIqmCCRERERFQFEyQiIiKiKpggEREREVXBBImIiIioCiZIRERERFUwQSIiIiKqggkSERERURVMkIiIiIiqYIJEBCAhIQEODg64ceOGRmOJ6qLQ0FCEhoa+6mEQ1WlMkIie4uuvv0ZCQsKrHgZRrWRnZyMqKgrp6emveihEWonPYiMCUF5eDoVCAQMDA+joVDzyceDAgTA3N8e2bdv+MpaorklPT8egQYOwYsUKDBkyRLKttLQUAGBgYPAqhkakFfRf9QCI6gI9PT3o6elpPJaoLmJiRPTXuMRGdVJUVBQcHBxw7do1zJw5Ey4uLvDw8MCqVatQVlYmxikUCkRHR6Nv377o1KkTfH19sWHDBpSXl0v6++GHHzBq1Ci4u7vDxcUF/v7+WLNmjbi9al2Rj48PLl68iFOnTsHBwQEODg5izUbV2Lfffhv+/v5qzyMwMBDjx48XXyuVSsTGxiIgIACdOnWCp6cnPvroIxQVFWnkfaOXp/IzmpWVhYULF8LNzQ1ubm4IDw9HcXGxJHb37t0YPHgwnJyc0L17dyxatAg5OTmSGKVSiaioKHh6esLZ2RmhoaG4fPkyfHx8EBYWJsbl5+dj5cqVCAoKgouLC1xdXTFp0iRcvHhRjDl58iQGDRoEAAgPDxc/w5VLxk/WIOXk5MDR0RGff/65yjmeP38eDg4O2LNnj9h2+/ZtLFy4EB4eHujUqROCgoKwb9++53w3ieoeziBRnTZz5kzY2dlh/vz5OHPmDGJjY/Hw4UMsXboUAPD+++8jMTERb775Jtzc3HD69GmsX78et2/fxrJlywAAf/zxB9555x24urpizpw50NXVRWZmJs6cOfPU47733nv4+OOPYWRkhClTpgAArK2t1cYGBAQgLCwMaWlp6NChg9h+6dIlXLlyRZIgLV68GElJSRg6dCjGjRuHzMxMbN++HZcvX8aWLVu4ZKeFZs6ciRYtWmDevHlIS0tDfHw8rKyssGDBAgBAdHQ0NmzYgDfffBMjRoxAdnY2tm7digsXLiAhIQFGRkYAgNWrV2PTpk3w8fGBp6cnLl68iIkTJ6KkpERyvKysLKSmpqJfv36wtbVFTk4Odu3ahZCQECQnJ6NJkyZo06YN5syZg7Vr1yI4OBhubm4AAFdXV5XxW1tbw93dHXK5HFOnTpVsk8vlMDQ0RN++fQEA9+7dw4gRI9CgQQOMHTsWjRo1wvfff4958+ahtLRUZSmPSKsJRHXQ+vV+b+neAAARiUlEQVTrBZlMJkyfPl3SHhYWJjg4OAjXr18X0tPTBZlMJvzzn/+UxCxZskSQyWRCenq6IAiCsHnzZsHV1VVQKBRPPd7u3bsFmUwmZGVliW0DBgwQQkJC/jK2sLBQ6NixoxARESGJW7t2rdChQwfh/v37giAIwk8//STIZDLhwIEDkrjk5GRBJpMJhw8f/qu3heqQys/okiVLJO3Tpk0TunXrJgiCIGRlZQmOjo5CbGysJObs2bOCg4ODEBcXJwiCIGRnZwsdOnQQZsyYIYmLiooSZDKZsGjRIrGtpKREKC8vl8RlZWUJnTp1EqKjo8W2tLQ0QSaTCbt371YZe0hIiOSzHRcXJ8hkMuHKlStim1KpFPr06SNMmzZNbAsPDxd69eolFBQUSPqbOHGi0LNnT5VxEWkzLrFRnTZ69GjJ6zFjxkAQBBw7dgyHDx8GALz11luSmMoZmyNHjgAAzM3NUVxcjKNHj76QMTZs2BCenp44cOCApP3AgQPw8PCApaWl+NrCwgJdu3bF/fv3xS93d3fo6enh1KlTL2R89GKNHDlS8trd3R35+fl48OABUlNTIQgC/Pz8JN9zOzs72NjYiN/z48ePQ6FQqHzeQ0JCVI5nYGAAXd2KH93l5eXIy8uDiYkJWrVqhbS0tFqdg7+/P/T09CCXy8W2c+fO4datWwgMDAQACIKAgwcPwsfHBwqFQnI+Xl5eyM7OxrVr12p1fKK6iEtsVKe1bNlS7eubN2+isLAQ+vr6sLOzk8TY29tDX18fN2/eBFBRBxQfH4933nkHNjY26NGjB/z8/ODr66uxJa3AwEAsWLAAFy5cQOfOnXHx4kVcu3YNkydPFmMyMzORn58PDw8PtX3cv39fI2Ohl6tp06aS1+bm5gCAgoICZGRkQKlUwtfXV+2+ld/zW7duAaj47D7JwsICjRo1krQplUps3boVcXFxuHHjhqTezsLColbnYGVlhe7du0Mul2PatGkAKpbXjI2N0adPH3GshYWFiIuLQ1xcnNp+8vLyanV8orqICRLVe0ZGRtixYwdOnjyJw4cP4+jRo9izZw969uyJmJgYjVyR5uPjA0NDQ8jlcnTu3BlyuRwNGjSAn5+fGKNUKmFjY4NVq1ap7aNx48bPPQ56+Z72+REEAUqlEnp6eoiJiVGbjFcmUzXxxRdfYN26dRg6dChmzZqFRo0aQVdXF8uXL4fwHHdtCQgIwJIlS3D58mW0adMGKSkp6N27N0xMTABUfH4BYMiQIQgKClLbR7t27Wp9fKK6hgkS1WkZGRmSf6FnZGQAAJo1a4aGDRtCoVDg+vXrkpmm69evQ6FQoHnz5mKbrq4uPDw84OHhgbCwMMTExCAiIgKnTp166oxOTWaXzMzM4OXlhQMHDmDhwoU4cOAAPD09Jb8A7ezscPLkSbi7u/My678JOzs7lJeXw97eHra2tk+Na9asGYCKWcYnP+95eXkoKCiQxKakpKB79+5Yvny5pL2wsFBczgVq9vkFAD8/P3zwwQeQy+Xo2bMn7ty5Iy6vARWzTKamphAEAT169KhR30TaiDVIVKdVncrfsWMHdHR04OXlhd69ewMAvvrqK0nM1q1bAUDcrm7a39HREQBUrhB6krGxMQoLC6s91sDAQNy8eRM7d+5ERkYGAgICJNv9/f1RVlaGL7/8UmXf0tJSPHjwoNrHIu3g5+cHXV1dbNiwQWWbUqlEfn4+AMDDwwP6+vpqP+9V6enpqcwUyeVy3L17V9JmbGwMANX+DFtaWuL111+HXC6HXC6HiYmJ+Heo8rh+fn7Yv38/rl69qrI/l4ipvuEMEtVpGRkZmDZtGnr06IEzZ84gOTkZwcHBaNGiBQBg8ODBiIuLQ2FhIVxdXfHzzz9j3759GDZsGBwcHAAAn332GU6fPo1evXrB1tYW9+/fR1xcHF577TXx8md1OnbsiO3bt+Ozzz6Dvb09rKysnjrbBADe3t4wMjLCqlWrJJdGV3r99dcxfPhwREVF4ddff4WHhwd0dXWRkZEBuVyOiIgI/su8nrG3t8fMmTMRGRmJrKwseHt7w9jYGFlZWUhJScHUqVMxfPhwWFtbY+zYsfj3v/+Nd999Fz179sSlS5dw5MgRWFpaSmaD+vTpgw0bNiA8PBwuLi74/fffkZSUJP6dqNS8eXNYWFhg586dMDU1hYmJCZycnFTinhQQEIDFixfjzp078PHxEW9BUGnevHk4efIkhg4diuDgYLRu3Rp5eXm4cOEC0tLScOjQIc2+gUSvEBMkqtPWr1+PNWvWICIiAoaGhpgwYQLmzp0rbl+2bBlsbW2RkJCAlJQUNG7cGDNnzhTvXQRU1AfdvHkTCQkJyMvLg6WlJbp164YZM2agYcOGTz321KlTcePGDWzatAlFRUXo1q3bMxOkyn9xp6SkwNfXF2ZmZioxH330ETp27IhvvvkGq1evhoGBAWxtbTF8+HC0b9++lu8S1WVTp06Fvb09tm7diqioKOjo6KBZs2bw9fWVJMTz58+HkZER4uPj8cMPP6BLly6IjY3F6NGjJUuyU6ZMQXFxMZKSkrB//3506NABGzduxOrVqyXH1dfXx8qVKxEREYGlS5dCoVBgxYoVz0yQ/Pz8sHTpUhQVFanMgAIVdXLx8fGIjo6GXC5Hbm4uLCws4ODggFmzZmng3SKqO/gsNqqToqKiEB0djZ9++qlWhaxE9UFhYSG6du2K2bNnq9zEkYheLNYgERHVAY8ePVJpq6yv69at28seDtHfHpfYiIjqgKSkJOzduxe9e/eGsbGxWHPn6en5zFo5InoxmCAREdUB7du3x759+xATE4OioiJYW1tj3LhxmD179qseGtHfEmuQiIiIiKpgDRIRERFRFUyQiIiIiKpggkRERERUBRMkor+RGzduwMHBAVFRUc9sq0vCwsLEu6L/FR8fH4SGhtb6WKGhofDx8an1/s/i4OCAsLCwF9I3EWker2IjesFOnjyJsWPHStpMTEzQqlUrDBw4ECEhIU99Inxdd+PGDSQmJsLX11d8vh0RUX3ABInoJenfvz969eoFQRBw7949JCYmYvny5bh8+TI++uijVzau5s2b45dffqlVknbz5k1ER0ejefPmTJCIqF5hgkT0knTo0AEDBw4UX48ePRoBAQGIj4/HrFmzYG1trXa/Bw8eqH2um6bo6OjA0NDwhfVPRKSNWINE9IqYmZnBxcUFgiAgKysLwOMamrS0NEycOBFubm4YMGCAuE9GRgYWLFgAT09PdOrUCT4+Pli5ciUePnyo0v/p06cxcuRIODk5oUePHvjwww/Vxj2rBiklJQWhoaFwd3eHs7Mz/P39sWzZMpSWliIhIUFcOgwPD4eDgwMcHBwkNUCCICAuLg5DhgyBs7MzXFxcEBoaihMnTqgcq6SkBCtXroSnpyecnJwwbNgwHDt2rOZvbBXHjh3D7Nmz0bdvXzg5OcHd3R0TJkzAqVOnnrpPVlYWpk6dCjc3N7i6umLatGni9+hJNTk/ItIunEEiekUEQUBmZiYAwNLSUmy/desWxo0bh379+uGNN94Qk5pff/0V48aNg7m5OYKDg9GkSRNcvHgR27Ztw9mzZ7Ft2zY0aNAAAHD+/Hm89dZbMDU1xeTJk9GwYUPs378fixYtqvb41q5diy+++AJt27bF+PHjYWNjg+vXr+O7777DzJkz0bVrV0yZMgVffPEFgoODxcdhPDkTtmDBAiQnJ8Pf3x9DhgxBaWkpkpKSMGHCBERFRaFv375i7Ny5c5Gamgpvb294eXnh+vXrmDFjBmxtbWv/JgNITExEQUEBBg0ahNdeew13795FfHw8xo8fj61bt8Ld3V0S//DhQ4SGhsLJyQlz585FZmYm4uLicP78eSQmJsLGxqZW50dEWkYgohfqxIkTgkwmE6KiooTc3FwhNzdXSE9PFxYvXizIZDJhxIgRYqy3t7cgk8mEb775RqWfoKAgwd/fX/jzzz8l7d99950gk8mE3bt3i23BwcFCx44dhatXr4ptJSUlwtChQwWZTCasX79ebM/KylJpO3/+vCCTyYTQ0FDh0aNHkuMplUpBqVRKzu3JY1cd186dOyXtZWVlwuDBgwVvb2+xn6NHjwoymUxYtGiRJPbgwYOCTCYTZDKZSv/qeHt7CyEhIZK2oqIilbjs7GyhW7duwqRJkyTtISEhgkwmE5YtW6b2XJYsWVKr8xMEQe35EVHdxSU2opckKioKHh4e8PDwwMCBA7F79274+Phgw4YNkjgLCwsMGTJE0nbp0iVcunQJ/fv3R2lpKe7fvy9+ubm5wcTEBD/88AMAIDc3F2fPnoWPjw9atWol9mFgYIDx48dXa6x79+4FAMybN0+lPklHRwc6OjrV6sPU1BS+vr6S8RYWFsLHxwc3b95ERkYGACA1NRUAMHHiREkfvr6+knOoDRMTE/HPRUVFyMvLg66uLpydnfHLL7+o3eftt9+WvPbz80OrVq3w/fff1+r8iEj7cImN6CUJDg5Gv379oKOjA2NjY7Rs2RIWFhYqcS1atFC5ouzKlSsAKpKsp92vKCcnBwDEWpnWrVurxLRt27ZaY83MzISOjg7at29frXh1rly5gqKiIvTo0eOpMbm5uWjVqhWysrKgq6uLli1bqsS0adMG165dq/U4rl+/jrVr1+LYsWMoLCyUbFOX6Jmbm0uW0Z4cR2pqKh4+fAgTE5ManR8RaR8mSEQvib29/TN/mVYyNjZ+6rYJEybAy8tL7TZzc/Naj02d6s4UPY0gCLCyssLq1aufGtOuXbta918dRUVFGDNmDIqLizFu3DjIZDKYmppCV1cXGzdufK5i6rpwfkT04jBBItIC9vb2AABdXd2/TLIqi5qvXr2qsu3y5cvVOl7Lli1x5MgRXLx4EU5OTk+Ne1YCZW9vj4yMDDg7O8PU1PSZx2vRogWUSiUyMjJUkorK2bPaOH78OO7du4fly5dj6NChkm2RkZFq9yksLER2drbKLNKVK1fwj3/8Q1yyq8n5EZH2YQ0SkRbo0KEDZDIZdu7cqfZyc4VCgfz8fAAVV5F16dIFhw4dkixNlZaWYsuWLdU6XlBQEABgzZo1KC0tVdkuCAKAx/U9BQUFKjGDBg2CUqnEmjVr1B6jckkQgHi1V2xsrCQmNTX1uZbXKpcqK8db6dixYzh//vxT9/vyyy8lrw8ePIhr167B19dXbKvJ+RGR9uEMEpEW0NHRwapVqzBu3DgMGDAAQ4cORdu2bfHo0SNkZmbi4MGDmDt3rljcHRYWhtDQUIwaNQpjxowRL/MvLy+v1vGcnJwwefJkxMTEYMiQIQgICICNjQ1u3LiBlJQUxMfHw9zcHG3btoWpqSni4uJgZGQEc3NzWFlZwcPDA/369cOQIUOwfft2/Pbbb/D29oalpSXu3LmDc+fOITMzUyx69vLygre3NxITE5Gfnw8vLy9kZWVh165dkMlk+P3332v1vrm5ucHGxgYrV67EzZs38dprryE9PR179ux5ar+WlpY4ePAg7t27h27duomX+VtbW2P69OliXE3Oj4i0DxMkIi3h6OiIxMREbNy4EYcOHcLOnTthamqK5s2bY/DgwfDw8BBjXVxcsHnzZqxevRpffvklGjZsCH9/f4waNUqcHfor8+fPR/v27bF9+3Zs2rQJgiDgtddeQ69evWBkZAQAMDIywtq1axEZGYnly5ejtLQU3bp1E8eyYsUKdO/eHd988w02btyIsrIy2NjYoEOHDpg3b57keJGRkYiMjERSUhJ+/PFHyGQyREVFYd++fbVOkMzNzbFp0yZ8+umn2L59OxQKBTp16oSYmBh8++23avs1MTHBV199heXLl2P16tUQBAFeXl4ICwtD48aNJbE1OT8i0i46QtW5ZyIiIqK/OdYgEREREVXBBImIiIioCiZIRERERFUwQSIiIiKqggkSERERURVMkIiIiIiqYIJEREREVAUTJCIiIqIqmCARERERVcEEiYiIiKiK/wdDodMh/PcyAgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Jh0ZGdNSiZ_"
      },
      "source": [
        "### 5.6.2 - Load Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hENlXi1hS8yK"
      },
      "source": [
        "And if we want to load a trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zV3ngDeyS321"
      },
      "source": [
        "'''model_dir = \"/content/drive/MyDrive/Degree/TFG/Models/BertForSentimentAnalysis\" #@param [\"sentiment_analysis\", \"/content/drive/MyDrive/Degree/TFG/Models/BertForSentimentAnalysis\"]\n",
        "model_dir = Path(model_dir)\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(model_dir)\n",
        "bert_classifier = BertClassifier()\n",
        "bert_classifier.load_state_dict(torch.load(model_dir/'pytorch_model.pth'))\n",
        "bert_classifier.to(device)'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HNmIMT68s2az"
      },
      "source": [
        "### 5.6.3 - Final Application for Sentiment Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6HysJTFYCToL"
      },
      "source": [
        "Finally, it is defined a pipeline to process and classify custom sentences."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4nAtUId96-N1"
      },
      "source": [
        "def bert_classification(model, sequence):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Load batch to device\n",
        "    b_input_ids, b_attn_mask = (sequence['input_ids'].to(device), \n",
        "                                sequence['attention_mask'].to(device))\n",
        "\n",
        "    # Compute logits\n",
        "    with torch.no_grad():\n",
        "        logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8byVLP5CCkJA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2821993a-290d-4161-b978-5e43ba969bad"
      },
      "source": [
        "#@title { vertical-output: true }\n",
        "\n",
        "#@markdown Insert here a sentence in English\n",
        "sentence = \"I love BERT!\" #@param {type:\"string\"}\n",
        "\n",
        "# Preprocess sentence\n",
        "encoded_sentence = dict()\n",
        "encoded_sentence['input_ids'], encoded_sentence['attention_mask'] = preprocessing_for_sa(pd.Series(sentence), tokenizer, len(tokenizer.encode(sentence)))\n",
        "\n",
        "# Compute probabilities\n",
        "final_probs = bert_classification(bert_classifier, encoded_sentence)\n",
        "\n",
        "# Print results\n",
        "print(f\"{'Sentiment':^11} | {'Probability':^13}\")\n",
        "print(27*'-')\n",
        "print(f\"{'Positive':^11} | {final_probs[0][1]:^13.5f}\")\n",
        "print(27*'-')\n",
        "print(f\"{'Negative':^11} | {final_probs[0][0]:^13.5f}\")\n",
        "print(27*'-')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Sentiment  |  Probability \n",
            "---------------------------\n",
            " Positive   |    0.99954   \n",
            "---------------------------\n",
            " Negative   |    0.00046   \n",
            "---------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oz6PyX0Enxw-"
      },
      "source": [
        "Finally, to download the whole folder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJkPGEDmn86P"
      },
      "source": [
        "# download_folder(sentiment_dir)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0zJIwTuBQKmU"
      },
      "source": [
        "# 6 - Question Answering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3_vCUke1zRjx"
      },
      "source": [
        "A more challenged task is **Question Answering**, which corresponds to extract the answer to a question from a given context. In this case the model answers the question by taking a substring of the context, not by generating new text.\n",
        "\n",
        "It is used [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) (Stanford Question Answering Dataset), a collection of 100K crowdsourced question/answer pairs. Given a question and a passage from Wikipedia containing the answer, the task is to predict the answer text span in the passage. This corresponds with SQuAD 1.1. In SQuAD 2.0 the problem definition is extended allowing for the possibility\n",
        "that no short answer exists in the provided paragraph, making the problem more realistic.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ReMEq5Ja3w9t"
      },
      "source": [
        "![Q&A](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/question_answering.png)\n",
        "\n",
        "> Image from [Hugging Face models](https://huggingface.co/models)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "egdBStwB4KQl"
      },
      "source": [
        "> For this task it is followed Hugging Face notebook's [question_answering](https://github.com/huggingface/notebooks/blob/master/examples/question_answering.ipynb). "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AICoRIG-pitu"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "I recommend to perform a *Factory reset runtime* after *Sentiment Analysis*, since sometimes there are problems in the part of evaluation due to cache's use of Hugging Face's functions. Running **Modules** and **Recurrent Functions** sections at the beginning is enough to load the required packages and functions.\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uHhkbyihMiix"
      },
      "source": [
        "## 6.1 - Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M7m050pfMpz6"
      },
      "source": [
        "As before, the option to change the runtime type to a GPU is a good option."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6pGcZbjzMyvX",
        "outputId": "8fb2bd83-6da3-4e02-cfb7-e64771e86a17"
      },
      "source": [
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2Oz5y8fM0ka"
      },
      "source": [
        "## 6.2 - Load SQuAD Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6hEJMkm3NNj_"
      },
      "source": [
        "We are going to use [Hugging Face Datasets](https://huggingface.co/docs/datasets/v1.4.1/index.html), so running next cell we install the libraries in Colab."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YenkcQigPzwK"
      },
      "source": [
        "With method [`load_dataset`](https://huggingface.co/docs/datasets/v1.4.1/package_reference/loading_methods.html#datasets.load_dataset) and specifying the name of the datasets we can load the required data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzRXM3NmXEAk"
      },
      "source": [
        "# This flag is the difference between SQUAD v1 or 2 \n",
        "# if you're using another dataset, it indicates if impossible answers are allowed or not).\n",
        "squad_v2 = False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "956e735c28474c379038bb250b0c268c",
            "a4ae947f220c4a79ba254915c4cf8ca4",
            "25ebe9caa03b471ea20f875965822373",
            "f8d533e04e2b4e8e94f8485a0a75e1fc",
            "98ad30ef82e94089a3f970380fece0a4",
            "9c209f1e20d34b5d983521d45e03237b",
            "36dd3cf4b0994a788e6978f847d85485",
            "cf2ed7ee41ab45e58cdc002a2e579f2d",
            "20e7c233366f4960af358647fc55fb2e",
            "a3b4aa5ac25444979dff08b340bcd477",
            "9af7e55b7f7f43f2a62e13536a9ddfa0",
            "51f139737b1b42aa9cf6bc7246cc757b",
            "7b0605b4683a4efda8a3a7b1b87e522b",
            "732f79f9ba6a4a899de7d7fd3c5cecd3",
            "c36f5698ca3c4bfb8ddee6185652b325",
            "b8fc4f18f2d241b5be0ea370f0f5cd89",
            "e00915c03da84b4db4817a41b5a70c1f",
            "11d7298002174fc9bf5868616db85ac3",
            "85073e02610c4af49fb9cacbed1974ec",
            "c463f96b9ada4620b1d4e74ea5796f01",
            "c47706f18edf471fa1e3333814098b85",
            "f2c7d32992294b87b652a1d5e6b2ce67",
            "f933a71a37824c0c94014cb630b33afd",
            "32dd4f881c78447aa66a64ccf3e9d8b3",
            "1e7b7253e8ae4f1896aacb4b025eff62",
            "cc9bf90f4b3b4d2da6d2c62e326421a3",
            "df6111ffb7f04d5ba91d755cd415f831",
            "713d8c1749c24169a42e74ce27319297",
            "909028f5595344609707fda09138ddc3",
            "b8c0d26c9aea4ac1bbe9f5588baaf97d",
            "23e2e22aa628434188a9395cf2bac22d",
            "6d19c3bb57814ae8887a374d450b50b1",
            "3536460505d743da970e10052fdb0374",
            "1d8f8d53a45d4347a3063efe89d34640",
            "23c8d9e8a9c94ae98bdc081a14b049db",
            "57545c835a6a4384a95c62290a6d1612",
            "e218ec4e93994b12976b772be77a2eaf",
            "7b73f6f50c714a6a9c7bacc478a85595",
            "d92a82801e454c07a8bd435de51b28ff",
            "80225e01fcd14fc5a3fe2839d5bfd659",
            "4fdd0db7c47242789d29db7d4596450c",
            "02d75e6ee1fc4d4894272d8dc82354a2",
            "eacc281c720f48dcac12e61e6effc5a2",
            "69a9da26012948a2b0bb60907a2d04f2",
            "e222a7989689479c99d5eeeae8553e95",
            "a10cf8317181497fb329ee2c9dd88e15",
            "ea386b7f33624dac885bfb75c59668c7",
            "7411ba313b3348489925f943bcc74785"
          ]
        },
        "id": "ZCsWZXIlOTTK",
        "outputId": "d47c7285-1589-4170-e054-a5d4793f58b2"
      },
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "original_datasets = load_dataset('squad_v2' if squad_v2 else 'squad')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "956e735c28474c379038bb250b0c268c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1947.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20e7c233366f4960af358647fc55fb2e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1021.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading and preparing dataset squad/plain_text (download: 33.51 MiB, generated: 85.63 MiB, post-processed: Unknown size, total: 119.14 MiB) to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/6b6c4172d0119c74515f44ea0b8262efe4897f2ddb6613e5e915840fdc309c16...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e00915c03da84b4db4817a41b5a70c1f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=8116577.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1e7b7253e8ae4f1896aacb4b025eff62",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1054280.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3536460505d743da970e10052fdb0374",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4fdd0db7c47242789d29db7d4596450c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset squad downloaded and prepared to /root/.cache/huggingface/datasets/squad/plain_text/1.0.0/6b6c4172d0119c74515f44ea0b8262efe4897f2ddb6613e5e915840fdc309c16. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cn_QnL6WQpOf"
      },
      "source": [
        "The loaded dataset is an object of [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict). It is a dictionary which contains train and validation samples, with the correspondig `context`, `question` and `answers`. As in the previous task, SQuAD is also used in competitions. In this case, there are not original test data. As before, one third of training set is going to be used as test."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDlay57Q0xL8"
      },
      "source": [
        "new_split = original_datasets['train'].train_test_split(test_size=0.33)\n",
        "original_datasets['train'] = new_split['train']\n",
        "original_datasets['test'] = new_split['test']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfGCw5Gk1o0I"
      },
      "source": [
        "With next function we can show random elements in the dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3j8APAoIrI3"
      },
      "source": [
        "from datasets import ClassLabel, Sequence\n",
        "from IPython.display import display, HTML\n",
        "\n",
        "def show_random_elements(dataset, num_examples):\n",
        "    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n",
        "    picks = []\n",
        "    for _ in range(num_examples):\n",
        "        pick = random.randint(0, len(dataset)-1)\n",
        "        while pick in picks:\n",
        "            pick = random.randint(0, len(dataset)-1)\n",
        "        picks.append(pick)\n",
        "    \n",
        "    df = pd.DataFrame(dataset[picks])\n",
        "    for column, typ in dataset.features.items():\n",
        "        if isinstance(typ, ClassLabel):\n",
        "            df[column] = df[column].transform(lambda i: typ.names[i])\n",
        "        elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):\n",
        "            df[column] = df[column].transform(lambda x: [typ.feature.names[i] for i in x])\n",
        "    display(HTML(df.to_html()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OGqxi4rE3j6e"
      },
      "source": [
        "For a fast intuition on the model, we can remove some data, since the original size of the dataset has more than 80k samples only in training. With `perc` we select the percentage of data we are going to use. Removed sample are selected randomly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 431
        },
        "id": "s4Ww8dtnQsSu",
        "outputId": "9c9a1d8d-c780-40de-bacd-26ef3940c19a"
      },
      "source": [
        "#@markdown Select a split\n",
        "split = \"train\" #@param [\"train\", \"validation\", \"test\"]\n",
        "#@markdown Select number of examples to visualize it\n",
        "num_examples = 3 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "print(f'Some sentences of {split} split, which has a length of {original_datasets[split].num_rows}\\n')\n",
        "show_random_elements(original_datasets[split], num_examples)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some sentences of train split, which has a length of 58691\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>answers</th>\n",
              "      <th>context</th>\n",
              "      <th>id</th>\n",
              "      <th>question</th>\n",
              "      <th>title</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>{'text': ['Hauts-de-Seine, Seine-Saint-Denis and Val-de-Marne'], 'answer_start': [318]}</td>\n",
              "      <td>The Métropole du Grand Paris, or Metropolis of Greater Paris, formally came into existence on January 1, 2016. It is an administrative structure for cooperation between the City of Paris and its nearest suburbs. It includes the City of Paris, plus the communes, or towns of the three departments of the inner suburbs; Hauts-de-Seine, Seine-Saint-Denis and Val-de-Marne; plus seven communes in the outer suburbs, including Argenteuil in Val d'Oise and Paray-Vieille-Poste in Essonne, which were added to include the major airports of Paris. The Metropole covers 814 square kilometers and has a population of 6.945 million persons.</td>\n",
              "      <td>5728ccfb4b864d1900164e61</td>\n",
              "      <td>What three departments of the inner suburbs are included in the metropole?</td>\n",
              "      <td>Paris</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>{'text': ['Government'], 'answer_start': [206]}</td>\n",
              "      <td>Connaught Place, one of North India's largest commercial and financial centres, is located in the northern part of New Delhi. Adjoining areas such as Barakhamba Road, ITO are also major commercial centres. Government and quasi government sector was the primary employer in New Delhi. The city's service sector has expanded due in part to the large skilled English-speaking workforce that has attracted many multinational companies. Key service industries include information technology, telecommunications, hotels, banking, media and tourism.</td>\n",
              "      <td>5706af7575f01819005e7d37</td>\n",
              "      <td>Prior to the expansion of the service sector, what sector was the largest employer in New Delhi?</td>\n",
              "      <td>New_Delhi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>{'text': ['age and accumulated experiences'], 'answer_start': [739]}</td>\n",
              "      <td>The inclusiveness of Weinreich's definition (above) directs attention to the totality of one's identity at a given phase in time, and assists in elucidating component aspects of one's total identity, such as one's gender identity, ethnic identity, occupational identity and so on. The definition readily applies to the young child, to the adolescent, to the young adult, and to the older adult in various phases of the life cycle. Depending on whether one is a young child or an adult at the height of one's powers, how one construes oneself as one was in the past will refer to very different salient experiential markers. Likewise, how one construes oneself as one aspires to be in the future will differ considerably according to one's age and accumulated experiences. (Weinreich &amp; Saunderson, (eds) 2003, pp 26–34).</td>\n",
              "      <td>570961b2ed30961900e84037</td>\n",
              "      <td>How one construes oneself now and in the future differs considerably because of what 2 things?</td>\n",
              "      <td>Identity_(social_science)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBP-0Z5w5R3t"
      },
      "source": [
        "As before, there is the option to remove some data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNZtBVZxA-SQ"
      },
      "source": [
        "from datasets import DatasetDict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dj7Kp7aAaBTO",
        "outputId": "d5f5e223-4753-49aa-ae68-e1a0915639c5"
      },
      "source": [
        "#@title 6.2.1 Resize Dataset { vertical-output: true }\n",
        "#@markdown Percentage of dataset to use %\n",
        "perc = 100 #@param {type:\"slider\", min:1, max:100, step:1}\n",
        "datasets = DatasetDict()\n",
        "\n",
        "for key, dataset in original_datasets.shuffle().items():\n",
        "    if perc != 100:\n",
        "        datasets[key] = dataset.select(np.arange(0, round(dataset.num_rows*(perc/100)+1)))\n",
        "        print(f'{key:<10}\\t', f'From: {original_datasets[key].num_rows:<10}', f'To: {datasets[key].num_rows:<10}')\n",
        "    else:\n",
        "        datasets[key] = original_datasets[key]\n",
        "        print(f'{key:<10}\\t', f'size: {len(original_datasets[key]):<10}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train     \t size: 58691     \n",
            "validation\t size: 10570     \n",
            "test      \t size: 28908     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R-LZ1teNUeFY"
      },
      "source": [
        "## 6.3 - Preprocessing SQuAD Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c8q68IuPU1_l"
      },
      "source": [
        "In this task the structure of the input differs from sentiment analysis. Now there are two sentences:\n",
        "\n",
        "\\begin{matrix}\n",
        "    \\text{[CLS]} & \\text{Sentence A} & \\text{[SEP]} & \\text{Sentence B}\n",
        "\\end{matrix}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZInR1JVXVn4V"
      },
      "source": [
        "It is employed [`BertTokenizerFast`](https://huggingface.co/transformers/model_doc/bert.html#transformers.BertTokenizerFast). This one is a [Fast Tokenizer](https://huggingface.co/transformers/main_classes/tokenizer.html#tokenizer), which according to Hugging Face implies \n",
        "\n",
        "- A significant speed-up in particular when doing batched tokenization.\n",
        "- Additional methods to map between the original string (character and words) and the token space.\n",
        "\n",
        "As before,[`bert-base-uncased`](https://huggingface.co/bert-base-uncased) is used as pre-trained model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vI6VYeQWRs0S"
      },
      "source": [
        "from transformers import BertTokenizerFast"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XBnVEhkRsBS"
      },
      "source": [
        "MODEL_NAME = 'bert-base-uncased'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PG9XpSxvRuWJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b65f52ac-3e08-49ef-9a44-5ee9ddae6f40"
      },
      "source": [
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...\\n')\n",
        "tokenizer = BertTokenizerFast.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ul-6neG4ki-Q"
      },
      "source": [
        "As before, we can check its attributes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "phbxf9K_kvXW",
        "outputId": "da3683da-473d-4c73-f721-b4d7f8531695"
      },
      "source": [
        "#@markdown Select an option to visualize it\n",
        "attribute = \"vocab_size\" #@param [\"vocab_size\", \"model_max_length\", \"padding_side\", \"sep_token\", \"pad_token\", \"cls_token\", \"mask_token\", \"unk_token\"]\n",
        "\n",
        "print(getattr(tokenizer, attribute))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30522\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hD1KrbgIAknD"
      },
      "source": [
        "One problem is how to manage long documents. The \n",
        "usual truncation process could carry in losing the answer we are looking for. To deal with it, it is truncated the context into features, text shorter than maximum length allowed, and allowed some overlap between them in case the answer lies at the point it is splitted a long context\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H4lwrtz5C3QR"
      },
      "source": [
        "MAX_LENGTH = 384 # The maximum length of a feature (question and context)\n",
        "DOC_STRIDE = 128 # The authorized overlap between two parts of the context when splitting it is needed."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UohhB81oTEIh"
      },
      "source": [
        "Let's find an example whose length is longer than `MAX_LENGTH`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wgCCZ_EGC354",
        "outputId": "54919344-3802-44c3-a255-4631626999ee"
      },
      "source": [
        "for i, example in enumerate(datasets[\"train\"]):\n",
        "    if len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"]) > MAX_LENGTH:\n",
        "        break\n",
        "example = datasets[\"train\"][i]\n",
        "example"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answers': {'answer_start': [56], 'text': ['1998 Winter Olympics']},\n",
              " 'context': \"Japan: The event was held in Nagano, which hosted the 1998 Winter Olympics, on April 26. Japanese Buddhist temple Zenkō-ji, which was originally scheduled to be the starting point for the Olympic torch relay in Nagano, refused to host the torch and pulled out of the relay plans, amid speculation that monks there sympathized with anti-Chinese government protesters. as well as the risk of disruption by violent protests. Parts of Zenkō-ji temple's main building (Zenkō-ji Hondō), reconstructed in 1707 and one of the National Treasures of Japan, was then vandalized with spraypaint. A new starting point, previously the site of a municipal building and now a parking lot, was chosen by the city. An event the city had planned to hold at the Minami Nagano Sports Park following the torch relay was also canceled out of concern about disruptions caused by demonstrators protesting against China's recent crackdown in Tibet. Thousands of riot police were mobilized to protect the torch along its route. The show of force kept most protesters in check, but slogans shouted by pro-China or pro-Tibet demonstrators, Japanese nationalists, and human rights organizations flooded the air. Five men were arrested and four injured amidst scenes of mob violence. The torch route was packed with mostly peaceful demonstrators. The public was not allowed at the parking lot where the relay started. After the Zenkoji monks held a prayer ceremony for victims of the recent events in Tibet. More than 100 police officers ran with the torch and riot police lined the streets while three helicopters flew above. Only two Chinese guards were allowed to accompany the torch because of Japan's concern over their treatment of demonstrators at previous relays. A man with a Tibetan flag tried to stop the torch at the beginning of the relay but was dragged off by police. Some raw eggs were also thrown from the crowd.\",\n",
              " 'id': '56db6d76e7c41114004b50e2',\n",
              " 'question': 'Which Olympics did Nagano host?',\n",
              " 'title': '2008_Summer_Olympics_torch_relay'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aKprbGPFgPF6",
        "outputId": "447914c2-c037-4e10-f5a7-9a35b076a2ac"
      },
      "source": [
        "print(f'Original length \\t{len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"])}')\n",
        "print(f'Truncated length \\t{len(tokenizer(example[\"question\"], example[\"context\"], max_length=MAX_LENGTH, truncation=\"only_second\")[\"input_ids\"])}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original length \t392\n",
            "Truncated length \t384\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2yGs3fWPVsUO"
      },
      "source": [
        "Only the context should be truncated, specifying `truncation=only_second`. The tokenizer will return a lisf of features capped by a certain maximum length, with the overlap we talked above. We just have to declare `return_overflowing_tokens=True` and `stride=doc_stride` to pass the stride.\n",
        "\n",
        "Mention we need to find in which of those features the answer actually is, and where exactly in that feature, the start and end positions. The tokenizer we're using can help us with that by returning an `offset_mapping`:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0_Y6IdUoC8_W"
      },
      "source": [
        "tokenized_example = tokenizer(\n",
        "    example[\"question\"],\n",
        "    example[\"context\"],\n",
        "    max_length=MAX_LENGTH,\n",
        "    truncation=\"only_second\",\n",
        "    return_overflowing_tokens=True,\n",
        "    return_offsets_mapping=True,\n",
        "    stride=DOC_STRIDE\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rmrd83ymaf_s"
      },
      "source": [
        "We can plot each of the features in order to see the overlapping part:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y7Jy6J3XadOn",
        "outputId": "c83bff7e-e358-462e-ed7e-92746f16df55"
      },
      "source": [
        "for i, x in enumerate(tokenized_example[\"input_ids\"]):\n",
        "    print(f'Feature {i+1} with {len(x)} tokens')\n",
        "    print(tokenizer.convert_ids_to_tokens(x), '\\n') # or tokenizer.decode(x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Feature 1 with 384 tokens\n",
            "['[CLS]', 'which', 'olympics', 'did', 'naga', '##no', 'host', '?', '[SEP]', 'japan', ':', 'the', 'event', 'was', 'held', 'in', 'naga', '##no', ',', 'which', 'hosted', 'the', '1998', 'winter', 'olympics', ',', 'on', 'april', '26', '.', 'japanese', 'buddhist', 'temple', 'zen', '##ko', '-', 'ji', ',', 'which', 'was', 'originally', 'scheduled', 'to', 'be', 'the', 'starting', 'point', 'for', 'the', 'olympic', 'torch', 'relay', 'in', 'naga', '##no', ',', 'refused', 'to', 'host', 'the', 'torch', 'and', 'pulled', 'out', 'of', 'the', 'relay', 'plans', ',', 'amid', 'speculation', 'that', 'monks', 'there', 'sy', '##mp', '##athi', '##zed', 'with', 'anti', '-', 'chinese', 'government', 'protesters', '.', 'as', 'well', 'as', 'the', 'risk', 'of', 'disruption', 'by', 'violent', 'protests', '.', 'parts', 'of', 'zen', '##ko', '-', 'ji', 'temple', \"'\", 's', 'main', 'building', '(', 'zen', '##ko', '-', 'ji', 'hon', '##do', ')', ',', 'reconstructed', 'in', '1707', 'and', 'one', 'of', 'the', 'national', 'treasures', 'of', 'japan', ',', 'was', 'then', 'van', '##dal', '##ized', 'with', 'spray', '##pa', '##int', '.', 'a', 'new', 'starting', 'point', ',', 'previously', 'the', 'site', 'of', 'a', 'municipal', 'building', 'and', 'now', 'a', 'parking', 'lot', ',', 'was', 'chosen', 'by', 'the', 'city', '.', 'an', 'event', 'the', 'city', 'had', 'planned', 'to', 'hold', 'at', 'the', 'mina', '##mi', 'naga', '##no', 'sports', 'park', 'following', 'the', 'torch', 'relay', 'was', 'also', 'canceled', 'out', 'of', 'concern', 'about', 'disruption', '##s', 'caused', 'by', 'demonstrators', 'protesting', 'against', 'china', \"'\", 's', 'recent', 'crack', '##down', 'in', 'tibet', '.', 'thousands', 'of', 'riot', 'police', 'were', 'mobilized', 'to', 'protect', 'the', 'torch', 'along', 'its', 'route', '.', 'the', 'show', 'of', 'force', 'kept', 'most', 'protesters', 'in', 'check', ',', 'but', 'slogan', '##s', 'shouted', 'by', 'pro', '-', 'china', 'or', 'pro', '-', 'tibet', 'demonstrators', ',', 'japanese', 'nationalists', ',', 'and', 'human', 'rights', 'organizations', 'flooded', 'the', 'air', '.', 'five', 'men', 'were', 'arrested', 'and', 'four', 'injured', 'amidst', 'scenes', 'of', 'mob', 'violence', '.', 'the', 'torch', 'route', 'was', 'packed', 'with', 'mostly', 'peaceful', 'demonstrators', '.', 'the', 'public', 'was', 'not', 'allowed', 'at', 'the', 'parking', 'lot', 'where', 'the', 'relay', 'started', '.', 'after', 'the', 'zen', '##ko', '##ji', 'monks', 'held', 'a', 'prayer', 'ceremony', 'for', 'victims', 'of', 'the', 'recent', 'events', 'in', 'tibet', '.', 'more', 'than', '100', 'police', 'officers', 'ran', 'with', 'the', 'torch', 'and', 'riot', 'police', 'lined', 'the', 'streets', 'while', 'three', 'helicopters', 'flew', 'above', '.', 'only', 'two', 'chinese', 'guards', 'were', 'allowed', 'to', 'accompany', 'the', 'torch', 'because', 'of', 'japan', \"'\", 's', 'concern', 'over', 'their', 'treatment', 'of', 'demonstrators', 'at', 'previous', 'relay', '##s', '.', 'a', 'man', 'with', 'a', 'tibetan', 'flag', 'tried', 'to', 'stop', 'the', 'torch', 'at', 'the', 'beginning', 'of', 'the', 'relay', 'but', 'was', 'dragged', 'off', 'by', 'police', '.', 'some', 'raw', '[SEP]'] \n",
            "\n",
            "Feature 2 with 146 tokens\n",
            "['[CLS]', 'which', 'olympics', 'did', 'naga', '##no', 'host', '?', '[SEP]', 'men', 'were', 'arrested', 'and', 'four', 'injured', 'amidst', 'scenes', 'of', 'mob', 'violence', '.', 'the', 'torch', 'route', 'was', 'packed', 'with', 'mostly', 'peaceful', 'demonstrators', '.', 'the', 'public', 'was', 'not', 'allowed', 'at', 'the', 'parking', 'lot', 'where', 'the', 'relay', 'started', '.', 'after', 'the', 'zen', '##ko', '##ji', 'monks', 'held', 'a', 'prayer', 'ceremony', 'for', 'victims', 'of', 'the', 'recent', 'events', 'in', 'tibet', '.', 'more', 'than', '100', 'police', 'officers', 'ran', 'with', 'the', 'torch', 'and', 'riot', 'police', 'lined', 'the', 'streets', 'while', 'three', 'helicopters', 'flew', 'above', '.', 'only', 'two', 'chinese', 'guards', 'were', 'allowed', 'to', 'accompany', 'the', 'torch', 'because', 'of', 'japan', \"'\", 's', 'concern', 'over', 'their', 'treatment', 'of', 'demonstrators', 'at', 'previous', 'relay', '##s', '.', 'a', 'man', 'with', 'a', 'tibetan', 'flag', 'tried', 'to', 'stop', 'the', 'torch', 'at', 'the', 'beginning', 'of', 'the', 'relay', 'but', 'was', 'dragged', 'off', 'by', 'police', '.', 'some', 'raw', 'eggs', 'were', 'also', 'thrown', 'from', 'the', 'crowd', '.', '[SEP]'] \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nUvWR31SbRIH"
      },
      "source": [
        "And how the tokenizer can provide us the positions of each token:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2IDygx3Cb4Vb",
        "outputId": "8a8fb8d3-1a9a-421a-cd8c-d71b5fc372b0"
      },
      "source": [
        "rnd_index = random.randint(0, len(tokenized_example[\"input_ids\"][0])-1)\n",
        "token_id = tokenized_example[\"input_ids\"][0][rnd_index]\n",
        "offsets = tokenized_example[\"offset_mapping\"][0][rnd_index]\n",
        "\n",
        "print(f'Token {rnd_index} \\t{tokenizer.convert_ids_to_tokens(token_id)}\\n')\n",
        "\n",
        "print(f'Start position \\t{offsets[0]}')\n",
        "print(f'End position \\t{offsets[1]}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token 379 \tpolice\n",
            "\n",
            "Start position \t1844\n",
            "End position \t1850\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LyKmuAv3wz79"
      },
      "source": [
        "To distinguish which parts of the offsets correspond to the question and which part correspond to the context, we can use [`sequence_ids`](https://huggingface.co/transformers/main_classes/tokenizer.html#transformers.BatchEncoding.sequence_ids) method. It returns `None` for the special tokens, then 0 or 1 depending on whether the corresponding token comes from the first sentence past (the question) or the second (the context)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7dLSn6Z4ej8N",
        "outputId": "8a910281-4a76-41b5-fdb8-b985cdf0772c"
      },
      "source": [
        "sequence_ids = tokenized_example.sequence_ids()\n",
        "print(sequence_ids)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[None, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-FZScaFfq92"
      },
      "source": [
        "With all of this, the last step is to check if the answer is in the feature or in the next one."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n_T4P94NfqPa",
        "outputId": "342dcc5d-7850-4d8e-8987-bb29ca723a0c"
      },
      "source": [
        "answers = example[\"answers\"]\n",
        "start_char = answers[\"answer_start\"][0]\n",
        "end_char = start_char + len(answers[\"text\"][0])\n",
        "\n",
        "# Start token index of the current span in the text.\n",
        "token_start_index = 0\n",
        "while sequence_ids[token_start_index] != 1:\n",
        "    token_start_index += 1\n",
        "\n",
        "# End token index of the current span in the text.\n",
        "token_end_index = len(tokenized_example[\"input_ids\"][0]) - 1\n",
        "while sequence_ids[token_end_index] != 1:\n",
        "    token_end_index -= 1\n",
        "\n",
        "# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "offsets = tokenized_example[\"offset_mapping\"][0]\n",
        "if (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n",
        "    # Move the token_start_index and token_end_index to the two ends of the answer.\n",
        "    # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n",
        "        token_start_index += 1\n",
        "    start_position = token_start_index - 1\n",
        "    while offsets[token_end_index][1] >= end_char:\n",
        "        token_end_index -= 1\n",
        "    end_position = token_end_index + 1\n",
        "    print(f'Original answer \\t{answers[\"text\"][0]}', '\\n')\n",
        "    print(f'Start position \\t\\t{start_position}')\n",
        "    print(f'End position \\t\\t{end_position}')\n",
        "    print(f'In feature \\t\\t{tokenizer.decode(tokenized_example[\"input_ids\"][0][start_position: end_position+1])}')\n",
        "else:\n",
        "    print(\"The answer is not in this feature.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original answer \t1998 Winter Olympics \n",
            "\n",
            "Start position \t\t22\n",
            "End position \t\t26\n",
            "In feature \t\t1998 winter olympics, on\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pt4b2eCbrz-Y"
      },
      "source": [
        "Once all steps are explained, we can integrate them in a whole function to preprocess whole data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQz2mF7fwz8A"
      },
      "source": [
        "def preprocessing_for_qa(examples):\n",
        "    # Tokenize our examples with truncation and padding, but keep the overflows using a stride. This results\n",
        "    # in one example possible giving several features when a context is long, each of those features having a\n",
        "    # context that overlaps a bit the context of the previous feature.\n",
        "    tokenized_examples = tokenizer(\n",
        "        examples[\"question\"],               #tokenize question\n",
        "        examples[\"context\"],                #tokenize context\n",
        "        truncation=\"only_second\",           #truncate only context if necessary\n",
        "        max_length=MAX_LENGTH,              #variable max_length\n",
        "        stride=DOC_STRIDE,                  #overlap to stride\n",
        "        return_overflowing_tokens=True,     #return overflowing tokens\n",
        "        return_offsets_mapping=True,        #return offsets mapping\n",
        "        padding=\"max_length\"                #pad to max length\n",
        "    )\n",
        "\n",
        "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "    # its corresponding example. This key gives us just that.\n",
        "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
        "\n",
        "    # The offset mappings will give us a map from token to character position in the original context. This will\n",
        "    # help us compute the start_positions and end_positions.\n",
        "    offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n",
        "\n",
        "    # Let's label those examples!\n",
        "    tokenized_examples[\"start_positions\"] = []\n",
        "    tokenized_examples[\"end_positions\"] = []\n",
        "\n",
        "    for i, offsets in enumerate(offset_mapping):\n",
        "        # We will label impossible answers with the index of the CLS token.\n",
        "        input_ids = tokenized_examples[\"input_ids\"][i]\n",
        "        cls_index = input_ids.index(tokenizer.cls_token_id)\n",
        "\n",
        "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
        "\n",
        "        # One example can give several spans, this is the index of the example containing this span of text.\n",
        "        sample_index = sample_mapping[i]\n",
        "        answers = examples[\"answers\"][sample_index]\n",
        "\n",
        "        # If no answers are given, set the cls_index as answer.\n",
        "        if len(answers[\"answer_start\"]) == 0:\n",
        "            tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "            tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "        else:\n",
        "            # Start/end character index of the answer in the text.\n",
        "            start_char = answers[\"answer_start\"][0]\n",
        "            end_char = start_char + len(answers[\"text\"][0])\n",
        "\n",
        "            # Start token index of the current span in the text.\n",
        "            token_start_index = 0\n",
        "            while sequence_ids[token_start_index] != 1:\n",
        "                token_start_index += 1\n",
        "\n",
        "            # End token index of the current span in the text.\n",
        "            token_end_index = len(input_ids) - 1\n",
        "            while sequence_ids[token_end_index] != 1:\n",
        "                token_end_index -= 1\n",
        "\n",
        "            # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "            if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n",
        "                tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "                tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "            else:\n",
        "                # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n",
        "                # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "                while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n",
        "                    token_start_index += 1\n",
        "                tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n",
        "                while offsets[token_end_index][1] >= end_char:\n",
        "                    token_end_index -= 1\n",
        "                tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n",
        "\n",
        "    return tokenized_examples"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hd0323JyKkoZ"
      },
      "source": [
        "Now, we only have to apply the function to the different splits. We can use the [`map`](https://huggingface.co/docs/datasets/v1.4.1/package_reference/main_classes.html?highlight=map#datasets.Dataset.map) method:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "15e62ce495804af88f80f2f9e7f032c4",
            "c0bd8ac8c42845789aea8f58ddace4d5",
            "3075a65030f244ca8c02cb26666cf332",
            "32a6faab02e94528b9bf64d40530e707",
            "1d879a06d7a44ac0a2bd16e65637a873",
            "fd32131fc363401a9b2731ed4b22cb91",
            "d04886221d1f4e7fa213c6429cf60d63",
            "35b4fb0fce2445118b82867891414ee5",
            "bf27d5db4f5c41b5af5d07b167a7f8cf",
            "e0c87ba45fa84041919c77cd205a4e59",
            "f906513a8bad4a7d9f0bf6617312eae7",
            "42c70a3119fa49d7a331c8d6b90e3070",
            "7370f6dc45d24fcaa95607b5b93f136e",
            "1cfecd15308b4fdab7720d1ef5c09286",
            "0aa8493b90d845b0bdcf010d4f5bfb24",
            "5b06933e172549dd865b18ba2642bbd4",
            "1421fbd7041149d3a2f1ff9e9f2fc4aa",
            "5937d42f58be45beb3aebd5edbbef105",
            "ecc48df63fce411081ffcebdcc8bb065",
            "4c64c064a688465d852fbc2a57ef087c",
            "40c630a6187c48cf9cd7011b4574a585",
            "176318f0f80541acb0631eceedef9ef4",
            "a311b116567a4d44a840cf97d9516838",
            "4e8155fd6c4e4156928eb915f7cbb190"
          ]
        },
        "id": "fUC0DNfzNJIW",
        "outputId": "00e97ce3-5a42-4df4-c0b1-6224d3937381"
      },
      "source": [
        "tokenized_datasets = datasets.map(preprocessing_for_qa, \n",
        "                                  batched=True,\n",
        "                                  load_from_cache_file=False,\n",
        "                                  remove_columns=datasets[\"train\"].column_names)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "15e62ce495804af88f80f2f9e7f032c4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=59.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bf27d5db4f5c41b5af5d07b167a7f8cf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1421fbd7041149d3a2f1ff9e9f2fc4aa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=29.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "voWiw8C7IrJV"
      },
      "source": [
        "The results are automatically cached by the [Datasets](https://huggingface.co/docs/datasets/v1.4.1/index.html) library to avoid spending time on this step the next time you run your notebook. With `load_from_cache_file=False` in the call to [`map`](https://huggingface.co/docs/datasets/v1.4.1/package_reference/main_classes.html?highlight=map#datasets.Dataset.map), the cached files are not used and the preprocessing is applied again.\n",
        "\n",
        "Note that we passed `batched=True` to encode the texts by batches together. This is to leverage the full benefit of the fast tokenizer we loaded earlier, which will use multi-threading to treat the texts in a batch concurrently."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iGeN3Ju3WZzQ"
      },
      "source": [
        "Once the original datasets are tokenizer, we get another [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), but with different columns than the original one: `attention_mask`, used to indicate words from additional padding, `input_ids`, tokens as integers, and `token_type_ids`, indicating to which sentence words belongs. Finally the start and end positions of the answer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 499
        },
        "id": "OauAJlyHWtpQ",
        "outputId": "34ce1d3b-7ca3-4b36-b6bf-89c380e51836"
      },
      "source": [
        "#@markdown Select a split\n",
        "split = \"train\" #@param [\"train\", \"validation\", \"test\"]\n",
        "#@markdown Select number of examples to visualize it\n",
        "num_examples = 3 #@param {type:\"slider\", min:1, max:10, step:1}\n",
        "print(f'Some sentences of {split} split, which has a length of {tokenized_datasets[split].num_rows}\\n')\n",
        "show_random_elements(tokenized_datasets[split], num_examples)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some sentences of train split, which has a length of 59327\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>attention_mask</th>\n",
              "      <th>end_positions</th>\n",
              "      <th>input_ids</th>\n",
              "      <th>start_positions</th>\n",
              "      <th>token_type_ids</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...]</td>\n",
              "      <td>110</td>\n",
              "      <td>[101, 2054, 2095, 2106, 10692, 2707, 2037, 3171, 4125, 2241, 2006, 2844, 14338, 1029, 102, 2138, 1997, 1996, 3795, 3171, 19396, 2008, 2211, 1999, 2289, 1010, 1996, 14230, 1997, 10692, 10548, 2011, 1015, 1012, 1018, 1003, 1999, 1996, 3416, 4284, 1997, 2263, 1010, 2058, 1017, 1003, 1999, 1996, 3822, 4284, 1997, 2263, 1010, 1998, 2058, 1023, 1003, 1999, 1996, 4343, 4284, 1997, 2263, 1012, 1996, 12029, 2231, 2081, 1037, 26215, 4997, 5166, 1010, 2029, 2001, 2979, 2011, 15544, 8004, 12676, 12193, 1012, 1996, 6599, 1997, 1996, 5166, 2001, 10548, 2005, 2263, 2011, 25212, 2243, 1020, 1012, 1015, 4551, 1998, 1996, ...]</td>\n",
              "      <td>110</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...]</td>\n",
              "      <td>42</td>\n",
              "      <td>[101, 1999, 2289, 1010, 2054, 2001, 1996, 2988, 5813, 1997, 1996, 21388, 26797, 2291, 1029, 102, 1999, 2289, 1010, 1996, 2880, 8418, 25311, 6692, 2739, 4034, 2988, 2008, 1996, 5813, 1997, 1996, 21388, 26797, 2291, 2001, 2004, 2152, 2004, 1014, 1012, 1019, 3620, 1012, 2007, 1996, 4493, 5310, 17703, 2009, 3544, 2008, 1996, 10250, 12322, 9250, 10640, 2003, 2322, 2213, 1006, 2531, 2213, 1010, 4895, 9289, 12322, 9250, 1007, 1012, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...]</td>\n",
              "      <td>36</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...]</td>\n",
              "      <td>178</td>\n",
              "      <td>[101, 2054, 12763, 2024, 3378, 2007, 25353, 21850, 6856, 1029, 102, 1999, 5184, 29393, 8458, 8189, 11233, 1010, 1996, 2034, 12553, 3424, 1011, 1999, 25969, 3512, 4319, 1010, 2001, 2764, 2011, 2703, 15501, 12190, 7033, 1998, 15535, 6152, 17064, 12112, 1997, 1996, 2820, 1997, 6388, 7242, 1999, 4068, 1012, 1996, 4319, 2001, 2445, 1996, 3293, 2171, 16183, 10755, 8791, 1012, 15501, 12190, 7033, 1010, 9073, 2119, 1996, 2236, 22423, 1997, 29596, 1998, 1996, 13228, 16326, 1997, 3056, 18554, 2015, 2011, 10327, 1010, 1044, 22571, 14573, 2229, 3550, 2008, 2019, 29596, 1011, 4820, 18554, 2007, 2714, 13228, 16326, 5144, 2071, 2022, ...]</td>\n",
              "      <td>168</td>\n",
              "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tQBsd8ivQ8cX"
      },
      "source": [
        "## 6.4 - Generation of the Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-_mjlVIIncTf"
      },
      "source": [
        "To fine tune on **Question Answering**, it is introduced a start vector $ S \\in \\mathbb R^{H} $ and an end vector $ E \\in \\mathbb R^{H} $."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPARYgkTSI7X"
      },
      "source": [
        "![Fine tune Q&A](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/question_answering_fine_tune.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jb12e17xqRSr"
      },
      "source": [
        "To compute the probability of a word $ i $ being the start or the end of the answer, we compute the dot product between $ S $ and $ T_i $ followed by a softmax over all the words in the sequence.\n",
        "\n",
        "\\begin{equation}\n",
        "    P_i = \\frac{e^{S \\cdot T_i}}{\\sum_{j} e^{S \\cdot T_j}}\n",
        "\\end{equation} \n",
        "\n",
        "The score for a candidate is defined as $ S \\cdot T_i \\ + \\ E \\cdot T_j $, where $ i $ is the start position and $ j $ the final position. The maximum scoring span: $ \\hat{s_{i, j}} = max_{j \\geq i} S \\cdot T_i \\ + \\ E \\cdot T_j $ is used as prediction, where the training objective is the sum of the log-likelihoods of the correct start and end positions."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZwmIElx_DKBV"
      },
      "source": [
        "![Model Q&A](https://raw.githubusercontent.com/ion-bueno/bert-from-inside/main/images/bert-qa.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URbe2r4LuNkG"
      },
      "source": [
        "To extend to SQuAD 2.0, questions which do not have an answer have an span with start and end at the $ [CLS] $ token. It is compared the score of the null answer span: $ s_{null} = S \\cdot C \\ + E \\cdot C $ with $ \\hat{s_{i, j}} $. It is predicted a non-null answer if $ \\hat{s_{i, j}} > s_{null} \\ + \\ \\tau$, where threshold $ \\tau $ i selected on the validation set to maximize F1."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B4iHhaveC1HX"
      },
      "source": [
        "In this task we are going to use the model designed by Hugging Face [`BertForQuestionAnswering`](https://huggingface.co/transformers/model_doc/bert.html#transformers.BertForQuestionAnswering)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eddDHjFJ_6zq",
        "outputId": "1abbf49a-01a2-44b2-881b-7783fc6d0d8f"
      },
      "source": [
        "from transformers import BertForQuestionAnswering\n",
        "\n",
        "bert_qa = BertForQuestionAnswering.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VtD_B4OnAGJP",
        "outputId": "998a6295-ef53-4bb5-cf98-c87a872ffac6"
      },
      "source": [
        "#@markdown Select whole model, only pretrained BERT or the question & answer outputs\n",
        "block = \"model\" #@param [\"model\", \"bert\", \"qa_outputs\"]\n",
        "if block == 'model':\n",
        "    print(bert_qa)\n",
        "else:\n",
        "    print(getattr(model, block))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertForQuestionAnswering(\n",
            "  (bert): BertModel(\n",
            "    (embeddings): BertEmbeddings(\n",
            "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (token_type_embeddings): Embedding(2, 768)\n",
            "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (encoder): BertEncoder(\n",
            "      (layer): ModuleList(\n",
            "        (0): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (1): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (2): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (3): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (4): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (5): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (6): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (7): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (8): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (9): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (10): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (11): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KvBPQ9ytQLkS"
      },
      "source": [
        "## 6.5 - Fine-Tune on Question Answering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CKMFi-Oy3JHL"
      },
      "source": [
        "Now it is the moment to train the model for our specific task. We can use an instance of [`Trainer`](https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainer#transformers.Trainer) class in order to avoid many lines of code. The most important is the [`TrainingArguments`](https://huggingface.co/transformers/main_classes/trainer.html#transformers.TrainingArguments), which is a class that contains all the attributes to customize the training. It requires one folder name, which will be used to save the checkpoints of the model, and all other arguments are optional."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUKFtfhi58BY"
      },
      "source": [
        "from transformers import TrainingArguments"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNvT62_G3SMm"
      },
      "source": [
        "#@title 6.5.1 - Hyperparameters Selection\n",
        "batch_size =  16#@param {type:\"integer\"}\n",
        "learning_rate = 5e-5 #@param {type:\"number\"}\n",
        "epochs =  3#@param {type:\"integer\"}\n",
        "epsilon = 1e-8 #@param {type:\"number\"}\n",
        "weight_decay=0.01 #@param {type:\"number\"}\n",
        "qa_dir = Path('question_answering')\n",
        "strategy = 'epoch'\n",
        "\n",
        "args = TrainingArguments(\n",
        "    qa_dir,\n",
        "    evaluation_strategy = strategy,\n",
        "    learning_rate=learning_rate,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    num_train_epochs=epochs,\n",
        "    weight_decay=weight_decay,\n",
        "    logging_dir= 'squad_logs',\n",
        "    logging_strategy=strategy,\n",
        "    load_best_model_at_end=True\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gzb8iZf5LQQg"
      },
      "source": [
        "For example, in this case we set the evaluation to be done at the end of each epoch in `evaluation_strategy`. Customize batch size per GPU core for training `per_device_train_batch_size` and evaluation `per_device_eval_batch_size`. In addition to give values for the total number of epochs to perform `num_train_epohs`, `learning_rate` and `weight_decay`."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-DUzYVcWPV5Z"
      },
      "source": [
        "Then it is only needed to pass the arguments and dataset to [`Trainer`](https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainer#transformers.Trainer). The data collator will batch our processed examples together. The tokenizer is used again in order to perfom padding according to the model's preferences: rigth or left and with which token."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lRjT5yTnZDI6"
      },
      "source": [
        "from transformers import Trainer, default_data_collator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcSaL_qe6HI9"
      },
      "source": [
        "trainer = Trainer(\n",
        "    bert_qa,\n",
        "    args,\n",
        "    train_dataset=tokenized_datasets[\"train\"],\n",
        "    eval_dataset=tokenized_datasets[\"validation\"],\n",
        "    data_collator=default_data_collator,\n",
        "    tokenizer=tokenizer\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UDFqOt-4QW8q"
      },
      "source": [
        "In order to fine-tune the model it is only required to call the [`train`](https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainer#transformers.Trainer.train) method for [`Trainer`](https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainer#transformers.Trainer)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "-kiqohIHQc2R",
        "outputId": "05a8d11e-b449-470d-e2e8-1c7bd4a25d08"
      },
      "source": [
        "info_device()    \n",
        "out = trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Using GPU Tesla P100-PCIE-16GB. \n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='11124' max='11124' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [11124/11124 2:07:15, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.336300</td>\n",
              "      <td>1.053371</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.727300</td>\n",
              "      <td>1.065886</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.409800</td>\n",
              "      <td>1.289591</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RFsuoranIfjQ"
      },
      "source": [
        "Now it is necesary to perform some operation to get the final results. We need to load the saved checkpoint and extract the information. Mention Hugging Face does not provide the training time per epoch, only total one. Due to that, we assume an equal distribution of time in each epoch."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rHbVFmrCCOqc"
      },
      "source": [
        "checkpoint = qa_dir/f'checkpoint-{out.global_step}/trainer_state.json'\n",
        "\n",
        "with open(checkpoint) as json_file:\n",
        "    train_dic = json.load(json_file)\n",
        "\n",
        "total_train_time = out.metrics['train_runtime']\n",
        "training_stats = []\n",
        "for i in range(epochs):\n",
        "    training_stats.append(\n",
        "                {\n",
        "                    'epoch': i + 1,\n",
        "                    'Training Loss': train_dic['log_history'][i*2]['loss'],\n",
        "                    'Validation Loss': train_dic['log_history'][i*2+1]['eval_loss'],\n",
        "                    'Validation Time':  format_time(train_dic['log_history'][i*2+1]['eval_runtime'])\n",
        "                }\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IfjbYPTqDt4w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "0c778796-3289-42cf-a730-ddc825b2cbe6"
      },
      "source": [
        "# Create a DataFrame from our training statistics.\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "df_stats['Training Time'] = format_time(total_train_time/epochs)\n",
        "\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats = df_stats[['Training Loss', 'Validation Loss', 'Training Time', 'Validation Time']]\n",
        "\n",
        "# Save data frame\n",
        "df_stats.to_excel(qa_dir/'training_stats.xlsx')\n",
        "\n",
        "df_stats"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.3363</td>\n",
              "      <td>1.053371</td>\n",
              "      <td>0:42:26</td>\n",
              "      <td>0:02:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.7273</td>\n",
              "      <td>1.065886</td>\n",
              "      <td>0:42:26</td>\n",
              "      <td>0:02:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.4098</td>\n",
              "      <td>1.289591</td>\n",
              "      <td>0:42:26</td>\n",
              "      <td>0:02:20</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Validation Loss Training Time Validation Time\n",
              "epoch                                                              \n",
              "1             1.3363         1.053371       0:42:26         0:02:20\n",
              "2             0.7273         1.065886       0:42:26         0:02:20\n",
              "3             0.4098         1.289591       0:42:26         0:02:20"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DzyvvSLKHdGF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "outputId": "e8701177-a497-4566-a305-245fb5e99be7"
      },
      "source": [
        "plot_loss_curves(df_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1hUZ9oG8HuGqvQqSmeQAREQFbAltqhYYowluvaWtkk2m2yaG5NsdJPdTdm4Sdb4RV1LbEHEGjUqosaoYC8REAYQsACCdAVm5nx/ABNHUBkcODNw/64rV8KZOWceRvNy885z3lciCIIAIiIiIiISjVTsAoiIiIiI2juGciIiIiIikTGUExERERGJjKGciIiIiEhkDOVERERERCJjKCciIiIiEhlDORG1Wbm5uZDL5fjmm2+afY333nsPcrlcj1W1XQ96v+VyOd57770mXeObb76BXC5Hbm6u3uuLi4uDXC5HYmKi3q9NRPS4TMUugIjaD13CbXx8PDw8PFqwGuNTWVmJZcuWYffu3cjPz4ejoyN69eqFP/7xj5DJZE26xp/+9Cf8/PPP2LZtG4KCghp9jiAIGDp0KEpLS3H06FFYWlrq89toUYmJiUhKSsKsWbNga2srdjkN5ObmYujQoZg2bRo+/PBDscshIgPCUE5Ereazzz7T+vr06dP48ccfMXnyZPTq1UvrMUdHx8d+PXd3d1y4cAEmJibNvsbixYvx8ccfP3Yt+rBw4UL89NNPGDNmDCIjI1FQUICDBw/i/PnzTQ7lEydOxM8//4wtW7Zg4cKFjT7nxIkTuHbtGiZPnqyXQH7hwgVIpa3zwWxSUhK+/fZbPPvssw1C+TPPPIPRo0fDzMysVWohItIFQzkRtZpnnnlG62uVSoUff/wRPXr0aPDY/crLy2Ftba3T60kkElhYWOhc570MJcDduXMHe/fuxYABA/Dll19qjr/66quorq5u8nUGDBiAzp07Y+fOnXjnnXdgbm7e4DlxcXEAagO8Pjzun4G+mJiYPNYvaERELYk95URkcIYMGYIZM2bg8uXLmDdvHnr16oWxY8cCqA3nX331FSZNmoSoqCh0794dw4YNwxdffIE7d+5oXaexHud7jyUkJGDChAkICQnBgAED8K9//QtKpVLrGo31lNcfKysrw0cffYS+ffsiJCQEU6ZMwfnz5xt8P7dv38aCBQsQFRWF8PBwzJw5E5cvX8aMGTMwZMiQJr0nEokEEomk0V8SGgvWDyKVSvHss8+iuLgYBw8ebPB4eXk59u3bh4CAAISGhur0fj9IYz3larUa//d//4chQ4YgJCQEY8aMwY4dOxo9X6FQ4G9/+xtGjx6N8PBwhIWFYfz48di8ebPW89577z18++23AIChQ4dCLpdr/fk/qKe8qKgIH3/8MQYOHIju3btj4MCB+Pjjj3H79m2t59Wff/z4caxcuRJPPfUUunfvjhEjRmDr1q1Nei90kZKSgldeeQVRUVEICQnBqFGjsHz5cqhUKq3n3bhxAwsWLMDgwYPRvXt39O3bF1OmTNGqSa1WY/Xq1Xj66acRHh6Onj17YsSIEfjrX/+KmpoavddORLrjTDkRGaTr169j1qxZiI6OxvDhw1FZWQkAyMvLQ2xsLIYPH44xY8bA1NQUSUlJWLFiBZKTk7Fy5comXf/w4cPYsGEDpkyZggkTJiA+Ph7/+9//YGdnh5deeqlJ15g3bx4cHR3xyiuvoLi4GKtWrcILL7yA+Ph4zax+dXU15syZg+TkZIwfPx4hISFITU3FnDlzYGdn1+T3w9LSEuPGjcOWLVuwa9cujBkzpsnn3m/8+PH47rvvEBcXh+joaK3HfvrpJ9y9excTJkwAoL/3+37/+Mc/sHbtWkRERGD27NkoLCzEokWL4Onp2eC5SUlJOHXqFAYNGgQPDw/NpwYLFy5EUVERXnzxRQDA5MmTUV5ejv3792PBggVwcHAA8PB7GcrKyvCHP/wBV69exYQJE9CtWzckJydj48aNOHHiBDZv3tzgE5qvvvoKd+/exeTJk2Fubo6NGzfivffeg5eXV4M2rOa6ePEiZsyYAVNTU0ybNg3Ozs5ISEjAF198gZSUFM2nJUqlEnPmzEFeXh6mTp0KHx8flJeXIzU1FadOncKzzz4LAPjuu+/w9ddfY/DgwZgyZQpMTEyQm5uLgwcPorq62mA+ESJq1wQiIpFs2bJFCAgIELZs2aJ1fPDgwUJAQIAQExPT4Jyqqiqhurq6wfGvvvpKCAgIEM6fP685lpOTIwQEBAhff/11g2NhYWFCTk6O5rharRZGjx4t9O/fX+u67777rhAQENDosY8++kjr+O7du4WAgABh48aNmmPr1q0TAgIChKVLl2o9t/744MGDG3wvjSkrKxOef/55oXv37kK3bt2En376qUnnPcjMmTOFoKAgIS8vT+v4c889JwQHBwuFhYWCIDz++y0IghAQECC8++67mq8VCoUgl8uFmTNnCkqlUnP80qVLglwuFwICArT+bCoqKhq8vkqlEqZPny707NlTq76vv/66wfn16v++nThxQnPs3//+txAQECCsW7dO67n1fz5fffVVg/OfeeYZoaqqSnP85s2bQnBwsPDGG280eM371b9HH3/88UOfN3nyZCEoKEhITk7WHFOr1cKf/vQnISAgQDh27JggCIKQnJwsBAQECN9///1Drzdu3Dhh5MiRj6yPiMTD9hUiMkj29vYYP358g+Pm5uaaWT2lUomSkhIUFRWhX79+ANBo+0hjhg4dqrW6i0QiQVRUFAoKClBRUdGka8yePVvr6z59+gAArl69qjmWkJAAExMTzJw5U+u5kyZNgo2NTZNeR61W4/XXX0dKSgr27NmDJ598Em+99RZ27typ9bwPPvgAwcHBTeoxnzhxIlQqFbZt26Y5plAocO7cOQwZMkRzo62+3u97xcfHQxAEzJkzR6vHOzg4GP3792/w/I4dO2r+u6qqCrdv30ZxcTH69++P8vJyZGRk6FxDvf3798PR0RGTJ0/WOj558mQ4OjriwIEDDc6ZOnWqVstQp06d4Ovri6ysrGbXca/CwkKcPXsWQ4YMQWBgoOa4RCLByy+/rKkbgObvUGJiIgoLCx94TWtra+Tl5eHUqVN6qZGI9I/tK0RkkDw9PR94U9769euxadMmpKenQ61Waz1WUlLS5Ovfz97eHgBQXFwMKysrna9R3y5RXFysOZabmwtXV9cG1zM3N4eHhwdKS0sf+Trx8fE4evQoPv/8c3h4eOA///kPXn31VbzzzjtQKpWaFoXU1FSEhIQ0qcd8+PDhsLW1RVxcHF544QUAwJYtWwBA07pSTx/v971ycnIAAH5+fg0ek8lkOHr0qNaxiooKfPvtt9izZw9u3LjR4JymvIcPkpubi+7du8PUVPvHoampKXx8fHD58uUG5zzo7861a9eaXcf9NQGAv79/g8f8/PwglUo176G7uzteeuklfP/99xgwYACCgoLQp08fREdHIzQ0VHPem2++iVdeeQXTpk2Dq6srIiMjMWjQIIwYMUKnexKIqOUwlBORQerQoUOjx1etWoV//vOfGDBgAGbOnAlXV1eYmZkhLy8P7733HgRBaNL1H7YKx+Neo6nnN1X9jYkREREAagP9t99+i5dffhkLFiyAUqlEYGAgzp8/j08++aRJ17SwsMCYMWOwYcMGnDlzBmFhYdixYwfc3NzwxBNPaJ6nr/f7cfzlL3/BoUOH8NxzzyEiIgL29vYwMTHB4cOHsXr16ga/KLS01lresaneeOMNTJw4EYcOHcKpU6cQGxuLlStXYv78+Xj77bcBAOHh4di/fz+OHj2KxMREJCYmYteuXfjuu++wYcMGzS+kRCQehnIiMirbt2+Hu7s7li9frhWOjhw5ImJVD+bu7o7jx4+joqJCa7a8pqYGubm5Tdrgpv77vHbtGjp37gygNpgvXboUL730Ej744AO4u7sjICAA48aNa3JtEydOxIYNGxAXF4eSkhIUFBTgpZde0npfW+L9rp9pzsjIgJeXl9ZjCoVC6+vS0lIcOnQIzzzzDBYtWqT12LFjxxpcWyKR6FxLZmYmlEql1my5UqlEVlZWo7PiLa2+rSo9Pb3BYxkZGVCr1Q3q8vT0xIwZMzBjxgxUVVVh3rx5WLFiBebOnQsnJycAgJWVFUaMGIERI0YAqP0EZNGiRYiNjcX8+fNb+LsiokcxrF/3iYgeQSqVQiKRaM3QKpVKLF++XMSqHmzIkCFQqVRYu3at1vGYmBiUlZU16RoDBw4EULvqx7394hYWFvj3v/8NW1tb5ObmYsSIEQ3aMB4mODgYQUFB2L17N9avXw+JRNJgbfKWeL+HDBkCiUSCVatWaS3v99tvvzUI2vW/CNw/I5+fn99gSUTg9/7zprbVPPXUUygqKmpwrZiYGBQVFeGpp55q0nX0ycnJCeHh4UhISMCVK1c0xwVBwPfffw8AGDZsGIDa1WPuX9LQwsJC0xpU/z4UFRU1eJ3g4GCt5xCRuDhTTkRGJTo6Gl9++SWef/55DBs2DOXl5di1a5dOYbQ1TZo0CZs2bcKSJUuQnZ2tWRJx79698Pb2brAuemP69++PiRMnIjY2FqNHj8YzzzwDNzc35OTkYPv27QBqA9Z///tfyGQyjBw5ssn1TZw4EYsXL8Yvv/yCyMjIBjOwLfF+y2QyTJs2DevWrcOsWbMwfPhwFBYWYv369QgMDNTq47a2tkb//v2xY8cOWFpaIiQkBNeuXcOPP/4IDw8Prf59AAgLCwMAfPHFF3j66adhYWGBrl27IiAgoNFa5s+fj71792LRokW4fPkygoKCkJycjNjYWPj6+rbYDPKlS5ewdOnSBsdNTU3xwgsv4P3338eMGTMwbdo0TJ06FS4uLkhISMDRo0cxZswY9O3bF0Bta9MHH3yA4cOHw9fXF1ZWVrh06RJiY2MRFhamCeejRo1Cjx49EBoaCldXVxQUFCAmJgZmZmYYPXp0i3yPRKQbw/wpRkT0APPmzYMgCIiNjcUnn3wCFxcXjBw5EhMmTMCoUaPELq8Bc3NzrFmzBp999hni4+OxZ88ehIaGYvXq1Xj//fdx9+7dJl3nk08+QWRkJDZt2oSVK1eipqYG7u7uiI6Oxty5c2Fubo7Jkyfj7bffho2NDQYMGNCk6z799NP47LPPUFVV1eAGT6Dl3u/3338fzs7OiImJwWeffQYfHx98+OGHuHr1aoObKz///HN8+eWXOHjwILZu3QofHx+88cYbMDU1xYIFC7Se26tXL7z11lvYtGkTPvjgAyiVSrz66qsPDOU2NjbYuHEjvv76axw8eBBxcXFwcnLClClT8Nprr+m8i2xTnT9/vtGVa8zNzfHCCy8gJCQEmzZtwtdff42NGzeisrISnp6eeOuttzB37lzN8+VyOYYNG4akpCTs3LkTarUanTt3xosvvqj1vLlz5+Lw4cP44YcfUFZWBicnJ4SFheHFF1/UWuGFiMQjEVrjLh0iItKiUqnQp08fhIaGNnsDHiIiajvYU05E1MIamw3ftGkTSktLG12Xm4iI2h+2rxARtbCFCxeiuroa4eHhMDc3x9mzZ7Fr1y54e3vjueeeE7s8IiIyAGxfISJqYdu2bcP69euRlZWFyspKODk5YeDAgXj99dfh7OwsdnlERGQAGMqJiIiIiETGnnIiIiIiIpExlBMRERERiYw3eta5fbsCanXrdvI4OVmjsLC8VV+TiEhfOIYRkbESa/ySSiVwcLBq9DGG8jpqtdDqobz+dYmIjBXHMCIyVoY2frF9hYiIiIhIZAzlREREREQiYygnIiIiIhIZQzkRERERkcgYyomIiIiIRMZQTkREREQkMoZyIiIiIiKRMZQTEREREYmMoZyIiIiISGTc0VMEx3+7ibjDChSVVsHR1gLjB8rQN9hN7LKIiIiISCQM5a3s+G83sWZPCqqVagBAYWkV1uxJAQAGcyIiIqJ2iu0rrSzusEITyOtVK9WIO6wQqSIiIiIiEhtDeSsrLK3S6TgRERERtX0M5a3Mydai0eNSCXA+/VYrV0NEREREhoChvJWNHyiDuan2225qIoGtlTn+E3sBy7ZfQmlFtUjVEREREZEYeKNnK6u/mfP+1VciAl2x+/hV7Dqehd8yizBlaFf06+4GiUQibsFERERE1OIkgiAIYhdhCAoLy6FWt+5b4eJig4KCMq1j125VYM2eFKRfK0E3HwfMjA6Eq32HVq2LiKgpGhvDiIiMgVjjl1QqgZOTdeOPtXIt9AjuzlZ4b3pPTB8egIzrpfhwRSL2JmZDpVY/+mQiIiIiMkpsXzFAUokEQ3p6oIe/M9btu4KYhHQkJudhzshAeHWyEbs8IiIiItIzzpQbMEdbS7w2IQQvj+uO26V3sWj1KWw+lI7qGpXYpRERERGRHnGm3MBJJBJEBLoiyNsBMQnp2HMiG6dTCzArOhBB3g5il0dEREREesCZciNh3cEMc0cF4a0pPSAIAj7feBardiej4m6N2KURERER0WNiKDcy3XwcsWheFEZGeeHXizfx/vJEnErJBxfRISIiIjJeDOVGyMLMBJMG++ODWb1hb22Opdsu4du4i7hdViV2aURERETUDFynvI6hrFOuK5VajX0nc7Dtl0yYmkgwcZA/BvboAik3HSKiFsZ1yonI2CTdPIMdir0oriqGvYU9xsqiEenWs9Ve/2HrlDOU1zHWUF4v/3Yl1uxNRfLV2+jqYYfZIwPR2clKL9cmImoMQzkRGZOkm2ewIWULatS/349nJjXD1MAJrRbMuXlQO+Dq0BFvTemBOaMCcf1WBT76XxJ2/poJpYqbDhEREVH7Vq2qQVzaLq1ADgA16hrsUOwVqSptXBKxDZFIJHgitAtC/Zyw4UAatv6SiaSUfMweGQhZFzuxyyMiIiJqFeU1FcgozoKiJAuK4ixkl+VCJTS+z8vtquJWrq5xDOVtkJ21BV4e1x19027hh32p+HTtaQzt5YHxA/1gac4/ciIiImo7BEFA4d3bUBRn1obwkizcrMgDAJhITOBt64Ehnk/g+I2TKK+paHC+g4V9a5fcKCa0NqxHV2fIveyx5bAC8adzcTatADNGBCJU5iR2aURERETNohbUuFZ+E4qSTGQUZyG9OBMl1aUAgA6mlvC180Zkp3DI7H3hZeMBcxMzAEAXa7dGe8rHyqJF+T7uxxs96xj7jZ6PkpZbjNV7UnCjsBJ9unXClKe6wrajeau8NhG1TbzRk4haQ7WqBldLszWtKBklV3FXdRcAYG9hB397X8jsfCCz90Vnq06QSh58yyRXXzECbT2UA0CNUo2fjmfhp+NX0cHCFFOG+qNvsBskXD6RiJqBoZyIWoJ2P3gmssuuafrBu1i5wc/epzaE2/nC0dK+WTlGrPGLobwJ2kMor3etoByr96RAcb0Uwb6OmDlCDhf7Dq1eBxEZN4ZyInpc2v3gmVAUZ+FmZT4AwFRiAi9bz7pZcB/42fnAyqyjXl6XodyAtadQDgBqtYCEs9cQe1gBQRAw/gk/PNXbE1IpZ82JqGkYyolIV/f2gyuKa0P4vf3gfnY+mlYUbxsPmNX1g+ubIYZy3ujZTkmlEgzt5YHwrs5Y+3MqNh1MR2JyHmaPDIKna+N/WYiIiIh0Ud8Pnl6cBUVJJjJLruKuqgpAbT94Vwe/JveDt3WcKa/T3mbK7yUIApKS87HhwBVU3lUiOsoLY/v7wMzUROzSiMiAGcoYRkSGo7y6AhklWUivWxnlQf3g/va+cLR0EK1OzpSTQZJIJIjq1gnBvo74MT4NPx2/ilOpBZgdLYfcS7z/YYiIiMhwNaUffIjnE3rvB2+rOFNepz3PlN/vt8wirNmbglsld/FkWBc8N1iGjpYt09NFRMbLUMcwImoZtf3gN6Coa0URqx9cHzhTTkYh2NcRi+dFYfvRTPx8MhvnFbcwfZgcveQuYpdGREREraRaVY2s0hxNCGc/eMviTHkdzpQ3LutmKVbtTkFOfjl6Bbhg6rAAONhYiF0WERkAYxjDiKjpyqsr6rapf3A/uL+dL2T2PqL2g+uDIc6UM5TXYSh/MKVKjX0nc7D9aCZMTaSYNFiGJ8O6QMpNh4jaNWMZw4ioodp+8CKtVpT7+8Hrd8r0tfNuc/3ghhjK2b5Cj2RqIsWoPt7oFeCCNXtTsHZvKk78lodZ0XJ0drISuzwiIiJ6hKb0g0e59YKfvY/B94O3VZwpr8OZ8qYRBAG/XLiBmIPpqFaqMba/D6KjvGBqwj4yovbGGMcwovbiUf3gtbPgta0o7bEfnDPlZPQkEgmeDOuCMJkT1h9IQ9yRDCQl52HOqCD4drYVuzwiIqJ26UH94BJI0NmqEyLcemq2qzf2fvC2ijPldThT3jxnrxTgh32pKKmoxrDennj2CT9YmHPTIaL2oC2MYUTG6P5+8PTiLOTd0w/ubesJWV0/uJ+dNzq2sX5wfeBMObU54QEukHs5IPawAvtO5uDMlQLMHCFHdz8nsUsjIiJqE+7tB6/dKTMTJdW1gbKDaQfI7LzRh/3gRo8z5XU4U/74ruQUY/WeFNwsqkTfYDdMGeoPm47mYpdFRC2krY1hRIbiYf3gDhb2kNn7tOt+cH0wxJlyhvI6DOX6UaNUYdexq9h94io6WJhi6lNdEdWtEyRcPpGozWmLYxiRGDT94MWZUJRkIbssF2pBrekHr29FYT+4/jCUGzCGcv3KzS/H6r0pyLheihA/J8wYEQBnuw5il0VEetSWxzCilnJvP3h6XQhnP3jrYyg3YAzl+qdWC4g/k4u4wxkAgPFP+mFoLw9IpZw1J2oL2voYRqQP9f3g9QG8sX5wmZ0v+8FbmSGGclFv9MzPz8fatWtx/vx5XLp0CZWVlVi7di2ioqIeep5arcbWrVuxf/9+JCcno6SkBB4eHhgzZgzmzp0Lc3P2MRsCqVSCYb09Ed7VGT/8fAUb49Nw4nIe5owMhIdr438hiYiIjFltP3h2XT94VoN+8K4OMsjsfOFv7ws3K1f2g5OGqDPliYmJmDlzJry9veHo6IizZ882KZRXVFSgZ8+e6NGjBwYNGgQnJyecPXsW27ZtQ1RUFFavXq1zLZwpb1mCICDxch42HEjDnSolRvbxxtP9vGFmyuUTiYxVexrDiB6kth88UxPC2Q9uHDhTfp/g4GCcOHECDg4OOHDgAF555ZUmnWdmZoaNGzeiZ8+emmPPPfcc3N3d8c033yAxMfGRwZ5al0QiQZ9gNwT7OmJTfDp2HcvCqZR8zB4ZiABPe7HLIyIieiRBEHDrTpFWCL+/H/wpr4HsB6dmETWUW1s3r4XB3NxcK5DXGzZsGL755hsoFAqGcgNl09Eczz/dDX2DO2HN3lT8c/0ZDAp3x8SBMnS05LL5RERkOFRqFa5V3NAE8EbXB+/cCzI7X3jZuLMfnB5Lm0pBt27dAgA4OPDjIUPX3c8Ji+dHYtsvmdh/Kgfn0gowY7gc4QEuYpdGRETtVFP6wf3tfSGzYz846V+bCuUrVqyAjY0NBgwYIHYp1ASW5qaYMrR2HfNVu1PwTdxF9Ja7YOqwANhbW4hdHhERtXFl1eXIKMnS7JSZU3ZNqx88wq0n/O184Md+cGoFbSaUL1u2DMeOHcOiRYtgY2Oj8/kParpvaS4uutfa1ri42KBncGdsPZSOjftSkZydhLlPB2NYpBc3HSIycBzDyFgIgoC8iltIKUhHyi0FUgrScb0sDwBgJjWFv5MPxroPQ6CzPwKcfWFtbiVyxdTSDG38ahOhfPfu3ViyZAkmT56MyZMnN+saXH1FfINCOyPQww5r9qTgm5hz2H8iC7OiA9HJkTfKEBkijmFkyLT6wevWCC/V6gf3QYSsZ6P94HdK1LgD/t1uy7j6Sgv49ddf8c4772Dw4MH46KOPxC6HHpObY0e8PTUcv5y/jpgEBT78XxLG9vfBiEgvmJqwd4+IiBpXparG1dLs2k16irOQWXoVVapqALX94HIHf8jsfdgPTgbLqEP5+fPn8eqrryIkJARfffUVTEy45nVbIJVIMLCHO0Jlztiw/wq2HM5AUnLt8om+nW3FLo+IiAxAfT94/U6Z9/aDd7F2Q5Rbr9qlCdkPTkbCKEJ5dnY2AMDLy0tzTKFQ4IUXXoC7uzuWLVsGS0tLscqjFuJgY4FXxofgdGo+1u2/gr+vPYXhEZ4YN8APFub8BYyIqL2oXx88vSQTGXUhPK+yAABgKjWFtw3XByfjJ3ooX7p0KYDakA0A27dvx+nTp2Fra4vp06cDAGbPng0AOHjwIACgvLwc8+bNQ2lpKebNm4dDhw5pXVMulyMwMLB1vgFqcb3krgjydsDmQwr8nJSD06kFmBUdiGBfR7FLIyKiFvCwfvCOph3gZ+eDPp171/aD23rATCp6nCF6bBJBEFr37sb7yOXyRo+7u7trQviQIUMA/B7Kc3NzMXTo0Ade89VXX8Vrr72mUx280dM4pGbfxuo9Kci7fQf9u7th8tCusO7AzRqIxMAxjPSlSlWNrJJszU6Z9/aDO1o6aLapZz846Ysh3ugpeig3FAzlxqNGqcKOX7OwNzEbHS1NMfWpAEQGuXL5RKJWxjGMmqusuhyKkt9nwe/vB5fZ+dQFcV84WNqLXS61QQzlBoyh3Pjk5Jdj9Z5kZN4oQ6jMCTOGy+Fkx3sLiFoLxzBqCkEQUHCnULNNfWP94LWz4D7ws/NBR7MOIldM7QFDuQFjKDdOarWAA6dzEXdEAYlEgokDZRgc7g6plLPmRC2NYxg1RqVW4Vr5Da2Z8Pv7wWX2PvC394WnDfvBSRyGGMr5fwIZNalUguERnujZ1Rlrf07F+v1XcOLyTcyODoS7izi7tBIRtSeP6gevXR/cFzI7H/aDEz0EZ8rrcKbc+AmCgBO/5WFjfBruVCkxuq83Rvf1gZkpfwAQtQSOYe2TVj94cRZyyhvpB68L4ewHJ0PFmXKiFiSRSNC3uxuC/RyxKT4NO37NwsmU2k2HunrwBwMRka7u7wdPL8lEfuUtAL/3gw/zGgSZvQ98bb3ZD070GDhTXocz5W3PBUUhfvg5BYWlVRjc0x0TB8rQwYK/hxLpC8ewtudR/eD1yxLK7NXEbXoAACAASURBVH3YD05GjTPlRK0oVOaExfOjEHckA/GncnEu7RZmDJejR1dnsUsjIjII9/eDZ5ReRbVWP3hXzcoo7AcnalmcKa/DmfK2TXG9BKv3pOBaQQUiAl0xdVgA7KzMxS6LyKhxDDM+j+4H99WEcPaDU1tmiDPlDOV1GMrbPqVKjT2J2dj5ayYszEzw3BB/DAjpzE2HiJqJY5hh0/SD17WhKO7rB/ex9dSEcPaDU3vDUG7AGMrbjxuFFVizJwVXcksQ5O2AWdFyuDp0FLssIqPDMcyw1PeDp9e1oihKMlFWXQ6A/eBE92MoN2AM5e2LWhBw5Nx1bD6UDqVKwLgnfDE8whMmUvZLEjUVxzBx1feDp5dkIuO+fnAnSwf43dOKwn5wIm2GGMr5azK1S1KJBIPC3RHm74x1+1KxOUGBxMt5mDMyCN5uNmKXR0TUQGl1GTKKs+p6whv2g/dx681+cCIjxpnyOpwpb78EQcDp1AKs338FZZU1GB7piWcG+MLCzETs0ogMGsewllPbD36rrg3l4f3gfnbe6GDKfnAiXXCmnMgASSQS9A50RZCPAzYnpGNvYjZOp+ZjVnQguvk4il0eEbUDKrUKueXXNbPgjfWD9+scyX5wojaMM+V1OFNO9ZKv3saavSnIv30HA0I647kh/rDuYCZ2WUQGh2NY891VViGrNLtup8wH94P72/uiU0cX9oMT6ZkhzpQzlNdhKKd7VdeosOPXLOxNzIZ1B1NMHRaAiEBXLp9IdA+OYU33qH5wmZ0v/O194Md+cKJWwVBuwBjKqTHZeWVYtScFV2+WoYe/M6YPD4CjraXYZREZBI5hjWvQD16cifw7jfWD+8LPzov94EQiYCg3YAzl9CAqtRoHTuVi65EMSKUSTBwkw6Bwd0g5a07tHMewWg/rB7cy7Qg/e29NCPe0cWc/OJEBMMRQzpGB6BFMpFKMiPRCeIALftibgnX7ruDEb3mYNTIQ7s5WYpdHRK3s3n5wRXEmMkuztfrBgxwDILPzgYz94ESkA86U1+FMOTWFIAg4dukmNsWnoapGhTF9fTCqrzdMTfhDl9qf9jKG3dsPnl6cidzy61r94P72vpDZsR+cyJhwppzIyEkkEvQP6YwQPydsjE/DtqOZOJmSj1kjA+Hvbid2eUT0mO7tB6/fKbO+H9xMagofWy8M9xoEP/aDE5Gecaa8DmfKqTnOp9/CD/tScbu0CkN6eWD8k37oYMHfdal9aAtjmKYfvDhT0xNeVnNvP7iPphWF/eBEbQdnyonamDB/ZwR42iPuSAYOns7F2bQCzBguR5i/s9ilEVEjNP3gdSFcux/cEUFO7AcnInFwprwOZ8rpcaVfK8HqPSm4fqsCkUGumPpUAGytzMUui6jFGMMYVlpdplkRRVGcpdUP7m7dGbK6mXD2gxO1L4Y4U85QXoehnPRBqVJj9/Gr2HU8CxZmJpgytCv6dXfjpkPUJhnaGCYIAvI164M33g8us/NhPzgRMZQbMoZy0qfrtyqwem8K0nNL0M3HATOjA+FqzwBAbYvYY5gu/eBeNu4wZT84EdVhKDdgDOWkb2pBwOGz17D5kAJqtYBxT/hhWIQHTKTsUaW2obXHsEf1g9e3ovjb+8KV/eBE9BCGGMo5bUDUQqQSCQb39ECYvzPW7buCmIR0JCbnYc7IQHh1shG7PCKD96h+8L6de9ftlOkDewsuSUpExo0z5XU4U04tSRAEnEotwPp9qSi/o0R0lBfG9veBuZmJ2KURNZs+x7Cm9oPL7H3hy35wInpMnCknaqckEgkiAl0R5O2AmIR07D5xFadS8zErOhBB3g5il0fU6prSD97fPQoyOx94sh+ciNoBzpTX4Uw5tabLWUVYuzcV+cV38GRYZ0wa7A8rSzOxyyLSiS5j2L394OklWcgquYpqdQ2A3/vB/etaUdgPTkQtzRBnyhnK6zCUU2urqlFhx9FM/JyUA+uOZpg+LAC95C5cPpGMxsPGsJKqMmSU1PeDZyK3/MZ964P71rWjsB+ciFofQ7kBYygnsVy9WYZVe5KRnVeO8K7OmD5cDgcbC7HLInqgpJtnsEOxF8VVxbC3sMdYvxHwtvOq7QcvzoSiJBMFdwoB3NMPXhfC2Q9ORIaAodyAMZSTmFRqNfadzMG2XzJhaiLBxEH+GNijC6ScNSc9UwtqCIIAtaCGSlBDgBrquq+1/xGgFlRQQ/uxS7dSsO9qApSCstHrW5l11KyIwn5wIjJUDOUGjKGcDEH+7Uqs2ZuK5Ku30dXDDrNHBqKzk5XYZRm0xw2Zvz9W99+457mCAKH+unX/VkP7mNZ11fXnP+y6D67r/tdq/PtSNbyuWt3g+xIEoa5e7WMCWmac62jaAX/p9Uf2gxORUWAoN2AM5WQoVGoVjl68js2H0lGtVCI6yhNDe3tAKgVDZiuGTH2QSqSQQlL773v+kUgkMJGYQFL3mIlEConm3/c8H1JIJQ3Prz1m0uDatdetvZYUv1/PRGKiw3XrH//9WH1tUokUyy6sfuD3+98hn7Xem0tE9BgMMZTzM0URNOjHlEUj0q2n2GW1GM5kNjNkhgBmAOKrgfhjov4Ravwe6hoLg7qGTDOdQqZJ3fn6CJlSiYn283FfoJXcF2gbBOv7vkbD60ogaZM37TpY2ON2VXGjx4mIqPkYyltZ0s0z2JCyBTV1S4HdrirG+uTNuFZ2A/4Ovm07ZBqgpoRME82xxw+ZDWYeHxIy61/rRuEdnE4pwJ27Ksi9HNErwBUWpqY6hUyTRr4vhkxqjrGyaK0xDADMpGYYK4sWsSoiIuPH9pU6rdW+svDXTxudZdIX/YTMxmYdmxcyG/soXR8zmY8KmQ8Oz8YZMu9UKbHlsAIJZ67B0dYCM0YEIlTmJHZZ1E61t0/7iKjtMcT2FYbyOq0Vyl85+M4DH3un92uPFTJ5c1Xbl5ZbjNV7UnCjsBJ9gjthytCusO1oLnZZ1E7xvhgiMlaGGMrZvtLKHtaP6W3rKUJFZEy6etjjb3Mi8dPxLPx0/CouZRRhylB/9A12M5pZfyIiImqIU6utbKwsGmZS7e3U2Y9JujAzlWLcE37425wIdHLogBW7kvHvmPO4VXxH7NKIiIiomdi+Uqc1l0RkPybpi1otIOHsNcQeVkAQBIx/wg9P9faEVMpZc2p5bF8hImNliO0rDOV1uE45GbOi0rtY+3MqLigK4dvZBrNHBsHTtfH/6Yn0hWMYERkrQwzlbF8hagMcbS3x+sRQvDg2GLdK7mLR6pPYcliBGqVK7NKIiIioCXijJ1EbIZFIENWtE4J9HfFjfBp+On4Vp1ILMDtaDrmXg9jlERER0UNwppyojbHuYIZ5Y7rhL5N7QKVS418bzmLN3hRU3q159MlEREQkCoZyojYq2NcRi+dFITrSC0fOX8f7KxJxOrVA7LKIiIioEQzlRG2YhbkJnhvijw9m9YZdR3P8d+tF/DfuIm6XVYldGhEREd2DoZyoHfBxs8XCWb0xcZAMFzIKsXBFIg6duwY1F18iIiIyCKKG8vz8fHzxxReYMWMGwsPDIZfLkZiY2OTzFQoF5s2bh/DwcERGRuLdd99FUVFRC1ZMZLxMTaQY1ccbi+ZGwruTNdbuTcXnG87iZlGl2KURERG1e6KG8szMTCxfvhx5eXmQy+U6nXvz5k1MmzYNOTk5eOONNzB37lwkJCRg3rx5qKnhDW1ED9LJsSPe/kM4Zo8MRE5+OT5cmYRdx7KgVKnFLo2IiKjdEnVJxODgYJw4cQIODg44cOAAXnnllSafu2zZMlRVVeGHH35Ap06dAAChoaGYM2cOtm/fjokTJ7ZU2URGTyKR4MmwLgiTOWH9gTTEHclAUnI+5owKhG9nW7HLIyIiandEnSm3traGg0Pz1k/et28fhgwZognkANCvXz/4+Phgz549+iqRqE2zs7bAH8d1x2vjQ1B+pxp/X3sKm+LTUFXNTYeIiIhak1FuHpSXl4fCwkJ07969wWOhoaH49ddfRaiKyHiFB7hA7uWALYcV2HcyB2euFGDmCDm6+zmJXRoREVG7YJSrr+Tn5wMAXFxcGjzm4uKCwsJCqFSc6SPSRUdLU8wYIcd703rC1ESKf8ecx/Kdl1FWWS12aURERG2eUc6UV1XVrrFsbm7e4DELCwsAwN27d2FlZdXkazo5WeunOB25uNiI8rpED+LiYoOIkC6Iib+C2Pg0/JZVhOfHhWBguDskEonY5ZGB4RhGRMbK0MYvowzl9cG7urrhDF59YLe0tNTpmoWF5VCrW3fNZhcXGxQUlLXqaxI11YheHgj2tMfqvSn4cv1p7DuehRkjAuBs10Hs0shAcAwjImMl1vgllUoeOBFslO0rrq6uAICCgoZbhhcUFMDJyQkmJiatXRZRm+Phao2/Tu+FPzzVFVdyivHBiiTsP5nT6r/AEhERtXVGGco7deoER0dHXLp0qcFjFy5cQFBQkAhVEbVNUqkEw3p7YvH8SAR42mNjfBo+XXcauQXlYpdGRETUZhhFKM/OzkZ2drbWseHDh+PgwYPIy8vTHDt+/DiysrIQHR3d2iUStXnOdh3w50mheOHpbsi/fQcfrzqJrUcyUKPkTdVERESPSyIIgqifQy9duhQAoFAosGvXLkyYMAEeHh6wtbXF9OnTAQBDhgwBABw8eFBz3o0bNzBu3DjY29tj+vTpqKysxMqVK9G5c2ds3ry50ZtAH4Y95URNV1ZZjU3x6Tj+2024OXbE7JGBCPC0F7ssamUcw4jIWBliT7nooVwulzd63N3dXRPCGwvlAJCWloZ//vOfOH36NMzMzDBo0CAsWLAAjo6OOtfBUE6ku0uZhVi7NxW3Su5iULg7Jg6UoaOlUd4/Ts3AMYyIjBVDuQFjKCdqnqpqFbb+koH9p3JgZ2WOGcPlCA9ouIcAtT0cw4jIWBliKDeKnnIiMlwW5iaYMrQrFs7sDesO5vgm7iKWbr2IkvIqsUsjIiIyGgzlRKQXvp1t8eHs3pgw0A/n0gvx/vJEHDl/HfwwjoiI6NEYyolIb0xNpBjd1weL5kXC09Uaq/ek4PONZ5FXVCl2aURERAaNoZyI9M7NsSPenhqOWdFyXM0rx4f/S8JPx7OgVKnFLo2IiMggcZkEImoRUokEA3u4I1TmjA37r2DL4QycTM7H7FGB8HGzFbs8IiIig8KZciJqUQ42FnhlfAheebY7SiqrsXjNKcQcTEdVDTcdIiIiqseZciJqFb3krgjydsDmQwrsTcrGqdR8zIoORLCv7vsKEBERtTWcKSeiVtPR0gyzogPx7tRwmJhI8eWP57By12WU36kRuzQiIiJRMZQTUauTezlg0dwIjOnnjROX8/D+8hNIvJzH5ROJiKjdYignIlGYmZpg/JMyfDg7As52lvi/Hb/hP7EXUFhyV+zSiIiIWh1DORGJytPVGu/P6I0pQ7siJfs2Fq5MRPzpXKjVnDUnIqL2g6GciEQnlUowPMITf58Xha7udli//wr+sf40rhWUi10aERFRq2AoJyKD4WzfAW88F4bnx3RDXtEd/G3VSWz7JQM1Sm46REREbRuXRCQigyKRSNC3uxuC/RyxKT4NO37NwsmUfMwZGQR/DzuxyyMiImoRnCknIoNk29EcLzwdjD9PCkN1jQr/WHca6/al4k6VUuzSiIiI9I6hnIgMWqjMCYvnR2Fobw8knLmGhSsScS7tlthlERER6ZVeQrlSqcTPP/+MmJgYFBQU6OOSREQaluammPpUAP46sxc6Wpri6y0X8N22SyipqBa7NCIiIr3Quaf8s88+Q2JiIrZs2QIAEAQBc+bMwalTpyAIAuzt7RETEwMvLy+9F0tE7Zusix0+mh2BPYnZ2PlrJi5nFeG5If4YENIZEolE7PKIiIiaTeeZ8l9++QW9e/fWfH3w4EGcPHkS8+bNw5dffgkA+P777/VXIRHRPUxNpHi6nw8+nhsJd2crrNqdgi82nUP+7UqxSyMiImo2nWfKb968CW9vb83XCQkJ8PDwwFtvvQUASEtLw86dO/VXIRFRIzo7WeGdaT1x5Nx1bD6Ujg9XJuGZJ3wxPMITJlLeLkNERMZF51BeU1MDU9PfT0tMTES/fv00X3t6erKvnIhahVQiwaBwd4T5O2PdvlRsTlAg8XIe5owMgrebjdjlERERNZnO00lubm44e/YsgNpZ8ZycHERERGgeLywsRMeOHfVXIRHRIzjYWODV8SH447juKCmvxuI1pxCTkI6qGpXYpRERETWJzjPlo0ePxtKlS1FUVIS0tDRYW1tj4MCBmseTk5N5kycRtTqJRILega4I8nHA5oR07E3MxpnUAsyKliPIx1Hs8oiIiB5K55nyF198Ec8++yzOnTsHiUSCf/3rX7C1tQUAlJWV4eDBg+jbt6/eCyUiagorSzPMHhmEt/8QDkiAzzedw/92J6P8To3YpRERET2QRBAEQV8XU6vVqKiogKWlJczMzPR12VZRWFgOtVpvb0WTuLjYoKCgrFVfk6g9qa5RYcevWdibmA3rDqaYOiwAEYGuXD5RTziGEZGxEmv8kkolcHKybvwxfb6QUqmEjY2N0QVyImqbzM1MMHGQDB/O7g0HW0ss2/4bvtlyEUWld8UujYiISIvOofzw4cP45ptvtI6tX78ePXv2RI8ePfCXv/wFNTX8mJiIDIdXJxssnNkLk4f443JWERauSMTBM7lQ6++DQiIioseicyhfuXIlMjIyNF8rFAp8+umncHV1Rb9+/bB7926sX79er0USET0uE6kUIyK9sGh+FGRdbLFu3xX8c90ZXL9VIXZpREREuofyjIwMdO/eXfP17t27YWFhgdjYWKxYsQKjRo3Ctm3b9FokEZG+uNp3wJuTe2De6CDcKKzA31YlYcfRTChVarFLIyKidkznUF5SUgIHBwfN18eOHUOfPn1gbV3btB4ZGYnc3Fz9VUhEpGcSiQT9Qzrjk+f7oJfcFduOZuLjVSeRfq1E7NKIiKid0jmUOzg44Pr16wCA8vJyXLx4Eb1799Y8rlQqoVJxww4iMny2VuZ4cWwwXp8YijvVSvzjh9NYv/8K7lQpxS6NiIjaGZ03D+rRowc2bdoEf39/HDlyBCqVCk8++aTm8atXr8LV1VWvRRIRtaQwf2cEeNoj7kgGDp7Oxdm0AswYLkeYv7PYpRERUTuh80z5n/70J6jVavz5z39GXFwcxo0bB39/fwCAIAg4cOAAevbsqfdCiYhaUgcLU0wbFoAFM3rB0twU/4m9gP/b8RtKK6rFLo2IiNqBZm0eVFxcjDNnzsDGxgYRERGa4yUlJdi2bRuioqIQGBio10JbGjcPIqJ6SpUau49fxa7jWbAwM8GUoV3Rr7sbNx26D8cwIjJWhrh5kF539DRmDOVEdL/rtyqwem8K0nNL0M3HATOjA+Fq30HssgwGxzAiMlZtKpRnZ2cjPj4eOTk5AABPT08MHToUXl5eza9URAzlRNQYtSDg8Nlr2HxIAbVawLgn/DAswgMmUr1uiGyUOIYRkbFqM6F8yZIlWL58eYNVVqRSKV588UW8/vrrzatURAzlRPQwRaV3sW7fFZxLvwVvNxvMGRkIr042YpclKo5hRGSsDDGU67z6SmxsLJYtW4bw8HDMnz8fXbt2BQCkpaVh5cqVWLZsGTw9PTF+/PjHq5qIyIA42lritQkhOJVagPX7UrFo9SlER3lhbH8fmJuZiF0eEREZOZ1nysePHw8zMzOsX78epqbamV6pVGLatGmoqalBXFycXgttaZwpJ6KmKr9Tg5iEdBy9cAOuDh0wKzoQQd4Ojz6xjeEYRkTGyhBnynVuilQoFBg1alSDQA4ApqamGDVqFBQKhe5VEhEZCesOZpg7KghvTekBCMDnG89i9Z5kVNytEbs0IiIyUjqHcjMzM1RWVj7w8YqKCpiZmT1WUURExqCbjyM+nheJkVFeOHrhJhYuT8SplHxwUSsiItKVzqE8JCQEP/74I27dutXgscLCQsTExCAsLEwvxRERGToLMxNMGuyPD2b1hp21OZZuu4Rv4y7idlmV2KUREZER0bmn/OTJk5g9ezasrKwwYcIEzW6e6enpiIuLQ0VFBVavXo3evXu3SMEthT3lRPS4VGo19p3MwbZfMmFqIsHEQf4Y2KMLpG100yGOYURkrAyxp7xZSyIePHgQixcvxo0bN7SOd+nSBR9++CEGDRrUrELFxFBORPqSf7sSa/amIvnqbXT1sMPskYHo7GQldll6xzGMiIxVmwnlAKBWq3Hp0iXk5uYCqN08KDg4GDExMVi7di12797d/IpFwFBORPokCAJ+vXgTPx5MQ1WNCk/388HIPt4wNWk7mw5xDCMiY2WIoVzndcp/v6gUoaGhCA0N1Tp++/ZtZGZmNveyRERtgkQiwYDQzgiROWHjgSvY+ksmklLyMXtkIGRd7MQuj4iIDEzbmbIhIjJAdlbmeOmZ7vjThFBU3lXi07WnseHAFdytVopdGhERGZBmz5QTEVHT9ejqDLmXPbYcViD+VC7OXinAjBGBCJU5iV0aEREZAM6UExG1kg4Wppg+XI73pveEuZkJlmw+j+93/obSymqxSyMiIpExlBMRtbKuHvb425xIjO3vg5PJ+Vi4PBHHL93kpkNERO1Yk9pXVq1a1eQLnjlzpsnPra6uxn/+8x9s374dpaWlCAwMxBtvvIG+ffs+8txjx47hu+++w5UrV6BWq+Hn54dZs2Zh1KhRTX59IiKxmJlKMe4JP0QEumL1nhQs33UZx3+7iZkj5HC27yB2eURE1MqatCRiYGCgbheVSJCcnPzI57355pvYt28fZs6cCW9vb2zduhWXLl3CDz/8gPDw8Aeel5CQgJdffhnh4eEYPXo0AOCnn37CmTNn8Pe//x2TJk3SqV6ASyISkXjUagEJZ68h9rACgiBg/BN+eKq3J6RSw950iGMYERkrQ1wSsUmhPCkpSecXjYyMfOjjFy5cwKRJk7BgwQLMnj0bAFBVVYUxY8bA1dUV69evf+C58+fPR2pqKuLj42Fubg6gdtZ96NCh8Pb2xrp163Sul6GciMRWVHoXa39OxQVFIXw722D2yCB4ujY+eBsCjmFEZKwMMZQ3qX3lUQG7Ofbu3QszMzOtWW0LCwtMnDgRX331FfLz8+Hq6troueXl5bCzs9MEcgAwNzeHnZ0dLCws9F4rEVFrcLS1xOsTQ3EyJR/r91/BotUnER3lhbH9fWBmaiJ2eURE1IJEu9EzOTkZvr6+sLLS3no6NDQUgiA8tP0lMjISaWlpWLJkCbKzs5GdnY0lS5YgKysLc+fObenSiYhajEQiQWRQJ3zyfB/0Ce6En45fxYf/O4nU7Ntil0ZERC1ItHXKCwoK0KlTpwbHXVxcAAD5+fkPPPell15CdnY2li1bhu+++w4A0LFjRyxduhT9+/dvmYKJiFqRdQczzBvdDX26uWHN3hT8a8NZDOzRBZMGydDR0kzs8oiISM9EC+V3796FmVnDHyz17SdVVVUPPNfc3Bw+Pj6Ijo7GsGHDoFKpEBMTgz//+c9YvXo1QkNDda7nQf09Lc3FxUaU1yUi4zDIxQZ9wtyxYV8qth9Ox8WMQrw0PhR9Q7qIXRoAjmFEZLwMbfwSLZRbWlqipqamwfH6MP6w3vDFixfj4sWLiI2NhVRa24EzcuRIjBkzBp9++ik2bdqkcz280ZOIDNnTfbwQ4mOP1btT8Onqk+gV4IKpwwLgYCPefTQcw4jIWBnijZ6i9ZS7uLg02qJSUFAAAA+8ybO6uhqxsbEYNGiQJpADgJmZGZ544glcvHgRSqWyZYomIhKRj5stFs7qjYmDZLiQUYiFKxJx+Nw1qLnpEBGR0RMtlAcGBiIzMxMVFRVax8+fP695vDHFxcVQKpVQqVQNHlMqlVAqldwVj4jaLFMTKUb18caiuZHw7mSNNXtT8fmGs7hZVCl2aURE9BhEC+XR0dGoqanB5s2bNceqq6sRFxeHnj17am4CvX79OhQKheY5Tk5OsLW1xf79+7XaXyoqKpCQkICAgIBGe9WJiNqSTo4d8fYfwjF7ZCBy8svx4cok7DqWBaVKLXZpRETUDKL1lIeFhSE6OhpffPEFCgoK4OXlha1bt+L69ev4xz/+oXneu+++i6SkJKSmpgIATExMMHfuXCxZsgSTJ0/G2LFjoVarERsbi5s3b+Ldd98V61siImpVEokET4Z1QZjMCesPpCHuSAaSkvMxZ1QgfDvbil0eERHpoEk7eraUqqoqLFmyBDt37kRJSQnkcjnefPNN9OvXT/OcGTNmaIXyejt37sTatWuRlZWF6upqyOVyPP/88xg2bFizauGNnkRk7M5eKcAP+1JRUlGNYb098ewTfrAwb7lNhziGEZGxMsQbPUUN5YaEoZyI2oLKu0psOaxAwtlrcLazxMwRcnT3c2qR1+IYRkTGyhBDuWg95UREpH8dLU0xY4Qc703rCVMTKf4dcx7Ld15GWWW12KUREdFDMJQTEbVBAZ72+HhuBJ7u54Ok5Dy8vzwRJ367ydWpiIgMFEM5EVEbZWZqgmef9MNHsyPg6tAB3++8jCWbL+BWyR2xSyMiovswlBMRtXEertb46/Re+MNTXXElpxgfrEjC/lM5rX4fDRERPRhDORFROyCVSjCstycWz49EgKc9Nh5Iw6frTiO3oFzs0oiICAzlRETtirNdB/x5UiheeLob8m/fwcerTmLrkQzUKBvukkxERK1HtM2DiIhIHBKJBH2C3RDs64hN8enYeSwLJ1PyMXtkIAI87cUuj4ioXeJMORFRO2XT0RzPP90Nb04Og1Klxj/Xn8Han1NReVcpdmlERO0OQzkRUTvX3dcJi+dFYXiEJw6fu4YPVibi7JUCscsiImpXGMqJiAgW5iaYMrQrFs7sDStLM3wTdxFLt15ESXmV2KUREbULDOVERKTh29kWH87uQp6HLwAAIABJREFUjQkD/XAuvRDvL0/EkfPXuekQEVELYygnIiItpiZSjO7rg0XzIuHpao3Ve1Lw+cazyCuqFLs0IqI2SyJw+gMAUFhY3uobabi42KCgoKxVX5OISBdqQcAv568jJkEBpUqNZwb4ws7KHNt+yUBRaRUcbS0wfqAMfYPdxC6ViKjJxMpgUqkETk7WjT7GJRGJiOiBpBIJBvZwR6jMGRv2X0HsIQUkAOqnMApLq7BmTwoAMJgTET0Gtq8QEdEjOdhY4JXxIbDuYIb7P1OsVqoRd1ghSl1ERG0FQzkRETVZ+Z2aRo8XllYhKTkPd6u5xjkRUXOwfYWIiJrMydYChaUNl0mUSIBl23+DuakUITInRAS6IlTmBEtz/pghImoKjpZERNRk4wfKsGZPCqqVas0xc1MpZkbL4WzXASeT83HqSj5OpxYwoBMR6YCrr9Th6itERE1z/LebiDuseODqK2q1gLTcYpxKKcCp1HyUVFRrBfQwmTMszE1E/A6IqL0zxNVXGMrrMJQTEemmKWNYfUA/mVI7e14f0ENlTujNgE5EImEoN2AM5UREutF1DGNAJyJDwVBuwBjKiYh08zhj2L0B/VRqAUrrA7q/c20Pup8TAzoRtRiGcgPGUE5EpBt9jWH1AT2pbga9tKIa5mZShMoY0ImoZTCUGzCGciIi3bTEGKZWC7iSU4yTqfk4nZKP0soaTUCPDHRFiMwJFmYM6ET0eBjKDRhDORGRblp6DNME9JR8nE79PaCH1c2gM6ATUXMxlBswhnIiIt205himVgtIzSnGKQZ0ItIDhnIDxlBORKQbscYwBnQielwM5QaMoZyISDeGMIbVB/STKfk4UxfQLcxMEObvhN5yBnQiahxDuQFjKCci0o2hjWEqtRpXsotxMrUAp1PzUXZPQI8IdEWInxPMGdCJCAzlBo2hnIhIN4Y8hmkCeko+Tl8pYEAnIi0M5QaMoZyISDfGMoap1GqkZtf1oDOgExEYyg0aQzkRkW6McQy7N6CfSi1A+Z0aWJibIEzmhIjATgjxc2RAJ2oHGMoNGEM5EZFujH0Mqw/oJ+t2Eq0P6D38nWtvEmVAJ2qzGMoNGEM5EZFu2tIYplKrkVLf4vL/7d15XJTl3j/wD8MsDLvCsOSWojKKyCJauJULSrl1CqNcK49lqaWesu2c57Hzy+plHpfIzPCcjpKnRYVQc8H1mMujIAgujCQuSSwzouzbwMzvj4GBcUYFBe4BPu//5rq3L7zw9sPF975uBnSido+h3IoxlBMRNU17vYfVBfTEdDWSM0wDuqEHvTMkYgZ0oraModyKMZQTETVNR7iH1eh0UF2vXQe9QUAP6u2OEAZ0ojaLodyKMZQTETVNR7uHVdfU96DXBXS7BjPoAxjQidoMhnIrxlBORNQ0HfkeVh/Q85CccbM+oPdxx2BfBnQia8dQbsUYyomImob3MIPqGh1Uv982PiRaWlHNgE5k5RjKrRhDORFR0/AeZq4uoNc9JGoS0JUeGNCTAZ3IGjCUWzGGciKipuE97N6qa3RQXb9t7EGvC+hBfQwPiTKgEwmHodyKMZQTETUN72GNZymgy2V1D4l6wq9nZ0jEIqHLJOowGMqtGEM5EVHT8B72YOoC+mmVGikmAV2BwUoPBnSiVmCNoVzcyrUQERF1aGJbEQb0csOAXm6oHu+L9NoZ9JQMDU5eyGVAJ+qgGMqJiIgEIrYVwb+XG/wbBvR0NVJ+uyOg9/OA36MM6ETtGUM5ERGRFTAJ6DW+uHjNsMxifUAXGx8SZUAnan8YyomIiKyM2FaEgT5uGOhjGtCTMzQ4cZ4Bnag94oOetfigJxFR0/Ae1vqqa3S4eO02ElV5SMm4ibLKamNAr+tBF9syoBPdDx/0JCIiogdmMoMersPFa7dql1m8aZxBD66bQWdAJ2pTOFNeizPlRERNw3uY9TDMoN8yvEn0t5sor6yGfd0Mej8P9H+UAZ2oIc6U36Gqqgpr165FfHw8ioqKoFQqsXjxYoSGhjbq+J07d2LTpk24fPkypFIp+vbti6VLl2LgwIEtXDkREZH1MMygu2Ogjztm1+hw4eotQw/6bzdx/HyuIaD3NbS4MKATWSdBQ/l7772HhIQEzJo1Cz169EBcXBzmzp2LmJgYBAUF3fPY1atXY+PGjZg8eTIiIyNRVlYGlUoFjUbTStUTERFZH7GtCAG93RHQ2x2zqg0z6Em1LS7HzzGgE1krwdpX0tLSMHXqVLz//vt46aWXAACVlZWYOHEiPDw8sGXLlrsem5ycjGnTpiEqKgphYWHNUg/bV4iImob3sLZFW13fg57SsMWlrzsGKz3R/9FODOjUYbB9pYG9e/dCIpFg6tSpxjGZTIaIiAisXr0aarUaHh4eFo/dvHkz/P39ERYWBp1Oh/Lycjg4OLRW6URERG2ORFw/g66t1uGCcQZdY5xBD+6rQIjSgwGdSACChfL09HT07NnTLEwPHDgQer0e6enpdw3lJ0+exIQJE7Bq1SrExMSgrKwMXbp0waJFizB58uTWKJ+IiKjNkohFCOztjsAGAT0xXY0zGWocO5cDBzsxgvoY3iTarwcDOlFrECyUazQaeHp6mo0rFAoAgFqttnhcYWEhCgoK8Msvv8DW1hZvv/02XF1dsWXLFrzzzjuQy+XN1tJCRETU3pkF9KuGFheTgN5XgcFKBnSiliRYKK+oqIBEIjEbl8lkAAz95ZaUlZUBAAoKCvDTTz8hICAAABAWFoawsDCsW7fugUL53fp7WppC4STIdYmImgPvYe3PI94uCBvaE9rqGqRc0uBY6h84dSEXx9Jy4GQvweMDvDEs4BEE9FEwoFObZm33L8FCuZ2dHbRardl4XRivC+d3qhvv2rWrMZADgFQqxfjx47F582aUlpY2ucecD3oSETUN72HtX08PB/QM64sXRvWunUHPw7HUP7D/9O/GGfQhSg8oOYNObQwf9GxAoVBYbFGpW9Lwbv3krq6ukEqlcHd3N9vm7u4OvV6PkpISPvhJRETUTCRiEQL7uCOwjzu01TU4X7sOepJKjWNphhaX4NoWFwZ0ogcjWChXKpWIiYkxm9VOTU01brdEJBKhX79+yMvLM9uWm5sLW1tbuLi4tEzRREREHZxEbIugPgoE9VEYA3qiSo1ElRq/MqATPTDB/qWEh4dDq9Vi69atxrGqqirExsYiODjY+BBodnY2MjMzzY7NycnB8ePHjWMlJSXYs2cPgoKCYGdn1zpfBBERUQdWF9BfneSHtW8Ox8Jn/eHv44ZElRqrfkrF4qhj+HZ3Os5fzUd1jU7ocomsmmAvDwKAt956CwcPHsTs2bPRvXt3xMXF4fz589i0aRMGDRoEAJg5cyZOnz6NS5cuGY8rLy/Hs88+i7y8PLz00ktwdnbG9u3bcfXqVZNjm4I95URETcN7GN2NtroG56/cQuIlw4uKKqtq4GAnxiBfwzroyu6cQSdhsaf8DitWrMCaNWsQHx+PwsJC+Pr64ptvvrlvqJbL5di8eTNWrFiB7777DhUVFfDz88O33377QIGciIiImo9EbIugvgoE9VXUB3SVGqfS1TiamgNHuQTBtW8SVfZwha2IAZ1I0Jlya8KZciKipuE9jJqqSlv/kGjKZcMMuiGg1/WgM6BT6+BMOREREXVYUoktgvsqENxXYQzohhn0PBxNzWZApw6NoZyIiIha3Z0B/dyVW0i6ZCGg9/OAsjsDOrV/bF+pdb/2Fa22CsXFBaiuroJOV9Ms1xSJRNDp+DR6e2JrK4ajoyvkcq6TT+0f21eoJTQM6Gd/u4lKraHFpf4hUQZ0enjW2L7CUF7rXqG8vLwUxcW34ejoAplMDpHIFjY2Ng99TbFYhOpqhvL2Qq/XQ6utQkGBBk5OnRjMqd1jKKeWVhfQE1V5SL2cz4BOzYah3IrdK5RrNNlwcekMqbR51z9nKG+fqqoqUVh4EwpFF6FLIWpRDOXUmgwBPR+JKrVJQA+pDei+DOjUBNYYytlT3gg1NVpIJDKhy6A2QiKRoqamWugyiIjaFanEFoN8PTDI1wOV2hqcrw3oJy/k4cjZbDjZSzCoLwM6tV0M5Y3UHO0q1DHwZ4WIqGXJ7gjo5zLzkXTJPKAPVnqgLwM6tREM5URERNRmySS2CFF6IERpGtBPXMitD+i+Hhjsq2BAJ6vGUE4tasGCVwEAX375TaseS0REHY+lgJ6oUuPE+RwcSfkDzvYSBNcGdN/unSAS8S+bZD0Yyjuo4cNDGrXf1q074O39SAtXQ0RE1LwaHdCVHvDt5sqAToLj6iu17rX6Sm7udXh59Wj2awq5+sq+fbtNPv/00/fIy8vBwoVLTMZHjhwFuVz+wNfRarUAAIlE0qrHCq2lfmaIrAlXX6G2qLLKsIrLaZUaaZk3UaXVwbm2xSWEAb3D4OorZDXGj3/a5PORIwdRWFhgNn6niooK2Nk1fmnIhwnUbTGMExGRdZNJG8ygV9UgrXYVl+Pnc3A45Q84O0jrHxJlQKdWxFBOd7VgwasoKSnB0qUfICpqNS5dUmH69FmYM+c1/PrrEezYEYeMjEsoKiqEQuGBp5+ehJkzX4atra3JOYD6vvDk5CS8+eY8LF++AlevXsHPP29HUVEh/P0D8M47H6Br127NciwAbN/+E374YQvy82/Cx8cHCxYsRnT0epNzEhFRxyWT2mKw0tDCYhLQzzUI6L4KDPZlQKeWx1AukJMXchF79AryCyvg5izDs0/4INTPS+iyzBQU3MbSpYsxblw4wsMnwNPTUOPu3bsgl9sjMnI67O3lOHMmCRs3fo3S0lLMn//Wfc+7adM/IRLZYtq0WSguLsL338fgo4/+iujoTc1ybFzcNqxevQKBgcGIjHwROTk5eP/9t+Hk5ASFwuPBvyFERNQuWQzo6Xk4npaDw8kM6NTyGMoFcPJCLjbtUaGqtp88v6gSm/aoAMDqgvnNmxq8997fMHHiFJPxZcs+hkxW38byzDMR+PzzTxAXtxVz574OqVR6z/NWV1fjX//aBLHY8CPo7OyCtWtX4sqVy+jVq/dDHavVarFx43r4+fljzZqvjPv17t0Hy5cvYygnIqJ7ujOgp2beRJJKbRbQhyg90KcrAzo1D4byh3D8XA6OpeU0+bjM7EJU15g+VFpVrcO3u9Nx9Gx2k883fKA3hvl7N/m4xrCzs0N4+ASz8YaBvKysFFVVWgQEBCE+PhbXr19Dnz5973neCRMmG8MyAAQEBAIAsrP/uG8ov9+xKtVFFBYW4o03/mSyX1hYOL74YtU9z01ERNSQTGqLIf08MaSfJyqqqpFWu4pLXUB3qZtBZ0Cnh8RQLoA7A/n9xoWkUHiYBNs6V65kIjp6PZKTE1FaWmqyrbS05L7nrWuDqePk5AwAKC6+/5PQ9zs2N9fwi9KdPeZisRje3i3zywsREbV/dlKxxYB+LC0HhxjQ6SExlD+EYf4PNkP9zlfHkV9UaTbu5izDu9ODm6O0ZtNwRrxOcXExFi58Ffb2jpgzZx66dOkKqVSKjAwV1q+Pgk53/2UeRSJbi+ONWaHzYY4lIiJqDncL6L82COghvh4IUSoY0KlRGMoF8OwTPiY95QAgFYvw7BM+AlbVeCkpZ1BYWIjlyz9HYGD9LxE5OU1vvWkJXl6GX5Sysm4gICDIOF5dXY2cnBz4+Ny7PYaIiKgpLAb0dDWOpmXjYHIWXBylCOnLgE73xlAugLqHOdvC6iuWiEQiAKYz01qtFnFxW4UqyYRS2R8uLi7YsSMO48c/bWy/2b9/L4qLiwSujoiI2rM7A3rq5XwkqcwD+uB+Hujd1QUiGwZ0MmAoF0ionxdGBDwi2Bs9H4a//0A4OTlj+fJliIiIhI2NDfbt2w1r6R6RSCR45ZVXsXr151i06A2MGjUGOTk52LNnJ7p06Qob3gCJiKgV2EnFeKy/Jx7r74nyyvoWF5OA7mtY5YUBnRjKqclcXFyxYsVqfPnlGkRHr4eTkzPGjXsKISFDsGTJAqHLAwA891wk9Ho9fvhhC9atWwsfnz747LNVWLNmJaRSmdDlERFRByOXmQZ0wzKLGhxNzcbBM1lwdZRiEAN6h2aj59NxAID8/BLodJa/Fbm51+Hl1aPZrykWi9rkTHlbpdPpMHFiGJ54YhTeffevLXqtlvqZIbImCoUTNJr7r5hERHfXMKCnZeajukbHgN4KhLp/iUQ2cHNztLiNM+XULlVWVkImM50R37v3FxQVFSIoaJBAVREREZmSy8R4vL8XHu/vZQzoielq/Pds/Qy6YRUXBvT2jqGc2qW0tLNYvz4KTz45Gs7OLsjIUOGXX3agVy8fjBo1VujyiIiIzJgF9Ms3kahS48jZbBw4k4VOTjIM6qvA4H4e8OnCgN7eMJRTu/TII13g7q7Atm0/oqioEM7OLggPn4B58xZAIpEIXR4REdE9yWViPO7nhcf97hHQa19UxIDePrCnvBZ7yqk5saecOgL2lBO1vvLKapy9fBNJKjXOXbmF6hodA/oDYE85ERERET0wuUyMUD8vhNbOoNcF9CMpf+BAUv0M+hClJ3p1cWZAb0MYyomIiIjaIEsBPTHdNKDXrYPOgG79GMqJiIiI2riGAb2sor4H/XBKFvYn3WBAbwMYyomIiIjaEXs7MUIHeCF0wN0D+mClYZnFXo8woFsLhnIiIiKidurOgH72sgZJKg0OJWchIfEGOjvLjOugM6ALi6GciIiIqAOwtxNj6ABvDB3gfc+APrg2oNswoLcqhnIiIiKiDsZSQE9MVzOgC4ihnIiIiKgDMw3oWqT8Zlhm8eAZQ0B3c5ZhEAN6ixMJXQC1D7t378Tw4SHIyck2jkVETMLy5cse6NiHlZychOHDQ5CcnNRs5yQiImrv7O0kGObvjbemBmDtm8MxZ0I/dFE44uCZLCyPOYOl60/gh4O/ITO7EHz/ZPPiTHkHtXTpYiQnJ2Lnzv2Qy+UW91myZAEuXDiHHTsSIJPJWrnCxjlwYB9u3crH889PE7oUIiKidqUuoA/zr59BT7xjBj2kbhUXb86gPyyG8g4qLGw8Tpz4FceO/RdhYeFm22/fvoUzZxIxbtxTDxzI//Of7RCJWvaPMQcPJuC33zLMQnlgYDAOHjwOiUTSotcnIiLqCO4W0A8kZWHf6Rtwc7ZDiFKBwUpP9PR2YkB/AAzlHdSIEU9CLrfHgQP7LIbyQ4cOoKamBuPGmW9rLKlU+jAlPhSRSGS1s/tERERtWcOAXlqhxVkLAb1uHXQG9MZjKO+g7OzsMGLEEzh8+ACKiorg7Oxssv3AgX1wc3NDt249sHLlZzhz5jTy8vJgZ2eH4OAQzJ//Fry9H7nnNSIiJiEoaBA+/HCZcezKlUysWfM5zp8/BxcXF0yZ8izc3RVmx/766xHs2BGHjIxLKCoqhELhgaefnoSZM1+Gra0tAGDBgldx9mwyAGD48BAAgJeXN7Zt24nk5CS8+eY8fPHF1wgODjGe9+DBBHz33b9x/fo12Ns7YNiwEXj99Tfh6upq3GfBgldRUlKC//mfv2PVqhVIT78AJydnTJ36AqZPn920bzQREVE75nBHQE/JuImkS2rsT7qBvad/Z0BvAoZygZzOTcbOK3txq6IAnWSumOwTjiFewa1aQ1hYOBIS9uDIkYOYPPlPxvHc3BycP5+GiIgXkJ5+AefPp2Hs2PFQKDyQk5ONn3/ejoULX8N3322FnZ1do6+Xn38Tb745DzqdDjNmzIadnRw7dsRZnNHevXsX5HJ7REZOh729HGfOJGHjxq9RWlqK+fPfAgDMnv0KysvLkZeXg4ULlwAA5HL7u15/9+6d+OSTj+Dn54/XX38TanUetm//EenpFxAdvdmkjqKiQvzlL29i1KgxGDNmHA4fPoD166PQq1dvhIYOa/TXTERE1FE42EkwfKA3hg+sD+iJqvqA7u5iZ1hmsZ8HHvViQL8TQ7kATucm4z+q7dDqtACA25UF+I9qOwC0ajAfPPgxuLp2woED+0xC+YED+6DX6xEWNh4+Pr0xatRYk+OGDRuJefNexpEjBxEePqHR19uyZRMKCwuwcWMMfH2VAICnnpqIF1/8k9m+y5Z9DJmsPvA/80wEPv/8E8TFbcXcua9DKpVi8ODHERu7FYWFBRg//ul7Xru6uhrr10ehd+++iIraYGyt8fVVYtmyD7FzZxwiIl4w7q9W5+F///djY2vPxIlTEBExEb/8Es9QTkREdB8NA3pJuRYpvxleVGQS0JWGZRYZ0A0Yyh/CqZwzOJmT2OTjrhb+jmp9tcmYVqfFlvRtOJF9usnnC/UejMe8BzX5OLFYjNGjx+Lnn7fj5s2bcHd3BwAcOJCArl27oX//ASb7V1dXo7S0BF27doOjoxMyMlRNCuUnTx6Hv3+AMZADQKdOnRAW9hTi4raa7NswkJeVlaKqSouAgCDEx8fi+vVr6NOnb5O+VpXqIm7fvmUM9HVGjw7DunVrceLEcZNQ7ujoiLFjxxs/SyQS9Ovnh+zsP5p0XSIioo7OUS7BiIGPYMTAR0wDeuIN7D3FgF6HoVwAdwby+423pLCwcMTGbsWhQwl4/vlpuHbtKi5fzsDLL88FAFRWViAm5t/YvXsnNBq1yZqkJSUlTbpWXl4u/P0DzMa7d+9hNnblSiaio9cjOTkRpaWlJttKS5t2XcDQkmPpWiKRCF27dkNeXo7JuIeHp9lNwcnJGZmZl5t8bSIiIjKwFNATVWqTgF7Xg97RAjpD+UN4zHvQA81Q//X4J7hdWWA23knmikXB85qjtEbz9w+At3cX7N+/F88/Pw379+8FAGPbxurVn2P37p2YOvVFDBjgD0dHRwA2WLbsgxZ7aUBxcTEWLnwV9vaOmDNnHrp06QqpVIqMDBXWr4+CTqdrkes2JBLZWhznixKIiIiah1lAz9Ag8ZIaCYk3sKcDBnSGcgFM9gk36SkHAIlIgsk+D7784MMYO3YcYmK+RVbWDRw8mABf337GGeW6vvGFCxcb96+srGzyLDkAeHp6ISvrhtn4779fN/mcknIGhYWFWL78cwQG1vfYW37jZ+P+gXp5eRuv1fCcer0eWVk30LOnT6POQ0RERM3PUS7BiIBHMCKgQUBXmQf0wf080MOzfQb0ln2zC1k0xCsY05TPobOdYRm+TjJXTFM+1+qrr9QZN+4pAMCXX65GVtYNk7XJLc0Yb9/+I2pqapp8ndDQYTh3LhWXLqmMY7dv38b+/XtM9qt74VDDWWmtVmvWdw4Acrm8Ub8gKJX90alTZ/z88zZotfW/DB0+fBAajRpDh/LhTSIiImtQF9CXRAZi9cLheOkpJbw62yMh8Qb+/u8kvLfhJLYeuYxruUXt6i/YnCkXyBCvYAztGoLq6pZvxbifnj17oXfvvjh27ChEIhHGjKl/wHHo0OHYt283HBwc8eijPXHhwjkkJZ2Gi4tLk68zbdps7Nu3G0uWzEdExAuQyeywY0ccPD29UVLym3E/f/+BcHJyxvLlyxAREQkbGxvs27cblv7d+foqkZCwB1FRq6BU9odcbo/hw0ea7ScWi/H66wvxyScfYeHC1zB27Dio1XnYtu1H9Orlg0mTzFeAISIiImE5yiUYGfAIRtbOoCdnaJCkUmPfqRvY83+/Q+Fa/5BoW59BZygnAMC4ceG4fDkDQUGDjKuwAMBbb70NkUiE/fv3oLKyCv7+AVizZh2WLFnY5Gu4u7vjiy82YPXqFYiJ+bfJy4M+++z/GfdzcXHFihWr8eWXaxAdvR5OTs4YN+4phIQMwZIlC0zOOWXKc8jIUGH37l348cf/wMvL22IoB4Cnn54EqVSKLVs2Yd26tXBwcEBYWDjmzVvIt38SERFZOUsBPdFCQB+i9ER3T8c2F9Bt9O1p3v8h5OeXQKez/K3Izb0OLy/zFUIellgssoqZcmp+LfUzQ2RNFAonaDTFQpdBRB1ccVkVUn4zvKgo/dpt6PR6KFztMFjpicFKD5OAfvJCLmL/m4lbRZXo7CzDs0/4INTPq9VqFYls4ObmaHGboDPlVVVVWLt2LeLj41FUVASlUonFixcjNDS0SeeZO3cujh49ilmzZuHDDz9soWqJiIiIyNo42UuNM+jGgJ6eh72nfsfu/7sOD1c5QpQesJOIsOvkdVTVTojmF1Vi0x7Dc26tGczvRtBQ/t577yEhIQGzZs1Cjx49EBcXh7lz5yImJgZBQUGNOseRI0eQlJTUwpUSERERkbW7M6DX9aDvPfU7dBaaQ6qqdYj9b6ZVhHLBVl9JS0vDL7/8grfffhtLly5FZGQkNm3aBG9vb6xcubJR56iqqsKnn36KOXPmtHC1RERERNSWONlL8URgF/zlhSCsXnj3Vdbyiypbsaq7EyyU7927FxKJBFOnTjWOyWQyRERE4MyZM1Cr1fc9x+bNm1FRUcFQTkRERER35WQvhZuz5UUd7jbe2gQL5enp6ejZsyccHBxMxgcOHAi9Xo/09PR7Hq/RaPDVV19h8eLFkMvlLVkqEREREbVxzz7hA6nYNPpKxSI8+4R1vEBQsJ5yjUYDT09Ps3GFQgEA950pX7VqFXr27IkpU6a0SH1ERERE1H7U9Y0LufrKvQgWyisqKiCRSMzG69aLrqy8e39PWloafv75Z8TExDTbGpR3W54GANRqEWxtbVpkvUuxmC9VbW/0ej1EIhEUCiehSyFqcfw5J6K2ZPKTTpj8ZB+hy7BIsFBuZ2dn8rrzOnVh/G4vc9Hr9Vi+fDnGjRuHkJCQZqvnXuuU29jYory8HFKpXbNdD+A65e1VVVUlbGxEXL+Z2j2uU05EbZVQ9y+rXKdcoVBYbFHRaDQAAA8PD4vH7d+/H2lpaVi8eDGysrJMtpWUlCArKwvu7u6ws2u+AO3o6IKCgptwcHCBnZ0cIpFtm3tLFLU8vV4PrbYKBQUaODl1ErocIiIiakMEC+VKpRIxMTEoLS01edjx+VCPAAAJ/ElEQVQzNTXVuN2S7Oxs6HQ6zJ4922xbbGwsYmNjER0djZEjLb9q/UHI5Q4QiyUoKSlAaWkhdLqaZjmvSCSCTseZ8vbE1lYMJ6dOkMsd7r8zERERUS3BQnl4eDj+9a9/YevWrXjppZcAGNYdj42NRXBwsPEh0OzsbJSXl8PHx/Bk7OjRo9G1a1ez882fPx+jRo1CREQE/Pz8mr1eiUSKTp0sz94/KP7pl4iIiIgAAUN5QEAAwsPDsXLlSmg0GnTv3h1xcXHIzs7Gp59+atzv3XffxenTp3Hp0iUAQPfu3dG9e3eL5+zWrRvGjh3bKvUTERERETUXwUI5AKxYsQJr1qxBfHw8CgsL4evri2+++QaDBg0SsiwiIiIiolZlo9frLS850sHca/WVlsL2FSJqy3gPI6K2yhpXX+Ei2UREREREAmMoJyIiIiISmKA95dZEJBJm3XGhrktE1Bx4DyOitkqI+9e9rsmeciIiIiIigbF9hYiIiIhIYAzlREREREQCYygnIiIiIhIYQzkRERERkcAYyomIiIiIBMZQTkREREQkMIZyIiIiIiKBMZQTEREREQmMoZyIiIiISGAM5UREREREAhMLXUBHo1arsXnzZqSmpuL8+fMoKyvD5s2b8dhjjwldGhHRPaWlpSEuLg6nTp1CdnY2XF1dERQUhEWLFqFHjx5Cl0dEdFfnzp3D119/jYsXLyI/Px9OTk5QKpWYP38+goODhS4PAEN5q7t69Sqio6PRo0cP+Pr6IiUlReiSiIgaZePGjUhOTkZ4eDh8fX2h0WiwZcsWPPPMM9i2bRt8fHyELpGIyKIbN26gpqYGU6dOhUKhQHFxMXbu3IkZM2YgOjoaw4YNE7pE2Oj1er3QRXQkJSUl0Gq16NSpEw4cOID58+dzppyI2oTk5GQMGDAAUqnUOHbt2jVMmjQJEyZMwGeffSZgdURETVNeXo6xY8diwIAB2LBhg9DlcKa8tTk6OgpdAhHRA7H0J95HH30Uffr0QWZmpgAVERE9OLlcjs6dO6OoqEjoUgDwQU8iInoIer0eN2/eRKdOnYQuhYjovkpKSnDr1i1cuXIFq1atQkZGBkJDQ4UuCwBnyomI6CHs2LEDeXl5WLx4sdClEBHd1wcffIB9+/YBACQSCV544QXMmzdP4KoMGMqJiOiBZGZm4u9//zsGDRqEKVOmCF0OEdF9zZ8/H5GRkcjNzUV8fDyqqqqg1WpNnpURCttXiIioyTQaDV577TW4uLhg7dq1EIn43wkRWT9fX18MGzYMzz33HP75z3/iwoULeP/994UuCwBDORERNVFxcTHmzp2L4uJibNy4EQqFQuiSiIiaTCKRYMyYMUhISEBFRYXQ5TCUExFR41VWVmLevHm4du0aNmzYgF69egldEhHRA6uoqIBer0dpaanQpTCUExFR49TU1GDRokU4e/Ys1q5di8DAQKFLIiJqlFu3bpmNlZSUYN++ffD29oabm5sAVZnig54C+OqrrwDAuK5vfHw8zpw5A2dnZ8yYMUPI0oiI7uqzzz7DoUOHMGrUKBQUFCA+Pt64zcHBAWPHjhWwOiKiu1u0aBFkMhmCgoKgUCiQk5OD2NhY5ObmYtWqVUKXB4Bv9BSEr6+vxfEuXbrg0KFDrVwNEVHjzJw5E6dPn7a4jfcvIrJm27ZtQ3x8PC5fvoyioiI4OTkhMDAQr7zyCoYMGSJ0eQAYyomIiIiIBMeeciIiIiIigTGUExEREREJjKGciIiIiEhgDOVERERERAJjKCciIiIiEhhDORERERGRwBjKiYiIiIgExlBORESCmTlzJkaPHi10GUREghMLXQARETWvU6dOYdasWXfdbmtri4sXL7ZiRUREdD8M5URE7dTEiRMxcuRIs3GRiH8kJSKyNgzlRETtVP/+/TFlyhShyyAiokbgdAkRUQeVlZUFX19fREVFYdeuXZg0aRL8/f3x5JNPIioqCtXV1WbHqFQqzJ8/H4899hj8/f3x9NNPIzo6GjU1NWb7ajQafPzxxxgzZgwGDBiA0NBQvPzyyzh+/LjZvnl5eViyZAkGDx6MgIAAzJkzB1evXm2Rr5uIyBpxppyIqJ0qLy/HrVu3zMalUikcHR2Nnw8dOoQbN25g+vTpcHd3x6FDh/Dll18iOzsbn376qXG/c+fOYebMmRCLxcZ9Dx8+jJUrV0KlUuEf//iHcd+srCy8+OKLyM/Px5QpUzBgwACUl5cjNTUVJ06cwLBhw4z7lpWVYcaMGQgICMDixYuRlZWFzZs344033sCuXbtga2vbQt8hIiLrwVBORNRORUVFISoqymz8ySefxIYNG4yfVSoVtm3bBj8/PwDAjBkzsGDBAsTGxiIyMhKBgYEAgOXLl6Oqqgo//PADlEqlcd9FixZh165diIiIQGhoKADgo48+glqtxsaNGzFixAiT6+t0OpPPt2/fxpw5czB37lzjWOfOnfH555/jxIkTZscTEbVHDOVERO1UZGQkwsPDzcY7d+5s8nno0KHGQA4ANjY2+POf/4wDBw5g//79CAwMRH5+PlJSUhAWFmYM5HX7vv7669i7dy/279+P0NBQFBQU4Ndff8WIESMsBuo7HzQViURmq8U8/vjjAIDr168zlBNRh8BQTkTUTvXo0QNDhw69734+Pj5mY7179wYA3LhxA4ChHaXheEO9evWCSCQy7vv7779Dr9ejf//+jarTw8MDMpnMZMzV1RUAUFBQ0KhzEBG1dXzQk4iIBHWvnnG9Xt+KlRARCYehnIiog8vMzDQbu3z5MgCgW7duAICuXbuajDd05coV6HQ6477du3eHjY0N0tPTW6pkIqJ2h6GciKiDO3HiBC5cuGD8rNfrsXHjRgDA2LFjAQBubm4ICgrC4cOHkZGRYbLvN998AwAICwsDYGg9GTlyJI4ePYoTJ06YXY+z30RE5thTTkTUTl28eBHx8fEWt9WFbQBQKpWYPXs2pk+fDoVCgYMHD+LEiROYMmUKgoKCjPt9+OGHmDlzJqZPn45p06ZBoVDg8OHDOHbsGCZOnGhceQUA/va3v+HixYuYO3cunnnmGfj5+aGyshKpqano0qUL3nnnnZb7womI2iCGciKidmrXrl3YtWuXxW0JCQnGXu7Ro0ejZ8+e2LBhA65evQo3Nze88cYbeOONN0yO8ff3xw8//IAvvvgC33//PcrKytCtWze8/fbbeOWVV0z27datG7Zv345169bh6NGjiI+Ph7OzM5RKJSIjI1vmCyYiasNs9Pw7IhFRh5SVlYUxY8ZgwYIFWLhwodDlEBF1aOwpJyIiIiISGEM5EREREZHAGMqJiIiIiATGnnIiIiIiIoFxppyIiIiISGAM5UREREREAmMoJyIiIiISGEM5EREREZHAGMqJiIiIiATGUE5EREREJLD/D4lVrZFrUvN8AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-mSjPuJZ-XTl"
      },
      "source": [
        "### 6.5.2 - Save Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w13kj3lHKLIv"
      },
      "source": [
        "Save the model and the corresponding state."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k4KKmHUcJeQv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b2deb9d1-04c4-486d-e24f-c344f4a435fa"
      },
      "source": [
        "print('Saving model, tokenizer and training arguments...')\n",
        "trainer.save_model()\n",
        "trainer.save_state()\n",
        "\n",
        "print('Saving training statistics..')\n",
        "save_json(qa_dir/'training_stats.json', training_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving model, tokenizer and training arguments...\n",
            "Saving training statistics..\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jv56bYwLH1IG"
      },
      "source": [
        "## 6.6 - Evaluation of the Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6xr8LoOJWdpV"
      },
      "source": [
        "Once the model is trained, in order to perform the predictions and get an intuition about the result, we need to map the predictions back to parts of the context. The model itself predicts logits for the start and end position of our answers. The output of the model is a dict-like object that contains the cross-entropy loss (since we provided labels), the start and end logits. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZ2CpVHdH5Ic",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4286a080-3dee-4ae6-8544-defffdb1473f"
      },
      "source": [
        "for batch in trainer.get_test_dataloader(tokenized_datasets['test']):\n",
        "    break\n",
        "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n",
        "with torch.no_grad():\n",
        "    output = bert_qa(**batch)\n",
        "\n",
        "print('Output of the model')\n",
        "print(output.keys())\n",
        "print('\\nDimension start and end logits')\n",
        "print(tuple(output.start_logits.shape))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Output of the model\n",
            "odict_keys(['loss', 'start_logits', 'end_logits'])\n",
            "\n",
            "Dimension start and end logits\n",
            "(16, 384)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e_dH31wtYt8U"
      },
      "source": [
        "There is one logit per feature and token. One idea to predict an answer is to take the index for the maximum of the start logits as a star position and the index of the maximum of the ends logits as an end position. The problem is if this prediction gives us an impossible result, as a greater start position than the end one.\n",
        "\n",
        "To classify the answer, we add the start and end logits to obtain a score. With hyper-parameter `n_best_size` we limit the possible answers. Best indices in the start and end logits are picked and gather all the answers this predicts. After checking if each one is valid, we can sort the `valid_answers` according to their `score` and only keep the best one."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8mwi1uEyE7Fo"
      },
      "source": [
        "The only point left is how to check a given span is inside the context (and not the question) and how to get back the text inside. To do this, we need to add two things to our validation features:\n",
        "- the ID of the example that generated the feature (since each example can generate several features, as seen before);\n",
        "- the offset mapping that will give us a map from token indices to character positions in the context.\n",
        "\n",
        "That's why we will re-process the validation set with the following function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQVj0dCbNJbl"
      },
      "source": [
        "def prepare_test_features(examples):\n",
        "    # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n",
        "    # in one example possible giving several features when a context is long, each of those features having a\n",
        "    # context that overlaps a bit the context of the previous feature.\n",
        "    tokenized_examples = tokenizer(\n",
        "        examples[\"question\"],\n",
        "        examples[\"context\"],\n",
        "        truncation=\"only_second\",\n",
        "        max_length=MAX_LENGTH,\n",
        "        stride=DOC_STRIDE,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "        padding='max_length'\n",
        "    )\n",
        "\n",
        "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "    # its corresponding example. This key gives us just that.\n",
        "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
        "\n",
        "    # We keep the example_id that gave us this feature and we will store the offset mappings.\n",
        "    tokenized_examples[\"example_id\"] = []\n",
        "\n",
        "    for i in range(len(tokenized_examples[\"input_ids\"])):\n",
        "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
        "\n",
        "        # One example can give several spans, this is the index of the example containing this span of text.\n",
        "        sample_index = sample_mapping[i]\n",
        "        tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n",
        "\n",
        "        # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n",
        "        # position is part of the context or not.\n",
        "        tokenized_examples[\"offset_mapping\"][i] = [\n",
        "            (o if sequence_ids[k] == 1 else None)\n",
        "            for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n",
        "        ]\n",
        "\n",
        "    return tokenized_examples"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P1oLfP50bSAU"
      },
      "source": [
        "And like before, we can apply that function to our test set easily."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ysa43PrQhhE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "b79f4abd93cf4694908f04f82100d8e3",
            "e5417262ec164e7c8afcc668810af42d",
            "0555bca300d1466aad538965818b01e6",
            "522fc8cf79b144c2a950c27523594972",
            "30b34b78dd3f4904ad4174bc335aac7f",
            "ac6c58d2182c450b9797c1cafb70bedb",
            "4ca84699247f4878a4665d2cc42938ad",
            "6ac76a04d4d54c90b0f80929030249fd"
          ]
        },
        "outputId": "4a4f03b2-625f-4ac2-c546-3a72f3085722"
      },
      "source": [
        "test_features = datasets['test'].map(\n",
        "    prepare_test_features,\n",
        "    batched=True,\n",
        "    load_from_cache_file=False,\n",
        "    remove_columns=datasets['test'].column_names\n",
        ")\n",
        "test_features = test_features.add_column('start_positions', tokenized_datasets['test']['start_positions'])\n",
        "test_features = test_features.add_column('end_positions', tokenized_datasets['test']['end_positions'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b79f4abd93cf4694908f04f82100d8e3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=29.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dnN3-DI6bXG2"
      },
      "source": [
        "Now we can get the predictions and the metrics for all features by using the [`predict`](https://huggingface.co/transformers/main_classes/trainer.html?highlight=trainer%20predict#transformers.Trainer.predict)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7aw7mq9RM0F",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "8c214949-cce2-4375-9281-8649a75d5b98"
      },
      "source": [
        "info_device()\n",
        "print('Testing\\n')\n",
        "raw_predictions = trainer.predict(test_features)\n",
        "\n",
        "test_stats = {\n",
        "    'test_loss': raw_predictions.metrics['test_loss'],\n",
        "    'test_runtime': format_time(raw_predictions.metrics['test_runtime'])\n",
        "}\n",
        "\n",
        "print('\\n')\n",
        "print(f\"{'Test Loss':^12} | {'Test time':^11}\")\n",
        "print(\"-\"*27)\n",
        "print(f\"{test_stats['test_loss']:^12.6f} | {test_stats['test_runtime']:^11}\")\n",
        "print(\"-\"*27)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Using GPU Tesla P100-PCIE-16GB. \n",
            "\n",
            "Testing\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1825' max='1825' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1825/1825 06:45]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            " Test Loss   |  Test time \n",
            "---------------------------\n",
            "  1.004139   |   0:06:46  \n",
            "---------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-tq29b5KZJQ9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f159030d-77b9-4646-d215-5a491b4b206d"
      },
      "source": [
        "print('Saving test statistics...')\n",
        "save_json(qa_dir/'test_stats.json', test_stats)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving test statistics...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qbG4KyzJbut_"
      },
      "source": [
        "The `Trainer` *hides* the columns that are not used by the model (here `example_id` and `offset_mapping` which we will need for our post-processing), so we set them back:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s0d33x7Me3OW"
      },
      "source": [
        "test_features.set_format(type=test_features.format[\"type\"], columns=list(test_features.features.keys()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkeV1CEqcHsW"
      },
      "source": [
        "We set `None` in the offset mappings when it corresponds to a part of the question, so it's easy to check if an answer is fully inside the context. We also eliminate very long answers from our considerations with hyper-parameter `max_answer_length`.\n",
        "\n",
        "We also need a map between examples and their corresponding features. Also, we gather together all the answers in all the features generated by a given example, then pick the best one.\n",
        "\n",
        "The last bit to deal with is the impossible answer (when `squad_v2 = True`). We need to also grab the score for the impossible answer (which has start and end indices corresponding to the index of the CLS token). When one example gives several features, we have to predict the impossible answer when all the features give a high score to the impossible answer, since one feature could predict the impossible answer just because the answer isn't in the part of the context it has access too. We then predict the impossible answer when that score is greater than the score of the best non-impossible answer. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpR-1V_cwz8O"
      },
      "source": [
        "import collections\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "def postprocess_predictions(examples, features, raw_predictions, n_best_size = 20, max_answer_length = 30):\n",
        "    m = nn.Softmax(dim=1)\n",
        "    all_start_logits, all_end_logits = raw_predictions\n",
        "    # Build a map example to its corresponding features.\n",
        "    example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n",
        "    features_per_example = collections.defaultdict(list)\n",
        "    for i, feature in enumerate(features):\n",
        "        features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)\n",
        "\n",
        "    # The dictionaries we have to fill.\n",
        "    predictions = collections.OrderedDict()\n",
        "\n",
        "    # Logging.\n",
        "    print(f\"Post-processing {len(examples)} example predictions split into {len(features)} features.\")\n",
        "\n",
        "    # Let's loop over all the examples!\n",
        "    for example_index, example in enumerate(tqdm(examples)):\n",
        "        # Those are the indices of the features associated to the current example.\n",
        "        feature_indices = features_per_example[example_index]\n",
        "\n",
        "        min_null_score = None # Only used if squad_v2 is True.\n",
        "        valid_answers = []\n",
        "        \n",
        "        context = example[\"context\"]\n",
        "        # Looping through all the features associated to the current example.\n",
        "        for feature_index in feature_indices:\n",
        "            # We grab the predictions of the model for this feature.\n",
        "            start_logits = all_start_logits[feature_index]\n",
        "            end_logits = all_end_logits[feature_index]\n",
        "            # This is what will allow us to map some the positions in our logits to span of texts in the original\n",
        "            # context.\n",
        "            offset_mapping = features[feature_index][\"offset_mapping\"]\n",
        "\n",
        "            # Update minimum null prediction.\n",
        "            cls_index = features[feature_index][\"input_ids\"].index(tokenizer.cls_token_id)\n",
        "            feature_null_score = start_logits[cls_index] + end_logits[cls_index]\n",
        "            if min_null_score is None or min_null_score < feature_null_score:\n",
        "                min_null_score = feature_null_score\n",
        "\n",
        "            # Go through all possibilities for the `n_best_size` greater start and end logits.\n",
        "            start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "            end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "            for start_index in start_indexes:\n",
        "                for end_index in end_indexes:\n",
        "                    # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
        "                    # to part of the input_ids that are not in the context.\n",
        "                    if (\n",
        "                        start_index >= len(offset_mapping)\n",
        "                        or end_index >= len(offset_mapping)\n",
        "                        or offset_mapping[start_index] is None\n",
        "                        or offset_mapping[end_index] is None\n",
        "                    ):\n",
        "                        continue\n",
        "                    # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
        "                    if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n",
        "                        continue\n",
        "\n",
        "                    start_char = offset_mapping[start_index][0]\n",
        "                    end_char = offset_mapping[end_index][1]\n",
        "                    valid_answers.append(\n",
        "                        {\n",
        "                            \"score\": start_logits[start_index] + end_logits[end_index],\n",
        "                            \"text\": context[start_char: end_char]\n",
        "                        }\n",
        "                    )\n",
        "        \n",
        "        if len(valid_answers) > 0:\n",
        "            best_answer = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[0]\n",
        "        else:\n",
        "            # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid\n",
        "            # failure.\n",
        "            best_answer = {\"text\": \"\", \"score\": 0.0}\n",
        "\n",
        "        # Let's pick our final answer: the best one or the null answer (only for squad_v2)\n",
        "        if not squad_v2:\n",
        "            predictions[example[\"id\"]] = best_answer[\"text\"]\n",
        "        else:\n",
        "            predictions[example[\"id\"]] = best_answer[\"text\"] if best_answer[\"score\"] > min_null_score else ''\n",
        "\n",
        "    return predictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MbIqiaakkbH"
      },
      "source": [
        "And we can apply our post-processing function to our raw predictions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trrh8PkOerpE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "9c420c3bfa424906bfa494b226e2907a",
            "20d6490579914d69b003a553cadf1644",
            "5acee8e2818646c18e6dc08d5dc3a94c",
            "5e52156ead384641aff0366d800c935e",
            "e4bdf4e8f4ee4eb19a0b5ac0ae4a2d96",
            "237ffb1f0cfc4375ab4d6a3c42e8cfa6",
            "fe4f75b3aa594325aeb80063c0bafb9d",
            "15d6ee18dbb74c75bb16c78e33484abc"
          ]
        },
        "outputId": "eca9a510-cdae-4f13-855d-df921c649386"
      },
      "source": [
        "final_predictions = postprocess_predictions(datasets['test'], test_features, raw_predictions.predictions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Post-processing 28908 example predictions split into 29197 features.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9c420c3bfa424906bfa494b226e2907a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=28908.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RHwf6B2iNKDI"
      },
      "source": [
        "Now it is necessary to indicate how to compute metrics from the predictions. We use a [`metric`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Metric) which is in charge of compute the corresponding metrics for each model. We load it using the class method [`load_metric`](https://huggingface.co/docs/datasets/v1.4.1/package_reference/loading_methods.html?highlight=load_metric#datasets.load_metric)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cu29AdtVe-kd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 577,
          "referenced_widgets": [
            "83c5de0bd35c480fb448c90f2c16c51f",
            "1a21f1dc66124e63a9b4952f30bd1166",
            "807de3b18c7d41c9b11e45bdcfe11935",
            "62a57b34b2fe4d1a8a2d128a8ba17cea",
            "ad944b55b2a24c34a3dafb9852202857",
            "ed80463428864e0fa11b67f034c08963",
            "ba1bf85fa0644758aa6abe5b807ea69a",
            "e63fa7ac974e4d23aec70a65252f618d",
            "ce5ba1af33684a328f83a33d7888841f",
            "cade3114c79e4bc692ca31e2eb1ee8ff",
            "dfeb105a607f49a59ffafdb08bdef022",
            "58e29c1f24b04521bde60aa8c9f6441b",
            "6b8fc7f02aad4816acdbe991251a052c",
            "505d0af9bbda41598afdffddc36095e4",
            "70dc6ac663a54e718adb1315ff36da0f",
            "caf02f85fb02418a850e3776b0709188"
          ]
        },
        "outputId": "d3daac41-5171-4673-d1d1-77e5d0a03695"
      },
      "source": [
        "from datasets import load_metric\n",
        "\n",
        "metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n",
        "metric"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "83c5de0bd35c480fb448c90f2c16c51f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1726.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ce5ba1af33684a328f83a33d7888841f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1119.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Metric(name: \"squad\", features: {'predictions': {'id': Value(dtype='string', id=None), 'prediction_text': Value(dtype='string', id=None)}, 'references': {'id': Value(dtype='string', id=None), 'answers': Sequence(feature={'text': Value(dtype='string', id=None), 'answer_start': Value(dtype='int32', id=None)}, length=-1, id=None)}}, usage: \"\"\"\n",
              "Computes SQuAD scores (F1 and EM).\n",
              "Args:\n",
              "    predictions: List of question-answers dictionaries with the following key-values:\n",
              "        - 'id': id of the question-answer pair as given in the references (see below)\n",
              "        - 'prediction_text': the text of the answer\n",
              "    references: List of question-answers dictionaries with the following key-values:\n",
              "        - 'id': id of the question-answer pair (see above),\n",
              "        - 'answers': a Dict in the SQuAD dataset format\n",
              "            {\n",
              "                'text': list of possible texts for the answer, as a list of strings\n",
              "                'answer_start': list of start positions for the answer, as a list of ints\n",
              "            }\n",
              "            Note that answer_start values are not taken into account to compute the metric.\n",
              "Returns:\n",
              "    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n",
              "    'f1': The F-score of predicted tokens versus the gold answer\n",
              "Examples:\n",
              "\n",
              "    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}]\n",
              "    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]\n",
              "    >>> squad_metric = datasets.load_metric(\"squad\")\n",
              "    >>> results = squad_metric.compute(predictions=predictions, references=references)\n",
              "    >>> print(results)\n",
              "    {'exact_match': 100.0, 'f1': 100.0}\n",
              "\"\"\", stored examples: 0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xrcNU7OykqAU"
      },
      "source": [
        "We just need to format predictions and labels a bit as it expects a list of dictionaries and not one big dictionary. In the case of squad_v2, we also have to set a `no_answer_probability` argument which we set to 0.0.\n",
        "\n",
        "Metrics computation returns:\n",
        "    \n",
        "- `exact_match`: exact match (the normalized answer exactly match the gold answer).\n",
        "\n",
        "- `f1`: the F-score of predicted tokens versus the gold answer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXzmbIchfLgv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4461bf04-6262-4684-da4a-9ce6a6406f99"
      },
      "source": [
        "if squad_v2:\n",
        "    formatted_predictions = [{\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in final_predictions.items()]\n",
        "else:\n",
        "    formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
        "references = [{\"id\": ex[\"id\"], \"answers\": ex[\"answers\"]} for ex in datasets['test']]\n",
        "scores = metric.compute(predictions=formatted_predictions, references=references)\n",
        "\n",
        "print(f\"{'Exact match':^13} | {'f1 score':^10}\")\n",
        "print(26*'-')\n",
        "print(f\"{scores['exact_match']:^13.5f} | {scores['f1']:^10.5f}\")\n",
        "print(26*'-')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Exact match  |  f1 score \n",
            "--------------------------\n",
            "  66.04400    |  79.93587 \n",
            "--------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z70BC9qnnH2I"
      },
      "source": [
        "def show_predictions(dataset, predictions, num_examples):\n",
        "    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n",
        "    picks = []\n",
        "    for _ in range(num_examples):\n",
        "        pick = random.randint(0, len(dataset)-1)\n",
        "        while pick in picks:\n",
        "            pick = random.randint(0, len(dataset)-1)\n",
        "        picks.append(pick)\n",
        "    \n",
        "    df = pd.DataFrame(dataset[picks])\n",
        "    df.drop(['id', 'title'], axis=1, inplace=True)\n",
        "    df['answers'] = df['answers'].map(lambda dic: dic['text'][0]).values\n",
        "    df['predicted answers'] = [predictions[idx]['prediction_text'] for idx in picks]\n",
        "    df = df[['context', 'question', 'answers', 'predicted answers']]\n",
        "    for column, typ in dataset.features.items():\n",
        "        if isinstance(typ, ClassLabel):\n",
        "            df[column] = df[column].transform(lambda i: typ.names[i])\n",
        "        elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):\n",
        "            df[column] = df[column].transform(lambda x: [typ.feature.names[i] for i in x])\n",
        "    display(HTML(df.to_html()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kojOMeuaoi5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "outputId": "62f07a6b-8269-4d62-8a1b-f7cd8e9f4f07"
      },
      "source": [
        "#@markdown Select number of examples to visualize it\n",
        "num_examples = 3 #@param {type:\"slider\", min:1, max:5, step:1}\n",
        "show_predictions(datasets['test'], formatted_predictions, num_examples)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>context</th>\n",
              "      <th>question</th>\n",
              "      <th>answers</th>\n",
              "      <th>predicted answers</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>As a result of continued warming, the polar ice caps melted and much of Gondwana became a desert. In Eastern Antarctica, seed ferns or pteridosperms became abundant and large amounts of sandstone and shale were laid down at this time. Synapsids, commonly known as \"mammal-like reptiles\", were common in Antarctica during the Early Triassic and included forms such as Lystrosaurus. The Antarctic Peninsula began to form during the Jurassic period (206–146 Ma), and islands gradually rose out of the ocean. Ginkgo trees, conifers, bennettites, horsetails, ferns and cycads were plentiful during this period. In West Antarctica, coniferous forests dominated through the entire Cretaceous period (146–66 Ma), though southern beech became more prominent towards the end of this period. Ammonites were common in the seas around Antarctica, and dinosaurs were also present, though only three Antarctic dinosaur genera (Cryolophosaurus and Glacialisaurus, from the Hanson Formation, and Antarctopelta) have been described to date. It was during this era that Gondwana began to break up.</td>\n",
              "      <td>When did the Antarctic peninsula form?</td>\n",
              "      <td>Jurassic period</td>\n",
              "      <td>Jurassic period</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>One way to divide hunter-gatherer groups is by their return systems. James Woodburn uses the categories \"immediate return\" hunter-gatherers for egalitarian and \"delayed return\" for nonegalitarian. Immediate return foragers consume their food within a day or two after they procure it. Delayed return foragers store the surplus food (Kelly, 31).</td>\n",
              "      <td>To what kind of group does the delayed return system belong?</td>\n",
              "      <td>nonegalitarian</td>\n",
              "      <td>nonegalitarian</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>New security features in Windows 8 include two new authentication methods tailored towards touchscreens (PINs and picture passwords), the addition of antivirus capabilities to Windows Defender (bringing it in parity with Microsoft Security Essentials). SmartScreen filtering integrated into Windows, Family Safety offers Parental controls, which allows parents to monitor and manage their children's activities on a device with activity reports and safety controls. Windows 8 also provides integrated system recovery through the new \"Refresh\" and \"Reset\" functions, including system recovery from USB drive. Windows 8's first security patches would be released on November 13, 2012; it would contain three fixes deemed \"critical\" by the company.</td>\n",
              "      <td>What new security features did Windows 8 provide?</td>\n",
              "      <td>PINs and picture passwords</td>\n",
              "      <td>two new authentication methods tailored towards touchscreens</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CqxEtUppTDNv"
      },
      "source": [
        "### 6.6.1 - Load Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cDnAaCuTTLzT"
      },
      "source": [
        "And if we want to load it again."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xu8hdE0jTF-r"
      },
      "source": [
        "'''model_dir = \"/content/drive/MyDrive/Degree/TFG/Models/BertForQuestionAnswering\" #@param [\"question_answering\", \"/content/drive/MyDrive/Degree/TFG/Models/BertForQuestionAnswering\"]\n",
        "\n",
        "tokenizer = BertTokenizerFast.from_pretrained(model_dir)\n",
        "bert_qa = BertForQuestionAnswering.from_pretrained(model_dir)\n",
        "trainer = Trainer(bert_qa)\n",
        "bert_qa.to(device)'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G5RmM7PStRs8"
      },
      "source": [
        "### 6.6.2 - Final Application for Question Answering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2UW-J8ZtZrZ"
      },
      "source": [
        "Finally, it is developed a pipeline to process and try to answer to a question from our own text."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jXsY72rh1NF_"
      },
      "source": [
        "from datasets import Dataset\n",
        "\n",
        "def bert_for_qa(context, question, trainer, id='0'):\n",
        "    # Generate dataset from input data\n",
        "    dataset = Dataset.from_dict(\n",
        "        {'id': [id], 'context': [context], 'question': [question]}\n",
        "    )\n",
        "    # Preprocess data\n",
        "    features = dataset.map(\n",
        "        prepare_test_features,\n",
        "        batched=True,\n",
        "        remove_columns=dataset.column_names\n",
        "    )\n",
        "    # Predictions\n",
        "    predictions = trainer.predict(features)\n",
        "    # Postprocess data\n",
        "    processed_predictions = postprocess_predictions(dataset, features, predictions.predictions)\n",
        "    # Return answer\n",
        "    return processed_predictions[id]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUQk4hhc1GYh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220,
          "referenced_widgets": [
            "87096b907d67440da538eb7f797b7848",
            "8c62b21898a04fc29511db53f032507e",
            "ec16172829e24547a80dd04789d709f0",
            "453876ce77004371987e12586fa6e410",
            "fc7ee193eae245f182a5a2b55c3cfd6f",
            "32df2363e0864fdaab1b3131a7e8204a",
            "7f609a3427ad4578bdcac0279f7f17d5",
            "6c28646616f3409ab97b1e20736f8399",
            "eeaf41eb4e194187bc1630b504a379ca",
            "452da70ac94b4c97aae5421860abc59e",
            "8b6b5ff3319e478d939d67359368aa20",
            "bd9bf739e5584fd8891e7115ff80661e",
            "8ac5a1e04cdf45f39739c5d706c09d96",
            "1f765ad233eb49c7a742d032b6924e87",
            "869509b4ddf74fefb7182836509b9f9f",
            "5a2ddcd374ce4cceaf1291587a2dd2ab"
          ]
        },
        "outputId": "13385467-35b1-40db-cfe3-06cb79f9a500"
      },
      "source": [
        "#@title { vertical-output: true }\n",
        "context = \"BERT is really easy to implement. Definitely, it is the best NLP model\" #@param {type:\"string\"}\n",
        "question = \"What is the best NLP model?\" #@param {type:\"string\"}\n",
        "answer = bert_for_qa(context, question, trainer)\n",
        "\n",
        "print('\\nThe predicted answer is\\n')\n",
        "print(answer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "87096b907d67440da538eb7f797b7848",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1826' max='1825' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1825/1825 08:54]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Post-processing 1 example predictions split into 1 features.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eeaf41eb4e194187bc1630b504a379ca",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "The predicted answer is\n",
            "\n",
            "BERT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NuyFHz59oCNt"
      },
      "source": [
        "To download the associated folder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lhOP8g9MoBzP"
      },
      "source": [
        "# download_folder(qa_dir)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4kAb-OLzm-Ux"
      },
      "source": [
        "# Cite"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kO9VRbFInDyU"
      },
      "source": [
        "Thanks for reading! If you find it useful, feel free to use it. You can cite it as:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qKqgXEO0nW5L"
      },
      "source": [
        "```\n",
        "@mastersthesis{bueno2021bert,\n",
        "  author    = {Bueno, Ion},\n",
        "  title     = {Inside a NLP multitasking neural network. BERT, one model to rule them all},\n",
        "  school    = {University Carlos III of Madrid},\n",
        "  year      = {2021},\n",
        "  month     = {jun},\n",
        "  address   = {Madrid, Spain},\n",
        "  url       = {https://github.com/ion-bueno/bert-from-inside}\n",
        "}\n",
        "```"
      ]
    }
  ]
}