Tokenizing_words.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Tokenizing words.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNecfSqjcTJKLoUG8q9ySKL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shelvi31/Natural-Language-Processing/blob/main/Tokenizing_words.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUSnLIEw0C0W"
      },
      "source": [
        "#importing\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "#getting tokenizer api from tensorflow keras\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S54K-Z5D0Tce"
      },
      "source": [
        "#defining sentences as a python array of strings     #feeding input\n",
        "\n",
        "sentences = [\n",
        "             \"i love my dog\",\n",
        "             \"i love myself\",\n",
        "             \"you love my dog\",\n",
        "             \"do you think my Dog is amazing?\"\n",
        "]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XzMH-0t80gmE"
      },
      "source": [
        "#create instance of tokenizer objects\n",
        "tokenizer = Tokenizer(num_words=100)     #num_words parameter is the max no. of words to keep"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1D7V--i1ehV"
      },
      "source": [
        "#asking tokenizer to go to text and fix them in sentences\n",
        "tokenizer.fit_on_texts(sentences)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xEvbvPXA1p7X"
      },
      "source": [
        "#indexing the words\n",
        "word_index = tokenizer.word_index"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "izsPAX3r16O2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c67ace2d-9ce0-4e0a-efcb-bd56b0f06e50"
      },
      "source": [
        "print(word_index)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'love': 1, 'my': 2, 'dog': 3, 'i': 4, 'you': 5, 'myself': 6, 'do': 7, 'think': 8, 'is': 9, 'amazing': 10}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNzIVNBc19wN"
      },
      "source": [
        "#turning sentences containing these words into sequences of Numbers(data)\n",
        "sequences = tokenizer.texts_to_sequences(sentences)\n",
        "\n",
        "#text_to_sequences method that creates sequencing of token thus representing each sentences"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5xwnKkEY8xQY",
        "outputId": "6a1a20a3-0f1e-4de5-d17e-0c2d781d72d4"
      },
      "source": [
        "print(sequences)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[4, 1, 2, 3], [4, 1, 6], [5, 1, 2, 3], [7, 5, 8, 2, 3, 9, 10]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9EfC6ZlC85JF",
        "outputId": "7d5ed370-d127-4c55-9eda-4c45afc3c550"
      },
      "source": [
        "#when neural network tends to classiy txt , what happen when it encounter words it has not seen before let's see\n",
        "\n",
        "test_data = [\n",
        "             \"i really love my dog\",\n",
        "             \"my dog loves my house\"\n",
        "]\n",
        "\n",
        "test_seq = tokenizer.texts_to_sequences(test_data)\n",
        "print(test_seq)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[4, 1, 2, 3], [2, 3, 2]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wd3dXTwh_DNU"
      },
      "source": [
        "#Result : Tokenizer ignored the unknown words , \n",
        "#\"i really love my dog\"  : 5 ---- 4 , unknown : really\n",
        "# \"my dog loves my house\" : 5 --- 3 , unknown : loves, house"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g-zxtnVQ_XIi"
      },
      "source": [
        "#USING OOV TOKEN PROPERTY\n",
        "#the tokenizer will create a token for an unknown word using \"<OOV>\", and replaced the unknown word with the token no.\n",
        "\n",
        "#this helps in making sequence length as the same length of sentence"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}