Sequencing_turning_sentences_into_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sequencing: turning sentences into data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO5S881MAsJf15zuY54UIQR",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shelvi31/Natural-Language-Processing/blob/main/Sequencing_turning_sentences_into_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gl-X-QJbAiI8",
        "outputId": "146f7d6a-87bc-4b0f-896d-4d4110cd6ce4"
      },
      "source": [
        "#importing\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "#getting tokenizer api from tensorflow keras\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "\n",
        "#defining sentences as a python array of strings     #feeding input\n",
        "sentences = [\n",
        "             \"i love my dog\",\n",
        "             \"i love myself\",\n",
        "             \"you love my dog\",\n",
        "             \"do you think my Dog is amazing?\"\n",
        "]\n",
        "\n",
        "\n",
        "#create instance of tokenizer objects\n",
        "tokenizer = Tokenizer(num_words=100,oov_token=\"<OOV>\")     #num_words parameter is the max no. of words to keep\n",
        "\n",
        "\n",
        "#asking tokenizer to go to text and fix them in sentences\n",
        "tokenizer.fit_on_texts(sentences)\n",
        "\n",
        "\n",
        "#indexing the words\n",
        "word_index = tokenizer.word_index\n",
        "print(\"word_index=\",word_index)\n",
        "\n",
        "\n",
        "#turning sentences containing these words into sequences of Numbers(data)\n",
        "sequences = tokenizer.texts_to_sequences(sentences)\n",
        "\n",
        "#text_to_sequences method that creates sequencing of token thus representing each sentences\n",
        "print(\"sequences\",sequences)\n",
        "\n",
        "#when neural network tends to classiy txt , what happen when it encounter words it has not seen before\n",
        "\n",
        "#Result : Tokenizer ignored the unknown words , \n",
        "#\"i really love my dog\"  : 5 ---- 4 , unknown : really\n",
        "# \"my dog loves my house\" : 5 --- 3 , unknown : loves, house\n",
        "\n",
        "test_data = [\n",
        "             \"i really love my dog\",\n",
        "             \"my dog loves my house\"\n",
        "]\n",
        "\n",
        "#USING OOV TOKEN PROPERTY : HANDLING UNKNOWN WORDS\n",
        "#the tokenizer will create a token for an unknown word using \"<OOV>\", and replaced the unknown word with the token no.\n",
        "#this helps in making sequence length as the same length of sentence\n",
        "test_seq = tokenizer.texts_to_sequences(test_data)\n",
        "print(\"test_seq after oov=\",test_seq)\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "word_index= {'<OOV>': 1, 'love': 2, 'my': 3, 'dog': 4, 'i': 5, 'you': 6, 'myself': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n",
            "sequences [[5, 2, 3, 4], [5, 2, 7], [6, 2, 3, 4], [8, 6, 9, 3, 4, 10, 11]]\n",
            "test_seq after oov= [[5, 1, 2, 3, 4], [3, 4, 1, 3, 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2AQxvSGqB4HX",
        "outputId": "c30d4664-df94-4b32-e8eb-e3f6b134b1c2"
      },
      "source": [
        "#Handling sentences of Different Length : USING PADDING\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "padded = pad_sequences(sequences)\n",
        "print(padded)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0  0  0  5  2  3  4]\n",
            " [ 0  0  0  0  5  2  7]\n",
            " [ 0  0  0  6  2  3  4]\n",
            " [ 8  6  9  3  4 10 11]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0FDX0rBJCpgV",
        "outputId": "c19a2c63-650a-427e-fd7c-ae9bec1e61c8"
      },
      "source": [
        "padded_post = pad_sequences(sequences, padding=\"post\")\n",
        "print(padded_post)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 5  2  3  4  0  0  0]\n",
            " [ 5  2  7  0  0  0  0]\n",
            " [ 6  2  3  4  0  0  0]\n",
            " [ 8  6  9  3  4 10 11]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g11Zb9deEkH0",
        "outputId": "014e0f17-8c45-487c-afe6-5cbd2fffa08a"
      },
      "source": [
        "padded_post2 = pad_sequences(sequences, padding=\"post\",truncating=\"post\",maxlen=5)\n",
        "print(padded_post2)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[5 2 3 4 0]\n",
            " [5 2 7 0 0]\n",
            " [6 2 3 4 0]\n",
            " [8 6 9 3 4]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}