scrappy_ai.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "9ba85b0f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ba85b0f",
        "outputId": "d0e63aad-6895-4af9-8edf-99525e5086a8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting autoscraper\n",
            "  Downloading autoscraper-1.1.14-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from autoscraper) (2.31.0)\n",
            "Collecting bs4 (from autoscraper)\n",
            "  Downloading bs4-0.0.2-py2.py3-none-any.whl (1.2 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from autoscraper) (4.9.4)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from bs4->autoscraper) (4.12.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->autoscraper) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->autoscraper) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->autoscraper) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->autoscraper) (2024.2.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->bs4->autoscraper) (2.5)\n",
            "Installing collected packages: bs4, autoscraper\n",
            "Successfully installed autoscraper-1.1.14 bs4-0.0.2\n"
          ]
        }
      ],
      "source": [
        "!pip install autoscraper"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "c32a7625",
      "metadata": {
        "id": "c32a7625"
      },
      "outputs": [],
      "source": [
        "from autoscraper import AutoScraper"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "id": "f1492bd8",
      "metadata": {
        "id": "f1492bd8"
      },
      "outputs": [],
      "source": [
        "amazon_url=\"https://www.amazon.in/s?k=headphones\"\n",
        "\n",
        "wanted_list=[\"₹7,790\",\"Sennheiser Professional Audio HD 280 PRO Wired Over Ear Headphones\",\"7,969\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "id": "11cc44b8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "11cc44b8",
        "outputId": "63f49cf7-41cf-45a4-9d1b-528dcf98aa67"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[]\n"
          ]
        }
      ],
      "source": [
        "scraper=AutoScraper()\n",
        "result=scraper.build(amazon_url,wanted_list)\n",
        "print(result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "id": "32cef483",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "32cef483",
        "outputId": "cb1b8e84-1a88-4f68-b780-dd899bb414ef"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "scraper.get_result_similar(amazon_url,grouped=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "id": "3b1d93c4",
      "metadata": {
        "id": "3b1d93c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "outputId": "985eb90f-c98c-4ab1-f420-a9fda0fa5fad"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "'rule_9joo'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-c5ad870cbc8b>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscraper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_rule_aliases\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'rule_9joo'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'Title'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rule_gvkw'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'Price'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rule_oycw'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'Ratings'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mscraper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeep_rules\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'rule_9joo'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rule_gvkw'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rule_oycw'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mscraper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'amazon-search'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/autoscraper/auto_scraper.py\u001b[0m in \u001b[0;36mset_rule_aliases\u001b[0;34m(self, rule_aliases)\u001b[0m\n\u001b[1;32m    717\u001b[0m         \u001b[0mid_to_stack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"stack_id\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mstack\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack_list\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    718\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mrule_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malias\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrule_aliases\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 719\u001b[0;31m             \u001b[0mid_to_stack\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mrule_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"alias\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    720\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgenerate_python_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'rule_9joo'"
          ]
        }
      ],
      "source": [
        "scraper.set_rule_aliases({'rule_9joo':'Title','rule_gvkw':'Price','rule_oycw': 'Ratings'})\n",
        "scraper.keep_rules(['rule_9joo','rule_gvkw','rule_oycw'])\n",
        "scraper.save('amazon-search')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e9f46ecb",
      "metadata": {
        "id": "e9f46ecb"
      },
      "outputs": [],
      "source": [
        "results=scraper.get_result_similar('https://www.amazon.in/s?k=mi+phones+under+15000',group_by_alias=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c6ddeb9a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "id": "c6ddeb9a",
        "outputId": "2970a86f-ed48-4630-da78-fcf04ec719ef"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "'Price'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-42-df15f2654548>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Price'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m: 'Price'"
          ]
        }
      ],
      "source": [
        "results['Price']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3ebf5d72",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ebf5d72",
        "outputId": "b2c487f8-5d47-4171-b3a8-f9d5adafe830"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Amazon\\'s Choicefor \"mi phones under 15000\"',\n",
              " 'Redmi 13C (Starfrost White, 6GB RAM, 128GB Storage) | Powered by 4G MediaTek Helio G85 | 90Hz Display | 50MP AI Triple Camera',\n",
              " 'Redmi 12 5G Jade Black 6GB RAM 128GB ROM',\n",
              " 'Redmi A2 (Aqua Blue, 2GB RAM, 64GB Storage)',\n",
              " 'Redmi 13C (Starshine Green, 4GB RAM, 128GB Storage) | Powered by 4G MediaTek Helio G85 | 90Hz Display | 50MP AI Triple Camera',\n",
              " 'Redmi 12 5G Moonstone Silver 8GB RAM 256GB ROM',\n",
              " 'Redmi A3 (Olive Green, 3GB RAM, 64GB Storage) | Premium Halo Design | 90Hz Display',\n",
              " 'Redmi 12 5G Pastel Blue 8GB RAM 256GB ROM',\n",
              " 'Redmi 12 5G Moonstone Silver 6GB RAM 128GB ROM',\n",
              " 'Redmi 12 5G Jade Black 8GB RAM 256GB ROM',\n",
              " 'Redmi 13C 5G (Starlight Black, 4GB RAM, 128GB Storage) | MediaTek Dimensity 6100+ 5G | 90Hz Display',\n",
              " 'Redmi 13C 5G (Starlight Black, 8GB RAM, 256GB Storage) | MediaTek Dimensity 6100+ 5G | 90Hz Display',\n",
              " 'Redmi 13C 5G (Startrail Silver, 4GB RAM, 128GB Storage) | MediaTek Dimensity 6100+ 5G | 90Hz Display',\n",
              " 'Redmi 13C 5G (Starlight Black, 6GB RAM, 128GB Storage) | MediaTek Dimensity 6100+ 5G | 90Hz Display',\n",
              " 'Redmi 12 5G Pastel Blue 4GB RAM 128GB ROM',\n",
              " 'Redmi 12 (Jade Black,6GB RAM, 128GB Storage)',\n",
              " 'Redmi 13C 5G (Startrail Silver, 6GB RAM, 128GB Storage) | MediaTek Dimensity 6100+ 5G | 90Hz Display']"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ],
      "source": [
        "results['Title']"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}