Approach 1 - preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Предобработка\n",
    "Код для первого подхода к улучшению классификации - предобработка данных для автоматического исправления маскировок  \n",
    "\n",
    "**Задачи:**\n",
    "* имплементировать правила замены букв и буквосочетаний\n",
    "* скомпилировать словари матерных, грубых и ругательных слов\n",
    "* имплементировать замену по расстоянию Левенштейна и словарю\n",
    "* собрать оба подхода в двухступенчатый алгоритм\n",
    "\n",
    "Ссылки на использованные словари:  \n",
    "https://gist.github.com/nestyme/8531fe4ec34cd2c8e9b306513cb8b59a (Zueva et al.) 89 слов  \n",
    "https://github.com/bohdan1/AbusiveLanguageDataset/blob/master/bad_words.txt (Andrusyak et al.) 623 слова  \n",
    "Из первого были удалены слова, не относящиеся к ругательным, из второго - повторы, имеющиеся в первом  \n",
    "Удалены искажённые формы, чтобы не сбивать Левенштейна  \n",
    "Добавлены слова: *хули, лахтодырка, пиздуй, школота, рашка, хуйло*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Правила замены:\n",
    "йо → ё  \n",
    "^мл → бл  \n",
    "ип → еб  \n",
    "п → б  \n",
    "к → х  \n",
    "т → д  \n",
    "а → о  \n",
    "с → з  \n",
    "и → е  \n",
    "\n",
    "к → г  \n",
    "ш → ж  \n",
    "ф → в  \n",
    "3.14 → пи  \n",
    "3,14 → пи"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import itertools\n",
    "import pymorphy2\n",
    "import pandas as pd\n",
    "from string import punctuation\n",
    "from Levenshtein import distance\n",
    "morph = pymorphy2.MorphAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_dict = {'йо': 'е', \n",
    "                'ип': 'еб',\n",
    "                'мл': 'бл',\n",
    "                'ау': 'ов',\n",
    "                'и': 'е',\n",
    "                'п': 'б',\n",
    "                'т': 'д',\n",
    "                'к': 'х',\n",
    "                'а': 'о',\n",
    "                'с': 'з',\n",
    "                'ш': 'ж',\n",
    "                'ф': 'в',\n",
    "                'у': 'в',\n",
    "                'ц': 'с',\n",
    "                'цц': 'тс',\n",
    "                '3.14': 'пи',\n",
    "                '3,14': 'пи'}\n",
    "\n",
    "with open('replacement.json', 'w', encoding='UTF-8') as f:\n",
    "    json.dump(replace_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('replacement.json', 'r', encoding='UTF-8') as f:\n",
    "    replace_dict = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('bad_wordlist.txt', encoding='UTF-8') as f:\n",
    "    bad_wordlist = [line.rstrip('\\n') for line in f.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "598"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bad_wordlist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Собирание замены букв по правилам"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_latin(text):\n",
    "    \"\"\"\n",
    "    replaces latin letters similar to cyrillic ones\n",
    "    \"\"\"\n",
    "    table = text.maketrans('wertyuiopahkxcbnm', 'шертуииоранкхсвпм')\n",
    "    return text.translate(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "nonletter_pat = re.compile('[^а-яё]')\n",
    "pi_pat = re.compile('^3[.,]14.+')\n",
    "\n",
    "def contains_nonletters(word):\n",
    "    \"\"\"\n",
    "    returns True, if given word contains any character that is not a cyrillic letter or\n",
    "    or a translatable latin letter or a \"3.14\" / \"3,14\" sequence\n",
    "    \"\"\"\n",
    "    if bool(re.search(nonletter_pat, word)) and not bool(re.search(pi_pat, word)):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "link_pat = re.compile('^\\[.*?|.*?\\], ')\n",
    "\n",
    "def remove_link(text):\n",
    "    \"\"\"\n",
    "    removes reply link\n",
    "    \"\"\"\n",
    "    return re.sub(link_pat, '', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(text):\n",
    "    \"\"\"\n",
    "    checks if comments is a reply\n",
    "    \"\"\"\n",
    "    if text.startswith('['):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_exists(word):\n",
    "    \"\"\"\n",
    "    checks whether given word is in OpenCorpora dictionary using PyMorphy2\n",
    "    \"\"\"\n",
    "    if morph.word_is_known(word.strip(punctuation)):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def closest_word_levestein(word):\n",
    "    \"\"\"\n",
    "    returns closest word from dictionary\n",
    "    \"\"\"\n",
    "    dists = [distance(word, candidate) for candidate in bad_wordlist]\n",
    "    closest_val = dists.index(min(dists))\n",
    "    return bad_wordlist[closest_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_by_letters(word, non_found_return):\n",
    "    \"\"\"\n",
    "    takes a word, replaces letters one pair at a time unless the result or its lemma is\n",
    "    found in the bad dictionary, otherwise returns the intial word\n",
    "    \"\"\"\n",
    "    for old, new in replace_dict.items():\n",
    "        if old in word:\n",
    "            new_word = word.replace(old, new)\n",
    "            if morph.parse(new_word)[0].normal_form in bad_wordlist or new_word in bad_wordlist:\n",
    "                return new_word\n",
    "    # if the word is not found, go through all possible combinations of rules\n",
    "    for l in range(1, len(replace_dict)):\n",
    "        for tple in itertools.combinations(replace_dict.keys(), l+1):\n",
    "            new_word = word\n",
    "            for key in tple:\n",
    "                new_word = new_word.replace(key, replace_dict[key])\n",
    "        if morph.parse(new_word)[0].normal_form in bad_wordlist or new_word in bad_wordlist:\n",
    "            return new_word\n",
    "    # if still not found, return the initial input or closest my levenstein\n",
    "    if non_found_return == 'initial':\n",
    "        return word\n",
    "    elif non_found_return == 'levenstein':\n",
    "        return closest_word_levestein(word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Собирание Левенштейна"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nonletter_pat = re.compile('[^а-яёА-ЯЁ]')\n",
    "\n",
    "def count_nonletters(word):\n",
    "    \"\"\"\n",
    "    counts symbols that are not letters\n",
    "    (latin letters are replaced by that time)\n",
    "    \"\"\"\n",
    "    return len(re.findall(nonletter_pat, word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_distances(word):\n",
    "    \"\"\"\n",
    "    counts all distances to dictionary words, returns a list of them\n",
    "    \"\"\"\n",
    "    return [distance(word, candidate) for candidate in bad_wordlist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_by_levenstein(word):\n",
    "    \"\"\"\n",
    "    three sequential approaches to find the masked word using edit distance\n",
    "    \"\"\"\n",
    "    n = count_nonletters(word)\n",
    "    dists = count_distances(word)\n",
    "    \n",
    "    if n in dists: # find the word with n == distance\n",
    "        idx = dists.index(n)\n",
    "        return bad_wordlist[idx]\n",
    "    else: # pick first word with closest distance from dist list\n",
    "        closest_val = min(dists, key=lambda x:abs(x-n))\n",
    "        idx = dists.index(closest_val)\n",
    "        return bad_wordlist[idx]\n",
    "    # might need to add threshold above that replacement does no happen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ебать'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_by_levenstein('еб***того')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Финальная функция:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_distortion(text, debug=False):\n",
    "    \"\"\"\n",
    "    performs all steps of checks and corrections for each token\n",
    "    \"\"\"\n",
    "    text = replace_latin(text) # this step is done for the whole text because it's fast,\n",
    "                               # does not require iterations and cannot produce non-existent words\n",
    "    if text.startswith('['):\n",
    "        tokens = remove_link(text).split()\n",
    "    else:\n",
    "        tokens = text.split()\n",
    "    \n",
    "    for token in tokens:\n",
    "        token_low = token.lower()\n",
    "        if word_exists(token_low) or token_low.isdigit() or all([char in punctuation for char in token_low]):\n",
    "            continue # skip the token if it is an existing word or a number or a piece of punctuation\n",
    "\n",
    "        if not contains_nonletters(token_low): # pass token further if it has non-cyrillic characters\n",
    "            if debug:\n",
    "                print('using replacement rules')\n",
    "                print(correct_by_letters(token_low, 'levenstein'))\n",
    "            text = text.replace(token, correct_by_letters(token_low, 'levenstein'))\n",
    "\n",
    "        else: # it has non-cyrillic characters and therefore is passed to Levenstein\n",
    "            if debug:\n",
    "                print('using Levenstein')\n",
    "                print(correct_by_levenstein(token_low))\n",
    "            text = text.replace(token, correct_by_levenstein(token_low))\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Уeище тупое\n",
      "using replacement rules\n",
      "уебище\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'уебище тупое'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_string = 'Уeище тупое'\n",
    "print(test_string)\n",
    "preprocess_distortion(test_string, debug=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Тестирование"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncorrected = pd.read_csv('uncorrected_data_NEW.tsv', encoding='UTF-8', sep='\\t')\n",
    "corrected = pd.read_csv('corrected_data_NEW.tsv', encoding='UTF-8', sep='\\t')\n",
    "test_x = uncorrected['comments']\n",
    "test_y = corrected['corrected']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preprocessing(test_x, test_y):\n",
    "    \"\"\"\n",
    "    prints data for visual assessment\n",
    "    \"\"\"\n",
    "    for x, y in zip(test_x, test_y):\n",
    "        edited = preprocess_distortion(x)\n",
    "        if x == y:\n",
    "            print('--------\\nсошлось')\n",
    "            print(f'\\nx: {x}')\n",
    "            print(f'\\nedited: {edited}')\n",
    "            print(f'\\ny: {y}')\n",
    "        else:\n",
    "            print('--------\\nне сошлось')\n",
    "            print(f'\\nx: {x}')\n",
    "            print(f'\\nedited: {edited}')\n",
    "            print(f'\\ny: {y}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_preprocessing(test_x[:5], test_y[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>comments</th>\n",
       "      <th>toxicity</th>\n",
       "      <th>preprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>[id695904995|Ярослав], как же обо%рался ОКР ко...</td>\n",
       "      <td>1</td>\n",
       "      <td>[иd695904995|Ярослав], как же обосранец очко к...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31</td>\n",
       "      <td>[id483002399|Максим], дол..еб, ты под каждым п...</td>\n",
       "      <td>1</td>\n",
       "      <td>[иd483002399|Максим], долбоеб ты под каждым по...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>[id483002399|Максим], лахтодырка-это твоя мама...</td>\n",
       "      <td>1</td>\n",
       "      <td>[иd483002399|Максим], лахтодырка твоя мама) А ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>[id238092031|?лег], от школоты иного ответа, к...</td>\n",
       "      <td>1</td>\n",
       "      <td>[иd238092031|?лег], от школота иного ответа, к...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>[id238092031|?лег], нет, лахтодырка ??, грамот...</td>\n",
       "      <td>1</td>\n",
       "      <td>[иd238092031|?лег], нет, лахтодырка ??, грамот...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                           comments  toxicity  \\\n",
       "0          25  [id695904995|Ярослав], как же обо%рался ОКР ко...         1   \n",
       "1          31  [id483002399|Максим], дол..еб, ты под каждым п...         1   \n",
       "2          33  [id483002399|Максим], лахтодырка-это твоя мама...         1   \n",
       "3          34  [id238092031|?лег], от школоты иного ответа, к...         1   \n",
       "4          36  [id238092031|?лег], нет, лахтодырка ??, грамот...         1   \n",
       "\n",
       "                                        preprocessed  \n",
       "0  [иd695904995|Ярослав], как же обосранец очко к...  \n",
       "1  [иd483002399|Максим], долбоеб ты под каждым по...  \n",
       "2  [иd483002399|Максим], лахтодырка твоя мама) А ...  \n",
       "3  [иd238092031|?лег], от школота иного ответа, к...  \n",
       "4  [иd238092031|?лег], нет, лахтодырка ??, грамот...  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uncorrected['preprocessed'] = uncorrected['comments'].apply(preprocess_distortion)\n",
    "uncorrected.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncorrected.to_csv('preprocessed_data_NEW.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}