CVLData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import Module, Embedding, Linear\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import graphnn\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "from time import time\n",
    "import json\n",
    "from glob import glob\n",
    "from pprint import pprint\n",
    "import io\n",
    "import os\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import unicodedata as ud\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "latin_letters= {}\n",
    "\n",
    "def is_latin(uchr):\n",
    "    try: return latin_letters[uchr]\n",
    "    except KeyError:\n",
    "         return latin_letters.setdefault(uchr, 'LATIN' in ud.name(uchr))\n",
    "\n",
    "def only_roman_chars(unistr):\n",
    "    return all(is_latin(uchr)\n",
    "           for uchr in unistr\n",
    "           if uchr.isalpha())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_roman_chars(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'18350465'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.basename(json_files[0])[:-5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03a6221020264b6cb8f3d2998d873135",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2327), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2327"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_files = glob('data/cvl/*.json')\n",
    "\n",
    "def load_file(fname):\n",
    "    return json.load(io.open(fname, 'r', encoding='utf-8-sig'))\n",
    "\n",
    "data = {os.path.basename(fname)[:-5]: load_file(fname) for fname in tqdm(json_files)}\n",
    "\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {key: value for key, value in data.items() if only_roman_chars(value.get('ARTICLE_DOMAIN', '') + value.get('ARTICLE_TITLE', ''))} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2273"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_domain(domain):\n",
    "    domain = domain.split(' | ')\n",
    "    if len(domain) >= 3:\n",
    "        return domain[-2].replace(':', ''), domain[-1].replace(':', '')\n",
    "    if len(domain) == 2:\n",
    "        return domain[-1].replace(':', ''), domain[-1].replace(':', '')\n",
    "    if len(domain) == 1:\n",
    "        return domain[0].replace(':', ''), domain[0].replace(':', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ced14daef5e4e658918302a127071f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2273), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "link_text_pre = \"www.capitaliq.com/CIQDotNet/CreditResearch/SPResearch.aspx?DocumentId=\"\n",
    "link_text_post = \"&From=SNP_CRS\"\n",
    "inner_link_pre = \"www.capitaliq.com\"\n",
    "# \"LINK\": \"www.capitaliq.com/CIQDotNet/CreditResearch/SPResearch.aspx?DocumentId=40944002&From=SNP_CRS\"\n",
    "\n",
    "c = Counter()\n",
    "d = Counter()\n",
    "cd = Counter()\n",
    "\n",
    "def convert(key, value):\n",
    "    \n",
    "    content = ' '.join(block['ANNOTATED_TEXT'] for block in value['EXTRACTED_CONTENT'])\n",
    "    refs = [{'LINK':f\"{inner_link_pre}{block['LINK']}\", \n",
    "              'LINK_TEXT':block['LINK_TEXT'], \n",
    "              'ART_OBJECTID':block['ART_OBJECTID']} for block in value['ALL_ARTICLE_LINKS']]\n",
    "    title = value.get('ARTICLE_TITLE', 'ERROR')\n",
    "    domain = value.get('ARTICLE_DOMAIN', 'ERROR')\n",
    "    domain1, domain2 = 'ERROR', 'ERROR'\n",
    "    if domain == 'ERROR':\n",
    "        if 'Table Of Contents' in title:\n",
    "            domain1 = 'Table Of Contents'\n",
    "            domain2 = 'Table Of Contents'\n",
    "    else:\n",
    "        domain2, domain1 = split_domain(domain)\n",
    "        \n",
    "    c[domain2] += 1\n",
    "    d[domain1] += 1\n",
    "    cd[(domain2, domain1)] += 1\n",
    "    \n",
    "    pub_date = value['ARTICLE_PUB_DATE']\n",
    "    link = f\"{link_text_pre}{key}{link_text_post}\"\n",
    "    article_id = key\n",
    "    return {'aid': article_id, 'title':title, 'top_domain':domain2, 'bottom_domain':domain1,  'pub_date':pub_date, 'link':link, 'content':content, 'refs':refs}\n",
    "\n",
    "\n",
    "new_data = {item[0] : convert(*item) for item in tqdm(data.items())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'Structured Finance': 993, 'Corporates': 362, 'Insurance': 250, 'Financial Institutions': 246, 'Governments': 231, 'General Criteria': 110, 'Legal Criteria': 65, 'Table Of Contents': 13, 'Guidance': 1, 'ERROR': 1, 'Credit FAQ': 1})\n",
      "11\n"
     ]
    }
   ],
   "source": [
    "print(c)\n",
    "print(len(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'General': 346, 'RMBS': 321, 'Request for Comment': 237, 'ABS': 187, 'CDOs': 184, 'Industrials': 144, 'U.S. Public Finance': 128, 'General Criteria': 91, 'Banks': 71, 'CMBS': 69, 'Legal Criteria': 65, 'Fixed-Income Funds': 49, 'Life': 44, 'Covered Bonds': 41, 'Specialty': 39, 'Project Finance': 35, 'Property/Casualty': 34, 'International Public Finance': 31, 'Sovereigns': 28, 'Utilities': 21, 'Health': 16, 'ABCP': 15, 'Table Of Contents': 13, 'Bond': 12, 'Other': 12, 'Finance Companies': 11, 'Recovery': 11, 'Servicer Evaluations': 7, 'Broker-Dealers': 6, 'Fundamentals': 2, 'Legal': 1, 'ERROR': 1, 'Credit FAQ': 1})\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "print(d)\n",
    "print(len(d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({('Structured Finance', 'RMBS'): 321, ('Structured Finance', 'ABS'): 187, ('Structured Finance', 'CDOs'): 184, ('Corporates', 'Industrials'): 144, ('Governments', 'U.S. Public Finance'): 128, ('Corporates', 'General'): 107, ('Structured Finance', 'Request for Comment'): 99, ('General Criteria', 'General Criteria'): 90, ('Insurance', 'General'): 89, ('Financial Institutions', 'General'): 73, ('Financial Institutions', 'Banks'): 71, ('Structured Finance', 'CMBS'): 69, ('Structured Finance', 'General'): 69, ('Legal Criteria', 'Legal Criteria'): 65, ('Financial Institutions', 'Fixed-Income Funds'): 49, ('Corporates', 'Request for Comment'): 44, ('Insurance', 'Life'): 44, ('Structured Finance', 'Covered Bonds'): 41, ('Insurance', 'Specialty'): 39, ('Governments', 'Request for Comment'): 36, ('Corporates', 'Project Finance'): 35, ('Insurance', 'Property/Casualty'): 34, ('Governments', 'International Public Finance'): 31, ('Governments', 'Sovereigns'): 28, ('Financial Institutions', 'Request for Comment'): 24, ('Corporates', 'Utilities'): 21, ('General Criteria', 'Request for Comment'): 20, ('Insurance', 'Health'): 16, ('Structured Finance', 'ABCP'): 15, ('Insurance', 'Request for Comment'): 14, ('Table Of Contents', 'Table Of Contents'): 13, ('Insurance', 'Bond'): 12, ('Financial Institutions', 'Other'): 12, ('Financial Institutions', 'Finance Companies'): 11, ('Corporates', 'Recovery'): 11, ('Governments', 'General'): 8, ('Structured Finance', 'Servicer Evaluations'): 7, ('Financial Institutions', 'Broker-Dealers'): 6, ('Insurance', 'Fundamentals'): 2, ('Guidance', 'General Criteria'): 1, ('Structured Finance', 'Legal'): 1, ('ERROR', 'ERROR'): 1, ('Credit FAQ', 'Credit FAQ'): 1})\n",
      "43\n"
     ]
    }
   ],
   "source": [
    "print(cd)\n",
    "print(len(cd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# new_data['18178692']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TITLE  18078955\n",
      "TDOMAIN 40509634\n",
      "BDOMAIN 40509634\n"
     ]
    }
   ],
   "source": [
    "for nd in new_data:\n",
    "    if new_data[nd]['top_domain'] == 'ERROR':\n",
    "        print('TDOMAIN', nd)\n",
    "    if new_data[nd]['bottom_domain'] == 'ERROR':\n",
    "        print('BDOMAIN', nd)\n",
    "    if new_data[nd]['title'] == 'ERROR':\n",
    "        print('TITLE ', nd)\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59095de814764840847fec3e9b7f1d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2273), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for key, value in tqdm(new_data.items()):\n",
    "    with open(f\"data/cvl_formatted/{key}.json\", 'w') as json_file:\n",
    "        json.dump(value, json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}