06_DL_for_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning for NLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understanding Deep Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle: Text Categorization Challenge\n",
    "\n",
    "# Getting the Data\n",
    "\n",
    "**Direct Download**: You can get the train and test data from the [data tab on challenge website](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/data).  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Solving environment: ...working... done\n",
      "\n",
      "## Package Plan ##\n",
      "\n",
      "  environment location: D:\\Miniconda3\\envs\\nlp\n",
      "\n",
      "  added / updated specs: \n",
      "    - pandas\n",
      "\n",
      "\n",
      "The following packages will be UPDATED:\n",
      "\n",
      "    pandas: 0.23.3-py36h830ac7b_0 --> 0.23.4-py36h830ac7b_0\n",
      "\n",
      "Preparing transaction: ...working... done\n",
      "Verifying transaction: ...working... done\n",
      "Executing transaction: ...working... done\n",
      "Solving environment: ...working... done\n",
      "\n",
      "## Package Plan ##\n",
      "\n",
      "  environment location: D:\\Miniconda3\\envs\\nlp\n",
      "\n",
      "  added / updated specs: \n",
      "    - numpy\n",
      "\n",
      "\n",
      "The following packages will be UPDATED:\n",
      "\n",
      "    numpy:      1.15.1-py36ha559c80_0 --> 1.15.4-py36ha559c80_0\n",
      "    numpy-base: 1.15.1-py36h8128ebf_0 --> 1.15.4-py36h8128ebf_0\n",
      "\n",
      "Preparing transaction: ...working... done\n",
      "Verifying transaction: ...working... done\n",
      "Executing transaction: ...working... done\n"
     ]
    }
   ],
   "source": [
    "!conda install -y pandas\n",
    "!conda install -y numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'data/train.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-81c8cd170b07>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtrain_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"data/train.csv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\Miniconda3\\envs\\nlp\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[0;32m    677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 678\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    679\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda3\\envs\\nlp\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    438\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 440\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    441\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda3\\envs\\nlp\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'has_index_names'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    786\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 787\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    788\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    789\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda3\\envs\\nlp\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1012\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'c'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1013\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'c'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1014\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1015\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1016\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'python'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda3\\envs\\nlp\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'usecols'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1707\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1708\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1709\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: File b'data/train.csv' does not exist"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"data/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.read_csv(\"data/valid.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiple Target Dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Why PyTorch? "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch and torchtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install -y pytorch cuda92 -c pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --upgrade git+https://github.com/pytorch/text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchtext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_gpu = True\n",
    "if use_gpu:\n",
    "    assert torch.cuda.is_available(), 'You either do not have a GPU or is not accessible to PyTorch'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see how many GPU devices are available to PyTorch on this machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loaders with torchtext\n",
    "\n",
    "### Conventions and Style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import Field "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL = Field(sequential=False, use_vocab=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize = lambda x: x.split()\n",
    "TEXT = Field(sequential=True, tokenize=tokenize, lower=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import TabularDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tv_datafields = [(\"id\", None), # we won't be needing the id, so we pass in None as the field\n",
    "                 (\"comment_text\", TEXT), (\"toxic\", LABEL),\n",
    "                 (\"severe_toxic\", LABEL), (\"threat\", LABEL),\n",
    "                 (\"obscene\", LABEL), (\"insult\", LABEL),\n",
    "                 (\"identity_hate\", LABEL)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn, vld = TabularDataset.splits(\n",
    "        path=\"data\", # the root directory where the data lies\n",
    "        train='train.csv', validation=\"valid.csv\",\n",
    "        format='csv',\n",
    "        skip_header=True, # if your csv header has a header, make sure to pass this to ensure it doesn't get proceesed as data!\n",
    "        fields=tv_datafields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst_datafields = [(\"id\", None), # we won't be needing the id, so we pass in None as the field\n",
    "                 (\"comment_text\", TEXT)\n",
    "                 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst = TabularDataset(\n",
    "        path=\"data/test.csv\", # the file path\n",
    "        format='csv',\n",
    "        skip_header=True, # if your csv header has a header, make sure to pass this to ensure it doesn't get proceesed as data!\n",
    "        fields=tst_datafields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring the Dataset Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<torchtext.data.dataset.TabularDataset at 0x176c45f30f0>,\n",
       " <torchtext.data.dataset.TabularDataset at 0x176c45f3908>,\n",
       " <torchtext.data.dataset.TabularDataset at 0x176c45f32b0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn, vld, tst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<torchtext.data.example.Example at 0x176c45f3978>,\n",
       " <torchtext.data.example.Example at 0x176c45f3940>,\n",
       " <torchtext.data.example.Example at 0x176c45f37f0>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0], vld[0], tst[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['comment_text', 'toxic', 'severe_toxic', 'threat', 'obscene', 'insult', 'identity_hate'])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0].__dict__.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['explanation', 'why', 'the', 'edits', 'made']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0].__dict__['comment_text'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.build_vocab(trn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torchtext.vocab.Vocab at 0x176c45f3400>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "collections.Counter"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(TEXT.vocab.freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 78), ('to', 41), ('you', 33), ('of', 30), ('and', 26)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.freqs.most_common(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(list, collections.defaultdict, 784, 784)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(TEXT.vocab.itos), type(TEXT.vocab.stoi), len(TEXT.vocab.itos), len(TEXT.vocab.stoi.keys()), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 'and')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.stoi['and'], TEXT.vocab.itos[7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Iterators!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import Iterator, BucketIterator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BucketIterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter, val_iter = BucketIterator.splits(\n",
    "        (trn, vld), # we pass in the datasets we want the iterator to draw data from\n",
    "        batch_sizes=(32, 32),\n",
    "        sort_key=lambda x: len(x.comment_text), # the BucketIterator needs to be told what function it should use to group the data.\n",
    "        sort_within_batch=False,\n",
    "        repeat=False # we pass repeat=False because we want to wrap this Iterator layer.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torchtext.data.iterator.BucketIterator at 0x176c45f30b8>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[torchtext.data.batch.Batch of size 25]\n",
       "\t[.comment_text]:[torch.LongTensor of size 494x25]\n",
       "\t[.toxic]:[torch.LongTensor of size 25]\n",
       "\t[.severe_toxic]:[torch.LongTensor of size 25]\n",
       "\t[.threat]:[torch.LongTensor of size 25]\n",
       "\t[.obscene]:[torch.LongTensor of size 25]\n",
       "\t[.insult]:[torch.LongTensor of size 25]\n",
       "\t[.identity_hate]:[torch.LongTensor of size 25]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = next(train_iter.__iter__())\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['batch_size', 'dataset', 'fields', 'comment_text', 'toxic', 'severe_toxic', 'threat', 'obscene', 'insult', 'identity_hate'])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.__dict__.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<torchtext.data.dataset.TabularDataset at 0x176c45f30f0>,\n",
       " <torchtext.data.dataset.TabularDataset at 0x176c45f30f0>,\n",
       " True)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.__dict__['dataset'], trn, batch.__dict__['dataset']==trn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_iter = Iterator(tst, batch_size=64, sort=False, sort_within_batch=False, repeat=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[torchtext.data.batch.Batch of size 33]\n",
       "\t[.comment_text]:[torch.LongTensor of size 158x33]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(test_iter.__iter__())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BatchWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchWrapper:\n",
    "    def __init__(self, dl, x_var, y_vars):\n",
    "        self.dl, self.x_var, self.y_vars = dl, x_var, y_vars # we pass in the list of attributes for x and y\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for batch in self.dl:\n",
    "            x = getattr(batch, self.x_var) # we assume only one input in this wrapper\n",
    "            \n",
    "            if self.y_vars is not None: # we will concatenate y into a single tensor\n",
    "                y = torch.cat([getattr(batch, feat).unsqueeze(1) for feat in self.y_vars], dim=1).float()\n",
    "            else:\n",
    "                y = torch.zeros((1))\n",
    "            if use_gpu:\n",
    "                yield (x.cuda(), y.cuda())\n",
    "            else:\n",
    "                yield (x, y)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = BatchWrapper(train_iter, \"comment_text\", [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"])\n",
    "valid_dl = BatchWrapper(val_iter, \"comment_text\", [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"])\n",
    "test_dl = BatchWrapper(test_iter, \"comment_text\", None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 453,   63,   15,  ...,  454,  660,  778],\n",
       "         [ 523,    4,  601,  ...,   78,   11,  650],\n",
       "         [  30,  664,  242,  ...,    8,    2,   22],\n",
       "         ...,\n",
       "         [   1,    1,    1,  ...,    1,    1,    1],\n",
       "         [   1,    1,    1,  ...,    1,    1,    1],\n",
       "         [   1,    1,    1,  ...,    1,    1,    1]], device='cuda:0'),\n",
       " tensor([[ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 1.,  1.,  0.,  1.,  1.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0'))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(train_dl.__iter__())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training a Text Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleLSTMBaseline(nn.Module):\n",
    "    def __init__(self, hidden_dim, emb_dim=300,\n",
    "                 spatial_dropout=0.05, recurrent_dropout=0.1, num_linear=2):\n",
    "        super().__init__() # don't forget to call this!\n",
    "        self.embedding = nn.Embedding(len(TEXT.vocab), emb_dim)\n",
    "        self.encoder = nn.LSTM(emb_dim, hidden_dim, num_layers=num_linear, dropout=recurrent_dropout)\n",
    "        self.linear_layers = []\n",
    "        for _ in range(num_linear - 1):\n",
    "            self.linear_layers.append(nn.Linear(hidden_dim, hidden_dim))\n",
    "        self.linear_layers = nn.ModuleList(self.linear_layers)\n",
    "        self.predictor = nn.Linear(hidden_dim, 6)\n",
    "    \n",
    "    def forward(self, seq):\n",
    "        hdn, _ = self.encoder(self.embedding(seq))\n",
    "        feature = hdn[-1, :, :]\n",
    "        for layer in self.linear_layers:\n",
    "            feature = layer(feature)\n",
    "        preds = self.predictor(feature)\n",
    "        return preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializing the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SimpleLSTMBaseline(\n",
      "  (embedding): Embedding(784, 300)\n",
      "  (encoder): LSTM(300, 500, num_layers=2, dropout=0.1)\n",
      "  (linear_layers): ModuleList(\n",
      "    (0): Linear(in_features=500, out_features=500, bias=True)\n",
      "  )\n",
      "  (predictor): Linear(in_features=500, out_features=6, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "em_sz = 300\n",
    "nh = 500\n",
    "model = SimpleLSTMBaseline(nh, emb_dim=em_sz)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.096706 million parameters\n"
     ]
    }
   ],
   "source": [
    "def model_size(model: torch.nn)->int:\n",
    "    \"\"\"\n",
    "    Calculates the number of trainable parameters in any model\n",
    "    \n",
    "    Returns:\n",
    "        params (int): the total count of all model weights\n",
    "    \"\"\"\n",
    "    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "#     model_parameters = model.parameters()\n",
    "    params = sum([np.prod(p.size()) for p in model_parameters])\n",
    "    return params\n",
    "\n",
    "print(f'{model_size(model)/10**6} million parameters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "if use_gpu:\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Putting together the pieces again**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import optim\n",
    "opt = optim.Adam(model.parameters(), lr=1e-2)\n",
    "loss_func = nn.BCEWithLogitsLoss().cuda()\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Training Loss: 13.5037, Validation Loss: 4.6498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  4.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2, Training Loss: 7.8243, Validation Loss: 24.5401\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3, Training Loss: 57.4577, Validation Loss: 4.0107\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "for epoch in range(1, epochs + 1):\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0\n",
    "    model.train() # turn on training mode\n",
    "    for x, y in tqdm(train_dl): # thanks to our wrapper, we can intuitively iterate over our data!\n",
    "        opt.zero_grad()\n",
    "        preds = model(x)\n",
    "        loss = loss_func(preds, y)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        \n",
    "        running_loss += loss.item() * x.size(0)\n",
    "        \n",
    "    epoch_loss = running_loss / len(trn)\n",
    "    \n",
    "    # calculate the validation loss for this epoch\n",
    "    val_loss = 0.0\n",
    "    model.eval() # turn on evaluation mode\n",
    "    for x, y in valid_dl:\n",
    "        preds = model(x)\n",
    "        loss = loss_func(preds, y)\n",
    "        val_loss += loss.item() * x.size(0)\n",
    "\n",
    "    val_loss /= len(vld)\n",
    "    print('Epoch: {}, Training Loss: {:.4f}, Validation Loss: {:.4f}'.format(epoch, epoch_loss, val_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  9.64it/s]\n"
     ]
    }
   ],
   "source": [
    "test_preds = []\n",
    "for x, y in tqdm(test_dl):\n",
    "    preds = model(x)\n",
    "    # if you're data is on the GPU, you need to move the data back to the cpu\n",
    "    # preds = preds.data.cpu().numpy()\n",
    "    preds = preds.data.cpu().numpy()\n",
    "    # the actual outputs of the model are logits, so we need to pass these values to the sigmoid function\n",
    "    preds = 1 / (1 + np.exp(-preds))\n",
    "    test_preds.append(preds)\n",
    "test_preds = np.hstack(test_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert predictions to a pandas dataframe\n",
    "\n",
    "This helps us convert the predictions to a more interpretable format. Let's insert the predictions in the correct column and then we can preview few rows of the dataframe: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"data/test.csv\")\n",
    "for i, col in enumerate([\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]):\n",
    "    test_df[col] = test_preds[:, i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00001cee341fdb12</td>\n",
       "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
       "      <td>0.629146</td>\n",
       "      <td>0.116721</td>\n",
       "      <td>0.438606</td>\n",
       "      <td>0.156848</td>\n",
       "      <td>0.139696</td>\n",
       "      <td>0.388736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000247867823ef7</td>\n",
       "      <td>== From RfC == \\r\\n\\r\\n The title is fine as i...</td>\n",
       "      <td>0.629146</td>\n",
       "      <td>0.116721</td>\n",
       "      <td>0.438606</td>\n",
       "      <td>0.156848</td>\n",
       "      <td>0.139696</td>\n",
       "      <td>0.388736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00013b17ad220c46</td>\n",
       "      <td>\" \\r\\n\\r\\n == Sources == \\r\\n\\r\\n * Zawe Ashto...</td>\n",
       "      <td>0.629146</td>\n",
       "      <td>0.116721</td>\n",
       "      <td>0.438606</td>\n",
       "      <td>0.156848</td>\n",
       "      <td>0.139696</td>\n",
       "      <td>0.388736</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  \\\n",
       "0  00001cee341fdb12  Yo bitch Ja Rule is more succesful then you'll...   \n",
       "1  0000247867823ef7  == From RfC == \\r\\n\\r\\n The title is fine as i...   \n",
       "2  00013b17ad220c46  \" \\r\\n\\r\\n == Sources == \\r\\n\\r\\n * Zawe Ashto...   \n",
       "\n",
       "      toxic  severe_toxic   obscene    threat    insult  identity_hate  \n",
       "0  0.629146      0.116721  0.438606  0.156848  0.139696       0.388736  \n",
       "1  0.629146      0.116721  0.438606  0.156848  0.139696       0.388736  \n",
       "2  0.629146      0.116721  0.438606  0.156848  0.139696       0.388736  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nlp]",
   "language": "python",
   "name": "conda-env-nlp-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}