ch06.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "badfa7a0",
   "metadata": {},
   "source": [
    "# Chapter 6: Reading and Writing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7f68ff54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------Version info---------\n",
      "Polars:               0.20.31\n",
      "Index type:           UInt32\n",
      "Platform:             macOS-12.5-arm64-arm-64bit\n",
      "Python:               3.11.9 (main, Apr  2 2024, 16:11:47) [Clang 14.0.0 (clang-1400.0.29.202)]\n",
      "\n",
      "----Optional dependencies----\n",
      "adbc_driver_manager:  0.8.0\n",
      "cloudpickle:          3.0.0\n",
      "connectorx:           0.3.2\n",
      "deltalake:            0.15.0\n",
      "fastexcel:            0.9.1\n",
      "fsspec:               2023.12.2\n",
      "gevent:               23.9.1\n",
      "hvplot:               0.9.2\n",
      "matplotlib:           3.8.4\n",
      "nest_asyncio:         1.6.0\n",
      "numpy:                1.26.4\n",
      "openpyxl:             3.1.2\n",
      "pandas:               2.2.2\n",
      "pyarrow:              14.0.2\n",
      "pydantic:             2.5.3\n",
      "pyiceberg:            0.5.1\n",
      "pyxlsb:               <not installed>\n",
      "sqlalchemy:           2.0.25\n",
      "torch:                <not installed>\n",
      "xlsx2csv:             0.8.2\n",
      "xlsxwriter:           3.2.0\n"
     ]
    }
   ],
   "source": [
    "import polars as pl\n",
    "pl.show_versions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8650578",
   "metadata": {},
   "source": [
    "## Reading CSV Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "16768c4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (344, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>rowid</th><th>species</th><th>island</th><th>bill_length_mm</th><th>bill_depth_mm</th><th>flipper_length_mm</th><th>body_mass_g</th><th>sex</th><th>year</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>&quot;39.1&quot;</td><td>&quot;18.7&quot;</td><td>&quot;181&quot;</td><td>&quot;3750&quot;</td><td>&quot;male&quot;</td><td>2007</td></tr><tr><td>2</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>&quot;39.5&quot;</td><td>&quot;17.4&quot;</td><td>&quot;186&quot;</td><td>&quot;3800&quot;</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>3</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>&quot;40.3&quot;</td><td>&quot;18&quot;</td><td>&quot;195&quot;</td><td>&quot;3250&quot;</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>4</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>&quot;NA&quot;</td><td>&quot;NA&quot;</td><td>&quot;NA&quot;</td><td>&quot;NA&quot;</td><td>&quot;NA&quot;</td><td>2007</td></tr><tr><td>5</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>&quot;36.7&quot;</td><td>&quot;19.3&quot;</td><td>&quot;193&quot;</td><td>&quot;3450&quot;</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>340</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>&quot;55.8&quot;</td><td>&quot;19.8&quot;</td><td>&quot;207&quot;</td><td>&quot;4000&quot;</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>341</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>&quot;43.5&quot;</td><td>&quot;18.1&quot;</td><td>&quot;202&quot;</td><td>&quot;3400&quot;</td><td>&quot;female&quot;</td><td>2009</td></tr><tr><td>342</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>&quot;49.6&quot;</td><td>&quot;18.2&quot;</td><td>&quot;193&quot;</td><td>&quot;3775&quot;</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>343</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>&quot;50.8&quot;</td><td>&quot;19&quot;</td><td>&quot;210&quot;</td><td>&quot;4100&quot;</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>344</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>&quot;50.2&quot;</td><td>&quot;18.7&quot;</td><td>&quot;198&quot;</td><td>&quot;3775&quot;</td><td>&quot;female&quot;</td><td>2009</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (344, 9)\n",
       "┌───────┬───────────┬───────────┬────────────────┬───┬───────────────┬─────────────┬────────┬──────┐\n",
       "│ rowid ┆ species   ┆ island    ┆ bill_length_mm ┆ … ┆ flipper_lengt ┆ body_mass_g ┆ sex    ┆ year │\n",
       "│ ---   ┆ ---       ┆ ---       ┆ ---            ┆   ┆ h_mm          ┆ ---         ┆ ---    ┆ ---  │\n",
       "│ i64   ┆ str       ┆ str       ┆ str            ┆   ┆ ---           ┆ str         ┆ str    ┆ i64  │\n",
       "│       ┆           ┆           ┆                ┆   ┆ str           ┆             ┆        ┆      │\n",
       "╞═══════╪═══════════╪═══════════╪════════════════╪═══╪═══════════════╪═════════════╪════════╪══════╡\n",
       "│ 1     ┆ Adelie    ┆ Torgersen ┆ 39.1           ┆ … ┆ 181           ┆ 3750        ┆ male   ┆ 2007 │\n",
       "│ 2     ┆ Adelie    ┆ Torgersen ┆ 39.5           ┆ … ┆ 186           ┆ 3800        ┆ female ┆ 2007 │\n",
       "│ 3     ┆ Adelie    ┆ Torgersen ┆ 40.3           ┆ … ┆ 195           ┆ 3250        ┆ female ┆ 2007 │\n",
       "│ 4     ┆ Adelie    ┆ Torgersen ┆ NA             ┆ … ┆ NA            ┆ NA          ┆ NA     ┆ 2007 │\n",
       "│ 5     ┆ Adelie    ┆ Torgersen ┆ 36.7           ┆ … ┆ 193           ┆ 3450        ┆ female ┆ 2007 │\n",
       "│ …     ┆ …         ┆ …         ┆ …              ┆ … ┆ …             ┆ …           ┆ …      ┆ …    │\n",
       "│ 340   ┆ Chinstrap ┆ Dream     ┆ 55.8           ┆ … ┆ 207           ┆ 4000        ┆ male   ┆ 2009 │\n",
       "│ 341   ┆ Chinstrap ┆ Dream     ┆ 43.5           ┆ … ┆ 202           ┆ 3400        ┆ female ┆ 2009 │\n",
       "│ 342   ┆ Chinstrap ┆ Dream     ┆ 49.6           ┆ … ┆ 193           ┆ 3775        ┆ male   ┆ 2009 │\n",
       "│ 343   ┆ Chinstrap ┆ Dream     ┆ 50.8           ┆ … ┆ 210           ┆ 4100        ┆ male   ┆ 2009 │\n",
       "│ 344   ┆ Chinstrap ┆ Dream     ┆ 50.2           ┆ … ┆ 198           ┆ 3775        ┆ female ┆ 2009 │\n",
       "└───────┴───────────┴───────────┴────────────────┴───┴───────────────┴─────────────┴────────┴──────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "penguins = pl.read_csv(\"data/penguins.csv\")\n",
    "penguins"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3866db12",
   "metadata": {},
   "source": [
    "## Parsing Missing Values Correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7c85ce21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (344, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>rowid</th><th>species</th><th>island</th><th>bill_length_mm</th><th>bill_depth_mm</th><th>flipper_length_mm</th><th>body_mass_g</th><th>sex</th><th>year</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>39.1</td><td>18.7</td><td>181</td><td>3750</td><td>&quot;male&quot;</td><td>2007</td></tr><tr><td>2</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>39.5</td><td>17.4</td><td>186</td><td>3800</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>3</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>40.3</td><td>18.0</td><td>195</td><td>3250</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>4</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>2007</td></tr><tr><td>5</td><td>&quot;Adelie&quot;</td><td>&quot;Torgersen&quot;</td><td>36.7</td><td>19.3</td><td>193</td><td>3450</td><td>&quot;female&quot;</td><td>2007</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>340</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>55.8</td><td>19.8</td><td>207</td><td>4000</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>341</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>43.5</td><td>18.1</td><td>202</td><td>3400</td><td>&quot;female&quot;</td><td>2009</td></tr><tr><td>342</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>49.6</td><td>18.2</td><td>193</td><td>3775</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>343</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>50.8</td><td>19.0</td><td>210</td><td>4100</td><td>&quot;male&quot;</td><td>2009</td></tr><tr><td>344</td><td>&quot;Chinstrap&quot;</td><td>&quot;Dream&quot;</td><td>50.2</td><td>18.7</td><td>198</td><td>3775</td><td>&quot;female&quot;</td><td>2009</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (344, 9)\n",
       "┌───────┬───────────┬───────────┬────────────────┬───┬───────────────┬─────────────┬────────┬──────┐\n",
       "│ rowid ┆ species   ┆ island    ┆ bill_length_mm ┆ … ┆ flipper_lengt ┆ body_mass_g ┆ sex    ┆ year │\n",
       "│ ---   ┆ ---       ┆ ---       ┆ ---            ┆   ┆ h_mm          ┆ ---         ┆ ---    ┆ ---  │\n",
       "│ i64   ┆ str       ┆ str       ┆ f64            ┆   ┆ ---           ┆ i64         ┆ str    ┆ i64  │\n",
       "│       ┆           ┆           ┆                ┆   ┆ i64           ┆             ┆        ┆      │\n",
       "╞═══════╪═══════════╪═══════════╪════════════════╪═══╪═══════════════╪═════════════╪════════╪══════╡\n",
       "│ 1     ┆ Adelie    ┆ Torgersen ┆ 39.1           ┆ … ┆ 181           ┆ 3750        ┆ male   ┆ 2007 │\n",
       "│ 2     ┆ Adelie    ┆ Torgersen ┆ 39.5           ┆ … ┆ 186           ┆ 3800        ┆ female ┆ 2007 │\n",
       "│ 3     ┆ Adelie    ┆ Torgersen ┆ 40.3           ┆ … ┆ 195           ┆ 3250        ┆ female ┆ 2007 │\n",
       "│ 4     ┆ Adelie    ┆ Torgersen ┆ null           ┆ … ┆ null          ┆ null        ┆ null   ┆ 2007 │\n",
       "│ 5     ┆ Adelie    ┆ Torgersen ┆ 36.7           ┆ … ┆ 193           ┆ 3450        ┆ female ┆ 2007 │\n",
       "│ …     ┆ …         ┆ …         ┆ …              ┆ … ┆ …             ┆ …           ┆ …      ┆ …    │\n",
       "│ 340   ┆ Chinstrap ┆ Dream     ┆ 55.8           ┆ … ┆ 207           ┆ 4000        ┆ male   ┆ 2009 │\n",
       "│ 341   ┆ Chinstrap ┆ Dream     ┆ 43.5           ┆ … ┆ 202           ┆ 3400        ┆ female ┆ 2009 │\n",
       "│ 342   ┆ Chinstrap ┆ Dream     ┆ 49.6           ┆ … ┆ 193           ┆ 3775        ┆ male   ┆ 2009 │\n",
       "│ 343   ┆ Chinstrap ┆ Dream     ┆ 50.8           ┆ … ┆ 210           ┆ 4100        ┆ male   ┆ 2009 │\n",
       "│ 344   ┆ Chinstrap ┆ Dream     ┆ 50.2           ┆ … ┆ 198           ┆ 3775        ┆ female ┆ 2009 │\n",
       "└───────┴───────────┴───────────┴────────────────┴───┴───────────────┴─────────────┴────────┴──────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "penguins = pl.read_csv(\"data/penguins.csv\", null_values=\"NA\")\n",
    "penguins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e09ca2a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>column</th><th>null_count</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>&quot;rowid&quot;</td><td>0</td></tr><tr><td>&quot;species&quot;</td><td>0</td></tr><tr><td>&quot;island&quot;</td><td>0</td></tr><tr><td>&quot;bill_length_mm&quot;</td><td>2</td></tr><tr><td>&quot;bill_depth_mm&quot;</td><td>2</td></tr><tr><td>&quot;flipper_length_mm&quot;</td><td>2</td></tr><tr><td>&quot;body_mass_g&quot;</td><td>2</td></tr><tr><td>&quot;sex&quot;</td><td>11</td></tr><tr><td>&quot;year&quot;</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌───────────────────┬────────────┐\n",
       "│ column            ┆ null_count │\n",
       "│ ---               ┆ ---        │\n",
       "│ str               ┆ u32        │\n",
       "╞═══════════════════╪════════════╡\n",
       "│ rowid             ┆ 0          │\n",
       "│ species           ┆ 0          │\n",
       "│ island            ┆ 0          │\n",
       "│ bill_length_mm    ┆ 2          │\n",
       "│ bill_depth_mm     ┆ 2          │\n",
       "│ flipper_length_mm ┆ 2          │\n",
       "│ body_mass_g       ┆ 2          │\n",
       "│ sex               ┆ 11         │\n",
       "│ year              ┆ 0          │\n",
       "└───────────────────┴────────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    penguins\n",
    "    .null_count()\n",
    "    .transpose(include_header=True, column_names=[\"null_count\"]) # <1>\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "023572a3",
   "metadata": {},
   "source": [
    "## Reading Files with Encodings Other than UTF-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f6fd3211",
   "metadata": {},
   "outputs": [
    {
     "ename": "ComputeError",
     "evalue": "could not parse `����` as dtype `str` at column 'name' (column number 1)\n\nThe current offset in the file is 19 bytes.\n\nYou might want to try:\n- increasing `infer_schema_length` (e.g. `infer_schema_length=10000`),\n- specifying correct dtype with the `dtypes` argument\n- setting `ignore_errors` to `True`,\n- adding `����` to the `null_values` list.\n\nOriginal error: ```invalid utf-8 sequence```",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mComputeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mpl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdata/directors.csv\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/python-polars-the-definitive-guide-b_q50VVY-py3.11/lib/python3.11/site-packages/polars/_utils/deprecation.py:135\u001b[0m, in \u001b[0;36mdeprecate_renamed_parameter.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m    132\u001b[0m     _rename_keyword_argument(\n\u001b[1;32m    133\u001b[0m         old_name, new_name, kwargs, function\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, version\n\u001b[1;32m    134\u001b[0m     )\n\u001b[0;32m--> 135\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/python-polars-the-definitive-guide-b_q50VVY-py3.11/lib/python3.11/site-packages/polars/_utils/deprecation.py:135\u001b[0m, in \u001b[0;36mdeprecate_renamed_parameter.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m    132\u001b[0m     _rename_keyword_argument(\n\u001b[1;32m    133\u001b[0m         old_name, new_name, kwargs, function\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, version\n\u001b[1;32m    134\u001b[0m     )\n\u001b[0;32m--> 135\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "    \u001b[0;31m[... skipping similar frames: deprecate_renamed_parameter.<locals>.decorate.<locals>.wrapper at line 135 (1 times)]\u001b[0m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/python-polars-the-definitive-guide-b_q50VVY-py3.11/lib/python3.11/site-packages/polars/_utils/deprecation.py:135\u001b[0m, in \u001b[0;36mdeprecate_renamed_parameter.<locals>.decorate.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(function)\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m    132\u001b[0m     _rename_keyword_argument(\n\u001b[1;32m    133\u001b[0m         old_name, new_name, kwargs, function\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, version\n\u001b[1;32m    134\u001b[0m     )\n\u001b[0;32m--> 135\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/python-polars-the-definitive-guide-b_q50VVY-py3.11/lib/python3.11/site-packages/polars/io/csv/functions.py:422\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(source, has_header, columns, new_columns, separator, comment_prefix, quote_char, skip_rows, schema, schema_overrides, null_values, missing_utf8_is_empty_string, ignore_errors, try_parse_dates, n_threads, infer_schema_length, batch_size, n_rows, encoding, low_memory, rechunk, use_pyarrow, storage_options, skip_rows_after_header, row_index_name, row_index_offset, sample_size, eol_char, raise_if_empty, truncate_ragged_lines, decimal_comma, glob)\u001b[0m\n\u001b[1;32m    410\u001b[0m         schema_overrides \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    411\u001b[0m             new_to_current\u001b[38;5;241m.\u001b[39mget(column_name, column_name): column_dtype\n\u001b[1;32m    412\u001b[0m             \u001b[38;5;28;01mfor\u001b[39;00m column_name, column_dtype \u001b[38;5;129;01min\u001b[39;00m schema_overrides\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m    413\u001b[0m         }\n\u001b[1;32m    415\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m prepare_file_arg(\n\u001b[1;32m    416\u001b[0m     source,\n\u001b[1;32m    417\u001b[0m     encoding\u001b[38;5;241m=\u001b[39mencoding,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    420\u001b[0m     storage_options\u001b[38;5;241m=\u001b[39mstorage_options,\n\u001b[1;32m    421\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m data:\n\u001b[0;32m--> 422\u001b[0m     df \u001b[38;5;241m=\u001b[39m \u001b[43m_read_csv_impl\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    423\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    424\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhas_header\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    425\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mprojection\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    426\u001b[0m \u001b[43m        \u001b[49m\u001b[43mseparator\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mseparator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    427\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcomment_prefix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcomment_prefix\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    428\u001b[0m \u001b[43m        \u001b[49m\u001b[43mquote_char\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquote_char\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    429\u001b[0m \u001b[43m        \u001b[49m\u001b[43mskip_rows\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mskip_rows\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    430\u001b[0m \u001b[43m        \u001b[49m\u001b[43mschema_overrides\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mschema_overrides\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    431\u001b[0m \u001b[43m        \u001b[49m\u001b[43mschema\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mschema\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    432\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnull_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnull_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    433\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmissing_utf8_is_empty_string\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmissing_utf8_is_empty_string\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    434\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    435\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtry_parse_dates\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtry_parse_dates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    436\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_threads\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_threads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    437\u001b[0m \u001b[43m        \u001b[49m\u001b[43minfer_schema_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minfer_schema_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    438\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    439\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_rows\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_rows\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    440\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf8-lossy\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf8\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    441\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlow_memory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlow_memory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    442\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrechunk\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrechunk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    443\u001b[0m \u001b[43m        \u001b[49m\u001b[43mskip_rows_after_header\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mskip_rows_after_header\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    444\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrow_index_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrow_index_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    445\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrow_index_offset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrow_index_offset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    446\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    447\u001b[0m \u001b[43m        \u001b[49m\u001b[43meol_char\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meol_char\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    448\u001b[0m \u001b[43m        \u001b[49m\u001b[43mraise_if_empty\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraise_if_empty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    449\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtruncate_ragged_lines\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_ragged_lines\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecimal_comma\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecimal_comma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    451\u001b[0m \u001b[43m        \u001b[49m\u001b[43mglob\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mglob\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    452\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    454\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_columns:\n\u001b[1;32m    455\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _update_columns(df, new_columns)\n",
      "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/python-polars-the-definitive-guide-b_q50VVY-py3.11/lib/python3.11/site-packages/polars/io/csv/functions.py:568\u001b[0m, in \u001b[0;36m_read_csv_impl\u001b[0;34m(source, has_header, columns, separator, comment_prefix, quote_char, skip_rows, schema, schema_overrides, null_values, missing_utf8_is_empty_string, ignore_errors, try_parse_dates, n_threads, infer_schema_length, batch_size, n_rows, encoding, low_memory, rechunk, skip_rows_after_header, row_index_name, row_index_offset, sample_size, eol_char, raise_if_empty, truncate_ragged_lines, decimal_comma, glob)\u001b[0m\n\u001b[1;32m    564\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg)\n\u001b[1;32m    566\u001b[0m projection, columns \u001b[38;5;241m=\u001b[39m parse_columns_arg(columns)\n\u001b[0;32m--> 568\u001b[0m pydf \u001b[38;5;241m=\u001b[39m \u001b[43mPyDataFrame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    569\u001b[0m \u001b[43m    \u001b[49m\u001b[43msource\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    570\u001b[0m \u001b[43m    \u001b[49m\u001b[43minfer_schema_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    571\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    572\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhas_header\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    573\u001b[0m \u001b[43m    \u001b[49m\u001b[43mignore_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    574\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_rows\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    575\u001b[0m \u001b[43m    \u001b[49m\u001b[43mskip_rows\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    576\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprojection\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    577\u001b[0m \u001b[43m    \u001b[49m\u001b[43mseparator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    578\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrechunk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    579\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    580\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    581\u001b[0m \u001b[43m    \u001b[49m\u001b[43mn_threads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    582\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    583\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdtype_list\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    584\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdtype_slice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    585\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlow_memory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    586\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcomment_prefix\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    587\u001b[0m \u001b[43m    \u001b[49m\u001b[43mquote_char\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    588\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprocessed_null_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    589\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmissing_utf8_is_empty_string\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    590\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtry_parse_dates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    591\u001b[0m \u001b[43m    \u001b[49m\u001b[43mskip_rows_after_header\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    592\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparse_row_index_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow_index_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrow_index_offset\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    593\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    594\u001b[0m \u001b[43m    \u001b[49m\u001b[43meol_char\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meol_char\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    595\u001b[0m \u001b[43m    \u001b[49m\u001b[43mraise_if_empty\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraise_if_empty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    596\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtruncate_ragged_lines\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtruncate_ragged_lines\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    597\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdecimal_comma\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecimal_comma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    598\u001b[0m \u001b[43m    \u001b[49m\u001b[43mschema\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mschema\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    599\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    600\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m wrap_df(pydf)\n",
      "\u001b[0;31mComputeError\u001b[0m: could not parse `����` as dtype `str` at column 'name' (column number 1)\n\nThe current offset in the file is 19 bytes.\n\nYou might want to try:\n- increasing `infer_schema_length` (e.g. `infer_schema_length=10000`),\n- specifying correct dtype with the `dtypes` argument\n- setting `ignore_errors` to `True`,\n- adding `����` to the `null_values` list.\n\nOriginal error: ```invalid utf-8 sequence```"
     ]
    }
   ],
   "source": [
    "pl.read_csv(\"data/directors.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "44902654",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (4, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>name</th><th>born</th><th>country</th></tr><tr><td>str</td><td>i64</td><td>str</td></tr></thead><tbody><tr><td>&quot;考侯&quot;</td><td>1930</td><td>&quot;泣塑&quot;</td></tr><tr><td>&quot;Verhoeven&quot;</td><td>1938</td><td>&quot;オランダ&quot;</td></tr><tr><td>&quot;弟宏&quot;</td><td>1942</td><td>&quot;泣塑&quot;</td></tr><tr><td>&quot;Tarantino&quot;</td><td>1963</td><td>&quot;势柜&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (4, 3)\n",
       "┌───────────┬──────┬──────────┐\n",
       "│ name      ┆ born ┆ country  │\n",
       "│ ---       ┆ ---  ┆ ---      │\n",
       "│ str       ┆ i64  ┆ str      │\n",
       "╞═══════════╪══════╪══════════╡\n",
       "│ 考侯      ┆ 1930 ┆ 泣塑     │\n",
       "│ Verhoeven ┆ 1938 ┆ オランダ │\n",
       "│ 弟宏      ┆ 1942 ┆ 泣塑     │\n",
       "│ Tarantino ┆ 1963 ┆ 势柜     │\n",
       "└───────────┴──────┴──────────┘"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.read_csv(\"data/directors.csv\", encoding=\"EUC-CN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c7a7207d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'EUC-JP'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import chardet\n",
    "\n",
    "def detect_encoding(filename: str) -> str:\n",
    "    \"\"\"Return the most probable character encoding for a file.\"\"\"\n",
    "\n",
    "    with open(filename, \"rb\") as f:\n",
    "        raw_data = f.read()\n",
    "        result = chardet.detect(raw_data)\n",
    "        return result[\"encoding\"]\n",
    "\n",
    "\n",
    "detect_encoding(\"data/directors.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d7e9be95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (4, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>name</th><th>born</th><th>country</th></tr><tr><td>str</td><td>i64</td><td>str</td></tr></thead><tbody><tr><td>&quot;深作&quot;</td><td>1930</td><td>&quot;日本&quot;</td></tr><tr><td>&quot;Verhoeven&quot;</td><td>1938</td><td>&quot;オランダ&quot;</td></tr><tr><td>&quot;宮崎&quot;</td><td>1942</td><td>&quot;日本&quot;</td></tr><tr><td>&quot;Tarantino&quot;</td><td>1963</td><td>&quot;米国&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (4, 3)\n",
       "┌───────────┬──────┬──────────┐\n",
       "│ name      ┆ born ┆ country  │\n",
       "│ ---       ┆ ---  ┆ ---      │\n",
       "│ str       ┆ i64  ┆ str      │\n",
       "╞═══════════╪══════╪══════════╡\n",
       "│ 深作      ┆ 1930 ┆ 日本     │\n",
       "│ Verhoeven ┆ 1938 ┆ オランダ │\n",
       "│ 宮崎      ┆ 1942 ┆ 日本     │\n",
       "│ Tarantino ┆ 1963 ┆ 米国     │\n",
       "└───────────┴──────┴──────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.read_csv(\"data/directors.csv\", encoding=\"EUC-JP\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9495ca2",
   "metadata": {},
   "source": [
    "## Reading Excel Spreadsheets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f418f6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2_000, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>positie</th><th>titel</th><th>artiest</th><th>jaar</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>1</td><td>&quot;Bohemian Rhapsody&quot;</td><td>&quot;Queen&quot;</td><td>1975</td></tr><tr><td>2</td><td>&quot;Roller Coaster&quot;</td><td>&quot;Danny Vera&quot;</td><td>2019</td></tr><tr><td>3</td><td>&quot;Hotel California&quot;</td><td>&quot;Eagles&quot;</td><td>1977</td></tr><tr><td>4</td><td>&quot;Piano Man&quot;</td><td>&quot;Billy Joel&quot;</td><td>1974</td></tr><tr><td>5</td><td>&quot;Fix You&quot;</td><td>&quot;Coldplay&quot;</td><td>2005</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>1996</td><td>&quot;Charlie Brown&quot;</td><td>&quot;Coldplay&quot;</td><td>2011</td></tr><tr><td>1997</td><td>&quot;Beast Of Burden&quot;</td><td>&quot;Bette Midler&quot;</td><td>1984</td></tr><tr><td>1998</td><td>&quot;It Was A Very Good Year&quot;</td><td>&quot;Frank Sinatra&quot;</td><td>1968</td></tr><tr><td>1999</td><td>&quot;Hou Van Mij&quot;</td><td>&quot;3JS&quot;</td><td>2008</td></tr><tr><td>2000</td><td>&quot;Drivers License&quot;</td><td>&quot;Olivia Rodrigo&quot;</td><td>2021</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2_000, 4)\n",
       "┌─────────┬─────────────────────────┬────────────────┬──────┐\n",
       "│ positie ┆ titel                   ┆ artiest        ┆ jaar │\n",
       "│ ---     ┆ ---                     ┆ ---            ┆ ---  │\n",
       "│ i64     ┆ str                     ┆ str            ┆ i64  │\n",
       "╞═════════╪═════════════════════════╪════════════════╪══════╡\n",
       "│ 1       ┆ Bohemian Rhapsody       ┆ Queen          ┆ 1975 │\n",
       "│ 2       ┆ Roller Coaster          ┆ Danny Vera     ┆ 2019 │\n",
       "│ 3       ┆ Hotel California        ┆ Eagles         ┆ 1977 │\n",
       "│ 4       ┆ Piano Man               ┆ Billy Joel     ┆ 1974 │\n",
       "│ 5       ┆ Fix You                 ┆ Coldplay       ┆ 2005 │\n",
       "│ …       ┆ …                       ┆ …              ┆ …    │\n",
       "│ 1996    ┆ Charlie Brown           ┆ Coldplay       ┆ 2011 │\n",
       "│ 1997    ┆ Beast Of Burden         ┆ Bette Midler   ┆ 1984 │\n",
       "│ 1998    ┆ It Was A Very Good Year ┆ Frank Sinatra  ┆ 1968 │\n",
       "│ 1999    ┆ Hou Van Mij             ┆ 3JS            ┆ 2008 │\n",
       "│ 2000    ┆ Drivers License         ┆ Olivia Rodrigo ┆ 2021 │\n",
       "└─────────┴─────────────────────────┴────────────────┴──────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_fixed = pl.read_excel(\n",
    "    \"data/top2000-2023.xlsx\", read_options={\"skip_rows_after_header\": 1}\n",
    ")\n",
    "songs_fixed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8146a25e",
   "metadata": {},
   "source": [
    "## Working with Multiple Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "78b81c2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (2_516, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>symbol</th><th>date</th><th>open</th><th>high</th><th>low</th><th>close</th><th>adj close</th><th>volume</th></tr><tr><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;NVDA&quot;</td><td>&quot;2010-01-04&quot;</td><td>4.6275</td><td>4.655</td><td>4.5275</td><td>4.6225</td><td>4.240429</td><td>80020400</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2010-01-05&quot;</td><td>4.605</td><td>4.74</td><td>4.605</td><td>4.69</td><td>4.30235</td><td>72864800</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2010-01-06&quot;</td><td>4.6875</td><td>4.73</td><td>4.6425</td><td>4.72</td><td>4.32987</td><td>64916800</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2010-01-07&quot;</td><td>4.695</td><td>4.715</td><td>4.5925</td><td>4.6275</td><td>4.245015</td><td>54779200</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2010-01-08&quot;</td><td>4.59</td><td>4.67</td><td>4.5625</td><td>4.6375</td><td>4.254189</td><td>47816800</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2019-12-24&quot;</td><td>59.549999</td><td>59.827499</td><td>59.205002</td><td>59.654999</td><td>59.422798</td><td>13886400</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2019-12-26&quot;</td><td>59.689999</td><td>60.080002</td><td>59.5</td><td>59.797501</td><td>59.564739</td><td>18285200</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2019-12-27&quot;</td><td>59.950001</td><td>60.084999</td><td>58.952499</td><td>59.217499</td><td>58.987</td><td>25464400</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2019-12-30&quot;</td><td>58.997501</td><td>59.049999</td><td>57.764999</td><td>58.080002</td><td>57.853928</td><td>25805600</td></tr><tr><td>&quot;NVDA&quot;</td><td>&quot;2019-12-31&quot;</td><td>57.724998</td><td>58.919998</td><td>57.532501</td><td>58.825001</td><td>58.596027</td><td>23100400</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (2_516, 8)\n",
       "┌────────┬────────────┬───────────┬───────────┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ symbol ┆ date       ┆ open      ┆ high      ┆ low       ┆ close     ┆ adj close ┆ volume   │\n",
       "│ ---    ┆ ---        ┆ ---       ┆ ---       ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n",
       "│ str    ┆ str        ┆ f64       ┆ f64       ┆ f64       ┆ f64       ┆ f64       ┆ i64      │\n",
       "╞════════╪════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ NVDA   ┆ 2010-01-04 ┆ 4.6275    ┆ 4.655     ┆ 4.5275    ┆ 4.6225    ┆ 4.240429  ┆ 80020400 │\n",
       "│ NVDA   ┆ 2010-01-05 ┆ 4.605     ┆ 4.74      ┆ 4.605     ┆ 4.69      ┆ 4.30235   ┆ 72864800 │\n",
       "│ NVDA   ┆ 2010-01-06 ┆ 4.6875    ┆ 4.73      ┆ 4.6425    ┆ 4.72      ┆ 4.32987   ┆ 64916800 │\n",
       "│ NVDA   ┆ 2010-01-07 ┆ 4.695     ┆ 4.715     ┆ 4.5925    ┆ 4.6275    ┆ 4.245015  ┆ 54779200 │\n",
       "│ NVDA   ┆ 2010-01-08 ┆ 4.59      ┆ 4.67      ┆ 4.5625    ┆ 4.6375    ┆ 4.254189  ┆ 47816800 │\n",
       "│ …      ┆ …          ┆ …         ┆ …         ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ NVDA   ┆ 2019-12-24 ┆ 59.549999 ┆ 59.827499 ┆ 59.205002 ┆ 59.654999 ┆ 59.422798 ┆ 13886400 │\n",
       "│ NVDA   ┆ 2019-12-26 ┆ 59.689999 ┆ 60.080002 ┆ 59.5      ┆ 59.797501 ┆ 59.564739 ┆ 18285200 │\n",
       "│ NVDA   ┆ 2019-12-27 ┆ 59.950001 ┆ 60.084999 ┆ 58.952499 ┆ 59.217499 ┆ 58.987    ┆ 25464400 │\n",
       "│ NVDA   ┆ 2019-12-30 ┆ 58.997501 ┆ 59.049999 ┆ 57.764999 ┆ 58.080002 ┆ 57.853928 ┆ 25805600 │\n",
       "│ NVDA   ┆ 2019-12-31 ┆ 57.724998 ┆ 58.919998 ┆ 57.532501 ┆ 58.825001 ┆ 58.596027 ┆ 23100400 │\n",
       "└────────┴────────────┴───────────┴───────────┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.read_csv(\"data/stock/nvda/201[0-9].csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "98f4402a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (18_476, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>symbol</th><th>date</th><th>open</th><th>high</th><th>low</th><th>close</th><th>adj close</th><th>volume</th></tr><tr><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;ASML&quot;</td><td>&quot;1999-01-04&quot;</td><td>11.765625</td><td>12.28125</td><td>11.765625</td><td>12.140625</td><td>7.522523</td><td>1801867</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;1999-01-05&quot;</td><td>11.859375</td><td>14.25</td><td>11.71875</td><td>13.96875</td><td>8.655257</td><td>8241600</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;1999-01-06&quot;</td><td>14.25</td><td>17.601563</td><td>14.203125</td><td>16.875</td><td>10.456018</td><td>16400267</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;1999-01-07&quot;</td><td>14.742188</td><td>17.8125</td><td>14.53125</td><td>16.851563</td><td>10.441495</td><td>17722133</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;1999-01-08&quot;</td><td>16.078125</td><td>16.289063</td><td>15.023438</td><td>15.796875</td><td>9.787995</td><td>10696000</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;TSM&quot;</td><td>&quot;2023-06-26&quot;</td><td>102.019997</td><td>103.040001</td><td>100.089996</td><td>100.110001</td><td>99.125954</td><td>8560000</td></tr><tr><td>&quot;TSM&quot;</td><td>&quot;2023-06-27&quot;</td><td>101.150002</td><td>102.790001</td><td>100.019997</td><td>102.080002</td><td>101.076591</td><td>9732000</td></tr><tr><td>&quot;TSM&quot;</td><td>&quot;2023-06-28&quot;</td><td>100.5</td><td>101.879997</td><td>100.220001</td><td>100.919998</td><td>99.927986</td><td>8160900</td></tr><tr><td>&quot;TSM&quot;</td><td>&quot;2023-06-29&quot;</td><td>101.339996</td><td>101.519997</td><td>100.019997</td><td>100.639999</td><td>99.650742</td><td>7383900</td></tr><tr><td>&quot;TSM&quot;</td><td>&quot;2023-06-30&quot;</td><td>101.400002</td><td>101.889999</td><td>100.410004</td><td>100.919998</td><td>99.927986</td><td>11701700</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (18_476, 8)\n",
       "┌────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬──────────┐\n",
       "│ symbol ┆ date       ┆ open       ┆ high       ┆ low        ┆ close      ┆ adj close  ┆ volume   │\n",
       "│ ---    ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---      │\n",
       "│ str    ┆ str        ┆ f64        ┆ f64        ┆ f64        ┆ f64        ┆ f64        ┆ i64      │\n",
       "╞════════╪════════════╪════════════╪════════════╪════════════╪════════════╪════════════╪══════════╡\n",
       "│ ASML   ┆ 1999-01-04 ┆ 11.765625  ┆ 12.28125   ┆ 11.765625  ┆ 12.140625  ┆ 7.522523   ┆ 1801867  │\n",
       "│ ASML   ┆ 1999-01-05 ┆ 11.859375  ┆ 14.25      ┆ 11.71875   ┆ 13.96875   ┆ 8.655257   ┆ 8241600  │\n",
       "│ ASML   ┆ 1999-01-06 ┆ 14.25      ┆ 17.601563  ┆ 14.203125  ┆ 16.875     ┆ 10.456018  ┆ 16400267 │\n",
       "│ ASML   ┆ 1999-01-07 ┆ 14.742188  ┆ 17.8125    ┆ 14.53125   ┆ 16.851563  ┆ 10.441495  ┆ 17722133 │\n",
       "│ ASML   ┆ 1999-01-08 ┆ 16.078125  ┆ 16.289063  ┆ 15.023438  ┆ 15.796875  ┆ 9.787995   ┆ 10696000 │\n",
       "│ …      ┆ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …        │\n",
       "│ TSM    ┆ 2023-06-26 ┆ 102.019997 ┆ 103.040001 ┆ 100.089996 ┆ 100.110001 ┆ 99.125954  ┆ 8560000  │\n",
       "│ TSM    ┆ 2023-06-27 ┆ 101.150002 ┆ 102.790001 ┆ 100.019997 ┆ 102.080002 ┆ 101.076591 ┆ 9732000  │\n",
       "│ TSM    ┆ 2023-06-28 ┆ 100.5      ┆ 101.879997 ┆ 100.220001 ┆ 100.919998 ┆ 99.927986  ┆ 8160900  │\n",
       "│ TSM    ┆ 2023-06-29 ┆ 101.339996 ┆ 101.519997 ┆ 100.019997 ┆ 100.639999 ┆ 99.650742  ┆ 7383900  │\n",
       "│ TSM    ┆ 2023-06-30 ┆ 101.400002 ┆ 101.889999 ┆ 100.410004 ┆ 100.919998 ┆ 99.927986  ┆ 11701700 │\n",
       "└────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴──────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stocks = pl.read_csv(\"data/stock/*/*.csv\")\n",
    "all_stocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4fd2750d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/stock/asml/2000.csv',\n",
       " 'data/stock/asml/2004.csv',\n",
       " 'data/stock/asml/2008.csv',\n",
       " 'data/stock/asml/2012.csv',\n",
       " 'data/stock/asml/2016.csv',\n",
       " 'data/stock/asml/2020.csv']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import calendar\n",
    "\n",
    "filenames = [\n",
    "    f\"data/stock/asml/{year}.csv\"\n",
    "    for year in range(1999, 2024)\n",
    "    if calendar.isleap(year)\n",
    "]\n",
    "\n",
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0cb5bf2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1_512, 8)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>symbol</th><th>date</th><th>open</th><th>high</th><th>low</th><th>close</th><th>adj close</th><th>volume</th></tr><tr><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;ASML&quot;</td><td>&quot;2000-01-03&quot;</td><td>43.875</td><td>43.875</td><td>41.90625</td><td>43.640625</td><td>27.040424</td><td>1121600</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2000-01-04&quot;</td><td>41.953125</td><td>42.5625</td><td>40.59375</td><td>40.734375</td><td>25.239666</td><td>968800</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2000-01-05&quot;</td><td>39.28125</td><td>39.703125</td><td>37.757813</td><td>39.609375</td><td>24.542597</td><td>1458133</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2000-01-06&quot;</td><td>36.75</td><td>37.59375</td><td>35.226563</td><td>37.171875</td><td>23.032274</td><td>3517867</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2000-01-07&quot;</td><td>36.867188</td><td>38.0625</td><td>36.65625</td><td>38.015625</td><td>23.555077</td><td>1631200</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2020-12-24&quot;</td><td>478.950012</td><td>484.600006</td><td>477.079987</td><td>483.089996</td><td>468.836365</td><td>271900</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2020-12-28&quot;</td><td>487.140015</td><td>488.720001</td><td>478.429993</td><td>480.23999</td><td>466.070496</td><td>449300</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2020-12-29&quot;</td><td>489.450012</td><td>489.450012</td><td>482.51001</td><td>484.01001</td><td>469.729218</td><td>377200</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2020-12-30&quot;</td><td>488.130005</td><td>492.660004</td><td>488.0</td><td>489.910004</td><td>475.455231</td><td>381900</td></tr><tr><td>&quot;ASML&quot;</td><td>&quot;2020-12-31&quot;</td><td>490.0</td><td>490.519989</td><td>483.450012</td><td>487.720001</td><td>473.329803</td><td>312700</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1_512, 8)\n",
       "┌────────┬────────────┬────────────┬────────────┬────────────┬────────────┬────────────┬─────────┐\n",
       "│ symbol ┆ date       ┆ open       ┆ high       ┆ low        ┆ close      ┆ adj close  ┆ volume  │\n",
       "│ ---    ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---        ┆ ---     │\n",
       "│ str    ┆ str        ┆ f64        ┆ f64        ┆ f64        ┆ f64        ┆ f64        ┆ i64     │\n",
       "╞════════╪════════════╪════════════╪════════════╪════════════╪════════════╪════════════╪═════════╡\n",
       "│ ASML   ┆ 2000-01-03 ┆ 43.875     ┆ 43.875     ┆ 41.90625   ┆ 43.640625  ┆ 27.040424  ┆ 1121600 │\n",
       "│ ASML   ┆ 2000-01-04 ┆ 41.953125  ┆ 42.5625    ┆ 40.59375   ┆ 40.734375  ┆ 25.239666  ┆ 968800  │\n",
       "│ ASML   ┆ 2000-01-05 ┆ 39.28125   ┆ 39.703125  ┆ 37.757813  ┆ 39.609375  ┆ 24.542597  ┆ 1458133 │\n",
       "│ ASML   ┆ 2000-01-06 ┆ 36.75      ┆ 37.59375   ┆ 35.226563  ┆ 37.171875  ┆ 23.032274  ┆ 3517867 │\n",
       "│ ASML   ┆ 2000-01-07 ┆ 36.867188  ┆ 38.0625    ┆ 36.65625   ┆ 38.015625  ┆ 23.555077  ┆ 1631200 │\n",
       "│ …      ┆ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …          ┆ …       │\n",
       "│ ASML   ┆ 2020-12-24 ┆ 478.950012 ┆ 484.600006 ┆ 477.079987 ┆ 483.089996 ┆ 468.836365 ┆ 271900  │\n",
       "│ ASML   ┆ 2020-12-28 ┆ 487.140015 ┆ 488.720001 ┆ 478.429993 ┆ 480.23999  ┆ 466.070496 ┆ 449300  │\n",
       "│ ASML   ┆ 2020-12-29 ┆ 489.450012 ┆ 489.450012 ┆ 482.51001  ┆ 484.01001  ┆ 469.729218 ┆ 377200  │\n",
       "│ ASML   ┆ 2020-12-30 ┆ 488.130005 ┆ 492.660004 ┆ 488.0      ┆ 489.910004 ┆ 475.455231 ┆ 381900  │\n",
       "│ ASML   ┆ 2020-12-31 ┆ 490.0      ┆ 490.519989 ┆ 483.450012 ┆ 487.720001 ┆ 473.329803 ┆ 312700  │\n",
       "└────────┴────────────┴────────────┴────────────┴────────────┴────────────┴────────────┴─────────┘"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.concat(pl.read_csv(f) for f in filenames)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7540b0cd",
   "metadata": {},
   "source": [
    "## Reading Parquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fed3857c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (39_656_098, 19)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>VendorID</th><th>tpep_pickup_datetime</th><th>tpep_dropoff_datetime</th><th>passenger_count</th><th>trip_distance</th><th>RatecodeID</th><th>store_and_fwd_flag</th><th>PULocationID</th><th>DOLocationID</th><th>payment_type</th><th>fare_amount</th><th>extra</th><th>mta_tax</th><th>tip_amount</th><th>tolls_amount</th><th>improvement_surcharge</th><th>total_amount</th><th>congestion_surcharge</th><th>airport_fee</th></tr><tr><td>i64</td><td>datetime[ns]</td><td>datetime[ns]</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>1</td><td>2022-01-01 00:35:40</td><td>2022-01-01 00:53:29</td><td>2.0</td><td>3.8</td><td>1.0</td><td>&quot;N&quot;</td><td>142</td><td>236</td><td>1</td><td>14.5</td><td>3.0</td><td>0.5</td><td>3.65</td><td>0.0</td><td>0.3</td><td>21.95</td><td>2.5</td><td>0.0</td></tr><tr><td>1</td><td>2022-01-01 00:33:43</td><td>2022-01-01 00:42:07</td><td>1.0</td><td>2.1</td><td>1.0</td><td>&quot;N&quot;</td><td>236</td><td>42</td><td>1</td><td>8.0</td><td>0.5</td><td>0.5</td><td>4.0</td><td>0.0</td><td>0.3</td><td>13.3</td><td>0.0</td><td>0.0</td></tr><tr><td>2</td><td>2022-01-01 00:53:21</td><td>2022-01-01 01:02:19</td><td>1.0</td><td>0.97</td><td>1.0</td><td>&quot;N&quot;</td><td>166</td><td>166</td><td>1</td><td>7.5</td><td>0.5</td><td>0.5</td><td>1.76</td><td>0.0</td><td>0.3</td><td>10.56</td><td>0.0</td><td>0.0</td></tr><tr><td>2</td><td>2022-01-01 00:25:21</td><td>2022-01-01 00:35:23</td><td>1.0</td><td>1.09</td><td>1.0</td><td>&quot;N&quot;</td><td>114</td><td>68</td><td>2</td><td>8.0</td><td>0.5</td><td>0.5</td><td>0.0</td><td>0.0</td><td>0.3</td><td>11.8</td><td>2.5</td><td>0.0</td></tr><tr><td>2</td><td>2022-01-01 00:36:48</td><td>2022-01-01 01:14:20</td><td>1.0</td><td>4.3</td><td>1.0</td><td>&quot;N&quot;</td><td>68</td><td>163</td><td>1</td><td>23.5</td><td>0.5</td><td>0.5</td><td>3.0</td><td>0.0</td><td>0.3</td><td>30.3</td><td>2.5</td><td>0.0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>2</td><td>2022-12-31 23:46:00</td><td>2023-01-01 00:11:00</td><td>null</td><td>11.56</td><td>null</td><td>null</td><td>16</td><td>36</td><td>0</td><td>39.55</td><td>0.0</td><td>0.5</td><td>8.21</td><td>0.0</td><td>1.0</td><td>49.26</td><td>null</td><td>null</td></tr><tr><td>2</td><td>2022-12-31 23:13:24</td><td>2022-12-31 23:29:08</td><td>null</td><td>5.06</td><td>null</td><td>null</td><td>75</td><td>50</td><td>0</td><td>26.23</td><td>0.0</td><td>0.5</td><td>0.0</td><td>0.0</td><td>1.0</td><td>30.23</td><td>null</td><td>null</td></tr><tr><td>2</td><td>2022-12-31 23:00:49</td><td>2022-12-31 23:26:57</td><td>null</td><td>13.35</td><td>null</td><td>null</td><td>168</td><td>197</td><td>0</td><td>47.73</td><td>0.0</td><td>0.5</td><td>9.85</td><td>0.0</td><td>1.0</td><td>59.08</td><td>null</td><td>null</td></tr><tr><td>1</td><td>2022-12-31 23:02:50</td><td>2022-12-31 23:16:05</td><td>null</td><td>0.0</td><td>null</td><td>null</td><td>238</td><td>116</td><td>0</td><td>12.74</td><td>0.0</td><td>0.5</td><td>0.0</td><td>0.0</td><td>1.0</td><td>16.74</td><td>null</td><td>null</td></tr><tr><td>2</td><td>2022-12-31 23:00:15</td><td>2022-12-31 23:22:56</td><td>null</td><td>4.24</td><td>null</td><td>null</td><td>85</td><td>37</td><td>0</td><td>19.6</td><td>0.0</td><td>0.5</td><td>4.0</td><td>0.0</td><td>1.0</td><td>25.1</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (39_656_098, 19)\n",
       "┌──────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬───────────┐\n",
       "│ VendorID ┆ tpep_pick ┆ tpep_drop ┆ passenger ┆ … ┆ improveme ┆ total_amo ┆ congestio ┆ airport_f │\n",
       "│ ---      ┆ up_dateti ┆ off_datet ┆ _count    ┆   ┆ nt_surcha ┆ unt       ┆ n_surchar ┆ ee        │\n",
       "│ i64      ┆ me        ┆ ime       ┆ ---       ┆   ┆ rge       ┆ ---       ┆ ge        ┆ ---       │\n",
       "│          ┆ ---       ┆ ---       ┆ f64       ┆   ┆ ---       ┆ f64       ┆ ---       ┆ f64       │\n",
       "│          ┆ datetime[ ┆ datetime[ ┆           ┆   ┆ f64       ┆           ┆ f64       ┆           │\n",
       "│          ┆ ns]       ┆ ns]       ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "╞══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n",
       "│ 1        ┆ 2022-01-0 ┆ 2022-01-0 ┆ 2.0       ┆ … ┆ 0.3       ┆ 21.95     ┆ 2.5       ┆ 0.0       │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 00:35:40  ┆ 00:53:29  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 1        ┆ 2022-01-0 ┆ 2022-01-0 ┆ 1.0       ┆ … ┆ 0.3       ┆ 13.3      ┆ 0.0       ┆ 0.0       │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 00:33:43  ┆ 00:42:07  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-01-0 ┆ 2022-01-0 ┆ 1.0       ┆ … ┆ 0.3       ┆ 10.56     ┆ 0.0       ┆ 0.0       │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 00:53:21  ┆ 01:02:19  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-01-0 ┆ 2022-01-0 ┆ 1.0       ┆ … ┆ 0.3       ┆ 11.8      ┆ 2.5       ┆ 0.0       │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 00:25:21  ┆ 00:35:23  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-01-0 ┆ 2022-01-0 ┆ 1.0       ┆ … ┆ 0.3       ┆ 30.3      ┆ 2.5       ┆ 0.0       │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 00:36:48  ┆ 01:14:20  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ …        ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …         │\n",
       "│ 2        ┆ 2022-12-3 ┆ 2023-01-0 ┆ null      ┆ … ┆ 1.0       ┆ 49.26     ┆ null      ┆ null      │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 23:46:00  ┆ 00:11:00  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-12-3 ┆ 2022-12-3 ┆ null      ┆ … ┆ 1.0       ┆ 30.23     ┆ null      ┆ null      │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 23:13:24  ┆ 23:29:08  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-12-3 ┆ 2022-12-3 ┆ null      ┆ … ┆ 1.0       ┆ 59.08     ┆ null      ┆ null      │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 23:00:49  ┆ 23:26:57  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 1        ┆ 2022-12-3 ┆ 2022-12-3 ┆ null      ┆ … ┆ 1.0       ┆ 16.74     ┆ null      ┆ null      │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 23:02:50  ┆ 23:16:05  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│ 2        ┆ 2022-12-3 ┆ 2022-12-3 ┆ null      ┆ … ┆ 1.0       ┆ 25.1      ┆ null      ┆ null      │\n",
       "│          ┆ 1         ┆ 1         ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "│          ┆ 23:00:15  ┆ 23:22:56  ┆           ┆   ┆           ┆           ┆           ┆           │\n",
       "└──────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trips = pl.read_parquet(\"data/taxi/yellow_tripdata_*.parquet\")\n",
    "trips"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "342865ad",
   "metadata": {},
   "source": [
    "## Reading JSON and NDJSON"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "024bf650",
   "metadata": {},
   "source": [
    "### JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b2230731",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>pokemon</th></tr><tr><td>list[struct[17]]</td></tr></thead><tbody><tr><td>[{1,&quot;001&quot;,&quot;Bulbasaur&quot;,&quot;http://www.serebii.net/pokemongo/pokemon/001.png&quot;,[&quot;Grass&quot;, &quot;Poison&quot;],&quot;0.71 m&quot;,&quot;6.9 kg&quot;,&quot;Bulbasaur Candy&quot;,&quot;2 km&quot;,0.69,69.0,&quot;20:00&quot;,[1.58],[&quot;Fire&quot;, &quot;Ice&quot;, … &quot;Psychic&quot;],25,[{&quot;002&quot;,&quot;Ivysaur&quot;}, {&quot;003&quot;,&quot;Venusaur&quot;}],null}, {2,&quot;002&quot;,&quot;Ivysaur&quot;,&quot;http://www.serebii.net/pokemongo/pokemon/002.png&quot;,[&quot;Grass&quot;, &quot;Poison&quot;],&quot;0.99 m&quot;,&quot;13.0 kg&quot;,&quot;Bulbasaur Candy&quot;,&quot;Not in Eggs&quot;,0.042,4.2,&quot;07:00&quot;,[1.2, 1.6],[&quot;Fire&quot;, &quot;Ice&quot;, … &quot;Psychic&quot;],100,[{&quot;003&quot;,&quot;Venusaur&quot;}],[{&quot;001&quot;,&quot;Bulbasaur&quot;}]}, … {151,&quot;151&quot;,&quot;Mew&quot;,&quot;http://www.serebii.net/pokemongo/pokemon/151.png&quot;,[&quot;Psychic&quot;],&quot;0.41 m&quot;,&quot;4.0 kg&quot;,&quot;None&quot;,&quot;Not in Eggs&quot;,0.0,0.0,&quot;N/A&quot;,null,[&quot;Bug&quot;, &quot;Ghost&quot;, &quot;Dark&quot;],null,null,null}]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 1)\n",
       "┌─────────────────────────────────┐\n",
       "│ pokemon                         │\n",
       "│ ---                             │\n",
       "│ list[struct[17]]                │\n",
       "╞═════════════════════════════════╡\n",
       "│ [{1,\"001\",\"Bulbasaur\",\"http://… │\n",
       "└─────────────────────────────────┘"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pokedex = pl.read_json(\"data/pokedex.json\")\n",
    "pokedex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bb5406d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (151, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>id</th><th>name</th><th>type</th><th>height</th><th>weight</th></tr><tr><td>i64</td><td>str</td><td>list[str]</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>1</td><td>&quot;Bulbasaur&quot;</td><td>[&quot;Grass&quot;, &quot;Poison&quot;]</td><td>&quot;0.71 m&quot;</td><td>&quot;6.9 kg&quot;</td></tr><tr><td>2</td><td>&quot;Ivysaur&quot;</td><td>[&quot;Grass&quot;, &quot;Poison&quot;]</td><td>&quot;0.99 m&quot;</td><td>&quot;13.0 kg&quot;</td></tr><tr><td>3</td><td>&quot;Venusaur&quot;</td><td>[&quot;Grass&quot;, &quot;Poison&quot;]</td><td>&quot;2.01 m&quot;</td><td>&quot;100.0 kg&quot;</td></tr><tr><td>4</td><td>&quot;Charmander&quot;</td><td>[&quot;Fire&quot;]</td><td>&quot;0.61 m&quot;</td><td>&quot;8.5 kg&quot;</td></tr><tr><td>5</td><td>&quot;Charmeleon&quot;</td><td>[&quot;Fire&quot;]</td><td>&quot;1.09 m&quot;</td><td>&quot;19.0 kg&quot;</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>147</td><td>&quot;Dratini&quot;</td><td>[&quot;Dragon&quot;]</td><td>&quot;1.80 m&quot;</td><td>&quot;3.3 kg&quot;</td></tr><tr><td>148</td><td>&quot;Dragonair&quot;</td><td>[&quot;Dragon&quot;]</td><td>&quot;3.99 m&quot;</td><td>&quot;16.5 kg&quot;</td></tr><tr><td>149</td><td>&quot;Dragonite&quot;</td><td>[&quot;Dragon&quot;, &quot;Flying&quot;]</td><td>&quot;2.21 m&quot;</td><td>&quot;210.0 kg&quot;</td></tr><tr><td>150</td><td>&quot;Mewtwo&quot;</td><td>[&quot;Psychic&quot;]</td><td>&quot;2.01 m&quot;</td><td>&quot;122.0 kg&quot;</td></tr><tr><td>151</td><td>&quot;Mew&quot;</td><td>[&quot;Psychic&quot;]</td><td>&quot;0.41 m&quot;</td><td>&quot;4.0 kg&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (151, 5)\n",
       "┌─────┬────────────┬──────────────────────┬────────┬──────────┐\n",
       "│ id  ┆ name       ┆ type                 ┆ height ┆ weight   │\n",
       "│ --- ┆ ---        ┆ ---                  ┆ ---    ┆ ---      │\n",
       "│ i64 ┆ str        ┆ list[str]            ┆ str    ┆ str      │\n",
       "╞═════╪════════════╪══════════════════════╪════════╪══════════╡\n",
       "│ 1   ┆ Bulbasaur  ┆ [\"Grass\", \"Poison\"]  ┆ 0.71 m ┆ 6.9 kg   │\n",
       "│ 2   ┆ Ivysaur    ┆ [\"Grass\", \"Poison\"]  ┆ 0.99 m ┆ 13.0 kg  │\n",
       "│ 3   ┆ Venusaur   ┆ [\"Grass\", \"Poison\"]  ┆ 2.01 m ┆ 100.0 kg │\n",
       "│ 4   ┆ Charmander ┆ [\"Fire\"]             ┆ 0.61 m ┆ 8.5 kg   │\n",
       "│ 5   ┆ Charmeleon ┆ [\"Fire\"]             ┆ 1.09 m ┆ 19.0 kg  │\n",
       "│ …   ┆ …          ┆ …                    ┆ …      ┆ …        │\n",
       "│ 147 ┆ Dratini    ┆ [\"Dragon\"]           ┆ 1.80 m ┆ 3.3 kg   │\n",
       "│ 148 ┆ Dragonair  ┆ [\"Dragon\"]           ┆ 3.99 m ┆ 16.5 kg  │\n",
       "│ 149 ┆ Dragonite  ┆ [\"Dragon\", \"Flying\"] ┆ 2.21 m ┆ 210.0 kg │\n",
       "│ 150 ┆ Mewtwo     ┆ [\"Psychic\"]          ┆ 2.01 m ┆ 122.0 kg │\n",
       "│ 151 ┆ Mew        ┆ [\"Psychic\"]          ┆ 0.41 m ┆ 4.0 kg   │\n",
       "└─────┴────────────┴──────────────────────┴────────┴──────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    pokedex.explode(\"pokemon\")\n",
    "    .unnest(\"pokemon\")\n",
    "    .select(\"id\", \"name\", \"type\", \"height\", \"weight\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b27113b5",
   "metadata": {},
   "source": [
    "### NDJSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "058d8eb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'$schema': '/mediawiki/recentchange/1.0.0',\n",
      " 'bot': False,\n",
      " 'comment': '/* League champions, runners-up and play-off finalists */',\n",
      " 'id': 1659529639,\n",
      " 'length': {'new': 91166, 'old': 91108},\n",
      " 'meta': {'domain': 'en.wikipedia.org',\n",
      "          'dt': '2023-07-29T07:51:39Z',\n",
      "          'id': '0416300b-980c-45bb-b0a2-c9d7a9e2b7eb',\n",
      "          'offset': 4820784717,\n",
      "          'partition': 0,\n",
      "          'request_id': 'ea0541fb-4e72-4fc3-82f0-6c26651b2043',\n",
      "          'stream': 'mediawiki.recentchange',\n",
      "          'topic': 'eqiad.mediawiki.recentchange',\n",
      "          'uri': 'https://en.wikipedia.org/wiki/EFL_Championship'},\n",
      " 'minor': False,\n",
      " 'namespace': 0,\n",
      " 'notify_url': 'https://en.wikipedia.org/w/index.php?diff=1167689309&oldid=1166824248',\n",
      " 'parsedcomment': '<span dir=\"auto\"><span class=\"autocomment\"><a '\n",
      "                  'href=\"/wiki/EFL_Championship#League_champions,_runners-up_and_play-off_finalists\" '\n",
      "                  'title=\"EFL Championship\">→\\u200eLeague champions, '\n",
      "                  'runners-up and play-off finalists</a></span></span>',\n",
      " 'revision': {'new': 1167689309, 'old': 1166824248},\n",
      " 'server_name': 'en.wikipedia.org',\n",
      " 'server_script_path': '/w',\n",
      " 'server_url': 'https://en.wikipedia.org',\n",
      " 'timestamp': 1690617099,\n",
      " 'title': 'EFL Championship',\n",
      " 'title_url': 'https://en.wikipedia.org/wiki/EFL_Championship',\n",
      " 'type': 'edit',\n",
      " 'user': '87.12.215.232',\n",
      " 'wiki': 'enwiki'}\n"
     ]
    }
   ],
   "source": [
    "from json import loads\n",
    "from pprint import pprint\n",
    "\n",
    "with open(\"data/wikimedia.ndjson\") as f:\n",
    "    pprint(loads(f.readline()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b80cb154",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (100, 20)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>$schema</th><th>meta</th><th>id</th><th>type</th><th>namespace</th><th>title</th><th>title_url</th><th>comment</th><th>timestamp</th><th>user</th><th>bot</th><th>notify_url</th><th>minor</th><th>length</th><th>revision</th><th>server_url</th><th>server_name</th><th>server_script_path</th><th>wiki</th><th>parsedcomment</th></tr><tr><td>str</td><td>struct[9]</td><td>i64</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>bool</td><td>str</td><td>bool</td><td>struct[2]</td><td>struct[2]</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/EFL_Championship&quot;,&quot;ea0541fb-4e72-4fc3-82f0-6c26651b2043&quot;,&quot;0416300b-980c-45bb-b0a2-c9d7a9e2b7eb&quot;,&quot;2023-07-29T07:51:39Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820784717}</td><td>1659529639</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;EFL Championship&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;/* League champions, runners-u…</td><td>1690617099</td><td>&quot;87.12.215.232&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{91108,91166}</td><td>{1166824248,1167689309}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&lt;span dir=&quot;auto&quot;&gt;&lt;span class=&quot;…</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Lim_Sang-choon&quot;,&quot;01a0f468-7553-48db-b553-7ac392b2187c&quot;,&quot;97e4dc39-fb32-4774-9c9a-b2caea391c9e&quot;,&quot;2023-07-29T07:51:42Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820784731}</td><td>1659529640</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Lim Sang-choon&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;&quot;</td><td>1690617102</td><td>&quot;Preferwiki&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{9807,10480}</td><td>{1167689034,1167689310}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&quot;</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Higher&quot;,&quot;5f053899-a2ab-4dec-8e98-1d01cd86093d&quot;,&quot;0da41aa2-ceb6-443f-8a0a-0633f83de6ec&quot;,&quot;2023-07-29T07:51:44Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820784748}</td><td>1659529642</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Higher&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;/* Albums */ add&quot;</td><td>1690617104</td><td>&quot;Ss112&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{5452,5548}</td><td>{1162509981,1167689312}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&lt;span dir=&quot;auto&quot;&gt;&lt;span class=&quot;…</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/International_Poker_Rules&quot;,&quot;10c4886c-95f5-4cd2-8db9-333cb45f041b&quot;,&quot;af9cc405-26b2-485d-b20c-edd21b2a2a4c&quot;,&quot;2023-07-29T07:51:44Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820784763}</td><td>1659529643</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;International Poker Rules&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;Nominated for deletion; see [[…</td><td>1690617104</td><td>&quot;Piotrus&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{2452,2896}</td><td>{1055827921,1167689313}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;Nominated for deletion; see &lt;a…</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Abdul_Hamid_Khan_Bhashani&quot;,&quot;198e47ae-fa33-4059-970a-550536e7bc7c&quot;,&quot;2012e36f-9fa1-49dc-ba68-5946de740cd4&quot;,&quot;2023-07-29T07:51:45Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820784800}</td><td>1659529653</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Abdul Hamid Khan Bhashani&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;Rescuing 1 sources and tagging…</td><td>1690617105</td><td>&quot;InternetArchiveBot&quot;</td><td>true</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{31503,31687}</td><td>{1163358967,1167689318}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;Rescuing 1 sources and tagging…</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Havering_Residents_Association&quot;,&quot;d563edd1-2a19-491b-9604-aea1b7c57a67&quot;,&quot;f5b5b53d-8024-4c4d-810b-e3c04b8f2392&quot;,&quot;2023-07-29T07:53:58Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820788373}</td><td>1659529961</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Havering Residents Association&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;/* 2018 election */ +map&quot;</td><td>1690617238</td><td>&quot;MRSC&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{7590,7707}</td><td>{1167689281,1167689476}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&lt;span dir=&quot;auto&quot;&gt;&lt;span class=&quot;…</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Olha_Kharlan&quot;,&quot;7caa4fe3-2ddf-4a57-b6d4-6eee505f5e49&quot;,&quot;b7b84f63-69f7-4317-82b2-df4a3e853723&quot;,&quot;2023-07-29T07:53:55Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820788404}</td><td>1659529963</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Olha Kharlan&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;Ce&quot;</td><td>1690617235</td><td>&quot;2603:7000:2101:AA00:2C88:EF86:…</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{65119,65323}</td><td>{1167666634,1167689477}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;Ce&quot;</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/Mukim_Kota_Batu&quot;,&quot;1032a354-1171-47a2-bdb3-cbcb78a070bf&quot;,&quot;a2b2601e-5adc-4ed2-b8fc-292d128009d7&quot;,&quot;2023-07-29T07:53:58Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820788418}</td><td>1659529964</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;Mukim Kota Batu&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;&quot;</td><td>1690617238</td><td>&quot;Pangalau&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{7178,7189}</td><td>{1150066841,1167689474}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&quot;</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/User:IDK1213safasx/sandbox&quot;,&quot;0f29a0a7-de5a-4197-a8dd-98b6a4eb5f6f&quot;,&quot;5c2219b5-7b3e-4720-944b-94afb5764b76&quot;,&quot;2023-07-29T07:53:59Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820788422}</td><td>1659529965</td><td>&quot;edit&quot;</td><td>2</td><td>&quot;User:IDK1213safasx/sandbox&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;&quot;</td><td>1690617239</td><td>&quot;94.101.29.27&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{2122,2122}</td><td>{1167356449,1167689478}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&quot;</td></tr><tr><td>&quot;/mediawiki/recentchange/1.0.0&quot;</td><td>{&quot;https://en.wikipedia.org/wiki/List_of_bus_routes_in_London&quot;,&quot;04c7fd52-5689-4905-a208-9a92900e8efe&quot;,&quot;2eea26f7-4a02-454e-b597-82d3cb78f2ea&quot;,&quot;2023-07-29T07:53:54Z&quot;,&quot;en.wikipedia.org&quot;,&quot;mediawiki.recentchange&quot;,&quot;eqiad.mediawiki.recentchange&quot;,0,4820788423}</td><td>1659529966</td><td>&quot;edit&quot;</td><td>0</td><td>&quot;List of bus routes in London&quot;</td><td>&quot;https://en.wikipedia.org/wiki/…</td><td>&quot;/* Non-TfL bus routes in Great…</td><td>1690617234</td><td>&quot;Pedroperezhumberto&quot;</td><td>false</td><td>&quot;https://en.wikipedia.org/w/ind…</td><td>false</td><td>{155160,155098}</td><td>{1167688737,1167689472}</td><td>&quot;https://en.wikipedia.org&quot;</td><td>&quot;en.wikipedia.org&quot;</td><td>&quot;/w&quot;</td><td>&quot;enwiki&quot;</td><td>&quot;&lt;span dir=&quot;auto&quot;&gt;&lt;span class=&quot;…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (100, 20)\n",
       "┌─────────────┬────────────┬────────────┬──────┬───┬────────────┬────────────┬────────┬────────────┐\n",
       "│ $schema     ┆ meta       ┆ id         ┆ type ┆ … ┆ server_nam ┆ server_scr ┆ wiki   ┆ parsedcomm │\n",
       "│ ---         ┆ ---        ┆ ---        ┆ ---  ┆   ┆ e          ┆ ipt_path   ┆ ---    ┆ ent        │\n",
       "│ str         ┆ struct[9]  ┆ i64        ┆ str  ┆   ┆ ---        ┆ ---        ┆ str    ┆ ---        │\n",
       "│             ┆            ┆            ┆      ┆   ┆ str        ┆ str        ┆        ┆ str        │\n",
       "╞═════════════╪════════════╪════════════╪══════╪═══╪════════════╪════════════╪════════╪════════════╡\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529639 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ <span dir= │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ \"auto\"><sp │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ an         │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ class=\"…   │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529640 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆            │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆            │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529642 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ <span dir= │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ \"auto\"><sp │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ an         │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ class=\"…   │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529643 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ Nominated  │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ for        │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ deletion;  │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ see <a…    │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529653 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ Rescuing 1 │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ sources    │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ and        │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ tagging…   │\n",
       "│ …           ┆ …          ┆ …          ┆ …    ┆ … ┆ …          ┆ …          ┆ …      ┆ …          │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529961 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ <span dir= │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ \"auto\"><sp │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ an         │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ class=\"…   │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529963 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ Ce         │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆            │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529964 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆            │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆            │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529965 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆            │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆            │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆            │\n",
       "│ /mediawiki/ ┆ {\"https:// ┆ 1659529966 ┆ edit ┆ … ┆ en.wikiped ┆ /w         ┆ enwiki ┆ <span dir= │\n",
       "│ recentchang ┆ en.wikiped ┆            ┆      ┆   ┆ ia.org     ┆            ┆        ┆ \"auto\"><sp │\n",
       "│ e/1.0.0     ┆ ia.org/wik ┆            ┆      ┆   ┆            ┆            ┆        ┆ an         │\n",
       "│             ┆ …          ┆            ┆      ┆   ┆            ┆            ┆        ┆ class=\"…   │\n",
       "└─────────────┴────────────┴────────────┴──────┴───┴────────────┴────────────┴────────┴────────────┘"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikimedia = pl.read_ndjson(\"data/wikimedia.ndjson\")\n",
    "wikimedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "861a007f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (100, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>timestamp</th><th>title</th><th>user</th><th>comment</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>1690617099</td><td>&quot;EFL Championship&quot;</td><td>&quot;87.12.215.232&quot;</td><td>&quot;/* League champions, runners-u…</td></tr><tr><td>1690617102</td><td>&quot;Lim Sang-choon&quot;</td><td>&quot;Preferwiki&quot;</td><td>&quot;&quot;</td></tr><tr><td>1690617104</td><td>&quot;Higher&quot;</td><td>&quot;Ss112&quot;</td><td>&quot;/* Albums */ add&quot;</td></tr><tr><td>1690617104</td><td>&quot;International Poker Rules&quot;</td><td>&quot;Piotrus&quot;</td><td>&quot;Nominated for deletion; see [[…</td></tr><tr><td>1690617105</td><td>&quot;Abdul Hamid Khan Bhashani&quot;</td><td>&quot;InternetArchiveBot&quot;</td><td>&quot;Rescuing 1 sources and tagging…</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>1690617238</td><td>&quot;Havering Residents Association&quot;</td><td>&quot;MRSC&quot;</td><td>&quot;/* 2018 election */ +map&quot;</td></tr><tr><td>1690617235</td><td>&quot;Olha Kharlan&quot;</td><td>&quot;2603:7000:2101:AA00:2C88:EF86:…</td><td>&quot;Ce&quot;</td></tr><tr><td>1690617238</td><td>&quot;Mukim Kota Batu&quot;</td><td>&quot;Pangalau&quot;</td><td>&quot;&quot;</td></tr><tr><td>1690617239</td><td>&quot;User:IDK1213safasx/sandbox&quot;</td><td>&quot;94.101.29.27&quot;</td><td>&quot;&quot;</td></tr><tr><td>1690617234</td><td>&quot;List of bus routes in London&quot;</td><td>&quot;Pedroperezhumberto&quot;</td><td>&quot;/* Non-TfL bus routes in Great…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (100, 4)\n",
       "┌────────────┬────────────────────────────┬────────────────────────────┬───────────────────────────┐\n",
       "│ timestamp  ┆ title                      ┆ user                       ┆ comment                   │\n",
       "│ ---        ┆ ---                        ┆ ---                        ┆ ---                       │\n",
       "│ i64        ┆ str                        ┆ str                        ┆ str                       │\n",
       "╞════════════╪════════════════════════════╪════════════════════════════╪═══════════════════════════╡\n",
       "│ 1690617099 ┆ EFL Championship           ┆ 87.12.215.232              ┆ /* League champions,      │\n",
       "│            ┆                            ┆                            ┆ runners-u…                │\n",
       "│ 1690617102 ┆ Lim Sang-choon             ┆ Preferwiki                 ┆                           │\n",
       "│ 1690617104 ┆ Higher                     ┆ Ss112                      ┆ /* Albums */ add          │\n",
       "│ 1690617104 ┆ International Poker Rules  ┆ Piotrus                    ┆ Nominated for deletion;   │\n",
       "│            ┆                            ┆                            ┆ see [[…                   │\n",
       "│ 1690617105 ┆ Abdul Hamid Khan Bhashani  ┆ InternetArchiveBot         ┆ Rescuing 1 sources and    │\n",
       "│            ┆                            ┆                            ┆ tagging…                  │\n",
       "│ …          ┆ …                          ┆ …                          ┆ …                         │\n",
       "│ 1690617238 ┆ Havering Residents         ┆ MRSC                       ┆ /* 2018 election */ +map  │\n",
       "│            ┆ Association                ┆                            ┆                           │\n",
       "│ 1690617235 ┆ Olha Kharlan               ┆ 2603:7000:2101:AA00:2C88:E ┆ Ce                        │\n",
       "│            ┆                            ┆ F86:…                      ┆                           │\n",
       "│ 1690617238 ┆ Mukim Kota Batu            ┆ Pangalau                   ┆                           │\n",
       "│ 1690617239 ┆ User:IDK1213safasx/sandbox ┆ 94.101.29.27               ┆                           │\n",
       "│ 1690617234 ┆ List of bus routes in      ┆ Pedroperezhumberto         ┆ /* Non-TfL bus routes in  │\n",
       "│            ┆ London                     ┆                            ┆ Great…                    │\n",
       "└────────────┴────────────────────────────┴────────────────────────────┴───────────────────────────┘"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    wikimedia.rename({\"id\": \"edit_id\"})\n",
    "    .unnest(\"meta\")\n",
    "    .select(\"timestamp\", \"title\", \"user\", \"comment\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54785c85",
   "metadata": {},
   "source": [
    "## Other File Formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e12d8997",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (62, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>abbreviation</th><th>Latin</th><th>translation</th><th>usage and notes</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;A.D.&quot;</td><td>&quot;anno Domini&quot;</td><td>&quot;&quot;in the year of the Lord&quot;&quot;</td><td>&quot;Used to label or number years …</td></tr><tr><td>&quot;A.I.&quot;</td><td>&quot;ad interim&quot;</td><td>&quot;&quot;temporarily&quot;&quot;</td><td>&quot;Used in business organizationa…</td></tr><tr><td>&quot;a.m.&quot;</td><td>&quot;ante meridiem&quot;</td><td>&quot;&quot;before midday&quot;[1]&quot;</td><td>&quot;Used on the twelve-hour clock …</td></tr><tr><td>&quot;ca./c.&quot;</td><td>&quot;circa&quot;</td><td>&quot;&quot;around&quot;, &quot;about&quot;, &quot;approximat…</td><td>&quot;Used with dates to indicate &quot;a…</td></tr><tr><td>&quot;Cap.&quot;</td><td>&quot;capitulus&quot;</td><td>&quot;&quot;chapter&quot;&quot;</td><td>&quot;Used before a chapter number o…</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;S.O.S.&quot;</td><td>&quot;si opus sit&quot;</td><td>&quot;&quot;if there is need&quot;, &quot;if occasi…</td><td>&quot;A prescription indication that…</td></tr><tr><td>&quot;sic&quot;</td><td>&quot;sic&quot;</td><td>&quot;&quot;thus&quot;&quot;</td><td>&quot;Used when quoting text that co…</td></tr><tr><td>&quot;stat.&quot;</td><td>&quot;statim&quot;</td><td>&quot;&quot;immediately&quot;&quot;</td><td>&quot;Often used in medical contexts…</td></tr><tr><td>&quot;viz.&quot;</td><td>&quot;videlicet&quot;</td><td>&quot;&quot;namely&quot;, &quot;to wit&quot;, &quot;precisely…</td><td>&quot;In contradistinction to &quot;i.e.&quot;…</td></tr><tr><td>&quot;vs. v.&quot;</td><td>&quot;versus&quot;</td><td>&quot;&quot;against&quot;&quot;</td><td>&quot;Sometimes is not abbreviated. …</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (62, 4)\n",
       "┌──────────────┬───────────────┬─────────────────────────────────┬─────────────────────────────────┐\n",
       "│ abbreviation ┆ Latin         ┆ translation                     ┆ usage and notes                 │\n",
       "│ ---          ┆ ---           ┆ ---                             ┆ ---                             │\n",
       "│ str          ┆ str           ┆ str                             ┆ str                             │\n",
       "╞══════════════╪═══════════════╪═════════════════════════════════╪═════════════════════════════════╡\n",
       "│ A.D.         ┆ anno Domini   ┆ \"in the year of the Lord\"       ┆ Used to label or number years … │\n",
       "│ A.I.         ┆ ad interim    ┆ \"temporarily\"                   ┆ Used in business organizationa… │\n",
       "│ a.m.         ┆ ante meridiem ┆ \"before midday\"[1]              ┆ Used on the twelve-hour clock … │\n",
       "│ ca./c.       ┆ circa         ┆ \"around\", \"about\", \"approximat… ┆ Used with dates to indicate \"a… │\n",
       "│ Cap.         ┆ capitulus     ┆ \"chapter\"                       ┆ Used before a chapter number o… │\n",
       "│ …            ┆ …             ┆ …                               ┆ …                               │\n",
       "│ S.O.S.       ┆ si opus sit   ┆ \"if there is need\", \"if occasi… ┆ A prescription indication that… │\n",
       "│ sic          ┆ sic           ┆ \"thus\"                          ┆ Used when quoting text that co… │\n",
       "│ stat.        ┆ statim        ┆ \"immediately\"                   ┆ Often used in medical contexts… │\n",
       "│ viz.         ┆ videlicet     ┆ \"namely\", \"to wit\", \"precisely… ┆ In contradistinction to \"i.e.\"… │\n",
       "│ vs. v.       ┆ versus        ┆ \"against\"                       ┆ Sometimes is not abbreviated. … │\n",
       "└──────────────┴───────────────┴─────────────────────────────────┴─────────────────────────────────┘"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "url = \"https://en.wikipedia.org/wiki/List_of_Latin_abbreviations\"\n",
    "pl.from_pandas(pd.read_html(url)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "144f83ef",
   "metadata": {},
   "source": [
    "## Querying Databases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "52404e85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (10, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>film_id</th><th>title</th><th>category</th><th>rating</th><th>length</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td><td>f64</td></tr></thead><tbody><tr><td>1</td><td>&quot;ACADEMY DINOSAUR&quot;</td><td>&quot;Documentary&quot;</td><td>&quot;PG&quot;</td><td>1.433333</td></tr><tr><td>2</td><td>&quot;ACE GOLDFINGER&quot;</td><td>&quot;Horror&quot;</td><td>&quot;G&quot;</td><td>0.8</td></tr><tr><td>3</td><td>&quot;ADAPTATION HOLES&quot;</td><td>&quot;Documentary&quot;</td><td>&quot;NC-17&quot;</td><td>0.833333</td></tr><tr><td>4</td><td>&quot;AFFAIR PREJUDICE&quot;</td><td>&quot;Horror&quot;</td><td>&quot;G&quot;</td><td>1.95</td></tr><tr><td>5</td><td>&quot;AFRICAN EGG&quot;</td><td>&quot;Family&quot;</td><td>&quot;G&quot;</td><td>2.166667</td></tr><tr><td>6</td><td>&quot;AGENT TRUMAN&quot;</td><td>&quot;Foreign&quot;</td><td>&quot;PG&quot;</td><td>2.816667</td></tr><tr><td>7</td><td>&quot;AIRPLANE SIERRA&quot;</td><td>&quot;Comedy&quot;</td><td>&quot;PG-13&quot;</td><td>1.033333</td></tr><tr><td>8</td><td>&quot;AIRPORT POLLOCK&quot;</td><td>&quot;Horror&quot;</td><td>&quot;R&quot;</td><td>0.9</td></tr><tr><td>9</td><td>&quot;ALABAMA DEVIL&quot;</td><td>&quot;Horror&quot;</td><td>&quot;PG-13&quot;</td><td>1.9</td></tr><tr><td>10</td><td>&quot;ALADDIN CALENDAR&quot;</td><td>&quot;Sports&quot;</td><td>&quot;NC-17&quot;</td><td>1.05</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (10, 5)\n",
       "┌─────────┬──────────────────┬─────────────┬────────┬──────────┐\n",
       "│ film_id ┆ title            ┆ category    ┆ rating ┆ length   │\n",
       "│ ---     ┆ ---              ┆ ---         ┆ ---    ┆ ---      │\n",
       "│ i64     ┆ str              ┆ str         ┆ str    ┆ f64      │\n",
       "╞═════════╪══════════════════╪═════════════╪════════╪══════════╡\n",
       "│ 1       ┆ ACADEMY DINOSAUR ┆ Documentary ┆ PG     ┆ 1.433333 │\n",
       "│ 2       ┆ ACE GOLDFINGER   ┆ Horror      ┆ G      ┆ 0.8      │\n",
       "│ 3       ┆ ADAPTATION HOLES ┆ Documentary ┆ NC-17  ┆ 0.833333 │\n",
       "│ 4       ┆ AFFAIR PREJUDICE ┆ Horror      ┆ G      ┆ 1.95     │\n",
       "│ 5       ┆ AFRICAN EGG      ┆ Family      ┆ G      ┆ 2.166667 │\n",
       "│ 6       ┆ AGENT TRUMAN     ┆ Foreign     ┆ PG     ┆ 2.816667 │\n",
       "│ 7       ┆ AIRPLANE SIERRA  ┆ Comedy      ┆ PG-13  ┆ 1.033333 │\n",
       "│ 8       ┆ AIRPORT POLLOCK  ┆ Horror      ┆ R      ┆ 0.9      │\n",
       "│ 9       ┆ ALABAMA DEVIL    ┆ Horror      ┆ PG-13  ┆ 1.9      │\n",
       "│ 10      ┆ ALADDIN CALENDAR ┆ Sports      ┆ NC-17  ┆ 1.05     │\n",
       "└─────────┴──────────────────┴─────────────┴────────┴──────────┘"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.read_database_uri(\n",
    "    query=\"\"\"\n",
    "    SELECT\n",
    "        f.film_id,\n",
    "        f.title,\n",
    "        c.name AS category,\n",
    "        f.rating,\n",
    "        f.length / 60.0 AS length\n",
    "    FROM\n",
    "        film AS f,\n",
    "        film_category AS fc,\n",
    "        category AS c\n",
    "    WHERE\n",
    "        fc.film_id = f.film_id\n",
    "        AND fc.category_id = c.category_id\n",
    "    LIMIT 10\n",
    "    \"\"\",\n",
    "    uri=\"sqlite:::data/sakila.db\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "42b08b13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (10, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>film_id</th><th>title</th><th>category</th><th>rating</th><th>length</th></tr><tr><td>i64</td><td>str</td><td>str</td><td>str</td><td>f64</td></tr></thead><tbody><tr><td>1</td><td>&quot;ACADEMY DINOSAUR&quot;</td><td>&quot;Documentary&quot;</td><td>&quot;PG&quot;</td><td>1.433333</td></tr><tr><td>2</td><td>&quot;ACE GOLDFINGER&quot;</td><td>&quot;Horror&quot;</td><td>&quot;G&quot;</td><td>0.8</td></tr><tr><td>3</td><td>&quot;ADAPTATION HOLES&quot;</td><td>&quot;Documentary&quot;</td><td>&quot;NC-17&quot;</td><td>0.833333</td></tr><tr><td>4</td><td>&quot;AFFAIR PREJUDICE&quot;</td><td>&quot;Horror&quot;</td><td>&quot;G&quot;</td><td>1.95</td></tr><tr><td>5</td><td>&quot;AFRICAN EGG&quot;</td><td>&quot;Family&quot;</td><td>&quot;G&quot;</td><td>2.166667</td></tr><tr><td>6</td><td>&quot;AGENT TRUMAN&quot;</td><td>&quot;Foreign&quot;</td><td>&quot;PG&quot;</td><td>2.816667</td></tr><tr><td>7</td><td>&quot;AIRPLANE SIERRA&quot;</td><td>&quot;Comedy&quot;</td><td>&quot;PG-13&quot;</td><td>1.033333</td></tr><tr><td>8</td><td>&quot;AIRPORT POLLOCK&quot;</td><td>&quot;Horror&quot;</td><td>&quot;R&quot;</td><td>0.9</td></tr><tr><td>9</td><td>&quot;ALABAMA DEVIL&quot;</td><td>&quot;Horror&quot;</td><td>&quot;PG-13&quot;</td><td>1.9</td></tr><tr><td>10</td><td>&quot;ALADDIN CALENDAR&quot;</td><td>&quot;Sports&quot;</td><td>&quot;NC-17&quot;</td><td>1.05</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (10, 5)\n",
       "┌─────────┬──────────────────┬─────────────┬────────┬──────────┐\n",
       "│ film_id ┆ title            ┆ category    ┆ rating ┆ length   │\n",
       "│ ---     ┆ ---              ┆ ---         ┆ ---    ┆ ---      │\n",
       "│ i64     ┆ str              ┆ str         ┆ str    ┆ f64      │\n",
       "╞═════════╪══════════════════╪═════════════╪════════╪══════════╡\n",
       "│ 1       ┆ ACADEMY DINOSAUR ┆ Documentary ┆ PG     ┆ 1.433333 │\n",
       "│ 2       ┆ ACE GOLDFINGER   ┆ Horror      ┆ G      ┆ 0.8      │\n",
       "│ 3       ┆ ADAPTATION HOLES ┆ Documentary ┆ NC-17  ┆ 0.833333 │\n",
       "│ 4       ┆ AFFAIR PREJUDICE ┆ Horror      ┆ G      ┆ 1.95     │\n",
       "│ 5       ┆ AFRICAN EGG      ┆ Family      ┆ G      ┆ 2.166667 │\n",
       "│ 6       ┆ AGENT TRUMAN     ┆ Foreign     ┆ PG     ┆ 2.816667 │\n",
       "│ 7       ┆ AIRPLANE SIERRA  ┆ Comedy      ┆ PG-13  ┆ 1.033333 │\n",
       "│ 8       ┆ AIRPORT POLLOCK  ┆ Horror      ┆ R      ┆ 0.9      │\n",
       "│ 9       ┆ ALABAMA DEVIL    ┆ Horror      ┆ PG-13  ┆ 1.9      │\n",
       "│ 10      ┆ ALADDIN CALENDAR ┆ Sports      ┆ NC-17  ┆ 1.05     │\n",
       "└─────────┴──────────────────┴─────────────┴────────┴──────────┘"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db = \"sqlite:::data/sakila.db\"\n",
    "films = pl.read_database_uri(\"SELECT * FROM film\", db)\n",
    "film_categories = pl.read_database_uri(\"SELECT * FROM film_category\", db)\n",
    "categories = pl.read_database_uri(\"SELECT * FROM category\", db)\n",
    "\n",
    "(\n",
    "    films.join(film_categories, on=\"film_id\", suffix=\"_fc\")\n",
    "    .join(categories, on=\"category_id\", suffix=\"_c\")\n",
    "    .select(\n",
    "        \"film_id\",\n",
    "        \"title\",\n",
    "        pl.col(\"name\").alias(\"category\"),\n",
    "        \"rating\",\n",
    "        pl.col(\"length\") / 60,\n",
    "    )\n",
    "    .limit(10)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca45ba84",
   "metadata": {},
   "source": [
    "## Writing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26ce227d",
   "metadata": {},
   "source": [
    "### CSV Format"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "364a9c6f",
   "metadata": {},
   "source": [
    "### Excel Format"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca2e87d5",
   "metadata": {},
   "source": [
    "### Parquet Format"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd6c6cbf",
   "metadata": {},
   "source": [
    "### Other Considerations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47aea7af",
   "metadata": {},
   "source": [
    "## Conclusion"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}