Finished MNIST downloading and caching modules

notebooks/dataloader.ipynb

@@ -1,198 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "collapsed": true,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/mnt/c/Users/rzimm/Workspace/data/zero-to-hero\n"
-     ]
-    }
-   ],
-   "source": [
-    "%cd ..\n",
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
-    "from datasets import downloader"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 56,
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import random\n",
-    "from glob import glob, escape\n",
-    "import imageio.v2 as imageio"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# download.download(\"cityscapes\", \"datasets/downloaded\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n",
-     "is_executing": true
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 41,
-   "outputs": [],
-   "source": [
-    "def load_dataset(name=\"gtFine\", path=\"datasets/downloads/\"):\n",
-    "    src = path+name\n",
-    "    test, train, val = [f\"{src}/{subpath}\" for subpath in [\"test\", \"train\", \"val\"]]\n",
-    "\n",
-    "    dataset = {\"test\": glob(test + \"/*/*\"), \"train\": glob(train + \"/*/*\"), \"val\": glob(val + \"/*/*\")}\n",
-    "\n",
-    "    return dataset"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "list"
-     },
-     "execution_count": 44,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "type(load_dataset()[\"train\"])"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 45,
-   "outputs": [],
-   "source": [
-    "a = [1, 2, 3]"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 143,
-   "outputs": [],
-   "source": [
-    "class DataLoader:\n",
-    "    def __init__(self, data):\n",
-    "        self.data = np.array(data)\n",
-    "        self.total = len(self.data)\n",
-    "        self.__items = self.data\n",
-    "        self.__remaining = len(self.data)\n",
-    "    def __next__(self, n=1):\n",
-    "        if n > self.total:\n",
-    "            raise ValueError(f\"Dataset doesn't have enough elements to suffice request of {n} elements.\")\n",
-    "        if self.__remaining > 0:\n",
-    "            indices = random.sample(range(self.__remaining), n)\n",
-    "            sampled = self.__items[indices]\n",
-    "            self.__items = np.delete(self.__items, indices)\n",
-    "            self.__remaining -= n\n",
-    "            return sampled\n",
-    "        else:\n",
-    "            self.__items = self.data\n",
-    "            self.__remaining = len(self.data)\n",
-    "            return self.__next__(n)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 144,
-   "outputs": [],
-   "source": [
-    "loader = DataLoader(a)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}

notebooks/dataset.ipynb

@@ -0,0 +1,187 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import gzip"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/rzimmerdev/conda/envs/data/lib/python3.9/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n",
+      "  warnings.warn(\"Setuptools is replacing distutils.\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "from src.downloader import download_dataset\n",
+    "download_dataset(\"mnist\", \"../datasets/mnist\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "b'\\x00\\x00\\x08\\x03\\x00\\x00\\xea`\\x00\\x00\\x00\\x1c\\x00\\x00\\x00\\x1c'"
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "f = gzip.open(\"../datasets/mnist/\" + os.listdir(\"../datasets/mnist/\")[0], 'r')\n",
+    "f.read(16)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from torch.utils.data import DataLoader, Dataset"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "outputs": [],
+   "source": [
+    "class DatasetMNIST(Dataset):\n",
+    "    def __init__(self, images, labels):\n",
+    "        with gzip.open(images, 'r') as f:\n",
+    "            f.read(4)\n",
+    "            self.total = int.from_bytes(f.read(4), 'big')\n",
+    "            rows = int.from_bytes(f.read(4), 'big')\n",
+    "            columns = int.from_bytes(f.read(4), 'big')\n",
+    "\n",
+    "            image_data = f.read()\n",
+    "            images = np.frombuffer(image_data, dtype=np.uint8)\\\n",
+    "                .reshape((self.total, rows, columns))\n",
+    "            self.images = images\n",
+    "        with gzip.open(labels, 'r') as f:\n",
+    "            f.read(4)\n",
+    "            total = int.from_bytes(f.read(4), 'big')\n",
+    "\n",
+    "            label_data = f.read()\n",
+    "            labels = np.frombuffer(label_data, dtype=np.uint8)\n",
+    "            self.labels = labels\n",
+    "        self.data = list(zip(self.images, self.labels))\n",
+    "    def __getitem__(self, n):\n",
+    "        if n > self.total:\n",
+    "            raise ValueError(f\"Dataset doesn't have enough elements to suffice request of {n} elements.\")\n",
+    "        return self.data[n]\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.data)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "outputs": [],
+   "source": [
+    "dataset_dir = \"../datasets/mnist/\"\n",
+    "loader = DatasetMNIST(dataset_dir + \"train_images\", dataset_dir + \"train_labels\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "<Figure size 432x288 with 1 Axes>",
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASY0lEQVR4nO3de9BU9X3H8ffHu4IGKZcgigZk2qi1JjKaTqjQ8VpnVNR6a42AmWBMYpNMbLXUqhm1STrV1GljJogWFJRg1HiJrTI2iqQpkaSoCBjUYkQIqEhUojHAt3+c83ROHnfPLnuX3+c1s/Ps7nfPOV+W/ew5e86e/SkiMLMd307dbsDMOsNhN0uEw26WCIfdLBEOu1kiHHazRDjstl0kXS1pTpXaRElr6pzPFEmLGuyh4WlT5rC3kKTHJL0hafcOLe8gSSFpl+2YZrWk49rZV6+RdIqkZZLelvRfkg7pdk/d4LC3iKSDgD8BAji1u91YH0ljgbnAZ4FBwAPA/dvzBrmjcNhb5wLgv4FZwORiQdIsSd+S9ANJb0laLGlMoR6SPitpVb5l8C1Jyms7SbpC0kuSNki6TdKH8kkX5n835WutP5Y0RtJ/Snpd0muS5koalM/rdmAU8ED++L/J7/9EvsbbJOkpSRMLvX1E0uN53wuAIfU+IZIul/RCPu1ySae//yH6F0m/krRS0rGFwock3SJpnaRXJF0raed6l11wIvBERCyKiC3AN4CRwIQG5vXBFhG+tOACPA98DjgS+C0wvFCbBWwEjgJ2IVvTzCvUA3iQbM0zCngVOCmvXZjPezQwELgHuD2vHZRPu0thXgcDxwO7A0PJ3hD+uVBfDRxXuD0SeB04mezN//j89tC8/mPghnx+xwBvAXOqPAcTgTWF22cB++XzPQfYDIzIa1OALcCXgV3z+q+AwXn9+8B3gAHAMOAnwEWFaRcVlvMgcHmVni4BHirc3hl4F/hit18zHX+NdruBHeECjM8DPiS/vRL4cqE+C5hZuH0ysLJwO4Dxhdvz+168wKPA5wq138+XtUulsFfobRLwP4Xb/cN+Wd+bR+G+h8m2TkblgRxQqN1Rb9gr1JcCp+XXpwBrARXqPwE+BQwHfgPsWaidB/ywMO2iasvpt8w/yN9kJgK7AX8PbAP+ttuvm05fvBnfGpOBRyLitfz2HfTblAd+Wbj+a7K1dD31/YCXCrWXyII+vFIjkoZJmpdv+r4JzKF80/tA4Kx8E36TpE1kb14j8mW/ERGb+y2/LpIukLS0MN/D+vXySuSJLMx7v7ynXYF1hWm/Q7aG3y4RsZLs/+JfgXX58pcDdR012JEkt5Oi1STtCZwN7CypL7C7A4Mk/VFEPNXkItaSvfj79K1t15Ntgvf3NbK1/eER8bqkSWQv9D79T3N8mWzN/pn+M5J0ILCvpAGFwI+qMI/3yae9GTgW+HFEbJW0FFDhYSMlqRD4UcD9eU+/IdtS2lJrWbVExPeA7+V9DSL7aPRks/P9oPGavXmTgK3AIcAR+eWjwBNkO+2adSfw5XxH2UDgH4Dv5iF4lWyTdHTh8XsDb5PttBsJ/HW/+a3v9/g5wCmSTpS0s6Q98uPl+0fES8AS4KuSdpM0Hjilzr4HkL0pvAogaSrZmr1oGPBXknaVdBbZ8/ZQRKwDHgGul7RPvpNyjKSGdqpJOjL/tw0l20J4IF/jJ8Vhb95k4N8i4hcR8cu+C9na9C9bcIjnVuB2sh1t/0u2c+kSgIj4NXAd8KN8c/cTwFeBj5Pt7PoB2Q69oq8BV+SPvzQiXgZOA6aTBfNlsjeIvtfGXwBHk+1gvAq4rZ6mI2I5cD3ZDr71wB8CP+r3sMXAWOC1/N/x5xHxel67gOwz9nLgDbI184hKy5L075Kml7RzI7AJeC7/+76tmBTodz8ymdmOymt2s0Q47GaJcNjNEuGwmyWio8fZJXlvoFmbRYQq3d/Uml3SSZKek/S8pMubmZeZtVfDh97yM5B+TnbixBqybySdlx9frTaN1+xmbdaONftRwPMR8WJEvAfMI/tyhpn1oGbCPpLs21Z91lDhu9qSpklaImlJE8sysyY1s4Ou0qbC+zbTI2IGMAO8GW/WTc2s2dcABxRu7092hpaZ9aBmwv4kMDY/G2s34Fyy0xPNrAc1vBkfEVskfYHsV012Bm6NiGdb1pmZtVRHz3rzZ3az9mvLl2rM7IPDYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q0PD47gKTVwFvAVmBLRIxrRVNm1npNhT33pxHxWgvmY2Zt5M14s0Q0G/YAHpH0U0nTKj1A0jRJSyQtaXJZZtYERUTjE0v7RcRaScOABcAlEbGw5PGNL8zM6hIRqnR/U2v2iFib/90A3Asc1cz8zKx9Gg67pAGS9u67DpwALGtVY2bWWs3sjR8O3Cupbz53RMR/tKQrM2u5pj6zb/fC/JndrO3a8pndzD44HHazRDjsZolw2M0S4bCbJaIVJ8JYDzv66KNL6+eff35pfcKECaX1Qw89dLt76nPppZeW1teuXVtaHz9+fGl9zpw5VWuLFy8unXZH5DW7WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIn/W2AzjnnHOq1m688cbSaYcMGVJaz09hruqxxx4rrQ8dOrRq7ZBDDimdtpZavd11111Va+eee25Ty+5lPuvNLHEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEz2fvAbvsUv7fMG5c+eC4N998c9XaXnvtVTrtwoVVB/AB4JprrimtL1q0qLS+++67V63Nnz+/dNoTTjihtF7LkiUecazIa3azRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBE+zt4Dav12+8yZMxue94IFC0rrZefCA7z55psNL7vW/Js9jr5mzZrS+uzZs5ua/46m5ppd0q2SNkhaVrhvsKQFklblf/dtb5tm1qx6NuNnASf1u+9y4NGIGAs8mt82sx5WM+wRsRDY2O/u04C+baTZwKTWtmVmrdboZ/bhEbEOICLWSRpW7YGSpgHTGlyOmbVI23fQRcQMYAb4ByfNuqnRQ2/rJY0AyP9uaF1LZtYOjYb9fmByfn0ycF9r2jGzdqn5u/GS7gQmAkOA9cBVwPeB+cAo4BfAWRHRfydepXkluRlf65zw6dOnl9Zr/R/ddNNNVWtXXHFF6bTNHkevZcWKFVVrY8eObWreZ555Zmn9vvvSXAdV+934mp/ZI+K8KqVjm+rIzDrKX5c1S4TDbpYIh90sEQ67WSIcdrNE+BTXFrjyyitL67UOrb333nul9Ycffri0ftlll1WtvfPOO6XT1rLHHnuU1mudpjpq1KiqtVpDLl977bWl9VQPrTXKa3azRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBE1T3Ft6cI+wKe4Dho0qGpt5cqVpdMOGTKktP7ggw+W1idNmlRab8bBBx9cWp87d25p/cgjj2x42XfffXdp/cILLyytb968ueFl78iqneLqNbtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulggfZ6/TsGFVR7hi7dq1Tc179OjRpfV33323tD516tSqtVNPPbV02sMOO6y0PnDgwNJ6rddPWf2MM84onfaBBx4orVtlPs5uljiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCx9nrVHY+e9mwxABDhw4trdf6/fR2/h/V+o5Ard5GjBhRWn/11VcbntYa0/Bxdkm3StogaVnhvqslvSJpaX45uZXNmlnr1bMZPws4qcL934yII/LLQ61ty8xarWbYI2IhsLEDvZhZGzWzg+4Lkp7ON/P3rfYgSdMkLZG0pIllmVmTGg37t4ExwBHAOuD6ag+MiBkRMS4ixjW4LDNrgYbCHhHrI2JrRGwDbgaOam1bZtZqDYVdUvGYyenAsmqPNbPeUHN8dkl3AhOBIZLWAFcBEyUdAQSwGriofS32hk2bNlWt1fpd91q/Cz948ODS+gsvvFBaLxunfNasWaXTbtxYvu913rx5pfVax8prTW+dUzPsEXFehbtvaUMvZtZG/rqsWSIcdrNEOOxmiXDYzRLhsJsloubeeKtt8eLFpfVap7h20zHHHFNanzBhQml927ZtpfUXX3xxu3uy9vCa3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhI+zJ27PPfcsrdc6jl7rZ659imvv8JrdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEh2y2Ulu3bi2t13r9lP3UdNlwzta4hodsNrMdg8NulgiH3SwRDrtZIhx2s0Q47GaJcNjNElHPkM0HALcBHwa2ATMi4kZJg4HvAgeRDdt8dkS80b5WrR1OPPHEbrdgHVLPmn0L8JWI+CjwCeDzkg4BLgcejYixwKP5bTPrUTXDHhHrIuJn+fW3gBXASOA0YHb+sNnApDb1aGYtsF2f2SUdBHwMWAwMj4h1kL0hAMNa3p2ZtUzdv0EnaSBwN/CliHhTqvj120rTTQOmNdaembVKXWt2SbuSBX1uRNyT371e0oi8PgLYUGnaiJgREeMiYlwrGjazxtQMu7JV+C3Aioi4oVC6H5icX58M3Nf69sysVerZjP8k8CngGUlL8/umA18H5kv6NPAL4Ky2dGhtNXr06G63YB1SM+wRsQio9gH92Na2Y2bt4m/QmSXCYTdLhMNulgiH3SwRDrtZIhx2s0R4yObEPfHEE6X1nXYqXx/UGtLZeofX7GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZInycPXHLli0rra9ataq0Xut8+DFjxlStecjmzvKa3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhCKicwuTOrcwa4kpU6aU1mfOnFlaf/zxx6vWLrnkktJply9fXlq3yiKi4k+/e81ulgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyWi5nF2SQcAtwEfBrYBMyLiRklXA58B+k5Knh4RD9WYl4+zf8Dss88+pfX58+eX1o877riqtXvuuad02qlTp5bWN2/eXFpPVbXj7PX8eMUW4CsR8TNJewM/lbQgr30zIv6pVU2aWfvUDHtErAPW5dffkrQCGNnuxsystbbrM7ukg4CPAYvzu74g6WlJt0rat8o00yQtkbSkuVbNrBl1h13SQOBu4EsR8SbwbWAMcATZmv/6StNFxIyIGBcR45pv18waVVfYJe1KFvS5EXEPQESsj4itEbENuBk4qn1tmlmzaoZdkoBbgBURcUPh/hGFh50OlP9MqZl1VT2H3sYDTwDPkB16A5gOnEe2CR/AauCifGde2bx86G0HU+vQ3HXXXVe1dvHFF5dOe/jhh5fWfQpsZQ0feouIRUCliUuPqZtZb/E36MwS4bCbJcJhN0uEw26WCIfdLBEOu1ki/FPSZjsY/5S0WeIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpaIen5dtpVeA14q3B6S39eLerW3Xu0L3FujWtnbgdUKHf1SzfsWLi3p1d+m69XeerUvcG+N6lRv3ow3S4TDbpaIbod9RpeXX6ZXe+vVvsC9NaojvXX1M7uZdU631+xm1iEOu1kiuhJ2SSdJek7S85Iu70YP1UhaLekZSUu7PT5dPobeBknLCvcNlrRA0qr8b8Ux9rrU29WSXsmfu6WSTu5SbwdI+qGkFZKelfTF/P6uPnclfXXkeev4Z3ZJOwM/B44H1gBPAudFRE/84r+k1cC4iOj6FzAkHQO8DdwWEYfl9/0jsDEivp6/Ue4bEZf1SG9XA293exjvfLSiEcVhxoFJwBS6+NyV9HU2HXjeurFmPwp4PiJejIj3gHnAaV3oo+dFxEJgY7+7TwNm59dnk71YOq5Kbz0hItZFxM/y628BfcOMd/W5K+mrI7oR9pHAy4Xba+it8d4DeETSTyVN63YzFQzvG2Yr/zusy/30V3MY707qN8x4zzx3jQx/3qxuhL3S72P10vG/T0bEx4E/Az6fb65afeoaxrtTKgwz3hMaHf68Wd0I+xrggMLt/YG1XeijoohYm//dANxL7w1Fvb5vBN3874Yu9/P/emkY70rDjNMDz103hz/vRtifBMZK+oik3YBzgfu70Mf7SBqQ7zhB0gDgBHpvKOr7gcn59cnAfV3s5Xf0yjDe1YYZp8vPXdeHP4+Ijl+Ak8n2yL8A/F03eqjS12jgqfzybLd7A+4k26z7LdkW0aeB3wMeBVblfwf3UG+3kw3t/TRZsEZ0qbfxZB8NnwaW5peTu/3clfTVkefNX5c1S4S/QWeWCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJeL/AHyD7vpJDzRWAAAAAElFTkSuQmCC\n"
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "X, y = loader[4]\n",
+    "plt.imshow(X, cmap=\"gray\")\n",
+    "plt.title(label=\"Annotated label: \" + str(y))\n",
+    "plt.show()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

src/dataset.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# coding: utf-8
+import gzip
+
+from src.downloader import download_dataset
+
+import numpy as np
+from torch.utils.data import Dataset
+
+
+def load_mnist(download_dir):
+    download_dataset("mnist", download_dir)
+
+    return {"train": (download_dir + "train_images", download_dir + "train_labels"),
+            "test": (download_dir + "test_images", download_dir + "test_labels")}
+
+
+class DatasetMNIST(Dataset):
+    def __init__(self, images, labels):
+        with gzip.open(images, 'r') as f:
+            f.read(4)
+            self.total = int.from_bytes(f.read(4), 'big')
+            rows = int.from_bytes(f.read(4), 'big')
+            columns = int.from_bytes(f.read(4), 'big')
+
+            image_data = f.read()
+            images = np.frombuffer(image_data, dtype=np.uint8).reshape((self.total, rows, columns))
+            self.images = images
+        with gzip.open(labels, 'r') as f:
+            f.read(8)
+
+            label_data = f.read()
+            labels = np.frombuffer(label_data, dtype=np.uint8)
+            self.labels = labels
+        self.data = list(zip(self.images, self.labels))
+
+    def __getitem__(self, n):
+        if n > self.total:
+            raise ValueError(f"Dataset doesn't have enough elements to suffice request of {n} elements.")
+        return self.data[n]
+
+    def __len__(self):
+        return len(self.data)
+
+
+if __name__ == "__main__":
+    download_dir = "../downloads/mnist/"
+    mnist = load_mnist(download_dir)
+
+    dataset = DatasetMNIST(*mnist["train"])
+
+    import matplotlib.pyplot as plt
+
+    X, y = dataset[4]
+    plt.imshow(X, cmap="gray")
+    plt.title(label="Annotated label: " + str(y))
+    plt.show()

{datasets → src}/downloader.py

@@ -4,26 +4,38 @@
 # To learn more about the dataset, access:
 # https://www.cityscapes-dataset.com/
 import os
-import sys
 import pip
+from urllib.request import urlretrieve
 
 
-# Download and cache dataset
-def main():
-    pass
-
-
-def download(name='cityscapes', path='datasets/downloads'):
-    """Select one of the available and implemented datasets to download:
+def download_dataset(name='cityscapes', path='downloads/downloads'):
+    """Select one of the available and implemented downloads to download:
     name=any(['cityscapes', 'camvid', 'labelme'])
     """
     if name == 'cityscapes':
         download_cityscapes(path)
+    elif name == "mnist":
+        pass
     else:
         raise NotImplementedError
 
 
-def download_cityscapes(path='datasets/downloads'):
+def download_mnist(path="downloads/mnist"):
+    remote_files = {"train_images": "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz",
+                    "train_labels": "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz",
+                    "test_images": "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz",
+                    "test_labels": "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz"}
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    for file in remote_files.keys():
+        if os.path.exists(path + "/" + file):
+            continue
+
+        urlretrieve(remote_files[file], path + "/" + file)
+
+
+def download_cityscapes(path='downloads/cityscapes'):
     if hasattr(pip, 'main'):
         pip.main(['install', 'cityscapesscripts'])
     else:
@@ -36,7 +48,3 @@ def download_cityscapes(path='datasets/downloads'):
         print("Invalid dataset name. Please try again.")
         ds_name = input()
     os.system(f"csDownload {ds_name} -d {path}/{ds_name}")
-
-
-if __name__ == "__main__":
-    main()