Created dataloader class

data/download.py → datasets/downloader.py

@@ -1,5 +1,5 @@
 # Script to automatically download and cache dataset
-# Usage: python download.py
+# Usage: python downloader.py
 #
 # To learn more about the dataset, access:
 # https://www.cityscapes-dataset.com/
@@ -13,7 +13,7 @@ def main():
     pass
 
 
-def download(name='cityscapes', path='data/downloads'):
+def download(name='cityscapes', path='datasets/downloads'):
     """Select one of the available and implemented datasets to download:
     name=any(['cityscapes', 'camvid', 'labelme'])
     """
@@ -23,11 +23,19 @@ def download(name='cityscapes', path='data/downloads'):
         raise NotImplementedError
 
 
-def download_cityscapes(path='data/downloads'):
+def download_cityscapes(path='datasets/downloads'):
     if hasattr(pip, 'main'):
-        pip.main(['install', 'cityscrapesscripts'])
+        pip.main(['install', 'cityscapesscripts'])
     else:
         raise EnvironmentError("pip is not installed")
+    print("Which dataset do you want to download?")
+    os.system("csDownload -l")
+    ds_name = input()
+    while ds_name not in ['gtFine_trainvaltest', 'gtFine_trainval', 'gtFine_test',
+                          'leftImg8bit_trainvaltest', 'leftImg8bit_trainval', 'leftImg8bit_test']:
+        print("Invalid dataset name. Please try again.")
+        ds_name = input()
+    os.system(f"csDownload {ds_name} -d {path}/{ds_name}")
 
 
 if __name__ == "__main__":

notebooks/dataloader.ipynb

@@ -0,0 +1,198 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/c/Users/rzimm/Workspace/data/zero-to-hero\n"
+     ]
+    }
+   ],
+   "source": [
+    "%cd ..\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "from datasets import downloader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import random\n",
+    "from glob import glob, escape\n",
+    "import imageio.v2 as imageio"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# download.download(\"cityscapes\", \"datasets/downloaded\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n",
+     "is_executing": true
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "outputs": [],
+   "source": [
+    "def load_dataset(name=\"gtFine\", path=\"datasets/downloads/\"):\n",
+    "    src = path+name\n",
+    "    test, train, val = [f\"{src}/{subpath}\" for subpath in [\"test\", \"train\", \"val\"]]\n",
+    "\n",
+    "    dataset = {\"test\": glob(test + \"/*/*\"), \"train\": glob(train + \"/*/*\"), \"val\": glob(val + \"/*/*\")}\n",
+    "\n",
+    "    return dataset"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "list"
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(load_dataset()[\"train\"])"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "outputs": [],
+   "source": [
+    "a = [1, 2, 3]"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 143,
+   "outputs": [],
+   "source": [
+    "class DataLoader:\n",
+    "    def __init__(self, data):\n",
+    "        self.data = np.array(data)\n",
+    "        self.total = len(self.data)\n",
+    "        self.__items = self.data\n",
+    "        self.__remaining = len(self.data)\n",
+    "    def __next__(self, n=1):\n",
+    "        if n > self.total:\n",
+    "            raise ValueError(f\"Dataset doesn't have enough elements to suffice request of {n} elements.\")\n",
+    "        if self.__remaining > 0:\n",
+    "            indices = random.sample(range(self.__remaining), n)\n",
+    "            sampled = self.__items[indices]\n",
+    "            self.__items = np.delete(self.__items, indices)\n",
+    "            self.__remaining -= n\n",
+    "            return sampled\n",
+    "        else:\n",
+    "            self.__items = self.data\n",
+    "            self.__remaining = len(self.data)\n",
+    "            return self.__next__(n)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 144,
+   "outputs": [],
+   "source": [
+    "loader = DataLoader(a)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}