Creating private tensorflow_datasets from tfds

[Medicmind]

The models in this repository require tensorflow_datasets to train on built with tfds. While the public list is quite large to create your own multiclass datasets you need to use 'tfds' aswell . The following is an example of how to create your own data using this retinal image data that has four classes CNV, DRUSEN, DME and normal. This is the link to the data:

https://www.kaggle.com/paultimothymooney/kermany2018#OCT2017.zip

It is in this format:

To create a tensorflow_dataset from the OCT2017.zip in the ~/tensorflow_data directory type:

tfds new oct2017

then

cd oct2017

You should see an oct2017.py file. Replace the 'oct2017.py' file with the following code replacing the '/home/ubuntu/efs/imagesets/oct2017' path with whatever directory you unzipped the OCT2017.zip file into:

"""oct2017 dataset."""

import tensorflow_datasets.public_api as tfds
from pathlib import Path
import random

# TODO(oct2017): Markdown description  that will appear on the catalog page.
_DESCRIPTION = """
Description is **formatted** as markdown.

It should also contain any processing which has been applied (if any),
(e.g. corrupted example skipped, images cropped,...):
"""

# TODO(oct2017): BibTeX citation
_CITATION = """
"""


class Oct2017(tfds.core.GeneratorBasedBuilder):
  """DatasetBuilder for oct2017 dataset."""

  VERSION = tfds.core.Version('1.0.0')
  RELEASE_NOTES = {
      '1.0.0': 'Initial release.',
  }

  def _info(self) -> tfds.core.DatasetInfo:
    """Returns the dataset metadata."""
    # TODO(oct2017): Specifies the tfds.core.DatasetInfo object
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            # These are the features of your dataset like images, labels ...
            'image': tfds.features.Image(shape=(None, None, 3)),
            'label': tfds.features.ClassLabel(names=['0','1','2','3']),
        }),
        # If there's a common (input, target) tuple from the
        # features, specify them here. They'll be used if
        # `as_supervised=True` in `builder.as_dataset`.
        supervised_keys=('image', 'label'),  # Set to `None` to disable
        homepage='https://dataset-homepage/',
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager: tfds.download.DownloadManager):
    """Returns SplitGenerators."""
    # TODO(oct2017): Downloads the data and defines the splits
    pathtrain = "/home/ubuntu/efs/imagesets/oct2017/OCT2017/train" #dl_manager.download_and_extract('https://todo-data-url')
    pathtest = "/home/ubuntu/efs/imagesets/oct2017/OCT2017/test" #dl_manager.download_and_extract('https://todo-data-url')
    #print("vooot",path)
    #die()
    # TODO(oct2017): Returns the Dict[split names, Iterator[Key, Example]]
    return {
        'train': self._generate_examples(Path(pathtrain)),
        'test': self._generate_examples(Path(pathtest)),
    }

  def _generate_examples(self, path):
    """Yields examples."""
    # TODO(oct2017): Yields (key, example) tuples from the dataset
    for f in path.glob('CNV/*.jpg'):
      image_id = random.getrandbits(512) #256)
      yield image_id, {  #'key'
          'image': f,
          'label': '0',
      }
    for f in path.glob('DME/*.jpg'):
      image_id = random.getrandbits(512) #256) 
      yield image_id, {
          'image': f,
          'label': '1',
      }      
    for f in path.glob('DRUSEN/*.jpg'):
      image_id = random.getrandbits(512) #256) 
      yield image_id, {
          'image': f,
          'label': '2',
      }      
    for f in path.glob('NORMAL/*.jpg'):
      image_id = random.getrandbits(512) #256) 
      yield image_id, {
          'image': f,
          'label': '3',
      }      

Then type

tfds build

to build the oct2017 dataset. Should be ready to use