Adding a field with instructions on how to prepare files in manual_dir.

cyfra · copybara-github · commit 37fc133c401e · 2019-11-28T01:08:29.000-08:00
PiperOrigin-RevId: 282898527
diff --git a/tensorflow_datasets/core/dataset_builder.py b/tensorflow_datasets/core/dataset_builder.py
@@ -151,6 +151,15 @@ class DatasetBuilder(object):
   # be available through tfds.{load, builder} or documented in overview.md.
   IN_DEVELOPMENT = False
 
+  # Must be set for datasets that use 'manual_dir' functionality - the ones
+  # that require users to do additional steps to download the data
+  # (this is usually due to some external regulations / rules).
+  #
+  # This field should contain a string with user instructions, including
+  # the list of files that should be present. It will be
+  # displayed in the dataset documentation.
+  MANUAL_DOWNLOAD_INSTRUCTIONS = None
+
 
   @api_utils.disallow_positional_args
   def __init__(self, data_dir=None, config=None, version=None):
@@ -649,19 +658,26 @@ def _as_dataset(self, split, decoders=None, shuffle_files=False):
     raise NotImplementedError
 
   def _make_download_manager(self, download_dir, download_config):
+    """Creates a new download manager object."""
     download_dir = download_dir or os.path.join(self._data_dir_root,
                                                 "downloads")
     extract_dir = (download_config.extract_dir or
                    os.path.join(download_dir, "extracted"))
-    manual_dir = (download_config.manual_dir or
-                  os.path.join(download_dir, "manual"))
-    manual_dir = os.path.join(manual_dir, self.name)
+
+    # Use manual_dir only if MANUAL_DOWNLOAD_INSTRUCTIONS are set.
+    if self.MANUAL_DOWNLOAD_INSTRUCTIONS:
+      manual_dir = (
+          download_config.manual_dir or os.path.join(download_dir, "manual"))
+      manual_dir = os.path.join(manual_dir, self.name)
+    else:
+      manual_dir = None
 
     return download.DownloadManager(
         dataset_name=self.name,
         download_dir=download_dir,
         extract_dir=extract_dir,
         manual_dir=manual_dir,
+        manual_dir_instructions=self.MANUAL_DOWNLOAD_INSTRUCTIONS,
         force_download=(download_config.download_mode == FORCE_REDOWNLOAD),
         force_extraction=(download_config.download_mode == FORCE_REDOWNLOAD),
         register_checksums=download_config.register_checksums,
diff --git a/tensorflow_datasets/core/download/download_manager.py b/tensorflow_datasets/core/download/download_manager.py
@@ -149,6 +149,7 @@ def __init__(self,
                download_dir,
                extract_dir=None,
                manual_dir=None,
+               manual_dir_instructions=None,
                dataset_name=None,
                force_download=False,
                force_extraction=False,
@@ -159,6 +160,8 @@ def __init__(self,
       download_dir: `str`, path to directory where downloads are stored.
       extract_dir: `str`, path to directory where artifacts are extracted.
       manual_dir: `str`, path to manually downloaded/extracted data directory.
+      manual_dir_instructions: `str`, human readable instructions on how to
+                         prepare contents of the manual_dir for this dataset.
       dataset_name: `str`, name of dataset this instance will be used for. If
         provided, downloads will contain which datasets they were used for.
       force_download: `bool`, default to False. If True, always [re]download.
@@ -171,6 +174,7 @@ def __init__(self,
     self._extract_dir = os.path.expanduser(
         extract_dir or os.path.join(download_dir, 'extracted'))
     self._manual_dir = manual_dir and os.path.expanduser(manual_dir)
+    self._manual_dir_instructions = manual_dir_instructions
     tf.io.gfile.makedirs(self._download_dir)
     tf.io.gfile.makedirs(self._extract_dir)
     self._force_download = force_download
@@ -372,10 +376,15 @@ def download_and_extract(self, url_or_urls):
   @property
   def manual_dir(self):
     """Returns the directory containing the manually extracted data."""
+    if not self._manual_dir:
+      raise AssertionError(
+          'Manual directory was enabled. '
+          'Did you set MANUAL_DOWNLOAD_INSTRUCTIONS in your dataset?')
     if not tf.io.gfile.exists(self._manual_dir):
       raise AssertionError(
           'Manual directory {} does not exist. Create it and download/extract '
-          'dataset artifacts in there.'.format(self._manual_dir))
+          'dataset artifacts in there. Additional instructions: {}'.format(
+              self._manual_dir, self._manual_dir_instructions))
     return self._manual_dir
 
 
diff --git a/tensorflow_datasets/image/abstract_reasoning.py b/tensorflow_datasets/image/abstract_reasoning.py
@@ -149,6 +149,12 @@ def __init__(self, split_type="neutral", **kwargs):
 
 class AbstractReasoning(tfds.core.BeamBasedBuilder):
   """Abstract reasoning dataset."""
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  Data can be downloaded from
+  https://console.cloud.google.com/storage/browser/ravens-matrices
+  Please put all the tar.gz files in manual_dir.
+  """
+
   BUILDER_CONFIGS = [
       AbstractReasoningConfig(
           name="neutral",
diff --git a/tensorflow_datasets/image/cbis_ddsm.py b/tensorflow_datasets/image/cbis_ddsm.py
@@ -134,6 +134,13 @@ def __init__(self, image_size=None, patch_size=None, **kwargs):
 class CuratedBreastImagingDDSM(tfds.core.GeneratorBasedBuilder):
   """Curated Breast Imaging Subset of DDSM."""
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  You can download the images from
+  https://wiki.cancerimagingarchive.net/display/Public/CBIS-DDSM
+  Please look at the source file (cbis_ddsm.py) to see the instructions
+  on how to conver them into png (using dcmj2pnm).
+  """
+
   BUILDER_CONFIGS = [
       CuratedBreastImagingDDSMConfig(
           name='patches',
diff --git a/tensorflow_datasets/image/celebahq.py b/tensorflow_datasets/image/celebahq.py
@@ -77,6 +77,13 @@ def __init__(self, resolution, **kwargs):
 class CelebAHq(tfds.core.GeneratorBasedBuilder):
   """Celeba_HQ Dataset."""
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  manual_dir should contain multiple tar files with images (data2x2.tar,
+  data4x4.tar .. data1024x1024.tar).
+  Detailed instructions are here:
+  https://github.com/tkarras/progressive_growing_of_gans#preparing-datasets-for-training
+  """
+
   VERSION = tfds.core.Version("0.1.0")
 
   BUILDER_CONFIGS = [
diff --git a/tensorflow_datasets/image/chexpert.py b/tensorflow_datasets/image/chexpert.py
@@ -80,6 +80,14 @@ class Chexpert(tfds.core.GeneratorBasedBuilder):
           "3.0.0", "New split API (https://tensorflow.org/datasets/splits)"),
   ]
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  You must register and agree to user agreement on the dataset page:
+  https://stanfordmlgroup.github.io/competitions/chexpert/
+  Afterwards, you have to put the CheXpert-v1.0-small directory in the
+  manual_dir. It should contain subdirectories: train/ and valid/ with images
+  and also train.csv and valid.csv files.
+  """
+
   def _info(self):
     return tfds.core.DatasetInfo(
         builder=self,
diff --git a/tensorflow_datasets/image/diabetic_retinopathy_detection.py b/tensorflow_datasets/image/diabetic_retinopathy_detection.py
@@ -78,6 +78,14 @@ def target_pixels(self):
 class DiabeticRetinopathyDetection(tfds.core.GeneratorBasedBuilder):
   """Diabetic retinopathy detection."""
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  You have to download this dataset from Kaggle.
+  https://www.kaggle.com/c/diabetic-retinopathy-detection/data
+  After downloading, unpack the test.zip file into test/ directory in manual_dir
+  and sample.zip to sample/. Also unpack the sampleSubmissions.csv and
+  trainLabels.csv.
+  """
+
   BUILDER_CONFIGS = [
       DiabeticRetinopathyDetectionConfig(
           name="original",
diff --git a/tensorflow_datasets/image/image_folder.py b/tensorflow_datasets/image/image_folder.py
@@ -74,6 +74,8 @@ class ImageLabelFolder(tfds.core.GeneratorBasedBuilder):
 
   """
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = "This is a 'template' dataset."
+
   VERSION = tfds.core.Version("1.0.0",
                               experiments={tfds.core.Experiment.S3: False})
   SUPPORTED_VERSIONS = [
diff --git a/tensorflow_datasets/image/imagenet.py b/tensorflow_datasets/image/imagenet.py
@@ -103,6 +103,13 @@ class Imagenet2012(tfds.core.GeneratorBasedBuilder):
           '5.0.0', 'New split API (https://tensorflow.org/datasets/splits)'),
   ]
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  manual_dir should contain two files: ILSVRC2012_img_train.tar and
+  ILSVRC2012_img_val.tar.
+  You need to register on http://www.image-net.org/download-images in order
+  to get the link to download the dataset.
+  """
+
   def _info(self):
     names_file = tfds.core.get_tfds_path(_LABELS_FNAME)
     return tfds.core.DatasetInfo(
diff --git a/tensorflow_datasets/image/resisc45.py b/tensorflow_datasets/image/resisc45.py
@@ -71,6 +71,12 @@ class Resisc45(tfds.core.GeneratorBasedBuilder):
   # 2.0.0: S3 with new hashing function (different shuffle).
   # 1.0.0: S3 (new shuffling, sharding and slicing mechanism).
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  Dataset can be downloaded from OneDrive:
+  https://1drv.ms/u/s!AmgKYzARBl5ca3HNaHIlzp_IXjs
+  After downloading the rar file, please extract it to the manual_dir.
+  """
+
   def _info(self):
     return tfds.core.DatasetInfo(
         builder=self,
diff --git a/tensorflow_datasets/summarization/newsroom.py b/tensorflow_datasets/summarization/newsroom.py
@@ -80,6 +80,12 @@ class Newsroom(tfds.core.GeneratorBasedBuilder):
   """NEWSROOM Dataset."""
 
   VERSION = tfds.core.Version("1.0.0")
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  You should download the dataset from https://summari.es/download/
+  The webpage requires registration.
+  After downloading, please put dev.jsonl, test.jsonl and train.jsonl
+  files in the manual_dir.
+  """
 
   def _info(self):
     features = {
diff --git a/tensorflow_datasets/summarization/wikihow.py b/tensorflow_datasets/summarization/wikihow.py
@@ -92,6 +92,11 @@ def __init__(self, filename=None, **kwargs):
 class Wikihow(tfds.core.GeneratorBasedBuilder):
   """WikiHow: A Large Scale Text Summarization Dataset."""
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  Links to files can be found on https://github.com/mahnazkoupaee/WikiHow-Dataset
+  Please download both wikihowAll.csv and wikihowSep.csv.
+  """
+
   BUILDER_CONFIGS = [
       WikihowConfig(
           name="all",
diff --git a/tensorflow_datasets/summarization/xsum.py b/tensorflow_datasets/summarization/xsum.py
@@ -68,6 +68,13 @@ class Xsum(tfds.core.GeneratorBasedBuilder):
   VERSION = tfds.core.Version("1.1.0")
   SUPPORTED_VERSIONS = [tfds.core.Version("1.0.0", "Dataset without cleaning.")]
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  Detailed download instructions (which require running a custom script) are
+  here:
+  https://github.com/EdinburghNLP/XSum/blob/master/XSum-Dataset/README.md#running-the-download-and-extraction-scripts
+  Afterwards, please put xsum-extracts-from-downloads.tar.gz file in the manual_dir.
+  """
+
   def _info(self):
     return tfds.core.DatasetInfo(
         builder=self,
diff --git a/tensorflow_datasets/testing/dataset_builder_testing.py b/tensorflow_datasets/testing/dataset_builder_testing.py
@@ -253,12 +253,21 @@ def test_download_and_prepare_as_dataset(self):
       self._download_and_prepare_as_dataset(self.builder)
 
   def _download_and_prepare_as_dataset(self, builder):
+    # Provide the manual dir only if builder has MANUAL_DOWNLOAD_INSTRUCTIONS
+    # set.
+
+    missing_dir_mock = absltest.mock.PropertyMock(
+        side_effect=Exception("Missing MANUAL_DOWNLOAD_INSTRUCTIONS"))
+
+    manual_dir = (
+        self.example_dir
+        if builder.MANUAL_DOWNLOAD_INSTRUCTIONS else missing_dir_mock)
     with absltest.mock.patch.multiple(
         "tensorflow_datasets.core.download.DownloadManager",
         download_and_extract=self._get_dl_extract_result,
         download=self._get_dl_download_result,
         download_checksums=lambda *_: None,
-        manual_dir=self.example_dir,
+        manual_dir=manual_dir,
     ):
       if isinstance(builder, dataset_builder.BeamBasedBuilder):
         import apache_beam as beam   # pylint: disable=g-import-not-at-top
@@ -377,4 +386,3 @@ def compare_shapes_and_types(tensor_info, output_types, output_shapes):
       expected_shape = feature_info.shape
       output_shape = output_shapes[feature_name]
       tf_utils.assert_shape_match(expected_shape, output_shape)
-
diff --git a/tensorflow_datasets/translate/wmt.py b/tensorflow_datasets/translate/wmt.py
@@ -618,6 +618,12 @@ def __init__(self,
 class WmtTranslate(tfds.core.GeneratorBasedBuilder):
   """WMT translation dataset."""
 
+  MANUAL_DOWNLOAD_INSTRUCTIONS = """\
+  Some of the wmt configs here, require a manual download.
+  Please look into wmt.py to see the exact path (and file name) that has to
+  be downloaded.
+  """
+
   def __init__(self, *args, **kwargs):
     if type(self) == WmtTranslate and "config" not in kwargs:   # pylint: disable=unidiomatic-typecheck
       raise ValueError(
