TFDS versioning: make it possible to mark a dataset version as readable but not preparable.

PiperOrigin-RevId: 281269279

Author: pierrot0

docs/datasets_versioning.md

@@ -55,8 +55,8 @@ class Imagenet2012(tfds.core.GeneratorBasedBuilder):
   VERSION = tfds.core.Version('2.0.1', 'Encoding fix. No changes from user POV')
   SUPPORTED_VERSIONS = [
       tfds.core.Version('3.0.0', 'S3: tensorflow.org/datasets/splits'),
-      tfds.core.Version('2.0.1', 'Encoding fix. No changes from user POV'),
       tfds.core.Version('1.0.0'),
+      tfds.core.Version('0.0.9', tfds_version_to_prepare="v1.0.0"),
   ]
 ```
 
@@ -70,6 +70,12 @@ Supported versions with a higher number than the canonical version number are
 considered experimental and might be broken. They will however eventually be
 made canonical.
 
+A version can specify `tfds_version_to_prepare`. This means this dataset version
+can only be used with current version of TFDS code if it has already been
+prepared by an older version of the code, but cannot be prepared. The
+value of `tfds_version_to_prepare` specifies the last known version of TFDS
+which can be used to download and prepare the dataset at this version.
+
 ## Loading a specific version
 
 When loading a dataset or a `DatasetBuilder`, you can specify the version to

tensorflow_datasets/core/dataset_builder.py

@@ -187,25 +187,36 @@ def __init__(self, data_dir=None, config=None, version=None):
       logging.info("Load pre-computed datasetinfo (eg: splits) from bucket.")
       self.info.initialize_from_bucket()
 
-  def _pick_version(self, requested_version):
-    """Returns utils.Version instance, or raise AssertionError."""
+  @utils.memoized_property
+  def canonical_version(self):
     if self._builder_config:
-      canonical_version = self._builder_config.version
-      supported_versions = self._builder_config.supported_versions
+      return self._builder_config.version
     else:
-      canonical_version = self.VERSION
-      supported_versions = self.SUPPORTED_VERSIONS
-    versions = [
+      return self.VERSION
+
+  @utils.memoized_property
+  def supported_versions(self):
+    if self._builder_config:
+      return self._builder_config.supported_versions
+    else:
+      return self.SUPPORTED_VERSIONS
+
+  @utils.memoized_property
+  def versions(self):
+    """Versions (canonical + availables), in preference order."""
+    return [
         utils.Version(v) if isinstance(v, six.string_types) else v
-        for v in [canonical_version] + supported_versions
+        for v in [self.canonical_version] + self.supported_versions
     ]
+
+  def _pick_version(self, requested_version):
+    """Returns utils.Version instance, or raise AssertionError."""
     if requested_version == "experimental_latest":
-      return max(versions)
-    for version in versions:
+      return max(self.versions)
+    for version in self.versions:
       if requested_version is None or version.match(requested_version):
         return version
-    available_versions = [str(v)
-                          for v in [canonical_version] + supported_versions]
+    available_versions = [str(v) for v in self.versions]
     msg = "Dataset {} cannot be loaded at version {}, only: {}.".format(
         self.name, requested_version, ", ".join(available_versions))
     raise AssertionError(msg)
@@ -253,9 +264,17 @@ def download_and_prepare(self, download_dir=None, download_config=None):
       logging.info("Reusing dataset %s (%s)", self.name, self._data_dir)
       return
 
-    dl_manager = self._make_download_manager(
-        download_dir=download_dir,
-        download_config=download_config)
+    if self.version.tfds_version_to_prepare:
+      available_to_prepare = ", ".join(str(v) for v in self.versions
+                                       if not v.tfds_version_to_prepare)
+      raise AssertionError(
+          "The version of the dataset you are trying to use ({}:{}) can only "
+          "be generated using TFDS code synced @ {} or earlier. Either sync to "
+          "that version of TFDS to first prepare the data or use another "
+          "version of the dataset (available for `download_and_prepare`: "
+          "{}).".format(
+              self.name, self.version, self.version.tfds_version_to_prepare,
+              available_to_prepare))
 
     # Currently it's not possible to overwrite the data because it would
     # conflict with versioning: If the last version has already been generated,
@@ -266,13 +285,18 @@ def download_and_prepare(self, download_dir=None, download_config=None):
           "the same version {} already exists. If the dataset has changed, "
           "please update the version number.".format(self.name, self._data_dir,
                                                      self.version))
+
     logging.info("Generating dataset %s (%s)", self.name, self._data_dir)
     if not utils.has_sufficient_disk_space(
         self.info.size_in_bytes, directory=self._data_dir_root):
       raise IOError("Not enough disk space. Needed: %s" %
                     units.size_str(self.info.size_in_bytes))
     self._log_download_bytes()
 
+    dl_manager = self._make_download_manager(
+        download_dir=download_dir,
+        download_config=download_config)
+
     # Create a tmp dir and rename to self._data_dir on successful exit.
     with file_format_adapter.incomplete_dir(self._data_dir) as tmp_data_dir:
       # Temporarily assign _data_dir to tmp_data_dir to avoid having to forward

tensorflow_datasets/core/dataset_builder_test.py

@@ -286,6 +286,18 @@ def test_previous_supported_version(self):
     older_builder = DummyDatasetSharedGenerator(version="0.0.*")
     self.assertEqual(str(older_builder.info.version), "0.0.9")
 
+  def test_non_preparable_version(self, *unused_mocks):
+    expected = (
+        "The version of the dataset you are trying to use ("
+        "dummy_dataset_shared_generator:0.0.7) can only be generated using TFDS"
+        " code synced @ v1.0.0 or earlier. Either sync to that version of TFDS "
+        "to first prepare the data or use another version of the dataset "
+        "(available for `download_and_prepare`: 1.0.0, 2.0.0, 0.0.9, 0.0.8).")
+    builder = DummyDatasetSharedGenerator(version="0.0.7")
+    self.assertIsNotNone(builder)
+    with self.assertRaisesWithPredicateMatch(AssertionError, expected):
+      builder.download_and_prepare()
+
   def test_invalid_split_dataset(self):
     with testing.tmp_dir(self.get_temp_dir()) as tmp_dir:
       with self.assertRaisesWithPredicateMatch(ValueError, "ALL is a special"):

tensorflow_datasets/core/utils/version.py

@@ -63,9 +63,22 @@ class Version(object):
       Experiment.S3: True,
   }
 
-  def __init__(self, version_str, description=None, experiments=None):
+  def __init__(self, version_str, description=None, experiments=None,
+               tfds_version_to_prepare=None):
+    """Version init.
+
+    Args:
+      version_str: string. Eg: "1.2.3".
+      description: string, a description of what is new in this version.
+      experiments: dict of experiments. See Experiment.
+      tfds_version_to_prepare: string, defaults to None. If set, indicates that
+        current version of TFDS cannot be used to `download_and_prepare` the
+        dataset, but that TFDS at version {tfds_version_to_prepare} should be
+        used instead.
+    """
     self.description = description
     self._experiments = self._DEFAULT_EXPERIMENTS.copy()
+    self.tfds_version_to_prepare = tfds_version_to_prepare
     if experiments:
       self._experiments.update(experiments)
     self.major, self.minor, self.patch = _str_to_version(version_str)

tensorflow_datasets/testing/test_utils.py

@@ -361,6 +361,7 @@ class DummyDatasetSharedGenerator(dataset_builder.GeneratorBasedBuilder):
       "2.0.0",
       "0.0.9",
       "0.0.8",
+      utils.Version("0.0.7", tfds_version_to_prepare="v1.0.0"),
   ]
 
   def _info(self):