sync changes.

Author: ChanchalKumarMaji

docs/api_docs/python/_toc.yaml

@@ -32,6 +32,8 @@ toc:
       path: /datasets/api_docs/python/tfds/core/DatasetBuilder
     - title: DatasetInfo
       path: /datasets/api_docs/python/tfds/core/DatasetInfo
+    - title: Experiment
+      path: /datasets/api_docs/python/tfds/core/Experiment
     - title: GeneratorBasedBuilder
       path: /datasets/api_docs/python/tfds/core/GeneratorBasedBuilder
     - title: get_tfds_path

docs/api_docs/python/index.md

@@ -10,6 +10,7 @@
 *   <a href="./tfds/core/BuilderConfig.md"><code>tfds.core.BuilderConfig</code></a>
 *   <a href="./tfds/core/DatasetBuilder.md"><code>tfds.core.DatasetBuilder</code></a>
 *   <a href="./tfds/core/DatasetInfo.md"><code>tfds.core.DatasetInfo</code></a>
+*   <a href="./tfds/core/Experiment.md"><code>tfds.core.Experiment</code></a>
 *   <a href="./tfds/core/GeneratorBasedBuilder.md"><code>tfds.core.GeneratorBasedBuilder</code></a>
 *   <a href="./tfds/core/Metadata.md"><code>tfds.core.Metadata</code></a>
 *   <a href="./tfds/core/MetadataDict.md"><code>tfds.core.MetadataDict</code></a>

docs/api_docs/python/tfds/_api_cache.json

@@ -10,11 +10,13 @@
     "tfds.core.BeamBasedBuilder.builder_config": "tfds.core.DatasetBuilder.builder_config", 
     "tfds.core.BeamBasedBuilder.data_dir": "tfds.core.DatasetBuilder.data_dir", 
     "tfds.core.BeamBasedBuilder.info": "tfds.core.DatasetBuilder.info", 
+    "tfds.core.BeamBasedBuilder.version": "tfds.core.DatasetBuilder.version", 
     "tfds.core.GeneratorBasedBuilder.BUILDER_CONFIGS": "tfds.core.DatasetBuilder.BUILDER_CONFIGS", 
     "tfds.core.GeneratorBasedBuilder.SUPPORTED_VERSIONS": "tfds.core.DatasetBuilder.SUPPORTED_VERSIONS", 
     "tfds.core.GeneratorBasedBuilder.builder_config": "tfds.core.DatasetBuilder.builder_config", 
     "tfds.core.GeneratorBasedBuilder.data_dir": "tfds.core.DatasetBuilder.data_dir", 
     "tfds.core.GeneratorBasedBuilder.info": "tfds.core.DatasetBuilder.info", 
+    "tfds.core.GeneratorBasedBuilder.version": "tfds.core.DatasetBuilder.version", 
     "tfds.core.MetadataDict.__contains__": "tfds.core.Metadata.__contains__", 
     "tfds.core.MetadataDict.__eq__": "tfds.core.Metadata.__eq__", 
     "tfds.core.MetadataDict.__ge__": "tfds.core.Metadata.__ge__", 
@@ -67,25 +69,9 @@
     "tfds.core.SplitDict.viewitems": "tfds.core.Metadata.viewitems", 
     "tfds.core.SplitDict.viewkeys": "tfds.core.Metadata.viewkeys", 
     "tfds.core.SplitDict.viewvalues": "tfds.core.Metadata.viewvalues", 
-    "tfds.core.Version.__init__": "tfds.core.SplitBase.__init__", 
     "tfds.features.Audio.dtype": "tfds.features.FeatureConnector.dtype", 
     "tfds.features.Audio.shape": "tfds.features.FeatureConnector.shape", 
-    "tfds.features.BBox.__add__": "tfds.core.Version.__add__", 
-    "tfds.features.BBox.__contains__": "tfds.core.Version.__contains__", 
-    "tfds.features.BBox.__eq__": "tfds.core.Version.__eq__", 
-    "tfds.features.BBox.__ge__": "tfds.core.Version.__ge__", 
-    "tfds.features.BBox.__getitem__": "tfds.core.Version.__getitem__", 
-    "tfds.features.BBox.__gt__": "tfds.core.Version.__gt__", 
     "tfds.features.BBox.__init__": "tfds.core.SplitBase.__init__", 
-    "tfds.features.BBox.__iter__": "tfds.core.Version.__iter__", 
-    "tfds.features.BBox.__le__": "tfds.core.Version.__le__", 
-    "tfds.features.BBox.__len__": "tfds.core.Version.__len__", 
-    "tfds.features.BBox.__lt__": "tfds.core.Version.__lt__", 
-    "tfds.features.BBox.__mul__": "tfds.core.Version.__mul__", 
-    "tfds.features.BBox.__ne__": "tfds.core.Version.__ne__", 
-    "tfds.features.BBox.__rmul__": "tfds.core.Version.__rmul__", 
-    "tfds.features.BBox.count": "tfds.core.Version.count", 
-    "tfds.features.BBox.index": "tfds.core.Version.index", 
     "tfds.features.BBoxFeature.dtype": "tfds.features.FeatureConnector.dtype", 
     "tfds.features.BBoxFeature.shape": "tfds.features.FeatureConnector.shape", 
     "tfds.features.ClassLabel.dtype": "tfds.features.FeatureConnector.dtype", 
@@ -110,11 +96,13 @@
     "tfds.testing.DummyDatasetSharedGenerator.builder_config": "tfds.core.DatasetBuilder.builder_config", 
     "tfds.testing.DummyDatasetSharedGenerator.data_dir": "tfds.core.DatasetBuilder.data_dir", 
     "tfds.testing.DummyDatasetSharedGenerator.info": "tfds.core.DatasetBuilder.info", 
+    "tfds.testing.DummyDatasetSharedGenerator.version": "tfds.core.DatasetBuilder.version", 
     "tfds.testing.DummyMnist.BUILDER_CONFIGS": "tfds.core.DatasetBuilder.BUILDER_CONFIGS", 
     "tfds.testing.DummyMnist.SUPPORTED_VERSIONS": "tfds.core.DatasetBuilder.SUPPORTED_VERSIONS", 
     "tfds.testing.DummyMnist.builder_config": "tfds.core.DatasetBuilder.builder_config", 
     "tfds.testing.DummyMnist.data_dir": "tfds.core.DatasetBuilder.data_dir", 
     "tfds.testing.DummyMnist.info": "tfds.core.DatasetBuilder.info", 
+    "tfds.testing.DummyMnist.version": "tfds.core.DatasetBuilder.version", 
     "tfds.testing.FeatureExpectationsTestCase.failureException": "tfds.testing.DatasetBuilderTestCase.failureException", 
     "tfds.testing.FeatureExpectationsTestCase.failureException.__getitem__": "tfds.testing.DatasetBuilderTestCase.failureException.__getitem__", 
     "tfds.testing.FeatureExpectationsTestCase.failureException.__init__": "tfds.testing.DatasetBuilderTestCase.failureException.__init__", 
@@ -151,8 +139,6 @@
     "tfds.core": false, 
     "tfds.core.BeamBasedBuilder": false, 
     "tfds.core.BeamBasedBuilder.BUILDER_CONFIGS": true, 
-    "tfds.core.BeamBasedBuilder.GOOGLE_DISABLED": true, 
-    "tfds.core.BeamBasedBuilder.IN_DEVELOPMENT": true, 
     "tfds.core.BeamBasedBuilder.SUPPORTED_VERSIONS": true, 
     "tfds.core.BeamBasedBuilder.VERSION": true, 
     "tfds.core.BeamBasedBuilder.__init__": true, 
@@ -163,6 +149,7 @@
     "tfds.core.BeamBasedBuilder.download_and_prepare": true, 
     "tfds.core.BeamBasedBuilder.info": true, 
     "tfds.core.BeamBasedBuilder.name": true, 
+    "tfds.core.BeamBasedBuilder.version": true, 
     "tfds.core.BuilderConfig": false, 
     "tfds.core.BuilderConfig.__init__": true, 
     "tfds.core.BuilderConfig.description": true, 
@@ -171,8 +158,6 @@
     "tfds.core.BuilderConfig.version": true, 
     "tfds.core.DatasetBuilder": false, 
     "tfds.core.DatasetBuilder.BUILDER_CONFIGS": true, 
-    "tfds.core.DatasetBuilder.GOOGLE_DISABLED": true, 
-    "tfds.core.DatasetBuilder.IN_DEVELOPMENT": true, 
     "tfds.core.DatasetBuilder.SUPPORTED_VERSIONS": true, 
     "tfds.core.DatasetBuilder.VERSION": true, 
     "tfds.core.DatasetBuilder.__init__": true, 
@@ -183,6 +168,7 @@
     "tfds.core.DatasetBuilder.download_and_prepare": true, 
     "tfds.core.DatasetBuilder.info": true, 
     "tfds.core.DatasetBuilder.name": true, 
+    "tfds.core.DatasetBuilder.version": true, 
     "tfds.core.DatasetInfo": false, 
     "tfds.core.DatasetInfo.__init__": true, 
     "tfds.core.DatasetInfo.as_json": true, 
@@ -205,10 +191,11 @@
     "tfds.core.DatasetInfo.urls": true, 
     "tfds.core.DatasetInfo.version": true, 
     "tfds.core.DatasetInfo.write_to_directory": true, 
+    "tfds.core.Experiment": false, 
+    "tfds.core.Experiment.DUMMY": true, 
+    "tfds.core.Experiment.S3": true, 
     "tfds.core.GeneratorBasedBuilder": false, 
     "tfds.core.GeneratorBasedBuilder.BUILDER_CONFIGS": true, 
-    "tfds.core.GeneratorBasedBuilder.GOOGLE_DISABLED": true, 
-    "tfds.core.GeneratorBasedBuilder.IN_DEVELOPMENT": true, 
     "tfds.core.GeneratorBasedBuilder.SUPPORTED_VERSIONS": true, 
     "tfds.core.GeneratorBasedBuilder.VERSION": true, 
     "tfds.core.GeneratorBasedBuilder.__init__": true, 
@@ -219,6 +206,7 @@
     "tfds.core.GeneratorBasedBuilder.download_and_prepare": true, 
     "tfds.core.GeneratorBasedBuilder.info": true, 
     "tfds.core.GeneratorBasedBuilder.name": true, 
+    "tfds.core.GeneratorBasedBuilder.version": true, 
     "tfds.core.Metadata": false, 
     "tfds.core.Metadata.__contains__": true, 
     "tfds.core.Metadata.__eq__": true, 
@@ -287,12 +275,14 @@
     "tfds.core.NamedSplit.__add__": true, 
     "tfds.core.NamedSplit.__eq__": true, 
     "tfds.core.NamedSplit.__init__": true, 
+    "tfds.core.NamedSplit.__ne__": true, 
     "tfds.core.NamedSplit.get_read_instruction": true, 
     "tfds.core.NamedSplit.subsplit": true, 
     "tfds.core.SplitBase": false, 
     "tfds.core.SplitBase.__add__": true, 
     "tfds.core.SplitBase.__eq__": true, 
     "tfds.core.SplitBase.__init__": true, 
+    "tfds.core.SplitBase.__ne__": true, 
     "tfds.core.SplitBase.get_read_instruction": true, 
     "tfds.core.SplitBase.subsplit": true, 
     "tfds.core.SplitDict": false, 
@@ -337,27 +327,16 @@
     "tfds.core.SplitInfo.get_proto": true, 
     "tfds.core.SplitInfo.num_examples": true, 
     "tfds.core.Version": false, 
-    "tfds.core.Version.LATEST": true, 
-    "tfds.core.Version.__add__": true, 
-    "tfds.core.Version.__contains__": true, 
     "tfds.core.Version.__eq__": true, 
     "tfds.core.Version.__ge__": true, 
-    "tfds.core.Version.__getitem__": true, 
     "tfds.core.Version.__gt__": true, 
     "tfds.core.Version.__init__": true, 
-    "tfds.core.Version.__iter__": true, 
     "tfds.core.Version.__le__": true, 
-    "tfds.core.Version.__len__": true, 
     "tfds.core.Version.__lt__": true, 
-    "tfds.core.Version.__mul__": true, 
     "tfds.core.Version.__ne__": true, 
-    "tfds.core.Version.__rmul__": true, 
-    "tfds.core.Version.count": true, 
-    "tfds.core.Version.index": true, 
-    "tfds.core.Version.major": true, 
+    "tfds.core.Version.implements": true, 
     "tfds.core.Version.match": true, 
-    "tfds.core.Version.minor": true, 
-    "tfds.core.Version.patch": true, 
+    "tfds.core.Version.tuple": true, 
     "tfds.core.get_tfds_path": false, 
     "tfds.core.lazy_imports": false, 
     "tfds.disable_progress_bar": false, 
@@ -462,6 +441,7 @@
     "tfds.features.FeatureConnector.save_metadata": true, 
     "tfds.features.FeatureConnector.shape": true, 
     "tfds.features.FeaturesDict": false, 
+    "tfds.features.FeaturesDict.__contains__": true, 
     "tfds.features.FeaturesDict.__getitem__": true, 
     "tfds.features.FeaturesDict.__init__": true, 
     "tfds.features.FeaturesDict.__iter__": true, 
@@ -600,8 +580,6 @@
     "tfds.file_adapter.TFRecordExampleAdapter.__init__": true, 
     "tfds.file_adapter.TFRecordExampleAdapter.dataset_from_filename": true, 
     "tfds.file_adapter.TFRecordExampleAdapter.filetype_suffix": true, 
-    "tfds.file_adapter.TFRecordExampleAdapter.parse_example": true, 
-    "tfds.file_adapter.TFRecordExampleAdapter.serialize_example": true, 
     "tfds.file_adapter.TFRecordExampleAdapter.write_from_generator": true, 
     "tfds.file_adapter.TFRecordExampleAdapter.write_from_pcollection": true, 
     "tfds.is_dataset_on_gcs": false, 
@@ -615,10 +593,10 @@
     "tfds.testing.DatasetBuilderTestCase.DATASET_CLASS": true, 
     "tfds.testing.DatasetBuilderTestCase.DL_EXTRACT_RESULT": true, 
     "tfds.testing.DatasetBuilderTestCase.EXAMPLE_DIR": true, 
-    "tfds.testing.DatasetBuilderTestCase.INTERNAL_DATASET": true, 
     "tfds.testing.DatasetBuilderTestCase.MOCK_MONARCH": true, 
     "tfds.testing.DatasetBuilderTestCase.MOCK_OUT_FORBIDDEN_OS_FUNCTIONS": true, 
     "tfds.testing.DatasetBuilderTestCase.OVERLAPPING_SPLITS": true, 
+    "tfds.testing.DatasetBuilderTestCase.VERSION": true, 
     "tfds.testing.DatasetBuilderTestCase.__call__": true, 
     "tfds.testing.DatasetBuilderTestCase.__eq__": true, 
     "tfds.testing.DatasetBuilderTestCase.__init__": true, 
@@ -760,8 +738,6 @@
     "tfds.testing.DatasetBuilderTestCase.test_session": true, 
     "tfds.testing.DummyDatasetSharedGenerator": false, 
     "tfds.testing.DummyDatasetSharedGenerator.BUILDER_CONFIGS": true, 
-    "tfds.testing.DummyDatasetSharedGenerator.GOOGLE_DISABLED": true, 
-    "tfds.testing.DummyDatasetSharedGenerator.IN_DEVELOPMENT": true, 
     "tfds.testing.DummyDatasetSharedGenerator.SUPPORTED_VERSIONS": true, 
     "tfds.testing.DummyDatasetSharedGenerator.VERSION": true, 
     "tfds.testing.DummyDatasetSharedGenerator.__init__": true, 
@@ -772,10 +748,9 @@
     "tfds.testing.DummyDatasetSharedGenerator.download_and_prepare": true, 
     "tfds.testing.DummyDatasetSharedGenerator.info": true, 
     "tfds.testing.DummyDatasetSharedGenerator.name": true, 
+    "tfds.testing.DummyDatasetSharedGenerator.version": true, 
     "tfds.testing.DummyMnist": false, 
     "tfds.testing.DummyMnist.BUILDER_CONFIGS": true, 
-    "tfds.testing.DummyMnist.GOOGLE_DISABLED": true, 
-    "tfds.testing.DummyMnist.IN_DEVELOPMENT": true, 
     "tfds.testing.DummyMnist.SUPPORTED_VERSIONS": true, 
     "tfds.testing.DummyMnist.VERSION": true, 
     "tfds.testing.DummyMnist.__init__": true, 
@@ -786,6 +761,7 @@
     "tfds.testing.DummyMnist.download_and_prepare": true, 
     "tfds.testing.DummyMnist.info": true, 
     "tfds.testing.DummyMnist.name": true, 
+    "tfds.testing.DummyMnist.version": true, 
     "tfds.testing.FeatureExpectationItem": false, 
     "tfds.testing.FeatureExpectationItem.__init__": true, 
     "tfds.testing.FeatureExpectationsTestCase": false, 
@@ -1212,7 +1188,6 @@
     "tfds.units.MiB": true, 
     "tfds.units.PiB": true, 
     "tfds.units.TiB": true, 
-    "tfds.units.absolute_import": true, 
     "tfds.units.division": true, 
     "tfds.units.print_function": true, 
     "tfds.units.size_str": false

docs/api_docs/python/tfds/core.md

@@ -22,6 +22,9 @@ Defined in [`core/__init__.py`](https://github.com/tensorflow/datasets/tree/mast
 
 [`class DatasetInfo`](../tfds/core/DatasetInfo.md): Information about a dataset.
 
+[`class Experiment`](../tfds/core/Experiment.md): Experiments which can be
+enabled/disabled on a per version basis.
+
 [`class GeneratorBasedBuilder`](../tfds/core/GeneratorBasedBuilder.md): Base class for datasets with data generation based on dict generators.
 
 [`class Metadata`](../tfds/core/Metadata.md): Abstract base class for

docs/api_docs/python/tfds/core/BeamBasedBuilder.md

@@ -4,12 +4,11 @@
 <meta itemprop="property" content="builder_config"/>
 <meta itemprop="property" content="data_dir"/>
 <meta itemprop="property" content="info"/>
+<meta itemprop="property" content="version"/>
 <meta itemprop="property" content="__init__"/>
 <meta itemprop="property" content="as_dataset"/>
 <meta itemprop="property" content="download_and_prepare"/>
 <meta itemprop="property" content="BUILDER_CONFIGS"/>
-<meta itemprop="property" content="GOOGLE_DISABLED"/>
-<meta itemprop="property" content="IN_DEVELOPMENT"/>
 <meta itemprop="property" content="SUPPORTED_VERSIONS"/>
 <meta itemprop="property" content="VERSION"/>
 <meta itemprop="property" content="builder_configs"/>
@@ -52,7 +51,9 @@ Callers must pass arguments as keyword arguments.
     subdirectories and versions.
 *   <b>`version`</b>: `str`. Optional version at which to load the dataset. An
     error is raised if specified version cannot be satisfied. Eg: '1.2.3',
-    '1.2.*'.
+    '1.2.*'. The special value "experimental_latest" will use the highest
+    version, even if not default. This is not recommended unless you know what
+    you are doing, as the version could be broken.
 
 ## Properties
 
@@ -66,16 +67,19 @@ Callers must pass arguments as keyword arguments.
 
 <a href="../../tfds/core/DatasetInfo.md"><code>tfds.core.DatasetInfo</code></a> for this builder.
 
+<h3 id="version"><code>version</code></h3>
+
 ## Methods
 
 <h3 id="as_dataset"><code>as_dataset</code></h3>
 
 ```python
 as_dataset(
     split=None,
-    batch_size=1,
+    batch_size=None,
     shuffle_files=None,
-    as_supervised=False
+    as_supervised=False,
+    in_memory=None
 )
 ```
 
@@ -90,16 +94,20 @@ Callers must pass arguments as keyword arguments.
     which subset(s) of the data to read. If None (default), returns all splits
     in a dict `<key: tfds.Split, value: tf.data.Dataset>`.
 *   <b>`batch_size`</b>: `int`, batch size. Note that variable-length features
-    will be 0-padded if `batch_size > 1`. Users that want more custom behavior
-    should use `batch_size=1` and use the `tf.data` API to construct a custom
-    pipeline. If `batch_size == -1`, will return feature dictionaries of the
-    whole dataset with `tf.Tensor`s instead of a `tf.data.Dataset`.
+    will be 0-padded if `batch_size` is set. Users that want more custom
+    behavior should use `batch_size=None` and use the `tf.data` API to construct
+    a custom pipeline. If `batch_size == -1`, will return feature dictionaries
+    of the whole dataset with `tf.Tensor`s instead of a `tf.data.Dataset`.
 *   <b>`shuffle_files`</b>: `bool`, whether to shuffle the input files. Defaults
     to `True` if `split == tfds.Split.TRAIN` and `False` otherwise.
 *   <b>`as_supervised`</b>: `bool`, if `True`, the returned `tf.data.Dataset`
     will have a 2-tuple structure `(input, label)` according to
     `builder.info.supervised_keys`. If `False`, the default, the returned
     `tf.data.Dataset` will have a dictionary with all the features.
+*   <b>`in_memory`</b>: `bool`, if `True`, loads the dataset in memory which
+    increases iteration speeds. Note that if `True` and the dataset has unknown
+    dimensions, the features will be padded to the maximum size across the
+    dataset.
 
 #### Returns:
 
@@ -137,8 +145,6 @@ Downloads and prepares dataset for reading.
 ## Class Members
 
 *   `BUILDER_CONFIGS` <a id="BUILDER_CONFIGS"></a>
-*   `GOOGLE_DISABLED = False` <a id="GOOGLE_DISABLED"></a>
-*   `IN_DEVELOPMENT = False` <a id="IN_DEVELOPMENT"></a>
 *   `SUPPORTED_VERSIONS` <a id="SUPPORTED_VERSIONS"></a>
 *   `VERSION = None` <a id="VERSION"></a>
 *   `builder_configs` <a id="builder_configs"></a>