Merge of e41e984

PiperOrigin-RevId: 257244747

Author: copybara-github

setup.py

@@ -112,6 +112,7 @@
     ],
     'groove': ['pretty_midi', 'pydub'],
     'librispeech': ['pydub'],  # and ffmpeg installed
+    'pet_finder': ['pandas'],
     'svhn': ['scipy'],
     'wikipedia': ['mwparserfromhell', 'apache_beam'],
 }

tensorflow_datasets/core/lazy_imports.py

@@ -71,7 +71,12 @@ def mwparserfromhell(cls):
 
   @utils.classproperty
   @classmethod
-  def PIL_Image(cls):   # pylint: disable=invalid-name
+  def pandas(cls):
+    return _try_import("pandas")
+
+  @utils.classproperty
+  @classmethod
+  def PIL_Image(cls):  # pylint: disable=invalid-name
     # TiffImagePlugin need to be activated explicitly on some systems
     # https://github.com/python-pillow/Pillow/blob/5.4.x/src/PIL/Image.py#L407
     _try_import("PIL.TiffImagePlugin")

tensorflow_datasets/core/lazy_imports_test.py

@@ -32,6 +32,7 @@ class LazyImportsTest(testing.TestCase, parameterized.TestCase):
       "matplotlib",
       "mwparserfromhell",
       "os",
+      "pandas",
       "pretty_midi",
       "pydub",
       "scipy",

tensorflow_datasets/image/__init__.py

@@ -53,6 +53,7 @@
 from tensorflow_datasets.image.oxford_flowers102 import OxfordFlowers102
 from tensorflow_datasets.image.oxford_iiit_pet import OxfordIIITPet
 from tensorflow_datasets.image.patch_camelyon import PatchCamelyon
+from tensorflow_datasets.image.pet_finder import PetFinder
 from tensorflow_datasets.image.quickdraw import QuickdrawBitmap
 from tensorflow_datasets.image.resisc45 import Resisc45
 from tensorflow_datasets.image.rock_paper_scissors import RockPaperScissors

tensorflow_datasets/image/pet_finder.py

@@ -0,0 +1,146 @@
+# coding=utf-8
+# Copyright 2019 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""PetFinder Dataset."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import tensorflow as tf
+import tensorflow_datasets.public_api as tfds
+
+# petfinder: BibTeX citation
+_CITATION = """
+@ONLINE {kaggle-petfinder-adoption-prediction,
+    author = "Kaggle and PetFinder.my",
+    title  = "PetFinder.my Adoption Prediction",
+    month  = "april",
+    year   = "2019",
+    url    = "https://www.kaggle.com/c/petfinder-adoption-prediction/data/"
+}
+"""
+
+_URL = ("https://storage.googleapis.com/petfinder_dataset/")
+_DATA_OPTIONS = [
+    "test_metadata", "test_images", "test_sentiment", "train_metadata",
+    "train_images", "train_sentiment"
+]
+_LABEL_OPTIONS = [
+    "test", "train", "breed_labels", "state_labels", "color_labels"
+]
+
+_DL_URLS = {name: _URL + name + ".zip" for name in _DATA_OPTIONS}
+_DL_URLS.update({label: _URL + label + ".csv" for label in _LABEL_OPTIONS})
+
+_INT_FEATS = [
+    "Type", "Age", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3",
+    "MaturitySize", "FurLength", "Vaccinated", "Dewormed", "Sterilized",
+    "Health", "Quantity", "Fee", "State", "VideoAmt"
+]
+_FLOAT_FEATS = ["PhotoAmt"]
+_OBJ_FEATS = ["name", "Type", "PetID", "RescurID"]
+_DESCRIPTION = ((
+    "A large set of images of cats and dogs."
+    "Together with the metadata information of sentiment information."))
+
+
+class PetFinder(tfds.core.GeneratorBasedBuilder):
+  """Pet Finder."""
+  VERSION = tfds.core.Version(
+      "1.0.0", experiments={tfds.core.Experiment.S3: True})
+
+  def _info(self):
+    return tfds.core.DatasetInfo(
+        builder=self,
+        description="Dataset with images from 5 classes (see config name for "
+        "information on the specific class)",
+        features=tfds.features.FeaturesDict({
+            "image": tfds.features.Image(),
+            "image/filename": tfds.features.Text(),
+            "PetID": tfds.features.Text(),
+            "attributes": {name: tf.int64 for name in _INT_FEATS},
+            "label": tfds.features.ClassLabel(num_classes=5),
+        }),
+        supervised_keys=("attributes", "label"),
+        urls=[_URL],
+        citation=_CITATION,
+    )
+
+  def _split_generators(self, dl_manager):
+    """Returns SplitGenerators."""
+    # petfinder: Downloads the data and defines the splits
+    # dl_manager is a tfds.download.DownloadManager that can be used to
+    # download and extract URLs
+    # dl_paths = dl_manager.download_kaggle_data(url)
+    dl_paths = dl_manager.download_and_extract(_DL_URLS)
+
+    return [
+        tfds.core.SplitGenerator(
+            name=tfds.Split.TRAIN,
+            num_shards=10,
+            gen_kwargs={
+                "csv_name": "train.csv",
+                "csv_paths": dl_paths["train"],
+                "img_paths": dl_paths["train_images"],
+            },
+        ),
+        tfds.core.SplitGenerator(
+            name=tfds.Split.TEST,
+            num_shards=10,
+            gen_kwargs={
+                "csv_name": "test.csv",
+                "csv_paths": dl_paths["test"],
+                "img_paths": dl_paths["test_images"],
+            },
+        ),
+    ]
+
+  def _generate_examples(self, csv_name, csv_paths, img_paths):
+    """Yields examples.
+
+    Args:
+      csv_name: file name for the csv file used in the split
+      csv_paths: Path to csv files containing the label and attributes
+        information.
+      img_paths: Path to images.
+    """
+    pd = tfds.core.lazy_imports.pandas
+
+    if not tf.io.gfile.exists(csv_paths):
+      raise AssertionError("{} not exist".format(csv_name))
+    with tf.io.gfile.GFile(csv_paths) as csv_file:
+      dataframe = pd.read_csv(csv_file)
+    # add a dummy label for test set
+    if csv_name == "test.csv":
+      dataframe["AdoptionSpeed"] = -1
+
+    images = tf.io.gfile.listdir(img_paths)
+    for image in images:
+      pet_id = image.split("-")[0]
+      image_path = os.path.join(img_paths, image)
+      attr_dict = dataframe.loc[dataframe["PetID"] == pet_id]
+      record = {
+          "image": image_path,
+          "image/filename": image,
+          "PetID": pet_id,
+          "attributes": attr_dict[_INT_FEATS].to_dict("records")[0],
+          "label": attr_dict["AdoptionSpeed"].values[0]
+      }
+      if self.version.implements(tfds.core.Experiment.S3):
+        yield image, record
+      else:
+        yield record

tensorflow_datasets/image/pet_finder_test.py

@@ -0,0 +1,42 @@
+# coding=utf-8
+# Copyright 2019 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test for PetFinder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow_datasets import testing
+from tensorflow_datasets.image import pet_finder
+
+
+class PetFinderTest(testing.DatasetBuilderTestCase):
+  # petfinder:
+  DATASET_CLASS = pet_finder.PetFinder
+  SPLITS = {
+      'train': 2,  # Number of fake train example
+      'test': 2,  # Number of fake test example
+  }
+  DL_EXTRACT_RESULT = {
+      'train': 'train.csv',
+      'train_images': 'train_images',
+      'test': 'test.csv',
+      'test_images': 'test_images',
+  }
+
+
+if __name__ == '__main__':
+  testing.test_main()

tensorflow_datasets/testing/test_data/fake_examples/pet_finder/test.csv

@@ -0,0 +1,3 @@
+Type,Name,Age,Breed1,Breed2,Gender,Color1,Color2,Color3,MaturitySize,FurLength,Vaccinated,Dewormed,Sterilized,Health,Quantity,Fee,State,RescuerID,VideoAmt,Description,PetID,PhotoAmt
+2,Dopey & Grey,8,266,266,1,2,6,7,1,1,1,1,2,1,2,0,41326,2ece3b2573dcdcebd774e635dca15fd9,0,"Dopey Age: 8mths old Male One half of a pair, Dopey is the reserved one compared to his brother Grey. However, he loves to be petted and is active by nature. Loves to chase balls and plays with anything that is mobile. Favourite hobby: Watching TV near the TV screen. Grey Age: 8mths old Male The wonder twin - Grey and Dopey are very brotherly and protects each other. Grey is more dominant than Dopey as he is the elder one and he is very playful. Favourite hobby: Loves to sit by the door and look outside",000000000,2.0
+2,Chi Chi,36,285,264,2,1,4,7,2,3,1,1,1,2,1,0,41326,2ece3b2573dcdcebd774e635dca15fd9,0,"Please note that Chichi has been neutered, therefore cannot breed. ChiChi is a Persian with a difference : She is a silent cat. She loves to be petted but needs regular grooming and cleaning. She has a defective tearduct on the right eye that requires daily cleaning. She has been neutered and goes through vaccinated routine regularly. Favourite hobby: Loves to roam and enjoys outside scenery. Please email if interested, comments are harder to keep track of.",aaaaaaaaa,1.0

tensorflow_datasets/testing/test_data/fake_examples/pet_finder/test_images/000000000-1.jpg

@@ -0,0 +1,3 @@
+Type,Name,Age,Breed1,Breed2,Gender,Color1,Color2,Color3,MaturitySize,FurLength,Vaccinated,Dewormed,Sterilized,Health,Quantity,Fee,State,RescuerID,VideoAmt,Description,PetID,PhotoAmt,AdoptionSpeed
+1,Alger,3,307,0,1,1,2,7,2,2,1,1,2,1,1,0,41326,fa90fa5b1ee11c86938398b60abc32cb,0,He is very intelligent and cute. Fluffy and looks much better in real life than in the photo. He deserves a good home. No tying or caging for long hours except for precautionary purposes Serious adopter pls call ,ddddddddd,7.0,2
+1,Terry,24,179,307,1,2,3,7,2,2,3,3,2,1,1,0,41326,719987dce7aeb027fdfa91b480800199,0,been at my place for a while..am hoping to find it a good home,666666666,0.0,4