Convert datasets to use kagglesdk (#636)

I have some `TODO`s that won't be done until Swagger is removed.

Ignore everything in `kagglesdk`.

The unit tests were not reformatted earlier.

```bash
$ yapf --version
yapf 0.40.2
```
The version is the same as specified in #634, so I don't know why some
indentation changed.

Author: stevemessick

kaggle/api/kaggle_api_extended.py

@@ -1,19 +1,19 @@
-#!/usr/bin/python
-#
-# Copyright 2024 Kaggle Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+#!/usr/bin/python
+#
+# Copyright 2024 Kaggle Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #!/usr/bin/python
 #
 # Copyright 2019 Kaggle Inc
@@ -131,9 +131,14 @@ def __repr__(self):
 class File(object):
 
   def __init__(self, init_dict):
-    parsed_dict = {k: parse(v) for k, v in init_dict.items()}
-    self.__dict__.update(parsed_dict)
-    self.size = File.get_size(self.totalBytes)
+    try: # TODO Remove try-block
+      parsed_dict = {k: parse(v) for k, v in init_dict.items()}
+      self.__dict__.update(parsed_dict)
+      self.size = File.get_size(self.totalBytes)
+    except AttributeError:
+      self.name = init_dict.name
+      self.creation_date = init_dict.creation_date
+      self.size = File.get_size(init_dict.total_bytes)
 
   def __repr__(self):
     return self.name
@@ -181,13 +186,18 @@ def __repr__(self):
 class ListFilesResult(object):
 
   def __init__(self, init_dict):
-    self.error_message = init_dict['errorMessage']
-    files = init_dict['datasetFiles']
+    try: # TODO Remove try-block
+      self.error_message = init_dict['errorMessage']
+      files = init_dict['datasetFiles']
+      token = init_dict['nextPageToken']
+    except TypeError:
+      self.error_message = init_dict.error_message
+      files = init_dict.dataset_files
+      token = init_dict.next_page_token
     if files:
       self.files = [File(f) for f in files]
     else:
       self.files = {}
-    token = init_dict['nextPageToken']
     if token:
       self.nextPageToken = token
     else:

kaggle/models/kaggle_models_extended.py

@@ -176,6 +176,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return '*'
+
 class ApiCreateDatasetResponse(KaggleObject):
   r"""
   Attributes:
@@ -310,6 +314,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return 'body'
+
 class ApiCreateDatasetVersionRequest(KaggleObject):
   r"""
   Attributes:
@@ -373,6 +381,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return 'body'
+
 class ApiCreateDatasetVersionRequestBody(KaggleObject):
   r"""
   Attributes:
@@ -2080,6 +2092,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return 'settings'
+
 class ApiUpdateDatasetMetadataResponse(KaggleObject):
   r"""
   Attributes:

kagglesdk/datasets/types/dataset_api_service.py

@@ -34,6 +34,16 @@ def _get_apikey_creds():
   api_key = api_key_data['key']
   return username, api_key
 
+def clean_data(data):
+  if isinstance(data, dict):
+    return {k: clean_data(v) for k, v in data.items() if v is not None}
+  if isinstance(data, list):
+    return [clean_data(v) for v in data if v is not None]
+  if data is True:
+    return 'true'
+  if data is False:
+    return 'false'
+  return data
 
 class KaggleHttpClient(object):
   _xsrf_cookie_name = 'XSRF-TOKEN'
@@ -75,6 +85,19 @@ def _prepare_request(self, service_name: str, request_name: str, request: Kaggle
         'Accept': 'application/json',
         'Content-Type': 'text/plain',
       })
+    elif method == 'POST':
+      self._session.headers.update({
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      })
+      if isinstance(data, dict):
+        fields = request.body_fields()
+        if fields is not None:
+          if fields != '*':
+            data = data[fields]
+        data = clean_data(data)
+        data = data.__str__().replace("'", '"')
+        # TODO Remove quotes from numbers.
     http_request = requests.Request(
       method=method,
       url=request_url,

kagglesdk/kaggle_http_client.py

@@ -205,6 +205,10 @@ class KaggleObject(object):
   def endpoint(self):
     raise 'Error: endpoint must be defined by the request object'
 
+  @staticmethod
+  def body_fields():
+    return None
+
   @classmethod
   def prepare_from(cls, http_response):
     return cls.from_json(http_response.text)
@@ -229,7 +233,7 @@ def to_dict(self, ignore_defaults=True):
   @staticmethod
   def to_field_map(self, ignore_defaults=True):
     kv_pairs = [(field.field_name, field.get_as_dict_item(self, ignore_defaults)) for field in self._fields]
-    return {k: v for (k, v) in kv_pairs if not ignore_defaults or v is not None}
+    return {k: str(v) for (k, v) in kv_pairs if not ignore_defaults or v is not None}
 
   @classmethod
   def from_dict(cls, json_dict):

kagglesdk/kaggle_object.py

@@ -1365,6 +1365,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return '*'
+
 class ApiSaveKernelResponse(KaggleObject):
   r"""
   Attributes:

kagglesdk/kernels/types/kernels_api_service.py

@@ -3,7 +3,7 @@
 from kagglesdk.datasets.types.dataset_api_service import ApiCategory, ApiDatasetNewFile, ApiUploadDirectoryInfo
 from kagglesdk.kaggle_object import *
 from kagglesdk.models.types.model_enums import ListModelsOrderBy, ModelFramework, ModelInstanceType
-from kagglesdk.models.types.model_types import BaseModelInstanceInformation
+from kagglesdk.models.types.model_types import BaseModelInstanceInformation, ModelLink
 from typing import Optional, List
 
 class ApiCreateModelInstanceRequest(KaggleObject):
@@ -69,6 +69,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return 'body'
+
 class ApiCreateModelInstanceRequestBody(KaggleObject):
   r"""
   Attributes:
@@ -357,6 +361,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return 'body'
+
 class ApiCreateModelInstanceVersionRequestBody(KaggleObject):
   r"""
   Attributes:
@@ -553,6 +561,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return '*'
+
 class ApiCreateModelResponse(KaggleObject):
   r"""
   Attributes:
@@ -1280,6 +1292,8 @@ class ApiListModelsRequest(KaggleObject):
       Page size.
     page_token (str)
       Page token used for pagination.
+    only_vertex_models (bool)
+      Only list models that have Vertex URLs
   """
 
   def __init__(self):
@@ -1288,6 +1302,7 @@ def __init__(self):
     self._owner = None
     self._page_size = None
     self._page_token = None
+    self._only_vertex_models = None
     self._freeze()
 
   @property
@@ -1363,6 +1378,20 @@ def page_token(self, page_token: str):
       raise TypeError('page_token must be of type str')
     self._page_token = page_token
 
+  @property
+  def only_vertex_models(self) -> bool:
+    """Only list models that have Vertex URLs"""
+    return self._only_vertex_models or False
+
+  @only_vertex_models.setter
+  def only_vertex_models(self, only_vertex_models: bool):
+    if only_vertex_models is None:
+      del self.only_vertex_models
+      return
+    if not isinstance(only_vertex_models, bool):
+      raise TypeError('only_vertex_models must be of type bool')
+    self._only_vertex_models = only_vertex_models
+
 
   def endpoint(self):
     path = '/api/v1/models/list'
@@ -1441,6 +1470,7 @@ class ApiModel(KaggleObject):
     publish_time (datetime)
     provenance_sources (str)
     url (str)
+    model_version_links (ModelLink)
   """
 
   def __init__(self):
@@ -1457,6 +1487,7 @@ def __init__(self):
     self._publish_time = None
     self._provenance_sources = ""
     self._url = ""
+    self._model_version_links = []
     self._freeze()
 
   @property
@@ -1633,6 +1664,21 @@ def url(self, url: str):
       raise TypeError('url must be of type str')
     self._url = url
 
+  @property
+  def model_version_links(self) -> Optional[List[Optional['ModelLink']]]:
+    return self._model_version_links
+
+  @model_version_links.setter
+  def model_version_links(self, model_version_links: Optional[List[Optional['ModelLink']]]):
+    if model_version_links is None:
+      del self.model_version_links
+      return
+    if not isinstance(model_version_links, list):
+      raise TypeError('model_version_links must be of type list')
+    if not all([isinstance(t, ModelLink) for t in model_version_links]):
+      raise TypeError('model_version_links must contain only items of type ModelLink')
+    self._model_version_links = model_version_links
+
 
 class ApiModelFile(KaggleObject):
   r"""
@@ -2139,6 +2185,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return '*'
+
 class ApiUpdateModelRequest(KaggleObject):
   r"""
   Attributes:
@@ -2292,6 +2342,10 @@ def endpoint(self):
   def method():
     return 'POST'
 
+  @staticmethod
+  def body_fields():
+    return '*'
+
 class ApiUpdateModelResponse(KaggleObject):
   r"""
   Attributes:
@@ -2587,6 +2641,7 @@ def create_url(self, create_url: str):
   FieldMetadata("owner", "owner", "_owner", str, None, PredefinedSerializer(), optional=True),
   FieldMetadata("pageSize", "page_size", "_page_size", int, None, PredefinedSerializer(), optional=True),
   FieldMetadata("pageToken", "page_token", "_page_token", str, None, PredefinedSerializer(), optional=True),
+  FieldMetadata("onlyVertexModels", "only_vertex_models", "_only_vertex_models", bool, None, PredefinedSerializer(), optional=True),
 ]
 
 ApiListModelsResponse._fields = [
@@ -2609,6 +2664,7 @@ def create_url(self, create_url: str):
   FieldMetadata("publishTime", "publish_time", "_publish_time", datetime, None, DateTimeSerializer()),
   FieldMetadata("provenanceSources", "provenance_sources", "_provenance_sources", str, "", PredefinedSerializer()),
   FieldMetadata("url", "url", "_url", str, "", PredefinedSerializer()),
+  FieldMetadata("modelVersionLinks", "model_version_links", "_model_version_links", ModelLink, [], ListSerializer(KaggleObjectSerializer())),
 ]
 
 ApiModelFile._fields = [

kagglesdk/models/types/model_api_service.py

@@ -42,3 +42,8 @@ class ModelInstanceType(enum.Enum):
   MODEL_INSTANCE_TYPE_KAGGLE_VARIANT = 2
   MODEL_INSTANCE_TYPE_EXTERNAL_VARIANT = 3
 
+class ModelVersionLinkType(enum.Enum):
+  MODEL_VERSION_LINK_TYPE_UNSPECIFIED = 0
+  MODEL_VERSION_LINK_TYPE_VERTEX_OPEN = 1
+  MODEL_VERSION_LINK_TYPE_VERTEX_DEPLOY = 2
+