Fix submit by custom handling of requests that use forms (#681)

Note: Some of the changes are due to other teams modifying proto files
(`ApiDownloadKernelOutputZipRequest`).

The list of requests that use forms should not be hard-coded like this,
but that's going to take more work than I have time for just now.

I'm guessing the changes in `__init__.py` are due to line endings.

The changes to `kaggle_http_client.py` have NOT yet been added to
`kapigen`.

Author: stevemessick

kaggle/__init__.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# coding=utf-8
-from __future__ import absolute_import
-from kaggle.api.kaggle_api_extended import KaggleApi
-
-api = KaggleApi()
-api.authenticate()
+# coding=utf-8
+from __future__ import absolute_import
+from kaggle.api.kaggle_api_extended import KaggleApi
+
+api = KaggleApi()
+api.authenticate()

kagglesdk/competitions/types/competition_api_service.py

@@ -771,7 +771,7 @@ def file_name(self, file_name: str):
 
 
   def endpoint(self):
-    path = '/api/v1/competitions/{competition_name}/submissions/url/{content_length}/{last_modified_epoch_seconds}'
+    path = '/api/v1/competitions/submission-url'
     return path.format_map(self.to_field_map(self))
 
 

kagglesdk/kaggle_http_client.py

@@ -1,8 +1,13 @@
+import binascii
+import codecs
 import json
 import os
 import urllib.parse
+from io import BytesIO
 
 import requests
+from urllib3.fields import RequestField
+
 from kagglesdk.kaggle_env import get_endpoint, get_env, KaggleEnv
 from kagglesdk.kaggle_object import KaggleObject
 from typing import Type
@@ -12,6 +17,8 @@
 # auth handling. The new client requires KAGGLE_API_TOKEN, so it is not
 # currently usable by the CLI.
 
+# TODO: Extend kapigen to add a boolean to these requests indicating that they use forms.
+REQUESTS_REQUIRING_FORMS = ['ApiUploadDatasetFileRequest', 'ApiCreateSubmissionRequest', 'ApiStartSubmissionUploadRequest', 'ApiUploadModelFileRequest']
 
 def _headers_to_str(headers):
   return '\n'.join(f'{k}: {v}' for k, v in headers.items())
@@ -117,18 +124,22 @@ def _prepare_request(self, service_name: str, request_name: str,
         'Content-Type': 'text/plain',
       })
     elif method == 'POST':
-      self._session.headers.update({
-        'Accept': 'application/json',
-        'Content-Type': 'application/json',
-      })
-      data = request.to_field_map(request, ignore_defaults=False)
+      data = request.to_field_map(request, ignore_defaults=True)
       if isinstance(data, dict):
         fields = request.body_fields()
         if fields is not None:
           if fields != '*':
             data = data[fields]
         data = clean_data(data)
-        data = json.dumps(data)
+        if self.requires_form(request):
+          data, content_type = self.make_form(data)
+        else:
+          content_type = 'application/json'
+          data = json.dumps(data)
+        self._session.headers.update({
+          'Accept': 'application/json',
+          'Content-Type': content_type,
+          })
     http_request = requests.Request(
       method=method,
       url=request_url,
@@ -270,3 +281,36 @@ def _try_fill_auth(self):
 
   def _get_request_url(self, request):
     return f'{self._endpoint}{request.endpoint()}'
+
+  @staticmethod
+  def make_form(fields):
+    body = BytesIO()
+    boundary = binascii.hexlify(os.urandom(16)).decode()
+    writer = codecs.lookup("utf-8")[3]
+
+    for field in fields.items():
+      field = RequestField.from_tuples(*field)
+      body.write(f"--{boundary}\r\n".encode("latin-1"))
+
+      writer(body).write(field.render_headers())
+      data = field.data
+
+      if isinstance(data, int):
+        data = str(data)
+
+      if isinstance(data, str):
+        writer(body).write(data)
+      else:
+        body.write(data)
+
+      body.write(b"\r\n")
+
+    body.write(f"--{boundary}--\r\n".encode("latin-1"))
+
+    content_type = f"multipart/form-data; boundary={boundary}"
+
+    return body.getvalue(), content_type
+
+  @staticmethod
+  def requires_form(request):
+    return type(request).__name__ in REQUESTS_REQUIRING_FORMS

kagglesdk/kernels/services/kernels_api_service.py

@@ -1,6 +1,7 @@
+from kagglesdk.common.types.file_download import FileDownload
 from kagglesdk.common.types.http_redirect import HttpRedirect
 from kagglesdk.kaggle_http_client import KaggleHttpClient
-from kagglesdk.kernels.types.kernels_api_service import ApiDownloadKernelOutputRequest, ApiGetKernelRequest, ApiGetKernelResponse, ApiGetKernelSessionStatusRequest, ApiGetKernelSessionStatusResponse, ApiListKernelFilesRequest, ApiListKernelFilesResponse, ApiListKernelSessionOutputRequest, ApiListKernelSessionOutputResponse, ApiListKernelsRequest, ApiListKernelsResponse, ApiSaveKernelRequest, ApiSaveKernelResponse
+from kagglesdk.kernels.types.kernels_api_service import ApiDownloadKernelOutputRequest, ApiDownloadKernelOutputZipRequest, ApiGetKernelRequest, ApiGetKernelResponse, ApiGetKernelSessionStatusRequest, ApiGetKernelSessionStatusResponse, ApiListKernelFilesRequest, ApiListKernelFilesResponse, ApiListKernelSessionOutputRequest, ApiListKernelSessionOutputResponse, ApiListKernelsRequest, ApiListKernelsResponse, ApiSaveKernelRequest, ApiSaveKernelResponse
 
 class KernelsApiClient(object):
 
@@ -92,3 +93,17 @@ def download_kernel_output(self, request: ApiDownloadKernelOutputRequest = None)
       request = ApiDownloadKernelOutputRequest()
 
     return self._client.call("kernels.KernelsApiService", "ApiDownloadKernelOutput", request, HttpRedirect)
+
+  def download_kernel_output_zip(self, request: ApiDownloadKernelOutputZipRequest = None) -> FileDownload:
+    r"""
+    Meant for use by Kaggle Hub (and DownloadKernelOutput above)
+
+    Args:
+      request (ApiDownloadKernelOutputZipRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiDownloadKernelOutputZipRequest()
+
+    return self._client.call("kernels.KernelsApiService", "ApiDownloadKernelOutputZip", request, FileDownload)

kagglesdk/kernels/types/kernels_api_service.py

@@ -85,6 +85,38 @@ def endpoint(self):
   def endpoint_path():
     return '/api/v1/kernels/output/download/{owner_slug}/{kernel_slug}'
 
+class ApiDownloadKernelOutputZipRequest(KaggleObject):
+  r"""
+  Attributes:
+    kernel_session_id (int)
+  """
+
+  def __init__(self):
+    self._kernel_session_id = 0
+    self._freeze()
+
+  @property
+  def kernel_session_id(self) -> int:
+    return self._kernel_session_id
+
+  @kernel_session_id.setter
+  def kernel_session_id(self, kernel_session_id: int):
+    if kernel_session_id is None:
+      del self.kernel_session_id
+      return
+    if not isinstance(kernel_session_id, int):
+      raise TypeError('kernel_session_id must be of type int')
+    self._kernel_session_id = kernel_session_id
+
+
+  def endpoint(self):
+    path = '/api/v1/kernels/output/download_zip/{kernel_session_id}'
+    return path.format_map(self.to_field_map(self))
+
+  @staticmethod
+  def endpoint_path():
+    return '/api/v1/kernels/output/download_zip/{kernel_session_id}'
+
 class ApiGetKernelRequest(KaggleObject):
   r"""
   Attributes:
@@ -1720,6 +1752,10 @@ def creation_date(self, creation_date: str):
   FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True),
 ]
 
+ApiDownloadKernelOutputZipRequest._fields = [
+  FieldMetadata("kernelSessionId", "kernel_session_id", "_kernel_session_id", int, 0, PredefinedSerializer()),
+]
+
 ApiGetKernelRequest._fields = [
   FieldMetadata("userName", "user_name", "_user_name", str, "", PredefinedSerializer()),
   FieldMetadata("kernelSlug", "kernel_slug", "_kernel_slug", str, "", PredefinedSerializer()),

kagglesdk/models/types/model_api_service.py

@@ -1768,6 +1768,7 @@ class ApiModelInstance(KaggleObject):
     model_instance_type (ModelInstanceType)
     base_model_instance_information (BaseModelInstanceInformation)
     external_base_model_url (str)
+    total_uncompressed_bytes (int)
   """
 
   def __init__(self):
@@ -1786,6 +1787,7 @@ def __init__(self):
     self._model_instance_type = ModelInstanceType.MODEL_INSTANCE_TYPE_UNSPECIFIED
     self._base_model_instance_information = None
     self._external_base_model_url = ""
+    self._total_uncompressed_bytes = 0
     self._freeze()
 
   @property
@@ -1985,6 +1987,19 @@ def external_base_model_url(self, external_base_model_url: str):
       raise TypeError('external_base_model_url must be of type str')
     self._external_base_model_url = external_base_model_url
 
+  @property
+  def total_uncompressed_bytes(self) -> int:
+    return self._total_uncompressed_bytes
+
+  @total_uncompressed_bytes.setter
+  def total_uncompressed_bytes(self, total_uncompressed_bytes: int):
+    if total_uncompressed_bytes is None:
+      del self.total_uncompressed_bytes
+      return
+    if not isinstance(total_uncompressed_bytes, int):
+      raise TypeError('total_uncompressed_bytes must be of type int')
+    self._total_uncompressed_bytes = total_uncompressed_bytes
+
 
 class ApiUpdateModelInstanceRequest(KaggleObject):
   r"""
@@ -3063,6 +3078,7 @@ def e(self, e: str):
   FieldMetadata("modelInstanceType", "model_instance_type", "_model_instance_type", ModelInstanceType, ModelInstanceType.MODEL_INSTANCE_TYPE_UNSPECIFIED, EnumSerializer()),
   FieldMetadata("baseModelInstanceInformation", "base_model_instance_information", "_base_model_instance_information", BaseModelInstanceInformation, None, KaggleObjectSerializer(), optional=True),
   FieldMetadata("externalBaseModelUrl", "external_base_model_url", "_external_base_model_url", str, "", PredefinedSerializer()),
+  FieldMetadata("totalUncompressedBytes", "total_uncompressed_bytes", "_total_uncompressed_bytes", int, 0, PredefinedSerializer()),
 ]
 
 ApiUpdateModelInstanceRequest._fields = [

tests/unit_tests.py

@@ -541,8 +541,11 @@ def test_dataset_i_create_new(self):
       new_dataset = api.dataset_create_new(dataset_directory)
       self.assertIsNotNone(new_dataset)
       if new_dataset.error is not None:
-        print(new_dataset.error)  # This is likely to happen, and that's OK.
-        # self.skip_create_version = True
+        if 'already in use' in new_dataset.error:
+          print(new_dataset.error)  # This is likely to happen, and that's OK.
+          self.skip_create_version = True
+        else:
+          self.fail(f"dataset_create_new failed: {new_dataset.error}")
     except ApiException as e:
       self.fail(f"dataset_create_new failed: {e}")
 
@@ -783,7 +786,7 @@ def test_model_instance_x_delete(self):
       self.assertIsNotNone(inst_update_resp)
       if len(inst_update_resp.error):
         print(inst_update_resp.error)
-      self.assertEquals(len(inst_update_resp.error), 0)
+      self.assertEqual(len(inst_update_resp.error), 0)
     except ApiException as e:
       self.fail(f"model_instance_delete failed: {e}")