Convert competitions to use kagglesdk (#625)

The competitions portion of the API now uses `kagglesdk`.

The HTTP client has ben updated to use prod, if needed.

Author: stevemessick

kaggle/api/kaggle_api_extended.py

@@ -136,7 +136,7 @@ def __init__(self, init_dict):
         self.size = File.get_size(self.totalBytes)
 
     def __repr__(self):
-        return self.ref
+        return self.name
 
     @staticmethod
     def get_size(size, precision=0):

kaggle/models/kaggle_models_extended.py

@@ -89,6 +89,10 @@ def content_length(self, content_length: int):
     self._content_length = content_length
 
 
+  @classmethod
+  def prepare_from(cls, http_response):
+    return http_response
+
 FileDownload._fields = [
   FieldMetadata("contentType", "content_type", "_content_type", str, "", PredefinedSerializer()),
   FieldMetadata("fileName", "file_name", "_file_name", str, "", PredefinedSerializer()),

kagglesdk/common/types/file_download.py

@@ -25,12 +25,14 @@ class Datasets(object):
     def __init__(self, http_client: KaggleHttpClient):
       self.dataset_api_client = DatasetApiClient(http_client)
 
-  def __init__(self, env: KaggleEnv = None, verbose: bool = False):
-    self._http_client = http_client = KaggleHttpClient(env, verbose, self._renew_iap_token)
+  def __init__(self, env: KaggleEnv = None, verbose: bool = False, username: str = None, password: str = None):
+    self._http_client = http_client = KaggleHttpClient(env, verbose, self._renew_iap_token, username=username, password=password)
     self.kernels = KaggleClient.Kernels(http_client)
     self.models = KaggleClient.Models(http_client)
     self.competitions = KaggleClient.Competitions(http_client)
     self.datasets = KaggleClient.Datasets(http_client)
+    self.username = username
+    self.password = password
 
   def _renew_iap_token(self):
     return self.admin.admin_client.renew_iap_token()

kagglesdk/kaggle_client.py

@@ -7,13 +7,17 @@ class KaggleEnv(Enum):
   STAGING = 1  # staging.kaggle.com
   ADMIN = 2  # admin.kaggle.com
   QA = 3 # qa.kaggle.com
+  # Direct prod access is not allowed to have IAP protection during testing, but we support basic auth.
+  PROD = 4  # www.kaggle.com
 
 
 _env_to_endpoint = {
   KaggleEnv.LOCAL: 'http://localhost',
   KaggleEnv.STAGING: 'https://staging.kaggle.com',
   KaggleEnv.ADMIN: 'https://admin.kaggle.com',
   KaggleEnv.QA: 'https://qa.kaggle.com',
+  # See the comment above in KaggleEnv enum.
+  KaggleEnv.PROD: "https://www.kaggle.com",
 }
 
 
@@ -33,4 +37,6 @@ def get_env():
     return KaggleEnv.STAGING
   if env == 'QA':
     return KaggleEnv.QA
+  if env == 'PROD':
+    return KaggleEnv.PROD
   raise Exception(f'Unrecognized value in KAGGLE_API_ENVIRONMENT: "{env}"')

kagglesdk/kaggle_env.py

@@ -1,5 +1,7 @@
 import os
 import json
+import urllib.parse
+
 import requests
 from kagglesdk.kaggle_env import get_endpoint, get_env, KaggleEnv
 from kagglesdk.kaggle_object import KaggleObject
@@ -42,12 +44,16 @@ class KaggleHttpClient(object):
   def __init__(self,
                env: KaggleEnv = None,
                verbose: bool = False,
-               renew_iap_token = None):
+               renew_iap_token = None,
+               username = None,
+               password = None):
     self._env = env or get_env()
     self._signed_in = None
     self._endpoint = get_endpoint(self._env)
     self._verbose = verbose
     self._session = None
+    self._username = username
+    self._password = password
 
   def call(self, service_name: str, request_name: str, request: KaggleObject, response_type: Type[KaggleObject]):
     self._init_session()
@@ -61,10 +67,14 @@ def call(self, service_name: str, request_name: str, request: KaggleObject, resp
   def _prepare_request(self, service_name: str, request_name: str, request: KaggleObject):
     request_url = self._get_request_url(request)
     method = request.method()
-    if method == 'POST':
-      data = request.__class__.to_dict(request)
-    else:
-      data = []
+    data = request.__class__.to_dict(request)
+    if method == 'GET':
+      request_url = f'{request_url}?{urllib.parse.urlencode(data)}'
+      data = ''
+      self._session.headers.update({
+        'Accept': 'application/json',
+        'Content-Type': 'text/plain',
+      })
     http_request = requests.Request(
       method=method,
       url=request_url,
@@ -86,6 +96,10 @@ def _get_xsrf_cookies(self):
   def _prepare_response(self, response_type, http_response):
     self._print_response(http_response)
     http_response.raise_for_status()
+    if 'application/json' in http_response.headers['Content-Type']:
+      resp = http_response.json()
+      if 'code' in resp and resp['code'] >= 400:
+        raise requests.exceptions.HTTPError(resp['message'], response=http_response)
     if response_type is None:  # Method doesn't have a return type
       return None
     return response_type.prepare_from(http_response)
@@ -184,7 +198,10 @@ def _try_fill_auth(self):
       self._signed_in = True
       return
 
-    apikey_creds = _get_apikey_creds()
+    if self._username and self._password:
+      apikey_creds = self._username, self._password
+    else:
+      apikey_creds = _get_apikey_creds()
     if apikey_creds is not None:
       self._session.auth = apikey_creds
       self._signed_in = True

kagglesdk/kaggle_http_client.py

@@ -65,7 +65,15 @@ def _from_str(cls, v):
     #   return enum_items[v[ix_start:]]
     # 
     # return enum_items[f'{enum_prefix}_{v}']
-    return cls[v]
+    try:
+      return cls[v]
+    except KeyError:
+      dct = vars(cls)
+      n = v.lower()
+      for key in dct.keys():
+        if key.lower() == n:
+          return dct[key]
+      raise
 
 
 class ListSerializer(ObjectSerializer):

kagglesdk/kaggle_object.py

@@ -33,7 +33,8 @@ dependencies = [
   "text-unidecode",
   "tqdm",
   "urllib3>=1.15.1",
-  "webencodings"
+  "webencodings", 
+  "protobuf"
 ]
 
 [project.scripts]

pyproject.toml

@@ -31,6 +31,7 @@
         'python-slugify',
         'urllib3',
         'bleach',
+        'protobuf'
     ],
     packages=find_packages(exclude=("src.*", "src")),
     license='Apache 2.0')