Update project management script (#663)

Author: stevemessick

README.md

@@ -22,8 +22,6 @@ Obviously, this depends on Kaggle services. When you're extending the API and mo
 or adding to those services, you should be working in your Kaggle mid-tier development
 environment. You'll run Kaggle locally, in the container, and test the Python code by
 running it in the container so it can connect to your local testing environment.
-However, do not try to create a release from within the container. The code formatter
-(`yapf3`) changes much more than intended.
 
 Also, run the following command to get `autogen.sh` installed:
 ```bash

kaggle/api/kaggle_api_extended.py

@@ -1,19 +1,19 @@
-#!/usr/bin/python
-#
-# Copyright 2024 Kaggle Inc
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+#!/usr/bin/python
+#
+# Copyright 2024 Kaggle Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #!/usr/bin/python
 #
 # Copyright 2024 Kaggle Inc
@@ -305,6 +305,8 @@ class KaggleApi:
   already_printed_version_warning = False
 
   args = {}  # DEBUG Add --local to use localhost
+  if os.environ.get('KAGGLE_API_ENVIRONMENT') == 'LOCALHOST':
+    args = {'--local'}
 
   # Kernels valid types
   valid_push_kernel_types = ['script', 'notebook']

pyproject.toml

@@ -61,11 +61,11 @@ install-autogen = """curl -fsSL --output /tmp/autogen.zip "https://github.com/mb
 # TODO: install in Mac/Windows
 install-yapf = """pip3 install yapf==0.40.2 --break-system-packages || echo 'yapf could not be installed'"""
 install-toml = """sudo apt-get install -y python3-toml || echo 'toml could not be installed'"""
-install-java = """sudo apt-get install -y default-jre || echo 'java could not be installed'"""
-install-deps = "hatch run install-unzip && hatch run install-autogen && hatch run install-yapf && hatch run install-toml && hatch run install-java"
+install-deps = "hatch run install-unzip && hatch run install-autogen && hatch run install-yapf && hatch run install-toml"
 
 integration-test = "pytest {args:integration_tests}"
 
 compile = "./tools/GeneratePythonLibrary.sh"
 install = "./tools/GeneratePythonLibrary.sh --install"
 watch = "./tools/GeneratePythonLibrary.sh --watch"
+test = "./tools/GeneratePythonLibrary.sh --test local"

src/kaggle/api/kaggle_api_extended.py

@@ -289,6 +289,8 @@ class KaggleApi:
   already_printed_version_warning = False
 
   args = {}  # DEBUG Add --local to use localhost
+  if os.environ.get('KAGGLE_API_ENVIRONMENT') == 'LOCALHOST':
+    args = {'--local'}
 
   # Kernels valid types
   valid_push_kernel_types = ['script', 'notebook']

tests/unit_tests.py

@@ -251,7 +251,6 @@ def test_kernels_c_push(self):
       self.fail(f"kernels_push failed: {e}")
 
   def test_kernels_d_status(self):
-    # AssertionError: kernels_status failed: 404 Client Error: Not Found for url: http://localhost/api/v1/kernels/status?username=stevemessick&kernelslug=testing
     if self.kernel_slug == '':
       self.test_kernels_c_push()
     try:
@@ -283,7 +282,6 @@ def test_kernels_e_list_files(self):
       self.fail(f"kernels_list_files failed: {e}")
 
   def test_kernels_f_output(self):
-    # AssertionError: kernels_output failed: 404 Client Error: Not Found for url: http://localhost/api/v1/kernels/output?username=stevemessick&kernelslug=testing
     fs = []
     if self.kernel_slug == '':
       self.test_kernels_c_push()

tools/GeneratePythonLibrary.sh

@@ -12,7 +12,7 @@ function usage {
   echo "         --editable (-e): Make the installed package always reference your latest"
   echo "                          source code. Implies \"-i|--install\". Be aware that changes to the \"src\""
   echo "                          directory won't be reflected. See the README for details."
-  echo "         --test (-t) [$LOCAL_ENV|$PROD_ENV]: Run tests (python_api_tests.py) against http://localhost" 
+  echo "         --test (-t) [$LOCAL_ENV|$PROD_ENV]: Run tests (unit_tests.py) against http://localhost" 
   echo "                                   or https://www.kaggle.com."
   echo "         --watch (-w): Run the script in watch mode. It will watch the files under the \"template\""
   echo "                       directory and KaggleSwagger* files, and regenerate the package when there is a change."
@@ -63,8 +63,6 @@ SELF_DIR=$(dirname $(realpath $0))
 SELF_DIR=${SELF_DIR%/*} # remove the last directory (tools) from the path
 cd $SELF_DIR
 
-SWAGGER_YAML=$SELF_DIR/src/KaggleSwagger.yaml
-SWAGGER_CONFIG=$SELF_DIR/src/KaggleSwaggerConfig.json
 KAGGLE_XDG_CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/kaggle"
 mkdir -p "$KAGGLE_XDG_CONFIG_DIR"
 KAGGLE_DEV_CONFIG_DIR=$(realpath "$KAGGLE_XDG_CONFIG_DIR/dev")
@@ -109,31 +107,6 @@ function create-local-creds {
   chmod 600 $kaggle_config_file
 }
 
-function generate-package {
-  # TODO Remove this.
-  if [[ -f "kaggle/api/__init__.py" ]]; then
-    sed -i -e 's/kaggle_api/kaggle_api_extended/g' kaggle/api/__init__.py
-  fi
-
-  if [[ -f "kaggle/configuration.py" ]]; then
-    # Replace the hard-coded endpoint URL with an environment variable in configuration.py
-    # to allow talking to localhost, staging, prod etc.
-    sed -i 's|self.host = "http|self.host = _get_endpoint_from_env() or "http|g' kaggle/configuration.py
-    echo -e "\n" >> kaggle/configuration.py
-    cat <<-END >> kaggle/configuration.py
-def _get_endpoint_from_env():
-    import os
-    endpoint = os.environ.get("KAGGLE_API_ENDPOINT")
-    if endpoint is None:
-        return None
-    endpoint = endpoint.rstrip("/")
-    if endpoint.endswith("/api/v1"):
-        return endpoint
-    return endpoint + "/api/v1"    
-END
-  fi
-}
-
 function copy-src {
   cp ./src/setup.py .
   cp ./src/setup.cfg .
@@ -151,16 +124,19 @@ function run-tests {
   fi
 
   if [[ "$TEST" == "$LOCAL_ENV" ]]; then
-    source ../use-localhost.sh
+    source tools/use-localhost.sh
   elif [[ "$TEST" == "$PROD_ENV" ]]; then
-    source ../use-prod.sh
+    source tools/use-prod.sh
   else
     return 0 # Nothing to do
   fi
 
-  cp ../python_api_tests.py ../sample_submission.csv ./
-  python3 python_api_tests.py
-  rm -f ./python_api_tests.py
+  cd tests
+  ln -s ../kagglesdk .
+  ln -s ../kaggle .
+  python3 unit_tests.py
+  rm kaggle kagglesdk
+  cd ..
 }
 
 function install-package {
@@ -177,10 +153,8 @@ function cleanup {
   rm -rf tox.ini \
     test-requirements.txt \
     test \
-    .swagger-codegen \
     .travis.yml \
     git_push.sh \
-    python_api_tests.py \
     sample_submission.csv \
     ds_salaries.csv \
     test.csv \
@@ -194,7 +168,6 @@ function cleanup {
 function run {
   reset
 
-  generate-package
   copy-src
   run-autogen
   install-package
@@ -205,28 +178,11 @@ function run {
 
 WATCHED_EVENTS="-e create -e modify -e delete"
 
-function watch-swagger {
-  local watched_paths="$SWAGGER_YAML $SWAGGER_CONFIG"
-
-  echo "Watching for changes to Swagger config..."
-  while inotifywait -q -r $WATCHED_EVENTS --format "%e %w%f" $watched_paths; do
-    echo "Deleting $SELF_DIR/kaggle/ $SELF_DIR/kaggle/"
-    rm -rf $SELF_DIR/kaggle/*
-    generate-package
-    run-autogen
-    copy-src
-    echo -e "\nWatching for changes to Swagger config..."
-  done
-}
-
 function watch-src {
   local watched_paths="$SELF_DIR/src"
 
   echo "Watching for changes under \"src\"..."
   while inotifywait -q -r $WATCHED_EVENTS --format "%e %w%f" $watched_paths; do
-    # Do not delete the output directory when there is no Swagger change to avoid
-    # having to run generate-package for each small code change as Swagger code
-    # generation is a bit slow (can take 2-3 seconds).
     echo "Copying changes..."
     copy-src
     echo "Done!"
@@ -244,9 +200,8 @@ function watch {
   TEST="no"
 
   echo
-  watch-swagger &
-  local pid=$!
   watch-src
+  local pid=$!
   wait $pid
 }