COMPUTE-1236: Add support for 429 response code from platform (#132)

* COMPUTE-1236: Add support for 429 response code from platform

* Fix test

Author: windok

DEVELOPING.md

@@ -15,7 +15,7 @@
     ```
    * Note that dxScala will compile with JDK8 or JDK11 and that JDK8 is used as the build target so the resulting JAR file can be executed with JRE8 or later.
 * Install [sbt](https://www.scala-sbt.org/), which also installs Scala. Sbt is a make-like utility that works with the ```scala``` language.
-   * On MacOS: `brew install sbt`
+   * On MacOS: `brew install sbt` or `brew install --ignore-dependencies sbt` (if you don't want to install the newest JDK)
    * On Linux:
     ```
     $ wget www.scala-lang.org/files/archive/scala-2.13.7.deb
@@ -48,7 +48,7 @@ If you want to make a change to dxScala, do the following:
 3. If the current snapshot version matches the release version, increment the snapshot version.
    - For example, if the current release is `1.0.0` and the current snapshot version is `1.0.0-SNAPSHOT`, increment the snapshot version to `1.0.1-SNAPSHOT`.
 4. Make your changes. Test locally using `sbt test`.
-5. Update the release notes under the top-most header (which should be "in develop").
+5. Update the release notes under the top-most header (which should be "unreleased").
 6. If the current snapshot version only differs from the release version by a patch, and you added any new functionality (vs just fixing a bug), increment the minor version instead.
    - For example, when you first created the branch you set the version to `1.0.1-SNAPSHOT`, but then you realized you needed to add a new function to the public API, change the version to `1.1.0-SNAPSHOT`. 
 7. When you are done, create a pull request against the `develop` branch.

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2024 DNAnexus
+   Copyright 2025 DNAnexus
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

api/RELEASE_NOTES.md

@@ -2,6 +2,10 @@
 
 ## unreleased
 
+* Support 429 Too Many Requests response code from platform to retry throttled requests.
+* Add identity tokens requests.
+* Add version number in user-agent string.
+
 ## 0.13.10 (2024-03-25)
 
 * Change to make the user-agent string for dxScala more distinctive.

api/src/main/java/com/dnanexus/DXAPI.java

@@ -21141,6 +21141,192 @@ public static JsonNode jobUpdate(String objectId, JsonNode inputParams, DXEnviro
                 RetryStrategy.SAFE_TO_RETRY);
     }
 
+    /**
+     * Invokes the jobGetIdentityToken method with an empty input, deserializing to an object of the specified class.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param outputClass class to deserialize the server reponse to
+     *
+     * @return Response object
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     */
+    public static <T> T jobGetIdentityToken(String objectId, Class<T> outputClass) {
+        return jobGetIdentityToken(objectId, mapper.createObjectNode(), outputClass);
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with the given input, deserializing to an object of the specified class.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param inputObject input object (to be JSON serialized to an input hash)
+     * @param outputClass class to deserialize the server reponse to
+     *
+     * @return Response object
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     */
+    public static <T> T jobGetIdentityToken(String objectId, Object inputObject, Class<T> outputClass) {
+        JsonNode input = mapper.valueToTree(inputObject);
+        return DXJSON.safeTreeToValue(
+                new DXHTTPRequest().request("/" + objectId + "/" + "getIdentityToken",
+                        input, RetryStrategy.SAFE_TO_RETRY), outputClass);
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with an empty input using the given environment, deserializing to an object of the specified class.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param outputClass class to deserialize the server reponse to
+     * @param env environment object specifying the auth token and remote server and protocol
+     *
+     * @return Response object
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     */
+    public static <T> T jobGetIdentityToken(String objectId, Class<T> outputClass, DXEnvironment env) {
+        return jobGetIdentityToken(objectId, mapper.createObjectNode(), outputClass, env);
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with the given input using the given environment, deserializing to an object of the specified class.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param inputObject input object (to be JSON serialized to an input hash)
+     * @param outputClass class to deserialize the server reponse to
+     * @param env environment object specifying the auth token and remote server and protocol
+     *
+     * @return Response object
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     */
+    public static <T> T jobGetIdentityToken(String objectId, Object inputObject, Class<T> outputClass, DXEnvironment env) {
+        JsonNode input = mapper.valueToTree(inputObject);
+        return DXJSON.safeTreeToValue(
+            new DXHTTPRequest(env).request("/" + objectId + "/" + "getIdentityToken",
+                    input, RetryStrategy.SAFE_TO_RETRY), outputClass);
+    }
+
+    /**
+     * Invokes the jobGetIdentityToken method.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     *
+     * @return Server response parsed from JSON
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     *
+     * @deprecated Use {@link #jobGetIdentityToken(String, Class)} instead and supply your own class to deserialize to.
+     */
+    @Deprecated
+    public static JsonNode jobGetIdentityToken(String objectId) {
+        return jobGetIdentityToken(objectId, mapper.createObjectNode());
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with the specified parameters.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param inputParams input parameters to the API call
+     *
+     * @return Server response parsed from JSON
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     *
+     * @deprecated Use {@link #jobGetIdentityToken(String, Object, Class)} instead and supply your own class to deserialize to.
+     */
+    @Deprecated
+    public static JsonNode jobGetIdentityToken(String objectId, JsonNode inputParams) {
+        return new DXHTTPRequest().request("/" + objectId + "/" + "getIdentityToken", inputParams,
+                RetryStrategy.SAFE_TO_RETRY);
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with the specified environment.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param env environment object specifying the auth token and remote server and protocol
+     *
+     * @return Server response parsed from JSON
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     *
+     * @deprecated Use {@link #jobGetIdentityToken(String, Class, DXEnvironment)} instead and supply your own class to deserialize to.
+     */
+    @Deprecated
+    public static JsonNode jobGetIdentityToken(String objectId, DXEnvironment env) {
+        return jobGetIdentityToken(objectId, mapper.createObjectNode(), env);
+    }
+    /**
+     * Invokes the jobGetIdentityToken method with the specified environment and parameters.
+     *
+     * <p>For more information about this method, see the <a href="https://documentation.dnanexus.com/developer/api/running-analyses/applets-and-entry-points#api-method-job-xxxx-getIdentityToken">API specification</a>.
+     *
+     * @param objectId ID of the object to operate on
+     * @param inputParams input parameters to the API call
+     * @param env environment object specifying the auth token and remote server and protocol
+     *
+     * @return Server response parsed from JSON
+     *
+     * @throws DXAPIException
+     *             If the server returns a complete response with an HTTP status
+     *             code other than 200 (OK).
+     * @throws DXHTTPException
+     *             If an error occurs while making the HTTP request or obtaining
+     *             the response (includes HTTP protocol errors).
+     *
+     * @deprecated Use {@link #jobGetIdentityToken(String, Object, Class, DXEnvironment)} instead and supply your own class to deserialize to.
+     */
+    @Deprecated
+    public static JsonNode jobGetIdentityToken(String objectId, JsonNode inputParams, DXEnvironment env) {
+        return new DXHTTPRequest(env).request("/" + objectId + "/" + "getIdentityToken", inputParams,
+                RetryStrategy.SAFE_TO_RETRY);
+    }
+
     /**
      * Invokes the jobNew method with an empty input, deserializing to an object of the specified class.
      *

api/src/main/java/com/dnanexus/DXHTTPRequest.java

@@ -287,6 +287,7 @@ private ParsedResponse requestImpl(String resource, String data, boolean parseRe
         // Retry with exponential backoff
         int timeoutSeconds = 1;
         int attempts = 0;
+        int attemptsWithThrottled = 0;
 
         while (true) {
             Integer statusCode = null;
@@ -356,6 +357,21 @@ private ParsedResponse requestImpl(String resource, String data, boolean parseRe
                     } else {
                         return new ParsedResponse(new String(value, Charset.forName("UTF-8")), null);
                     }
+                } else if (statusCode == 503 || statusCode == 429) {
+                    retryRequest = true;
+                    Header retryAfterHeader = response.getFirstHeader("retry-after");
+                    // Consume the response to avoid leaking resources
+                    EntityUtils.consume(entity);
+                    if (retryAfterHeader != null) {
+                        try {
+                            retryAfterSeconds = Integer.parseInt(retryAfterHeader.getValue());
+                        } catch (NumberFormatException e) {
+                            // Just fall back to the default
+                        }
+                    }
+
+                    // will be processed further in this file
+                    throw new ServiceUnavailableException("503 Service Unavailable", statusCode, retryAfterSeconds);
                 } else if (statusCode < 500) {
                     // 4xx errors should be considered not recoverable.
                     String responseStr = EntityUtils.toString(entity);
@@ -383,44 +399,45 @@ private ParsedResponse requestImpl(String resource, String data, boolean parseRe
                     throw DXAPIException.getInstance(errorType, errorMessage, statusCode);
                 } else {
                     // Propagate 500 error to caller
-                    if (this.disableRetry && statusCode != 503) {
+                    if (this.disableRetry) {
                         logError("POST " + resource + ": " + statusCode + " Internal Server Error, try "
                                 + String.valueOf(attempts + 1) + "/" + NUM_RETRIES
                                 + " Request ID: " +  requestId);
                         throw new InternalErrorException("Internal Server Error", statusCode);
                     }
                     // If retries enabled, 500 InternalError should get retried unconditionally
                     retryRequest = true;
-                    if (statusCode == 503) {
-                        Header retryAfterHeader = response.getFirstHeader("retry-after");
-                        // Consume the response to avoid leaking resources
-                        EntityUtils.consume(entity);
-                        if (retryAfterHeader != null) {
-                            try {
-                                retryAfterSeconds = Integer.parseInt(retryAfterHeader.getValue());
-                            } catch (NumberFormatException e) {
-                                // Just fall back to the default
-                            }
-                        }
-                        throw new ServiceUnavailableException("503 Service Unavailable", statusCode, retryAfterSeconds);
+                    String entityStr = EntityUtils.toString(entity);
+                    String msg;
+                    if (entityStr == null || "".equals(entityStr)) {
+                        // This can happen if the server was unable to send response back to us,
+                        // as happened with TIP-1743. Make this more clear in the exception.
+                        msg = "No response entity received";
+                    } else {
+                        msg = String.format("Response entity: %s", entityStr);
                     }
-                    throw new IOException(EntityUtils.toString(entity));
+                    throw new IOException(msg);
                 }
             } catch (ServiceUnavailableException e) {
-                int secondsToWait = retryAfterSeconds;
+                // increased backoff for throttled requests over attempts up to x5 times from the original
+                double increasedBackoff = retryAfterSeconds < 60
+                        ? retryAfterSeconds + 0.25 * Math.min(20, attemptsWithThrottled) * retryAfterSeconds
+                        : retryAfterSeconds;
+                timeoutSeconds = (int) increasedBackoff;
+
+                String logMessage = String.format("POST %s: %s, attempt %d, %s %d seconds. Request ID: %s",
+                                                  resource,
+                                                  statusCode == 429 ? "429 Too Many Requests" : "503 Service Unavailable",
+                                                  attemptsWithThrottled + 1,
+                                                  this.disableRetry ? "suggested wait " : "waiting for",
+                                                  timeoutSeconds,
+                                                  requestId);
+
+                logError(logMessage);
 
                 if (this.disableRetry) {
-                    logError("POST " + resource + ": 503 Service Unavailable, suggested wait "
-                            + secondsToWait + " seconds" + ". Request ID: " +  requestId);
                     throw e;
                 }
-
-                // Retries due to 503 Service Unavailable and Retry-After do NOT count against the
-                // allowed number of retries.
-                logError("POST " + resource + ": 503 Service Unavailable, waiting for "
-                        + Integer.toString(secondsToWait) + " seconds" + " Request ID: " +  requestId);
-                sleep(secondsToWait);
-                continue;
             } catch (IOException e) {
                 // Note, this catches both exceptions directly thrown from httpclient.execute (e.g.
                 // no connectivity to server) and exceptions thrown by our code above after parsing
@@ -434,12 +451,19 @@ private ParsedResponse requestImpl(String resource, String data, boolean parseRe
                     throw new InternalErrorException("Maximum number of retries reached, or unsafe to retry",
                             statusCode);
                 }
+
+                timeoutSeconds *= 2;
             }
 
             assert attempts < NUM_RETRIES;
             assert retryRequest;
 
-            attempts++;
+            attemptsWithThrottled++;
+            // Retries due to 429 or 503 Service Unavailable and Retry-After do NOT count against the
+            // allowed number of retries.
+            if (statusCode != 503 && statusCode != 429) {
+                attempts++;
+            }
 
             // The number of failed attempts is now no more than NUM_RETRIES, and the total number
             // of attempts allowed is NUM_RETRIES + 1 (the first attempt, plus up to NUM_RETRIES
@@ -452,7 +476,6 @@ private ParsedResponse requestImpl(String resource, String data, boolean parseRe
 
 
             sleep(timeoutSeconds);
-            timeoutSeconds *= 2;
         }
     }
 }