chore: add retry to pod exec (#12)

* lets'go

* empty commit for ci

* bump v

Author: nielstenboom

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hooks",
-  "version": "0.3.12",
+  "version": "0.3.13",
   "description": "Three projects are included - k8s: a kubernetes hook implementation that spins up pods dynamically to run a job - docker: A hook implementation of the runner's docker implementation  - A hook lib, which contains shared typescript definitions and utilities that the other packages consume",
   "main": "",
   "directories": {

packages/k8s/package-lock.json

@@ -18,6 +18,7 @@
     "@actions/io": "^1.1.2",
     "@kubernetes/client-node": "^0.16.3",
     "@types/lodash": "^4.14.191",
+    "exponential-backoff": "^3.1.1",
     "hooklib": "file:../hooklib"
   },
   "devDependencies": {

packages/k8s/package.json

@@ -14,6 +14,7 @@ import {
 } from '../hooks/constants'
 import { PodPhase } from './utils'
 import * as fs from 'fs'
+import { backOff } from 'exponential-backoff'
 
 const kc = new k8s.KubeConfig()
 
@@ -222,38 +223,70 @@ export async function execPodStep(
 ): Promise<void> {
   const exec = new k8s.Exec(kc)
   await new Promise(async function (resolve, reject) {
+    const backOffOptions = {
+      numOfAttempts: 3,
+      retry: (e, attemptNumber) => {
+        core.debug(e.toString())
+        core.debug(
+          `an error occurred trying to execute command in pod, retrying (${attemptNumber}/3)`
+        )
+        return true
+      }
+    }
     try {
-      await exec.exec(
-        namespace(),
-        podName,
-        containerName,
-        command,
-        process.stdout,
-        process.stderr,
-        stdin ?? null,
-        false /* tty */,
-        resp => {
-          // kube.exec returns an error if exit code is not 0, but we can't actually get the exit code
-          if (resp.status === 'Success') {
-            resolve(resp.code)
-          } else {
-            core.debug(
-              JSON.stringify({
-                message: resp?.message,
-                details: resp?.details
-              })
-            )
-            reject(resp?.message)
-          }
-        }
-      )
-    } catch (error) {
-      core.error(`Failed to exec pod step`)
-      core.error(error as Error)
+      await backOff(async () => {
+        await execInPod(
+          exec,
+          command,
+          podName,
+          containerName,
+          resolve,
+          reject,
+          stdin
+        )
+      }, backOffOptions)
+    } catch (e) {
+      core.debug('something went wrong in calling pod exec')
+      reject(e)
     }
   })
 }
 
+async function execInPod(
+  exec: k8s.Exec,
+  command: string[],
+  podName: string,
+  containerName: string,
+  resolve: (value?: number | PromiseLike<number> | undefined) => void,
+  reject: (reason?: any) => void,
+  stdin?: stream.Readable
+): Promise<void> {
+  await exec.exec(
+    namespace(),
+    podName,
+    containerName,
+    command,
+    process.stdout,
+    process.stderr,
+    stdin ?? null,
+    false /* tty */,
+    resp => {
+      // kube.exec returns an error if exit code is not 0, but we can't actually get the exit code
+      if (resp.status === 'Success') {
+        resolve(resp.code)
+      } else {
+        core.debug(
+          JSON.stringify({
+            message: resp?.message,
+            details: resp?.details
+          })
+        )
+        reject(resp?.message)
+      }
+    }
+  )
+}
+
 export async function waitForJobToComplete(jobName: string): Promise<void> {
   const backOffManager = new BackOffManager()
   while (true) {