Add util to download file over SFTP (#4032)

Author: dskloetc

.changeset/grumpy-buses-try.md

@@ -0,0 +1,5 @@
+---
+'@chainlink/ftse-sftp-adapter': patch
+---
+
+Add utility that's not used yet.

packages/sources/ftse-sftp/src/transport/utils.ts

@@ -0,0 +1,74 @@
+import { makeLogger } from '@chainlink/external-adapter-framework/util'
+import { AdapterInputError } from '@chainlink/external-adapter-framework/validation/error'
+import path from 'path'
+import SftpClient, { ConnectOptions, FileInfo } from 'ssh2-sftp-client'
+
+const logger = makeLogger('FTSE SFTP Utils')
+
+// Downloads the single file matching the provided regex from the specified
+// directory over SFTP.
+export const getFileContentsFromFileRegex = async ({
+  connectOptions,
+  directory,
+  filenameRegex,
+}: {
+  connectOptions: ConnectOptions
+  directory: string
+  filenameRegex: RegExp
+}): Promise<{
+  filename: string
+  fileContent: string | NodeJS.WritableStream | Buffer
+}> => {
+  const client = new SftpClient()
+  try {
+    await client.connect(connectOptions)
+    logger.debug('Successfully connected to SFTP server')
+
+    const filename = await getFilenameFromRegex({
+      client,
+      directory,
+      filenameRegex,
+    })
+    const filePath = path.join(directory, filename)
+
+    return {
+      filename,
+      fileContent: await client.get(filePath),
+    }
+  } finally {
+    client.end()
+    logger.debug('SFTP connection closed')
+  }
+}
+
+export const getFilenameFromRegex = async ({
+  client,
+  directory,
+  filenameRegex,
+}: {
+  client: SftpClient
+  directory: string
+  filenameRegex: RegExp
+}): Promise<string> => {
+  const fileList = await client.list(directory)
+  // Filter files based on the regex pattern
+  const matchingFiles = fileList
+    .map((file: FileInfo) => file.name)
+    .filter((fileName: string) => filenameRegex.test(fileName))
+
+  if (matchingFiles.length === 0) {
+    throw new AdapterInputError({
+      statusCode: 500,
+      message: `No files matching pattern ${filenameRegex} found in directory '${directory}'`,
+    })
+  } else if (matchingFiles.length > 1) {
+    throw new AdapterInputError({
+      statusCode: 500,
+      message: `Multiple files matching pattern ${filenameRegex} found in directory '${directory}': ${matchingFiles.join(
+        ', ',
+      )}`,
+    })
+  }
+
+  return matchingFiles[0]
+}

packages/sources/ftse-sftp/test/unit/utils.test.ts

@@ -0,0 +1,147 @@
+import { LoggerFactoryProvider } from '@chainlink/external-adapter-framework/util'
+import { makeStub } from '@chainlink/external-adapter-framework/util/testing-utils'
+import { getFileContentsFromFileRegex } from '../../src/transport/utils'
+
+const mockSftpClient = makeStub('mockSftpClient', {
+  connect: jest.fn(),
+  list: jest.fn(),
+  get: jest.fn(),
+  end: jest.fn(),
+})
+
+jest.mock(
+  'ssh2-sftp-client',
+  () =>
+    function () {
+      return mockSftpClient
+    },
+)
+
+LoggerFactoryProvider.set()
+
+describe('Transport utils', () => {
+  const host = 'sftp.test.com'
+  const port = 22
+  const username = 'testuser'
+  const password = 'testpassword'
+  const connectOptions = {
+    host,
+    port,
+    username,
+    password,
+  }
+  const directory = '/some/directory'
+  const filenameRegex = /^test_file_\d{3}\.csv/
+  const expectedFilename = 'test_file_123.csv'
+  const expectedFilename2 = 'test_file_456.csv'
+  const expectedFullPath = '/some/directory/test_file_123.csv'
+  const differentFilename1 = 'test_file_9999.csv'
+  const differentFilename2 = 'differentfile_9999.csv'
+  const differentFilename3 = 'test_file_123.txt'
+  const expectedContent = 'some,file,content'
+
+  beforeEach(async () => {
+    jest.resetAllMocks()
+    jest.useFakeTimers()
+
+    mockSftpClient.get.mockResolvedValue(Buffer.from(expectedContent))
+  })
+
+  describe('getFileContentsFromFileRegex', () => {
+    it('should return filename and content', async () => {
+      mockSftpClient.list.mockResolvedValue([{ name: expectedFilename }])
+
+      const { filename, fileContent } = await getFileContentsFromFileRegex({
+        connectOptions,
+        directory,
+        filenameRegex,
+      })
+
+      expect(filename).toBe(expectedFilename)
+      expect(fileContent.toString()).toBe(expectedContent)
+
+      expect(mockSftpClient.connect).toHaveBeenCalledWith(connectOptions)
+      expect(mockSftpClient.connect).toHaveBeenCalledTimes(1)
+      expect(mockSftpClient.list).toHaveBeenCalledWith(directory)
+      expect(mockSftpClient.list).toHaveBeenCalledTimes(1)
+      expect(mockSftpClient.get).toHaveBeenCalledWith(expectedFullPath)
+      expect(mockSftpClient.get).toHaveBeenCalledTimes(1)
+      expect(mockSftpClient.end).toHaveBeenCalledTimes(1)
+    })
+
+    it('should load correct file among multiple', async () => {
+      mockSftpClient.list.mockResolvedValue([
+        { name: differentFilename1 },
+        { name: differentFilename2 },
+        { name: expectedFilename },
+        { name: differentFilename3 },
+      ])
+
+      const { filename, fileContent } = await getFileContentsFromFileRegex({
+        connectOptions,
+        directory,
+        filenameRegex,
+      })
+
+      expect(filename).toBe(expectedFilename)
+      expect(fileContent.toString()).toBe(expectedContent)
+      expect(mockSftpClient.get).toHaveBeenCalledWith(expectedFullPath)
+      expect(mockSftpClient.get).toHaveBeenCalledTimes(1)
+    })
+
+    it('should throw if file not found', async () => {
+      mockSftpClient.list.mockResolvedValue([
+        { name: differentFilename1 },
+        { name: differentFilename2 },
+        { name: differentFilename3 },
+      ])
+
+      await expect(() =>
+        getFileContentsFromFileRegex({
+          connectOptions,
+          directory,
+          filenameRegex,
+        }),
+      ).rejects.toThrow(
+        `No files matching pattern ${filenameRegex} found in directory '${directory}'`,
+      )
+
+      expect(mockSftpClient.end).toHaveBeenCalledTimes(1)
+    })
+
+    it('should throw if multiple files match', async () => {
+      mockSftpClient.list.mockResolvedValue([
+        { name: differentFilename1 },
+        { name: expectedFilename },
+        { name: expectedFilename2 },
+      ])
+
+      await expect(() =>
+        getFileContentsFromFileRegex({
+          connectOptions,
+          directory,
+          filenameRegex,
+        }),
+      ).rejects.toThrow(
+        `Multiple files matching pattern ${filenameRegex} found in directory '${directory}': test_file_123.csv, test_file_456.csv`,
+      )
+
+      expect(mockSftpClient.end).toHaveBeenCalledTimes(1)
+    })
+
+    it('should end the connection if the client throws an error', async () => {
+      const errorMessage = 'error listing files'
+      mockSftpClient.list.mockRejectedValue(new Error(errorMessage))
+
+      await expect(() =>
+        getFileContentsFromFileRegex({
+          connectOptions,
+          directory,
+          filenameRegex,
+        }),
+      ).rejects.toThrow(errorMessage)
+
+      expect(mockSftpClient.end).toHaveBeenCalledTimes(1)
+    })
+  })
+})