-
Notifications
You must be signed in to change notification settings - Fork 41
[Test PR] SDK Multipart support #93
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
export type Part = { | ||
contentDispositionHeader: string; | ||
contentTypeHeader: string; | ||
part: number[]; | ||
}; | ||
|
||
type Input = { | ||
filename?: string; | ||
name?: string; | ||
type: string; | ||
data: Buffer; | ||
}; | ||
|
||
enum ParsingState { | ||
INIT, | ||
READING_HEADERS, | ||
READING_DATA, | ||
READING_PART_SEPARATOR, | ||
} | ||
|
||
export function parse(multipartBodyBuffer: Buffer, boundary: string): Input[] { | ||
let lastline = ""; | ||
let contentDispositionHeader = ""; | ||
let contentTypeHeader = ""; | ||
let state: ParsingState = ParsingState.INIT; | ||
let buffer: number[] = []; | ||
const allParts: Input[] = []; | ||
|
||
let currentPartHeaders: string[] = []; | ||
|
||
for (let i = 0; i < multipartBodyBuffer.length; i++) { | ||
const oneByte: number = multipartBodyBuffer[i]; | ||
const prevByte: number | null = i > 0 ? multipartBodyBuffer[i - 1] : null; | ||
// 0x0a => \n | ||
// 0x0d => \r | ||
const newLineDetected: boolean = oneByte === 0x0a && prevByte === 0x0d; | ||
const newLineChar: boolean = oneByte === 0x0a || oneByte === 0x0d; | ||
|
||
if (!newLineChar) lastline += String.fromCharCode(oneByte); | ||
if (ParsingState.INIT === state && newLineDetected) { | ||
// searching for boundary | ||
if ("--" + boundary === lastline) { | ||
state = ParsingState.READING_HEADERS; // found boundary. start reading headers | ||
} | ||
lastline = ""; | ||
} else if (ParsingState.READING_HEADERS === state && newLineDetected) { | ||
// parsing headers. Headers are separated by an empty line from the content. Stop reading headers when the line is empty | ||
if (lastline.length) { | ||
currentPartHeaders.push(lastline); | ||
} else { | ||
// found empty line. search for the headers we want and set the values | ||
for (const h of currentPartHeaders) { | ||
if (h.toLowerCase().startsWith("content-disposition:")) { | ||
contentDispositionHeader = h; | ||
} else if (h.toLowerCase().startsWith("content-type:")) { | ||
contentTypeHeader = h; | ||
} | ||
} | ||
state = ParsingState.READING_DATA; | ||
buffer = []; | ||
} | ||
lastline = ""; | ||
} else if (ParsingState.READING_DATA === state) { | ||
// parsing data | ||
if (lastline.length > boundary.length + 4) { | ||
lastline = ""; // mem save | ||
} | ||
if ("--" + boundary === lastline) { | ||
const j = buffer.length - lastline.length; | ||
const part = buffer.slice(0, j - 1); | ||
|
||
allParts.push( | ||
process({ contentDispositionHeader, contentTypeHeader, part }) | ||
); | ||
buffer = []; | ||
currentPartHeaders = []; | ||
lastline = ""; | ||
state = ParsingState.READING_PART_SEPARATOR; | ||
contentDispositionHeader = ""; | ||
contentTypeHeader = ""; | ||
} else { | ||
buffer.push(oneByte); | ||
} | ||
if (newLineDetected) { | ||
lastline = ""; | ||
} | ||
} else if (ParsingState.READING_PART_SEPARATOR === state) { | ||
if (newLineDetected) { | ||
state = ParsingState.READING_HEADERS; | ||
} | ||
} | ||
} | ||
return allParts; | ||
} | ||
|
||
export function getBoundary(str: string): string { | ||
const lines = str.replaceAll("\r\n", "\n").split("\n").reverse(); | ||
for (const line of lines) { | ||
if (line !== "") { | ||
return line.slice(0, -2).slice(2); | ||
} | ||
} | ||
return ""; | ||
} | ||
|
||
function process(part: Part): Input { | ||
// will transform this object: | ||
// { header: 'Content-Disposition: form-data; name="uploads[]"; filename="A.txt"', | ||
// info: 'Content-Type: text/plain', | ||
// part: 'AAAABBBB' } | ||
// into this one: | ||
// { filename: 'A.txt', type: 'text/plain', data: <Buffer 41 41 41 41 42 42 42 42> } | ||
const obj = function (str: string) { | ||
const k = str.split("="); | ||
const a = k[0].trim(); | ||
|
||
const b = JSON.parse(k[1].trim()); | ||
const o = {}; | ||
Object.defineProperty(o, a, { | ||
value: b, | ||
writable: true, | ||
enumerable: true, | ||
configurable: true, | ||
}); | ||
return o; | ||
}; | ||
const header = part.contentDispositionHeader.split(";"); | ||
|
||
const filenameData = header[2]; | ||
let input = {}; | ||
if (filenameData) { | ||
input = obj(filenameData); | ||
const contentType = part.contentTypeHeader.split(":")[1].trim(); | ||
Object.defineProperty(input, "type", { | ||
value: contentType, | ||
writable: true, | ||
enumerable: true, | ||
configurable: true, | ||
}); | ||
} | ||
// always process the name field | ||
Object.defineProperty(input, "name", { | ||
value: header[1].split("=")[1].replace(/"/g, ""), | ||
writable: true, | ||
enumerable: true, | ||
configurable: true, | ||
}); | ||
|
||
Object.defineProperty(input, "data", { | ||
value: Buffer.from(part.part), | ||
writable: true, | ||
enumerable: true, | ||
configurable: true, | ||
}); | ||
return input as Input; | ||
} |
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I dont see type change.. like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have added There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
import { realpathSync, readFileSync } from "fs"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we import |
||
export class NewPayload { | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
private data: Buffer; | ||
|
||
constructor(data: Buffer) { | ||
this.data = data; | ||
} | ||
|
||
public getData(): Buffer { | ||
return this.data; | ||
} | ||
|
||
public static fromBuffer(buffer: Buffer): Buffer { | ||
return Buffer.from(buffer); | ||
} | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
public toBuffer(): Buffer { | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return this.data; | ||
} | ||
|
||
public static fromString(string: string): Buffer { | ||
return Buffer.from(string); | ||
} | ||
|
||
public toString(encoding: BufferEncoding = "utf8"): string { | ||
return this.data.toString(encoding); | ||
} | ||
|
||
public static fromJson(json: object): Buffer { | ||
return Buffer.from(JSON.stringify(json)); | ||
} | ||
|
||
public toJson(): object { | ||
return JSON.parse(this.data.toString()); | ||
} | ||
|
||
public static fromPath(path: string): Buffer { | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
const realPath = realpathSync(path); | ||
const contents = readFileSync(realPath); | ||
return Buffer.from(contents); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Manually QA fromFile and fromPath. Upload a picture to Storage service. Then see the picture in Appwrite Console. Does it know it's PNG? Can you preview it? If not, we need to fix it. |
||
} | ||
|
||
public toFile(fileName: string): File { | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (!fileName) { | ||
fileName = "code.tar.gz"; | ||
} | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
const blob = new Blob([this.data]); | ||
return new File([blob], fileName); | ||
vermakhushboo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
} |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -1,6 +1,9 @@ | ||||||
import { fetch, FormData, File } from 'node-fetch-native-with-agent'; | ||||||
import { createAgent } from 'node-fetch-native-with-agent/agent'; | ||||||
import { Models } from './models'; | ||||||
import { NewPayload } from './NewPayload'; | ||||||
import * as multipart from 'parse-multipart-data'; | ||||||
const { buffer } = require('node:stream/consumers'); | ||||||
|
||||||
type Payload = { | ||||||
[key: string]: any; | ||||||
|
@@ -331,7 +334,28 @@ class Client { | |||||
data = await response.json(); | ||||||
} else if (responseType === 'arrayBuffer') { | ||||||
data = await response.arrayBuffer(); | ||||||
} else { | ||||||
} else if (response.headers.get('content-type')?.includes('multipart/form-data')) { | ||||||
const body = await buffer(response.body); | ||||||
const boundary = multipart.getBoundary( | ||||||
response.headers.get("content-type") || "" | ||||||
); | ||||||
const parts = multipart.parse(body, boundary); | ||||||
const partsObject: { [key: string]: Buffer | string } = {}; | ||||||
for (const part of parts) { | ||||||
if (part.name) { | ||||||
if (part.name === "responseBody") { | ||||||
partsObject[part.name] = part.data; | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
I think |
||||||
} else { | ||||||
partsObject[part.name] = part.data.toString(); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What if it's array or integer or boolean or object. Address when moving to sdk-generator |
||||||
} | ||||||
} | ||||||
} | ||||||
data = { | ||||||
...partsObject, | ||||||
responseBody: new NewPayload(partsObject.responseBody as Buffer), | ||||||
}; | ||||||
} | ||||||
else { | ||||||
data = { | ||||||
message: await response.text() | ||||||
}; | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -33,3 +33,5 @@ export { ImageGravity } from './enums/image-gravity'; | |
export { ImageFormat } from './enums/image-format'; | ||
export { PasswordHash } from './enums/password-hash'; | ||
export { MessagingProviderType } from './enums/messaging-provider-type'; | ||
export { NewPayload } from './NewPayload'; | ||
export { InputFile } from './inputFile'; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should remove all mention of InputFile. It should no longer exist. With that, we also need to update existing endpoints that used to need it. they all should now use payload class |
Uh oh!
There was an error while loading. Please reload this page.