Skip to content

Improvement/attachments and requests #147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 14 commits into from
Closed
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Backtrace.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -3814,7 +3814,7 @@
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
DEVELOPMENT_TEAM = JWKXD469L2;
DEVELOPMENT_TEAM = "";
ENABLE_BITCODE = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
Expand Down Expand Up @@ -3898,7 +3898,7 @@
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
DEVELOPMENT_TEAM = JWKXD469L2;
DEVELOPMENT_TEAM = "";
ENABLE_BITCODE = NO;
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
Expand Down
1 change: 0 additions & 1 deletion Examples/Example-iOS-ObjC/ViewController.m
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,4 @@ - (IBAction) crashAction: (id) sender {
array[1];
}


@end
4 changes: 2 additions & 2 deletions Podfile.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
PODS:
- Backtrace (2.0.6):
- Backtrace (2.0.7):
- PLCrashReporter (= 1.11.1)
- Nimble (10.0.0)
- PLCrashReporter (1.11.1)
Expand All @@ -22,7 +22,7 @@ EXTERNAL SOURCES:
:path: "./Backtrace.podspec"

SPEC CHECKSUMS:
Backtrace: 72bea7c55570b42ab18dd7218ce41ceef3edca14
Backtrace: a686b3bfb9a7a176080e259f7f8dd7a5f521d957
Nimble: 5316ef81a170ce87baf72dd961f22f89a602ff84
PLCrashReporter: 5d2d3967afe0efad61b3048d617e2199a5d1b787
Quick: 749aa754fd1e7d984f2000fe051e18a3a9809179
Expand Down
20 changes: 17 additions & 3 deletions Sources/Features/Error/BacktraceError.swift
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ enum NetworkError: BacktraceError {

enum HttpError: BacktraceError {
case malformedUrl(URL)
case fileCreationFailed(URL)
case fileWriteFailed
case attachmentError(String)
case multipartFormError(Error)
case unknownError
}

Expand All @@ -42,7 +46,7 @@ enum CodingError: BacktraceError {
extension HttpError {
var backtraceStatus: BacktraceReportStatus {
switch self {
case .malformedUrl:
case .malformedUrl, .fileCreationFailed, .fileWriteFailed, .attachmentError, .multipartFormError:
return .unknownError
case .unknownError:
return .serverError
Expand All @@ -62,8 +66,18 @@ extension NetworkError {
extension HttpError {
var localizedDescription: String {
switch self {
case .malformedUrl(let url): return "Provided URL cannot be parsed: \(url)."
case .unknownError: return "Unknown error occurred."
case .malformedUrl(let url):
return "Provided URL cannot be parsed: \(url)."

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

small nit: can we either get rid of trailing dots or put the URL in quotes?
this goes for every description that has some data displayed in the string

case .fileCreationFailed(let url):
return "File Error occurred: \(url)."
case .fileWriteFailed:
return "File Write Error occurred."
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion:

I think file is to generic and might be weird from the developer perspective that the file means a temporary file during HTTP submission. My recommendation is to make it more explicit by saying "Temporary submission file" instead.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, it looks out of place
also need to properly handle an 'attachmentError'.

case .attachmentError(let string):
return "Attachment Error occurred: \(string)."
case .multipartFormError(let error):
return "Multipart Form Error occurred: \(error.localizedDescription)."
case .unknownError:
return "Unknown error occurred."
}
}
}
Expand Down
100 changes: 72 additions & 28 deletions Sources/Features/Network/MultipartRequest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -48,41 +48,85 @@ extension MultipartRequest {
var multipartRequest = urlRequest
let boundary = generateBoundaryString()
multipartRequest.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")

let boundaryPrefix = "--\(boundary)\r\n"
let body = NSMutableData()
// attributes
for attribute in report.attributes {
body.appendString(boundaryPrefix)
body.appendString("Content-Disposition: form-data; name=\"\(attribute.key)\"\r\n\r\n")
body.appendString("\(attribute.value)\r\n")
}
// report file
body.appendString(boundaryPrefix)
body.appendString("Content-Disposition: form-data; name=\"upload_file\"; filename=\"upload_file\"\r\n")
body.appendString("Content-Type: application/octet-stream\r\n\r\n")
body.append(report.reportData)
body.appendString("\r\n")

// attachments
for attachment in Set(report.attachmentPaths).compactMap(Attachment.init(filePath:)) {
body.appendString(boundaryPrefix)
body.appendString("Content-Disposition: form-data; name=\"\(attachment.filename)\"; filename=\"\(attachment.filename)\"\r\n")
body.appendString("Content-Type: \(attachment.mimeType)\r\n\r\n")
body.append(attachment.data)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we stream data instead of reading it to memory?

Copy link
Collaborator Author

@melekr melekr Jan 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can use a Stream.

Benefits:

  • no longer holds attachment or report data in memory before writing since data is written directly to file handle as it's being read
  • Data is never loaded entirely into memory, great for multiple and/or large attachments

Caviats:

  • Memory and buffer management
  • Error handling (incomplete streams, stream cannot be opened
  • Small buffer chunks

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

comment is on original not the updated code

body.appendString("\r\n")

// temporary file to stream data
let tempURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happen to the file if the app is restarted while file is being created?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

file is created in Temp directory managed by OS, if the app is restarted there is no guarantee the file is persisted but :

  • OOM : MultipartRequest will be retried/reconstructed from the already generated and persisted live report.
  • BacktraceReport/Resource : MultipartRequest will be retried/reconstructed from repository
  • Watcher: MultipartRequest will be retried/reconstructed from repository

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unlike NSMutableData it's even possible to recover multipart request file! (not sure if necessary but can be a good excuse to consider for // TODO: T16698 - Add retry logic)


do {
let fileCreated = FileManager.default.createFile(atPath: tempURL.path, contents: nil, attributes: nil)
if !fileCreated {
throw HttpError.fileCreationFailed(tempURL)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it won't be captured by the L:111 - which means we will return a generic error rather this this one?

}

let fileHandle = try FileHandle(forWritingTo: tempURL)
defer {
if #available(iOS 13.0, tvOS 13.0, macOS 11.0, *) {
try? fileHandle.close()
} else {
fileHandle.closeFile()
}
}

// attributes
var attributesString = ""
for attribute in report.attributes {
attributesString += "--\(boundary)\r\n"
attributesString += "Content-Disposition: form-data; name=\"\(attribute.key)\"\r\n\r\n"
attributesString += "\(attribute.value)\r\n"
}
try writeToFile(fileHandle, attributesString)

// report
var reportString = "--\(boundary)\r\n"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be better if we start with report rather than with attributes - if the application stops during submission part, we can still submit a valid report.

reportString += "Content-Disposition: form-data; name=\"upload_file\"; filename=\"upload_file\"\r\n"
reportString += "Content-Type: application/octet-stream\r\n\r\n"
try writeToFile(fileHandle, reportString)
fileHandle.write(report.reportData)
try writeToFile(fileHandle, "\r\n")

// attachments
for attachmentPath in Set(report.attachmentPaths) {
guard let attachment = Attachment(filePath: attachmentPath) else {
BacktraceLogger.error("Failed to create attachment for path: \(attachmentPath)")
continue
}

try writeToFile(fileHandle, "--\(boundary)\r\n")
try writeToFile(fileHandle, "Content-Disposition: form-data; name=\"\(attachment.filename)\"; filename=\"\(attachment.filename)\"\r\n")
try writeToFile(fileHandle, "Content-Type: \(attachment.mimeType)\r\n\r\n")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should do it with a single write

fileHandle.write(attachment.data)
Copy link
Collaborator

@konraddysput konraddysput Jan 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you're making an attachment copy in the temporary file. WE cannot stream a file directly from the file available somewhere?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

possible but requires creating an InputStream for each attachments.
If we decide to keep files, i'll make the necessary improvements.

try writeToFile(fileHandle, "\r\n")
}

// Final boundary
try writeToFile(fileHandle, "--\(boundary)--\r\n")

// Set Content-Length
let fileSize = try FileManager.default.attributesOfItem(atPath: tempURL.path)[.size] as? Int ?? 0
multipartRequest.setValue("\(fileSize)", forHTTPHeaderField: "Content-Length")

// Attach file stream to HTTP body
multipartRequest.httpBodyStream = InputStream(url: tempURL)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of creating the file and setting it as httpBodyStream, couldn't you write directly to the request?

I'm not proficient in Swift, but I found this:
https://developer.apple.com/documentation/foundation/url_loading_system/uploading_streams_of_data

You can use bound streams to achieve that, it seems.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can, this is the the approach I was referencing:

Holding data in memory for attributes or attachments changes the purpose of writing into files. A hybrid approach would be buffering data in chunks, using a buffered stream.


} catch {
BacktraceLogger.error("Error during multipart form creation: \(error.localizedDescription)")
throw HttpError.multipartFormError(error)
}
body.appendString("\(boundaryPrefix)--")
multipartRequest.httpBody = body as Data

multipartRequest.setValue("\(body.length)", forHTTPHeaderField: "Content-Length")


return multipartRequest
}

private static func generateBoundaryString() -> String {
return "Boundary-\(NSUUID().uuidString)"
}

private static func writeToFile(_ fileHandle: FileHandle, _ string: String) throws {
if let data = string.data(using: .utf8) {
fileHandle.write(data)
} else {
throw HttpError.fileWriteFailed
}
}
}

private extension NSMutableData {
Expand Down
51 changes: 34 additions & 17 deletions Sources/Features/Repository/Model/Attachment.swift
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import Foundation
import UniformTypeIdentifiers
#if (os(iOS) || os(tvOS))
import MobileCoreServices
#elseif os(macOS)
Expand All @@ -7,42 +8,58 @@ import CoreServices

struct Attachment {
let data: Data
// file name with extension that represents real file name
/// File name with extension that represents the real file name
let filename: String

/// MIME type of the file
let mimeType: String

// Make sure attachments are not bigger than 10 MB.
/// Maximum allowed size for attachments (10 MB)
private static let maximumAttachmentSize = 10 * 1024 * 1024


/// Custom prefix for generated filenames
private static let filenamePrefix = "attachment_"

/// Initializes an Attachment from a file path
/// - Parameter filePath: Path to the file
init?(filePath: String) {
let fileURL = URL(fileURLWithPath: filePath)

// Don't allow too large attachments
// Ensure file size is within maximumAttachmentSize limits
guard let fileAttributes = try? FileManager.default.attributesOfItem(atPath: filePath),
let fileSize = fileAttributes[FileAttributeKey.size] as? UInt64,
fileSize < Attachment.maximumAttachmentSize else {
fileSize <= Attachment.maximumAttachmentSize else {
BacktraceLogger.warning("Skipping attachment because fileSize couldn't be read or is larger than 10MB: \(filePath)")
return nil
}

do {
data = try Data(contentsOf: fileURL, options: Data.ReadingOptions.mappedIfSafe)
data = try Data(contentsOf: fileURL, options: .mappedIfSafe)
} catch {
BacktraceLogger.warning("Skipping attachment: \(filePath): \(error.localizedDescription)")
return nil
}

mimeType = Attachment.mimeTypeForPath(fileUrl: fileURL)
filename = "attachment_" + fileURL.lastPathComponent
filename = Attachment.filenamePrefix + fileURL.lastPathComponent
}


/// Determines the MIME type for a file URL
/// - Parameter fileUrl: File URL
/// - Returns: MIME type as a string, or "application/octet-stream" as fallback
static private func mimeTypeForPath(fileUrl: URL) -> String {
guard let uti = UTTypeCreatePreferredIdentifierForTag(kUTTagClassFilenameExtension,
fileUrl.pathExtension as NSString, nil)?
.takeRetainedValue(),
let mimetype = UTTypeCopyPreferredTagWithClass(uti, kUTTagClassMIMEType)?
if #available(iOS 14.0, tvOS 14.0, macOS 11.0, *) {
if let type = UTType(filenameExtension: fileUrl.pathExtension)?.preferredMIMEType {
return type
}
} else {
guard let uti = UTTypeCreatePreferredIdentifierForTag(kUTTagClassFilenameExtension,
fileUrl.pathExtension as NSString, nil)?
.takeRetainedValue(),
let mimetype = UTTypeCopyPreferredTagWithClass(uti, kUTTagClassMIMEType)?
.takeRetainedValue() as String? else { return "application/octet-stream" }
return mimetype
return mimetype
}
return "application/octet-stream"
}
}