diff --git a/.gitmodules b/.gitmodules index e44e8f36..8318c42e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "JAMTests/jamtestvectors"] path = JAMTests/jamtestvectors url = https://github.com/open-web3-stack/jamtestvectors.git -[submodule "JAMTests/jamixir"] - path = JAMTests/jamixir - url = https://github.com/jamixir/jamtestnet.git [submodule "JAMTests/javajam"] path = JAMTests/javajam url = https://github.com/javajamio/javajam-trace.git diff --git a/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift b/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift index 5baa2e63..8b4d9522 100644 --- a/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift +++ b/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift @@ -727,12 +727,13 @@ extension ProtocolConfig { UInt16(recentHistorySize), UInt16(maxWorkItems), UInt16(maxDepsInWorkReport), + UInt16(maxTicketsPerExtrinsic), UInt32(maxLookupAnchorAge), + UInt16(ticketEntriesPerValidator), UInt16(maxAuthorizationsPoolItems), UInt16(slotPeriodSeconds), UInt16(maxAuthorizationsQueueItems), UInt16(coreAssignmentRotationPeriod), - UInt16(maxAccumulationQueueItems), UInt16(maxWorkPackageExtrinsics), UInt16(preimageReplacementPeriod), UInt16(totalNumberOfValidators), @@ -740,7 +741,6 @@ extension ProtocolConfig { UInt32(maxEncodedWorkPackageSize), UInt32(maxServiceCodeSize), UInt32(erasureCodedPieceSize), - UInt32(segmentSize), UInt32(maxWorkPackageImports), UInt32(erasureCodedSegmentSize), UInt32(maxWorkReportBlobSize), diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift index b83f5626..dc71ffc9 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift @@ -1,3 +1,4 @@ +import Codec import Foundation import Utils @@ -9,14 +10,14 @@ public struct OperandTuple: Codable { public var segmentRoot: Data32 /// a public var authorizerHash: Data32 - /// o - public var authorizerTrace: Data /// y public var payloadHash: Data32 /// g - public var gasLimit: Gas + @CodingAs> public var gasLimit: Gas /// d public var workResult: WorkResult + /// o + public var authorizerTrace: Data } public struct DeferredTransfers: Codable { diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift index 08dd01ac..d93bdec0 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift @@ -112,7 +112,7 @@ public struct AccountChanges { throw .duplicatedNewService } guard altered.isDisjoint(with: other.altered) else { - logger.debug("altered accounts have duplicates, self: \(altered), other: \(other.altered)") + logger.debug("same service being altered in parallel, self: \(altered), other: \(other.altered)") throw .duplicatedContributionToService } guard removed.isDisjoint(with: other.removed) else { @@ -170,10 +170,10 @@ extension Accumulation { packageHash: report.packageSpecification.workPackageHash, segmentRoot: report.packageSpecification.segmentRoot, authorizerHash: report.authorizerHash, - authorizerTrace: report.authorizerTrace, payloadHash: digest.payloadHash, gasLimit: digest.gasLimit, workResult: digest.result, + authorizerTrace: report.authorizerTrace, )) } } @@ -234,7 +234,7 @@ extension Accumulation { var accountsRef = ServiceAccountsMutRef(state.accounts.value) var servicePreimageSet = Set() - for service in services { + for service in Set(services) { let singleOutput = try await singleAccumulate( config: config, state: AccumulateState( @@ -527,7 +527,7 @@ extension Accumulation { transfersStats[service] = (count, gasUsed) } - self = accountsMutRef.value as! Self + self = accumulateOutput.state.accounts.value as! Self // update accumulation history let accumulated = accumulatableReports[0 ..< accumulateOutput.numAccumulated] diff --git a/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift b/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift index 9cc7e00b..f5e55c07 100644 --- a/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift +++ b/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift @@ -41,23 +41,17 @@ public class ServiceAccountsMutRef { public func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails) { ref.value.set(serviceAccount: index, account: account) - changes.addAlteration(index: index) { accounts in - accounts.set(serviceAccount: index, account: account) - } + changes.addAlteration(index: index) { $0.set(serviceAccount: index, account: account) } } public func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) { ref.value.set(serviceAccount: index, storageKey: key, value: value) - changes.addAlteration(index: index) { accounts in - accounts.set(serviceAccount: index, storageKey: key, value: value) - } + changes.addAlteration(index: index) { $0.set(serviceAccount: index, storageKey: key, value: value) } } public func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data?) { ref.value.set(serviceAccount: index, preimageHash: hash, value: value) - changes.addAlteration(index: index) { accounts in - accounts.set(serviceAccount: index, preimageHash: hash, value: value) - } + changes.addAlteration(index: index) { $0.set(serviceAccount: index, preimageHash: hash, value: value) } } public func set( @@ -67,9 +61,7 @@ public class ServiceAccountsMutRef { value: LimitedSizeArray? ) { ref.value.set(serviceAccount: index, preimageHash: hash, length: length, value: value) - changes.addAlteration(index: index) { accounts in - accounts.set(serviceAccount: index, preimageHash: hash, length: length, value: value) - } + changes.addAlteration(index: index) { $0.set(serviceAccount: index, preimageHash: hash, length: length, value: value) } } public func addNew(serviceAccount index: ServiceIndex, account: ServiceAccount) { diff --git a/Blockchain/Sources/Blockchain/State/State.swift b/Blockchain/Sources/Blockchain/State/State.swift index eaeb7529..1fdd10ce 100644 --- a/Blockchain/Sources/Blockchain/State/State.swift +++ b/Blockchain/Sources/Blockchain/State/State.swift @@ -390,6 +390,28 @@ extension State: ServiceAccounts { } public mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) { + // update footprint + let oldValue = layer[serviceAccount: index, storageKey: key] + let oldAccount = layer[serviceAccount: index] + if let oldValue { + if let value { + // replace: update byte count difference + layer[serviceAccount: index]?.totalByteLength = + max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count))) + (32 + UInt64(value.count)) + } else { + // remove: decrease count and bytes + layer[serviceAccount: index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 1) + layer[serviceAccount: index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count))) + } + } else { + if let value { + // add: increase count and bytes + layer[serviceAccount: index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 1 + layer[serviceAccount: index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 32 + UInt64(value.count) + } + } + + // update value layer[serviceAccount: index, storageKey: key] = value } @@ -403,6 +425,28 @@ extension State: ServiceAccounts { length: UInt32, value: StateKeys.ServiceAccountPreimageInfoKey.Value? ) { + // update footprint + let oldValue = layer[serviceAccount: index, preimageHash: hash, length: length] + let oldAccount = layer[serviceAccount: index] + if let oldValue { + if let value { + // replace: update byte count difference + layer[serviceAccount: index]?.totalByteLength = + max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count))) + (81 + UInt64(value.count)) + } else { + // remove: decrease count and bytes + layer[serviceAccount: index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 2) + layer[serviceAccount: index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count))) + } + } else { + if let value { + // add: increase count and bytes + layer[serviceAccount: index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 2 + layer[serviceAccount: index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 81 + UInt64(value.count) + } + } + + // update value layer[serviceAccount: index, preimageHash: hash, length: length] = value } } diff --git a/Blockchain/Sources/Blockchain/Types/CodeAndMeta.swift b/Blockchain/Sources/Blockchain/Types/CodeAndMeta.swift index f6bb1a54..cd4f687e 100644 --- a/Blockchain/Sources/Blockchain/Types/CodeAndMeta.swift +++ b/Blockchain/Sources/Blockchain/Types/CodeAndMeta.swift @@ -1,6 +1,9 @@ import Codec +import TracingUtils import Utils +private let logger = Logger(label: "CodeAndMeta") + /// account preimage data is: meta length + meta + code public struct CodeAndMeta: Sendable, Equatable { public enum Error: Swift.Error { @@ -15,6 +18,7 @@ public struct CodeAndMeta: Sendable, Equatable { let metaLength = slice.decode() guard let metaLength else { throw Error.invalidMetadataLength } metadata = data[slice.startIndex ..< slice.startIndex + Int(metaLength)] + logger.debug("Metadata: \(String(data: metadata, encoding: .utf8) ?? "nil")") codeBlob = data[slice.startIndex + Int(metaLength) ..< slice.endIndex] } } diff --git a/Blockchain/Sources/Blockchain/Types/PrivilegedServices.swift b/Blockchain/Sources/Blockchain/Types/PrivilegedServices.swift index a0850284..77a0d8ed 100644 --- a/Blockchain/Sources/Blockchain/Types/PrivilegedServices.swift +++ b/Blockchain/Sources/Blockchain/Types/PrivilegedServices.swift @@ -9,7 +9,7 @@ public struct PrivilegedServices: Sendable, Equatable, Codable { // v public var designate: ServiceIndex // g - @CodingAs> public var basicGas: [ServiceIndex: Gas] + public var basicGas: [ServiceIndex: Gas] public init(blessed: ServiceIndex, assign: ServiceIndex, designate: ServiceIndex, basicGas: [ServiceIndex: Gas]) { self.blessed = blessed @@ -17,4 +17,28 @@ public struct PrivilegedServices: Sendable, Equatable, Codable { self.designate = designate self.basicGas = basicGas } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + blessed = try container.decode(ServiceIndex.self, forKey: .blessed) + assign = try container.decode(ServiceIndex.self, forKey: .assign) + designate = try container.decode(ServiceIndex.self, forKey: .designate) + + let compactGas = try container.decode(SortedKeyValues>.self, forKey: .basicGas) + basicGas = compactGas.alias.mapValues { $0.alias } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(blessed, forKey: .blessed) + try container.encode(assign, forKey: .assign) + try container.encode(designate, forKey: .designate) + + let compactGas = SortedKeyValues(alias: basicGas.mapValues { Compact(alias: $0) }) + try container.encode(compactGas, forKey: .basicGas) + } + + private enum CodingKeys: String, CodingKey { + case blessed, assign, designate, basicGas + } } diff --git a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift index 3f6f6834..dacb4c68 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift @@ -63,15 +63,15 @@ public class Fetch: HostCall { } private func getWorkItemMeta(item: WorkItem) throws -> Data { - let encoder = JamEncoder(capacity: 4 + 4 + 8 + 8 + 2 + 2 + 2 + 4) + let encoder = JamEncoder(capacity: 4 + 32 + 8 + 8 + 2 + 2 + 2 + 4) try encoder.encode(item.serviceIndex) try encoder.encode(item.codeHash) try encoder.encode(item.refineGasLimit) try encoder.encode(item.accumulateGasLimit) try encoder.encode(item.exportsCount) - try encoder.encode(item.inputs.count) - try encoder.encode(item.outputs.count) - try encoder.encode(item.payloadBlob.count) + try encoder.encode(UInt16(item.inputs.count)) + try encoder.encode(UInt16(item.outputs.count)) + try encoder.encode(UInt32(item.payloadBlob.count)) return encoder.data } @@ -177,8 +177,12 @@ public class Fetch: HostCall { let first = min(Int(reg8), value?.count ?? 0) let len = min(Int(reg9), (value?.count ?? 0) - first) + logger.debug("writeAddr: \(writeAddr), first: \(first), len: \(len)") + let isWritable = state.isMemoryWritable(address: writeAddr, length: len) + logger.debug("isWritable: \(isWritable), value: \(value?.toDebugHexString() ?? "nil")") + if !isWritable { throw VMInvocationsError.panic } else if value == nil { diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift index 153693c4..8e2448cc 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift @@ -50,7 +50,7 @@ public func accumulate( ) ) let ctx = AccumulateContext(context: contextContent, config: config, timeslot: timeslot, operands: arguments) - let argument = try JamEncoder.encode(timeslot, serviceIndex, arguments.count) + let argument = try JamEncoder.encode(UInt(timeslot), UInt(serviceIndex), UInt(arguments.count)) let (exitReason, gas, output) = await invokePVM( config: config, diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift index 0a8a103d..28c79c2f 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift @@ -8,7 +8,7 @@ public func isAuthorized( package: WorkPackage, coreIndex: CoreIndex ) async throws -> (Result, Gas) { - let args = try JamEncoder.encode(package, coreIndex) + let args = try JamEncoder.encode(coreIndex) let codeBlob = try await package.authorizationCode(serviceAccounts: serviceAccounts) guard let codeBlob else { return (.failure(.invalidCode), Gas(0)) diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift index c2c96734..eabcdbb5 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift @@ -31,7 +31,7 @@ public func onTransfer( let contextContent = OnTransferContext.ContextType(serviceIndex: serviceIndex, accounts: serviceAccounts) let ctx = OnTransferContext(context: contextContent, config: config, entropy: entropy, transfers: transfers) let gasLimitSum = transfers.reduce(Balance(0)) { $0 + $1.gasLimit } - let argument = try JamEncoder.encode(timeslot, serviceIndex, transfers.count) + let argument = try JamEncoder.encode(UInt(timeslot), UInt(serviceIndex), UInt(transfers.count)) let (_, gas, _) = await invokePVM( config: config, diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift index d0f2f5af..4814dc24 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift @@ -37,8 +37,8 @@ public func refine( let codeBlob = try CodeAndMeta(data: preimage).codeBlob let argumentData = try JamEncoder.encode( - workItemIndex, - service, + UInt(workItemIndex), + UInt(service), workItem.payloadBlob, workPackage.hash(), ) diff --git a/Codec/Sources/Codec/Compact.swift b/Codec/Sources/Codec/Compact.swift new file mode 100644 index 00000000..b711631e --- /dev/null +++ b/Codec/Sources/Codec/Compact.swift @@ -0,0 +1,120 @@ +import Foundation + +/// Protocol for types that can be converted to/from UInt for compact encoding +public protocol CompactEncodable { + /// Convert the value to UInt for encoding + func toUInt() throws -> UInt + + /// Create an instance from UInt after decoding + static func fromUInt(_ value: UInt) throws -> Self +} + +/// A coding wrapper that converts CompactEncodable types to UInt for compact encoding/decoding. +/// This supports both Swift's built-in integer types and custom types that conform to CompactEncodable. +public struct Compact: Codable, CodableAlias { + public typealias Alias = T + + public var alias: T + + public init(alias: T) { + self.alias = alias + } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let uintValue = try container.decode(UInt.self) + + do { + alias = try T.fromUInt(uintValue) + } catch let error as CompactEncodingError { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: container.codingPath, + debugDescription: "Compact decoding failed: \(error.localizedDescription)" + ) + ) + } catch { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: container.codingPath, + debugDescription: "Value \(uintValue) cannot be converted to \(T.self): \(error)" + ) + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + + do { + let uintValue = try alias.toUInt() + try container.encode(uintValue) + } catch let error as CompactEncodingError { + throw EncodingError.invalidValue( + alias, + EncodingError.Context( + codingPath: encoder.codingPath, + debugDescription: "Compact encoding failed: \(error.localizedDescription)" + ) + ) + } catch { + throw EncodingError.invalidValue( + alias, + EncodingError.Context( + codingPath: encoder.codingPath, + debugDescription: "Cannot convert \(alias) to UInt: \(error)" + ) + ) + } + } +} + +/// Extension to provide default implementation for unsigned integer types +extension UnsignedInteger where Self: CompactEncodable { + public func toUInt() throws -> UInt { + guard let result = UInt(exactly: self) else { + throw CompactEncodingError.valueOutOfRange(value: "\(self)", sourceType: "\(Self.self)", targetType: "UInt") + } + return result + } + + public static func fromUInt(_ value: UInt) throws -> Self { + guard let result = Self(exactly: value) else { + throw CompactEncodingError.valueOutOfRange(value: "\(value)", sourceType: "UInt", targetType: "\(Self.self)") + } + return result + } +} + +extension UInt8: CompactEncodable {} +extension UInt16: CompactEncodable {} +extension UInt32: CompactEncodable {} +extension UInt64: CompactEncodable {} +extension UInt: CompactEncodable {} + +public enum CompactEncodingError: Error, LocalizedError, CustomStringConvertible { + case valueOutOfRange(value: String, sourceType: String, targetType: String) + case conversionFailed(value: String, fromType: String, toType: String, reason: String) + + public var description: String { + switch self { + case let .valueOutOfRange(value, sourceType, targetType): + "Value \(value) of type \(sourceType) is out of range for type \(targetType)" + case let .conversionFailed(value, fromType, toType, reason): + "Failed to convert value \(value) from \(fromType) to \(toType): \(reason)" + } + } + + public var errorDescription: String? { + description + } + + public var failureReason: String? { + switch self { + case .valueOutOfRange: + "Value exceeds the range of the target type" + case .conversionFailed: + "Type conversion failed during compact encoding/decoding" + } + } +} diff --git a/Codec/Tests/CodecTests/CompactTests.swift b/Codec/Tests/CodecTests/CompactTests.swift new file mode 100644 index 00000000..125ebf94 --- /dev/null +++ b/Codec/Tests/CodecTests/CompactTests.swift @@ -0,0 +1,173 @@ +@testable import Codec +import Testing + +struct CompactTests { + @Test + func testUInt8Compact() throws { + let value: UInt8 = 255 + let compact = Compact(alias: value) + + let encoded = try JamEncoder.encode(compact) + let decoded = try JamDecoder.decode(Compact.self, from: encoded) + + #expect(decoded.alias == value) + } + + @Test + func testUInt16Compact() throws { + let value: UInt16 = 65535 + let compact = Compact(alias: value) + + let encoded = try JamEncoder.encode(compact) + let decoded = try JamDecoder.decode(Compact.self, from: encoded) + + #expect(decoded.alias == value) + } + + @Test + func testUInt32Compact() throws { + let value: UInt32 = 4_294_967_295 + let compact = Compact(alias: value) + + let encoded = try JamEncoder.encode(compact) + let decoded = try JamDecoder.decode(Compact.self, from: encoded) + + #expect(decoded.alias == value) + } + + @Test + func testUInt64Compact() throws { + let value: UInt64 = 1_234_567_890 + let compact = Compact(alias: value) + + let encoded = try JamEncoder.encode(compact) + let decoded = try JamDecoder.decode(Compact.self, from: encoded) + + #expect(decoded.alias == value) + } + + @Test + func testUIntCompact() throws { + let value: UInt = 987_654_321 + let compact = Compact(alias: value) + + let encoded = try JamEncoder.encode(compact) + let decoded = try JamDecoder.decode(Compact.self, from: encoded) + + #expect(decoded.alias == value) + } + + @Test + func testValueOutOfRangeError() throws { + // Test decoding a value that's too large for UInt8 + let largeValue: UInt = 256 + let compact = Compact(alias: largeValue) + + let encoded = try JamEncoder.encode(compact) + + #expect(throws: DecodingError.self) { + try JamDecoder.decode(Compact.self, from: encoded) + } + } + + @Test + func testValueOutOfRangeErrorDetails() throws { + // Test decoding a value that's too large for UInt8 + let largeValue: UInt = 256 + let compact = Compact(alias: largeValue) + + let encoded = try JamEncoder.encode(compact) + + do { + _ = try JamDecoder.decode(Compact.self, from: encoded) + #expect(Bool(false), "Expected decoding to throw an error") + } catch let decodingError as DecodingError { + if case let .dataCorrupted(context) = decodingError { + #expect(context.debugDescription.contains("Value 256")) + #expect(context.debugDescription.contains("out of range")) + #expect(context.debugDescription.contains("UInt8")) + } else { + #expect(Bool(false), "Expected dataCorrupted decoding error") + } + } + } + + @Test + func testCompactEncodingErrorDirectly() throws { + // Test CompactEncodingError types directly + let outOfRangeError = CompactEncodingError.valueOutOfRange(value: "256", sourceType: "UInt", targetType: "UInt8") + #expect(outOfRangeError.description.contains("Value 256")) + #expect(outOfRangeError.description.contains("out of range")) + + let conversionError = CompactEncodingError.conversionFailed( + value: "123", + fromType: "String", + toType: "UInt", + reason: "invalid format" + ) + #expect(conversionError.description.contains("Failed to convert")) + #expect(conversionError.description.contains("invalid format")) + } + + struct TestPrivilegedServices: Codable { + var blessed: UInt32 + var assign: UInt32 + var designate: UInt32 + @CodingAs>> var basicGas: [UInt32: Compact] + + init(blessed: UInt32, assign: UInt32, designate: UInt32, basicGas: [UInt32: UInt64]) { + self.blessed = blessed + self.assign = assign + self.designate = designate + self.basicGas = basicGas.mapValues { Compact(alias: $0) } + } + } + + @Test + func testCodingAsPropertyWrapper() throws { + let original = TestPrivilegedServices( + blessed: 1, + assign: 2, + designate: 3, + basicGas: [1: 1000, 2: 2000, 3: 3000] + ) + + let encoded = try JamEncoder.encode(original) + let decoded = try JamDecoder.decode(TestPrivilegedServices.self, from: encoded) + + #expect(decoded.blessed == original.blessed) + #expect(decoded.assign == original.assign) + #expect(decoded.designate == original.designate) + #expect(decoded.basicGas.mapValues { $0.alias } == [1: 1000, 2: 2000, 3: 3000]) + } + + @Test + func testZeroValues() throws { + let zeroUInt8 = Compact(alias: UInt8(0)) + let zeroUInt32 = Compact(alias: UInt32(0)) + + let encodedUInt8 = try JamEncoder.encode(zeroUInt8) + let encodedUInt32 = try JamEncoder.encode(zeroUInt32) + + let decodedUInt8 = try JamDecoder.decode(Compact.self, from: encodedUInt8) + let decodedUInt32 = try JamDecoder.decode(Compact.self, from: encodedUInt32) + + #expect(decodedUInt8.alias == 0) + #expect(decodedUInt32.alias == 0) + } + + @Test + func testMaxValues() throws { + let maxUInt8 = Compact(alias: UInt8.max) + let maxUInt16 = Compact(alias: UInt16.max) + + let encodedUInt8 = try JamEncoder.encode(maxUInt8) + let encodedUInt16 = try JamEncoder.encode(maxUInt16) + + let decodedUInt8 = try JamDecoder.decode(Compact.self, from: encodedUInt8) + let decodedUInt16 = try JamDecoder.decode(Compact.self, from: encodedUInt16) + + #expect(decodedUInt8.alias == UInt8.max) + #expect(decodedUInt16.alias == UInt16.max) + } +} diff --git a/JAMTests/Package.swift b/JAMTests/Package.swift index e682008b..eadb16c6 100644 --- a/JAMTests/Package.swift +++ b/JAMTests/Package.swift @@ -35,7 +35,6 @@ let package = Package( .copy("../../jamtestvectors"), .copy("../../jamduna"), .copy("../../javajam"), - .copy("../../jamixir"), ], swiftSettings: [ .interoperabilityMode(.Cxx), diff --git a/JAMTests/Tests/JAMTests/jamtestnet/JamdunaTests.swift b/JAMTests/Tests/JAMTests/jamtestnet/JamdunaTests.swift index 45d469fe..54fdd5b3 100644 --- a/JAMTests/Tests/JAMTests/jamtestnet/JamdunaTests.swift +++ b/JAMTests/Tests/JAMTests/jamtestnet/JamdunaTests.swift @@ -5,7 +5,7 @@ import Utils struct JamdunaTests { @Test(arguments: try JamTestnet.loadTests(path: "data/generic/state_transitions", src: .jamduna)) - func safroleTests(_ input: Testcase) async throws { + func genericTests(_ input: Testcase) async throws { await withKnownIssue("TODO: debug", isIntermittent: true) { try await STFTests.test(input) } diff --git a/JAMTests/Tests/JAMTests/jamtestnet/JamixirTests.swift b/JAMTests/Tests/JAMTests/jamtestnet/JamixirTests.swift deleted file mode 100644 index def4b614..00000000 --- a/JAMTests/Tests/JAMTests/jamtestnet/JamixirTests.swift +++ /dev/null @@ -1,6 +0,0 @@ -import Testing -import Utils - -@testable import JAMTests - -struct JamixirTests {} diff --git a/JAMTests/Tests/JAMTests/jamtestnet/JavajamTests.swift b/JAMTests/Tests/JAMTests/jamtestnet/JavajamTests.swift index de367b40..5424fe3a 100644 --- a/JAMTests/Tests/JAMTests/jamtestnet/JavajamTests.swift +++ b/JAMTests/Tests/JAMTests/jamtestnet/JavajamTests.swift @@ -11,52 +11,53 @@ struct JavajamTests { try await STFTests.test(input) } - @Test(arguments: try JamTestnet.loadTests(path: "erasure_coding", src: .javajam, ext: "json")) - func erasureCodingTests(_ input: Testcase) async throws { - struct ECTestCase: Codable { - let data: String - let shards: [String] - } - - let isTiny = input.description.contains("tiny") - let config = isTiny ? TestVariants.tiny.config : TestVariants.full.config - - let decoder = JSONDecoder() - let testCase = try decoder.decode(ECTestCase.self, from: input.data) - - let originalData = Data(fromHexString: testCase.data)! - - let recoveryShards = testCase.shards.enumerated().map { index, hexString -> ErasureCoding.Shard in - return .init(data: Data(fromHexString: hexString)!, index: UInt32(index)) - } - - let originalCount = config.value.totalNumberOfCores - let basicSize = config.value.erasureCodedPieceSize - let recoveryCount = config.value.totalNumberOfValidators - - #expect(recoveryShards.count == recoveryCount) - - withKnownIssue("does not match", isIntermittent: true) { - let recoveredData = try ErasureCoding.reconstruct( - shards: recoveryShards, - basicSize: basicSize, - originalCount: originalCount, - recoveryCount: recoveryCount - ) - - #expect(recoveredData == originalData, "reconstructed data should match original data") - - let generatedShards = try ErasureCoding.chunk( - data: originalData, - basicSize: basicSize, - recoveryCount: recoveryCount - ) - - #expect(generatedShards.count == recoveryCount, "should generate expected number of recovery shards") - - for (index, shard) in recoveryShards.enumerated() where index < generatedShards.count { - #expect(generatedShards[index] == shard.data, "generated shard at index \(index) should match test data") - } - } - } + // TODO: try again after updating our erasure coding + // @Test(arguments: try JamTestnet.loadTests(path: "erasure_coding", src: .javajam, ext: "json")) + // func erasureCodingTests(_ input: Testcase) async throws { + // struct ECTestCase: Codable { + // let data: String + // let shards: [String] + // } + + // let isTiny = input.description.contains("tiny") + // let config = isTiny ? TestVariants.tiny.config : TestVariants.full.config + + // let decoder = JSONDecoder() + // let testCase = try decoder.decode(ECTestCase.self, from: input.data) + + // let originalData = Data(fromHexString: testCase.data)! + + // let recoveryShards = testCase.shards.enumerated().map { index, hexString -> ErasureCoding.Shard in + // return .init(data: Data(fromHexString: hexString)!, index: UInt32(index)) + // } + + // let originalCount = config.value.totalNumberOfCores + // let basicSize = config.value.erasureCodedPieceSize + // let recoveryCount = config.value.totalNumberOfValidators + + // #expect(recoveryShards.count == recoveryCount) + + // withKnownIssue("does not match", isIntermittent: true) { + // let recoveredData = try ErasureCoding.reconstruct( + // shards: recoveryShards, + // basicSize: basicSize, + // originalCount: originalCount, + // recoveryCount: recoveryCount + // ) + + // #expect(recoveredData == originalData, "reconstructed data should match original data") + + // let generatedShards = try ErasureCoding.chunk( + // data: originalData, + // basicSize: basicSize, + // recoveryCount: recoveryCount + // ) + + // #expect(generatedShards.count == recoveryCount, "should generate expected number of recovery shards") + + // for (index, shard) in recoveryShards.enumerated() where index < generatedShards.count { + // #expect(generatedShards[index] == shard.data, "generated shard at index \(index) should match test data") + // } + // } + // } } diff --git a/JAMTests/Tests/JAMTests/jamtestnet/PolkajamTests.swift b/JAMTests/Tests/JAMTests/jamtestnet/PolkajamTests.swift index dde185fd..e14cede3 100644 --- a/JAMTests/Tests/JAMTests/jamtestnet/PolkajamTests.swift +++ b/JAMTests/Tests/JAMTests/jamtestnet/PolkajamTests.swift @@ -24,4 +24,11 @@ struct PolkajamTests { // _ = try await STFTests.test(input) // } // } + + // @Test(arguments: try JamTestnet.loadTests(path: "traces/reports-l1", src: .w3f)) + // func reportsl0Tests(_ input: Testcase) async throws { + // await withKnownIssue("TODO: debug", isIntermittent: true) { + // _ = try await STFTests.test(input) + // } + // } } diff --git a/JAMTests/Tests/JAMTests/w3f/AccumulateTests.swift b/JAMTests/Tests/JAMTests/w3f/AccumulateTests.swift index 0d4f7c7a..110afbd4 100644 --- a/JAMTests/Tests/JAMTests/w3f/AccumulateTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/AccumulateTests.swift @@ -7,6 +7,8 @@ import Utils @testable import JAMTests +private let logger = Logger(label: "AccumulateTests") + private struct AccumulateInput: Codable { var timeslot: TimeslotIndex var reports: [WorkReport] @@ -17,8 +19,14 @@ private struct PreimageMapEntry: Codable, Equatable { var blob: Data } +private struct StorageMapEntry: Codable, Equatable { + var key: Data + var value: Data +} + private struct Account: Codable, Equatable { var service: ServiceAccountDetails + var storage: [StorageMapEntry] var preimages: [PreimageMapEntry] } @@ -44,8 +52,8 @@ private struct AccumulateState: Equatable, Codable { ProtocolConfig.EpochLength > var privilegedServices: PrivilegedServices - var accounts: [AccountsMapEntry] var serviceStatistics: [ServiceStatisticsMapEntry] + var accounts: [AccountsMapEntry] } private enum Output: Codable { @@ -127,6 +135,29 @@ private struct FullAccumulateState: Accumulation { } mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) { + // update footprint + let oldValue = storages[index]?[key] + let oldAccount = accounts[index] + if let oldValue { + if let value { + // replace: update byte count difference + accounts[index]?.totalByteLength = + max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count))) + (32 + UInt64(value.count)) + } else { + // remove: decrease count and bytes + accounts[index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 1) + accounts[index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count))) + } + } else { + if let value { + // add: increase count and bytes + accounts[index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 1 + accounts[index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 32 + UInt64(value.count) + } + } + logger.debug("storage footprint update: \(accounts[index]?.itemsCount ?? 0) items, \(accounts[index]?.totalByteLength ?? 0) bytes") + + // update value storages[index, default: [:]][key] = value } @@ -140,20 +171,41 @@ private struct FullAccumulateState: Accumulation { length _: UInt32, value: StateKeys.ServiceAccountPreimageInfoKey.Value? ) { + // update footprint + let oldValue = preimageInfo[index]?[hash] + let oldAccount = accounts[index] + if let oldValue { + if let value { + // replace: update byte count difference + accounts[index]?.totalByteLength = + max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count))) + (81 + UInt64(value.count)) + } else { + // remove: decrease count and bytes + accounts[index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 2) + accounts[index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count))) + } + } else { + if let value { + // add: increase count and bytes + accounts[index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 2 + accounts[index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 81 + UInt64(value.count) + } + } + logger.debug("preimage footprint update: \(accounts[index]?.itemsCount ?? 0) items, \(accounts[index]?.totalByteLength ?? 0) bytes") + + // update value preimageInfo[index, default: [:]][hash] = value } } struct AccumulateTests { - // init() { - // setupTestLogger() - // } - static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "accumulate/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/accumulate/\(variant)", extension: "bin") } func accumulateTests(_ testcase: Testcase, variant: TestVariants) async throws { + // setupTestLogger() + let config = variant.config let decoder = JamDecoder(data: testcase.data, config: config) let testcase = try decoder.decode(AccumulateTestcase.self) @@ -170,6 +222,10 @@ struct AccumulateTests { for entry in testcase.preState.accounts { fullState.accounts[entry.index] = entry.data.service + for storage in entry.data.storage { + let storageKey = Blake2b256.hash(entry.index.encode(), storage.key) + fullState.storages[entry.index, default: [:]][storageKey] = storage.value + } for preimage in entry.data.preimages { fullState.preimages[entry.index, default: [:]][preimage.hash] = preimage.blob } @@ -190,15 +246,18 @@ struct AccumulateTests { switch testcase.output { case let .ok(expectedRoot): // NOTE: timeslot and entropy are not changed by accumulate - #expect(content.root == expectedRoot, "root mismatch") + #expect(content.root == expectedRoot, "accumulate root mismatch") #expect(fullState.accumulationQueue == testcase.postState.accumulationQueue, "AccumulationQueue mismatch") #expect(fullState.accumulationHistory == testcase.postState.accumulationHistory, "AccumulationHistory mismatch") #expect(fullState.privilegedServices == testcase.postState.privilegedServices, "PrivilegedServices mismatch") - #expect(fullState.accounts.count == testcase.postState.accounts.count, "Accounts count mismatch") for entry in testcase.postState.accounts { let account = fullState.accounts[entry.index]! #expect(account == entry.data.service, "ServiceAccountDetail mismatch") + for storage in entry.data.storage { + let key = Blake2b256.hash(entry.index.encode(), storage.key) + #expect(fullState.storages[entry.index]?[key] == storage.value, "Storage mismatch") + } for preimage in entry.data.preimages { #expect(fullState.preimages[entry.index]?[preimage.hash] == preimage.blob, "Preimage mismatch") } @@ -219,15 +278,11 @@ struct AccumulateTests { @Test(arguments: try AccumulateTests.loadTests(variant: .tiny)) func tinyTests(_ testcase: Testcase) async throws { - await withKnownIssue("TODO: debug", isIntermittent: true) { - try await accumulateTests(testcase, variant: .tiny) - } + try await accumulateTests(testcase, variant: .tiny) } @Test(arguments: try AccumulateTests.loadTests(variant: .full)) func fullTests(_ testcase: Testcase) async throws { - await withKnownIssue("TODO: debug", isIntermittent: true) { - try await accumulateTests(testcase, variant: .full) - } + try await accumulateTests(testcase, variant: .full) } } diff --git a/JAMTests/Tests/JAMTests/w3f/AssurancesTests.swift b/JAMTests/Tests/JAMTests/w3f/AssurancesTests.swift index d30f718d..008f3df9 100644 --- a/JAMTests/Tests/JAMTests/w3f/AssurancesTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/AssurancesTests.swift @@ -27,7 +27,7 @@ struct AssurancesTestcase: Codable { struct AssurancesTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "assurances/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/assurances/\(variant)", extension: "bin") } func assurancesTests(_ testcase: Testcase, variant: TestVariants) throws { diff --git a/JAMTests/Tests/JAMTests/w3f/AuthorizationsTests.swift b/JAMTests/Tests/JAMTests/w3f/AuthorizationsTests.swift index 0c976571..f89134e2 100644 --- a/JAMTests/Tests/JAMTests/w3f/AuthorizationsTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/AuthorizationsTests.swift @@ -47,7 +47,7 @@ struct AuthorizationsTestcase: Codable { struct AuthorizationsTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "authorizations/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/authorizations/\(variant)", extension: "bin") } func authorizationsTests(_ testcase: Testcase, variant: TestVariants) throws { diff --git a/JAMTests/Tests/JAMTests/w3f/CodecTests.swift b/JAMTests/Tests/JAMTests/w3f/CodecTests.swift index 2ff4df4c..00b538cf 100644 --- a/JAMTests/Tests/JAMTests/w3f/CodecTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/CodecTests.swift @@ -7,20 +7,12 @@ import Utils @testable import JAMTests struct CodecTests { - static func config() -> ProtocolConfigRef { - var config = ProtocolConfigRef.mainnet.value - config.totalNumberOfValidators = 6 - config.epochLength = 12 - config.totalNumberOfCores = 2 - return Ref(config) - } - - static func test(_ type: (some Codable).Type, path: String) throws -> (JSON, JSON) { - let config = config() + static func test(_ type: (some Codable).Type, path: String, variant: TestVariants) throws -> (JSON, JSON) { + let config = variant.config - let jsonData = try TestLoader.getFile(path: "codec/data/\(path)", extension: "json") + let jsonData = try TestLoader.getFile(path: "codec/\(variant)/\(path)", extension: "json") let json = try JSONDecoder().decode(JSON.self, from: jsonData) - let bin = try TestLoader.getFile(path: "codec/data/\(path)", extension: "bin") + let bin = try TestLoader.getFile(path: "codec/\(variant)/\(path)", extension: "bin") let decoded = try JamDecoder.decode(type, from: bin, withConfig: config) let encoded = try JamEncoder.encode(decoded) @@ -37,6 +29,16 @@ struct CodecTests { return (transformed, json) } + func testBothVariants(_ type: (some Codable).Type, path: String) throws { + // Test with tiny variant + let (actualTiny, expectedTiny) = try Self.test(type, path: path, variant: .tiny) + #expect(actualTiny == expectedTiny) + + // Test with full variant + let (actualFull, expectedFull) = try Self.test(type, path: path, variant: .full) + #expect(actualFull == expectedFull) + } + static func transform(_ json: JSON, value: Any) -> JSON { if case Optional.none = value { return .null @@ -253,91 +255,76 @@ struct CodecTests { @Test func assurances_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicAvailability.self, path: "assurances_extrinsic") - #expect(actual == expected) + try testBothVariants(ExtrinsicAvailability.self, path: "assurances_extrinsic") } @Test func block() throws { - let (actual, expected) = try Self.test(Block.self, path: "block") - #expect(actual == expected) + try testBothVariants(Block.self, path: "block") } @Test func disputes_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicDisputes.self, path: "disputes_extrinsic") - #expect(actual == expected) + try testBothVariants(ExtrinsicDisputes.self, path: "disputes_extrinsic") } @Test func extrinsic() throws { - let (actual, expected) = try Self.test(Extrinsic.self, path: "extrinsic") - #expect(actual == expected) + try testBothVariants(Extrinsic.self, path: "extrinsic") } @Test func guarantees_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicGuarantees.self, path: "guarantees_extrinsic") - #expect(actual == expected) + try testBothVariants(ExtrinsicGuarantees.self, path: "guarantees_extrinsic") } @Test func header_0() throws { - let (actual, expected) = try Self.test(Header.self, path: "header_0") - #expect(actual == expected) + try testBothVariants(Header.self, path: "header_0") } @Test func header_1() throws { - let (actual, expected) = try Self.test(Header.self, path: "header_1") - #expect(actual == expected) + try testBothVariants(Header.self, path: "header_1") } @Test func preimages_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicPreimages.self, path: "preimages_extrinsic") - #expect(actual == expected) + try testBothVariants(ExtrinsicPreimages.self, path: "preimages_extrinsic") } @Test func refine_context() throws { - let (actual, expected) = try Self.test(RefinementContext.self, path: "refine_context") - #expect(actual == expected) + try testBothVariants(RefinementContext.self, path: "refine_context") } @Test func tickets_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicTickets.self, path: "tickets_extrinsic") - #expect(actual == expected) + try testBothVariants(ExtrinsicTickets.self, path: "tickets_extrinsic") } @Test func work_item() throws { - let (actual, expected) = try Self.test(WorkItem.self, path: "work_item") - #expect(actual == expected) + try testBothVariants(WorkItem.self, path: "work_item") } @Test func work_package() throws { - let (actual, expected) = try Self.test(WorkPackage.self, path: "work_package") - #expect(actual == expected) + try testBothVariants(WorkPackage.self, path: "work_package") } @Test func work_report() throws { - let (actual, expected) = try Self.test(WorkReport.self, path: "work_report") - #expect(actual == expected) + try testBothVariants(WorkReport.self, path: "work_report") } @Test func work_result_0() throws { - let (actual, expected) = try Self.test(WorkDigest.self, path: "work_result_0") - #expect(actual == expected) + try testBothVariants(WorkDigest.self, path: "work_result_0") } @Test func work_result_1() throws { - let (actual, expected) = try Self.test(WorkDigest.self, path: "work_result_1") - #expect(actual == expected) + try testBothVariants(WorkDigest.self, path: "work_result_1") } } diff --git a/JAMTests/Tests/JAMTests/w3f/DisputesTests.swift b/JAMTests/Tests/JAMTests/w3f/DisputesTests.swift index 16ca3d86..f8ee77ab 100644 --- a/JAMTests/Tests/JAMTests/w3f/DisputesTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/DisputesTests.swift @@ -35,7 +35,7 @@ struct DisputesTestcase: Codable { struct DisputesTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "disputes/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/disputes/\(variant)", extension: "bin") } func disputesTests(_ testcase: Testcase, variant: TestVariants) throws { diff --git a/JAMTests/Tests/JAMTests/w3f/PVMTests.swift b/JAMTests/Tests/JAMTests/w3f/PVMTests.swift index 55cf6b16..76529fcb 100644 --- a/JAMTests/Tests/JAMTests/w3f/PVMTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/PVMTests.swift @@ -117,6 +117,6 @@ struct PVMTests { #expect(value == byte) } } - // #expect(vmState.getGas() == testCase.expectedGas) + #expect(vmState.getGas().value == testCase.expectedGas.value) } } diff --git a/JAMTests/Tests/JAMTests/w3f/PreimagesTests.swift b/JAMTests/Tests/JAMTests/w3f/PreimagesTests.swift index a46f098d..c30a8fbb 100644 --- a/JAMTests/Tests/JAMTests/w3f/PreimagesTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/PreimagesTests.swift @@ -28,7 +28,6 @@ private struct AccountsMapEntry: Codable, Equatable { private struct PreimagesState: Equatable, Codable, Preimages { var accounts: [AccountsMapEntry] = [] - // NOTE: we are not using/updating stats in preimage stf, may need to check var serviceStatistics: [ServiceStatisticsMapEntry] func get(serviceAccount index: ServiceIndex, preimageHash hash: Data32) async throws -> Data? { @@ -79,8 +78,8 @@ private struct PreimagesTestcase: Codable { } struct PreimagesTests { - static func loadTests() throws -> [Testcase] { - try TestLoader.getTestcases(path: "preimages/data", extension: "bin") + static func loadTests(variant: TestVariants) throws -> [Testcase] { + try TestLoader.getTestcases(path: "stf/preimages/\(variant)", extension: "bin") } func preimagesTests(_ testcase: Testcase, variant: TestVariants) async throws { @@ -102,7 +101,7 @@ struct PreimagesTests { switch testcase.output { case .none: state.mergeWith(postState: postState) - // NOTE: we are not using/updating stats in preimage stf, may need to check + // NOTE: we are not updating stats in preimage stf, so not checking this state.serviceStatistics = testcase.postState.serviceStatistics #expect(state == testcase.postState) case .some: @@ -119,8 +118,13 @@ struct PreimagesTests { } } - @Test(arguments: try PreimagesTests.loadTests()) - func tests(_ testcase: Testcase) async throws { + @Test(arguments: try PreimagesTests.loadTests(variant: .tiny)) + func tinyTests(_ testcase: Testcase) async throws { + try await preimagesTests(testcase, variant: .tiny) + } + + @Test(arguments: try PreimagesTests.loadTests(variant: .full)) + func fullTests(_ testcase: Testcase) async throws { try await preimagesTests(testcase, variant: .full) } } diff --git a/JAMTests/Tests/JAMTests/w3f/RecentHistoryTests.swift b/JAMTests/Tests/JAMTests/w3f/RecentHistoryTests.swift index 9f17fcbe..efdb4351 100644 --- a/JAMTests/Tests/JAMTests/w3f/RecentHistoryTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/RecentHistoryTests.swift @@ -25,13 +25,12 @@ struct RecentHistoryTestcase: Codable { } struct RecentHistoryTests { - static func loadTests() throws -> [Testcase] { - try TestLoader.getTestcases(path: "history/data", extension: "bin") + static func loadTests(variant: TestVariants) throws -> [Testcase] { + try TestLoader.getTestcases(path: "stf/history/\(variant)", extension: "bin") } - @Test(arguments: try loadTests()) - func recentHistory(_ testcase: Testcase) throws { - let config = ProtocolConfigRef.mainnet + func recentHistory(_ testcase: Testcase, variant: TestVariants) throws { + let config = variant.config let testcase = try JamDecoder.decode(RecentHistoryTestcase.self, from: testcase.data, withConfig: config) var state = testcase.preState @@ -47,4 +46,14 @@ struct RecentHistoryTests { #expect(state == testcase.postState) } + + @Test(arguments: try RecentHistoryTests.loadTests(variant: .tiny)) + func tinyTests(_ testcase: Testcase) throws { + try recentHistory(testcase, variant: .tiny) + } + + @Test(arguments: try RecentHistoryTests.loadTests(variant: .full)) + func fullTests(_ testcase: Testcase) throws { + try recentHistory(testcase, variant: .full) + } } diff --git a/JAMTests/Tests/JAMTests/w3f/ReportsTests.swift b/JAMTests/Tests/JAMTests/w3f/ReportsTests.swift index a0f965b4..dc5d61b9 100644 --- a/JAMTests/Tests/JAMTests/w3f/ReportsTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/ReportsTests.swift @@ -20,7 +20,6 @@ struct ReportsTestcaseState: Codable, Equatable { ProtocolConfig.TotalNumberOfCores > @CodingAs> var services: [ServiceIndex: ServiceAccountDetails] - // NOTE: we are not updating stats in guaranteeing STF var coresStatistics: ConfigFixedSizeArray @CodingAs> var servicesStatistics: [ServiceIndex: Statistics.Service] } @@ -73,7 +72,7 @@ struct ReportsTestcase: Codable { struct ReportsTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "reports/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/reports/\(variant)", extension: "bin") } func reportsTests(_ testcase: Testcase, variant: TestVariants) throws { @@ -119,7 +118,7 @@ struct ReportsTests { recentHistory: state.recentHistory, coreAuthorizationPool: state.coreAuthorizationPool, services: state.services, - // NOTE: just use testcase postState since we don't udpate stats in guaranteeing STF + // NOTE: just use testcase postState since we don't update stats in guaranteeing STF coresStatistics: testcase.postState.coresStatistics, servicesStatistics: testcase.postState.servicesStatistics ) diff --git a/JAMTests/Tests/JAMTests/w3f/SafroleTests.swift b/JAMTests/Tests/JAMTests/w3f/SafroleTests.swift index 7bb358f2..b91aa53a 100644 --- a/JAMTests/Tests/JAMTests/w3f/SafroleTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/SafroleTests.swift @@ -86,7 +86,7 @@ struct SafroleTestcase: Codable { struct SafroleTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "safrole/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/safrole/\(variant)", extension: "bin") } func safroleTests(_ input: Testcase, variant: TestVariants) throws { diff --git a/JAMTests/Tests/JAMTests/w3f/StatisticsTests.swift b/JAMTests/Tests/JAMTests/w3f/StatisticsTests.swift index 408c5125..809d60e8 100644 --- a/JAMTests/Tests/JAMTests/w3f/StatisticsTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/StatisticsTests.swift @@ -6,36 +6,52 @@ import Utils @testable import JAMTests -struct StatisticsState: Equatable, Codable, ActivityStatistics { - var activityStatistics: Statistics +// NOTE: the statistics tests only test the validator stats +struct TestStatsState: Equatable, Codable { + var current: ConfigFixedSizeArray + var previous: ConfigFixedSizeArray var timeslot: TimeslotIndex var currentValidators: ConfigFixedSizeArray } -struct StatisticsInput: Codable { +struct StatsInput: Codable { var timeslot: TimeslotIndex var author: ValidatorIndex var extrinsic: Extrinsic } -struct StatisticsTestcase: Codable { - var input: StatisticsInput - var preState: StatisticsState - var postState: StatisticsState +struct StatsTestcase: Codable { + var input: StatsInput + var preState: TestStatsState + var postState: TestStatsState +} + +struct StatsState: ActivityStatistics { + var activityStatistics: Statistics + var timeslot: TimeslotIndex + var currentValidators: ConfigFixedSizeArray } struct StatisticsTests { static func loadTests(variant: TestVariants) throws -> [Testcase] { - try TestLoader.getTestcases(path: "statistics/\(variant)", extension: "bin") + try TestLoader.getTestcases(path: "stf/statistics/\(variant)", extension: "bin") } func statisticsTests(_ testcase: Testcase, variant: TestVariants) throws { let config = variant.config let decoder = JamDecoder(data: testcase.data, config: config) - let testcase = try decoder.decode(StatisticsTestcase.self) + let testcase = try decoder.decode(StatsTestcase.self) - var state = testcase.preState - let result = try state.update( + var testStatsState = testcase.preState + var activityStatistics = Statistics.dummy(config: config) + activityStatistics.accumulator = testStatsState.current + activityStatistics.previous = testStatsState.previous + let fullStatsState = StatsState( + activityStatistics: activityStatistics, + timeslot: testStatsState.timeslot, + currentValidators: testStatsState.currentValidators + ) + let result = try fullStatsState.update( config: config, newTimeslot: testcase.input.timeslot, extrinsic: testcase.input.extrinsic, @@ -44,22 +60,19 @@ struct StatisticsTests { accumulateStats: [:], transfersStats: [:] ) - state.activityStatistics = result + testStatsState.current = result.accumulator + testStatsState.previous = result.previous - #expect(state == testcase.postState) + #expect(testStatsState == testcase.postState) } @Test(arguments: try StatisticsTests.loadTests(variant: .tiny)) func tinyTests(_ testcase: Testcase) throws { - withKnownIssue("https://github.com/w3f/jamtestvectors/pull/28#issuecomment-2762410106", isIntermittent: true) { - try statisticsTests(testcase, variant: .tiny) - } + try statisticsTests(testcase, variant: .tiny) } @Test(arguments: try StatisticsTests.loadTests(variant: .full)) func fullTests(_ testcase: Testcase) throws { - withKnownIssue("https://github.com/w3f/jamtestvectors/pull/28#issuecomment-2762410106", isIntermittent: true) { - try statisticsTests(testcase, variant: .full) - } + try statisticsTests(testcase, variant: .full) } } diff --git a/JAMTests/Tests/JAMTests/w3f/TrieTests.swift b/JAMTests/Tests/JAMTests/w3f/TrieTests.swift index ae20ffd5..6e71202d 100644 --- a/JAMTests/Tests/JAMTests/w3f/TrieTests.swift +++ b/JAMTests/Tests/JAMTests/w3f/TrieTests.swift @@ -22,10 +22,11 @@ struct TrieTests { let decoder = JSONDecoder() let testcase = try decoder.decode(TrieTestCase.self, from: testcase.data) for element in testcase { - let kv = element.input.reduce(into: [Data31: Data]()) { _, entry in + let kv = element.input.reduce(into: [Data31: Data]()) { result, entry in let keyData = Data(fromHexString: entry.key) let valueData = Data(fromHexString: entry.value) // result[Data31(keyData!)!] = valueData + result[Data31()] = valueData } let result = try stateMerklize(kv: kv) diff --git a/JAMTests/jamduna b/JAMTests/jamduna index f1e3e7b0..d5ba406f 160000 --- a/JAMTests/jamduna +++ b/JAMTests/jamduna @@ -1 +1 @@ -Subproject commit f1e3e7b0a88cac9d176fa742411ba2c0b37f4a76 +Subproject commit d5ba406f98e7b6b7b36453a891cd48a96b265345 diff --git a/JAMTests/jamixir b/JAMTests/jamixir deleted file mode 160000 index 57e796e2..00000000 --- a/JAMTests/jamixir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57e796e2d3f155d9846188155f2dbda94d52dc8d diff --git a/JAMTests/jamtestvectors b/JAMTests/jamtestvectors index bb1b6851..2ff14efd 160000 --- a/JAMTests/jamtestvectors +++ b/JAMTests/jamtestvectors @@ -1 +1 @@ -Subproject commit bb1b6851fcb1ee38a09300f7c5edb20d72433ba8 +Subproject commit 2ff14efd614ce54dec7ae0ca73e3a232f78c9d4d diff --git a/JAMTests/javajam b/JAMTests/javajam index 2be7fa79..23f9c97e 160000 --- a/JAMTests/javajam +++ b/JAMTests/javajam @@ -1 +1 @@ -Subproject commit 2be7fa79bc8e4ea74a838145d39e6d5354cbde7e +Subproject commit 23f9c97e4d06bf6041492d739951ee1e30191300 diff --git a/PolkaVM/Sources/PolkaVM/Engine.swift b/PolkaVM/Sources/PolkaVM/Engine.swift index 839efb7c..03941a60 100644 --- a/PolkaVM/Sources/PolkaVM/Engine.swift +++ b/PolkaVM/Sources/PolkaVM/Engine.swift @@ -2,11 +2,12 @@ import Foundation import TracingUtils import Utils -private let logger = Logger(label: "Engine ") +private let logger = Logger(label: "Engine") public class Engine { let config: PvmConfig let invocationContext: (any InvocationContext)? + private var stepCounter: Int = 0 public init(config: PvmConfig, invocationContext: (any InvocationContext)? = nil) { self.config = config @@ -59,10 +60,8 @@ public class Engine { func step(program: ProgramCode, context: ExecutionContext) -> ExecOutcome { let pc = context.state.pc let skip = program.skip(pc) - logger.trace("======== pc(\(pc)) skip(\(skip)) =========") let inst = program.getInstructionAt(pc: pc) - guard let inst else { return .exit(.panic(.invalidInstructionIndex)) } @@ -72,8 +71,24 @@ public class Engine { context.state.consumeGas(blockGas) } - logger.trace("exec \(inst)") + logStep(pc: pc, instruction: inst, context: context) return inst.execute(context: context, skip: skip) } + + private func logStep(pc: UInt32, instruction: any Instruction, context: ExecutionContext) { + stepCounter += 1 + + let gas = context.state.getGas() + let regArray = (0 ..< 13).map { context.state.readRegister(Registers.Index(raw: $0)) as UInt64 } + let instructionName = getInstructionName(instruction).padding(toLength: 20, withPad: " ", startingAt: 0) + + logger.trace("\(String(format: "%4d", stepCounter)): PC \(String(format: "%6d", pc)) \(instructionName) g=\(gas) reg=\(regArray)") + } + + private func getInstructionName(_ inst: any Instruction) -> String { + let typeName = String(describing: type(of: inst)) + let cleanName = typeName.replacingOccurrences(of: "Instructions.", with: "") + return cleanName.replacingOccurrences(of: "([a-z])([A-Z])", with: "$1_$2", options: .regularExpression).uppercased() + } } diff --git a/PolkaVM/Sources/PolkaVM/Instruction.swift b/PolkaVM/Sources/PolkaVM/Instruction.swift index 418effa3..b85bb10f 100644 --- a/PolkaVM/Sources/PolkaVM/Instruction.swift +++ b/PolkaVM/Sources/PolkaVM/Instruction.swift @@ -47,7 +47,7 @@ extension Instruction { } public func gasCost() -> Gas { - Gas(0) + Gas(1) } public func updatePC(context: ExecutionContext, skip: UInt32) -> ExecOutcome { diff --git a/PolkaVM/Sources/PolkaVM/InstructionTable.swift b/PolkaVM/Sources/PolkaVM/InstructionTable.swift index 20fab50e..04ff14e3 100644 --- a/PolkaVM/Sources/PolkaVM/InstructionTable.swift +++ b/PolkaVM/Sources/PolkaVM/InstructionTable.swift @@ -155,17 +155,17 @@ public class InstructionTable { }() public static func parse(_ data: Data) -> (any Instruction)? { - logger.trace("parsing \(data)") + // logger.trace("parsing \(data)") guard data.count >= 1 else { return nil } let opcode = data[data.startIndex] - logger.trace("parsed opcode: \(opcode)") + // logger.trace("parsed opcode: \(opcode)") guard let instType = table[Int(opcode)] else { return nil } - logger.trace("initializing \(instType)") + // logger.trace("initializing \(instType)") return try? instType.init(data: data[relative: 1...]) } } diff --git a/PolkaVM/Sources/PolkaVM/Instructions/BranchInstructionBase.swift b/PolkaVM/Sources/PolkaVM/Instructions/BranchInstructionBase.swift index 62052d00..132cb620 100644 --- a/PolkaVM/Sources/PolkaVM/Instructions/BranchInstructionBase.swift +++ b/PolkaVM/Sources/PolkaVM/Instructions/BranchInstructionBase.swift @@ -1,7 +1,7 @@ import Foundation import TracingUtils -private let logger = Logger(label: "Branch ") +private let logger = Logger(label: "Branch") protocol Branch: Instruction { var offset: UInt32 { get } diff --git a/PolkaVM/Sources/PolkaVM/Instructions/Instructions+Helpers.swift b/PolkaVM/Sources/PolkaVM/Instructions/Instructions+Helpers.swift index 99bad5cf..f4fbc116 100644 --- a/PolkaVM/Sources/PolkaVM/Instructions/Instructions+Helpers.swift +++ b/PolkaVM/Sources/PolkaVM/Instructions/Instructions+Helpers.swift @@ -1,7 +1,7 @@ import Foundation import TracingUtils -private let logger = Logger(label: "Insts ") +private let logger = Logger(label: "Insts ") extension Instructions { enum Constants { diff --git a/PolkaVM/Sources/PolkaVM/Instructions/Instructions.swift b/PolkaVM/Sources/PolkaVM/Instructions/Instructions.swift index f32502c4..857589b1 100644 --- a/PolkaVM/Sources/PolkaVM/Instructions/Instructions.swift +++ b/PolkaVM/Sources/PolkaVM/Instructions/Instructions.swift @@ -2,7 +2,7 @@ import Foundation import TracingUtils import Utils -private let logger = Logger(label: "Insts ") +private let logger = Logger(label: "Insts ") public let BASIC_BLOCK_INSTRUCTIONS: Set = [ Instructions.Trap.opcode, diff --git a/PolkaVM/Sources/PolkaVM/VMStateInterpreter.swift b/PolkaVM/Sources/PolkaVM/VMStateInterpreter.swift index f6092e81..50934ad0 100644 --- a/PolkaVM/Sources/PolkaVM/VMStateInterpreter.swift +++ b/PolkaVM/Sources/PolkaVM/VMStateInterpreter.swift @@ -1,9 +1,6 @@ import Foundation -import TracingUtils import Utils -private let logger = Logger(label: "VMState ") - public class VMStateInterpreter: VMState { public let program: ProgramCode @@ -68,14 +65,12 @@ public class VMStateInterpreter: VMState { public func readMemory(address: some FixedWidthInteger) throws -> UInt8 { try validateAddress(address) let res = try memory.read(address: UInt32(truncatingIfNeeded: address)) - logger.trace("read \(address) (\(res))") return res } public func readMemory(address: some FixedWidthInteger, length: Int) throws -> Data { try validateAddress(address) let res = try memory.read(address: UInt32(truncatingIfNeeded: address), length: length) - logger.trace("read \(address)..+\(length) (\(res))") return res } @@ -85,14 +80,14 @@ public class VMStateInterpreter: VMState { public func writeMemory(address: some FixedWidthInteger, value: UInt8) throws { try validateAddress(address) - logger.trace("write \(address) (\(value))") try memory.write(address: UInt32(truncatingIfNeeded: address), value: value) } public func writeMemory(address: some FixedWidthInteger, values: some Sequence) throws { + let data = Data(values) + guard !data.isEmpty else { return } try validateAddress(address) - logger.trace("write \(address) (\(values))") - try memory.write(address: UInt32(truncatingIfNeeded: address), values: Data(values)) + try memory.write(address: UInt32(truncatingIfNeeded: address), values: data) } public func sbrk(_ increment: UInt32) throws -> UInt32 { @@ -101,38 +96,31 @@ public class VMStateInterpreter: VMState { public func consumeGas(_ amount: Gas) { gas -= GasInt(amount) - logger.trace("gas - \(amount) => \(gas)") } public func increasePC(_ amount: UInt32) { // using wrapped add // so that it can also be used for jumps which are negative pc &+= amount - logger.trace("pc &+ \(amount) => \(pc)") } public func updatePC(_ newPC: UInt32) { pc = newPC - logger.trace("pc => \(pc)") } public func readRegister(_ index: Registers.Index) -> T { - logger.trace("read w\(index.value) (\(registers[index]))") - return T(truncatingIfNeeded: registers[index]) + T(truncatingIfNeeded: registers[index]) } public func readRegister(_ index: Registers.Index, _ index2: Registers.Index) -> (T, T) { - logger.trace("read w\(index.value) (\(registers[index])) w\(index2.value) (\(registers[index2]))") - return (T(truncatingIfNeeded: registers[index]), T(truncatingIfNeeded: registers[index2])) + (T(truncatingIfNeeded: registers[index]), T(truncatingIfNeeded: registers[index2])) } public func readRegisters(in range: Range) -> [T] { - _ = range.map { logger.trace("read w\($0) (\(T(truncatingIfNeeded: registers[Registers.Index(raw: $0)])))") } - return range.map { T(truncatingIfNeeded: registers[Registers.Index(raw: $0)]) } + range.map { T(truncatingIfNeeded: registers[Registers.Index(raw: $0)]) } } public func writeRegister(_ index: Registers.Index, _ value: some FixedWidthInteger) { - logger.trace("write w\(index.value) (\(value))") registers[index] = UInt64(truncatingIfNeeded: value) } diff --git a/Utils/Sources/Utils/SaturatingNumber.swift b/Utils/Sources/Utils/SaturatingNumber.swift index 4b3147e1..19e03540 100644 --- a/Utils/Sources/Utils/SaturatingNumber.swift +++ b/Utils/Sources/Utils/SaturatingNumber.swift @@ -145,3 +145,19 @@ extension SaturatingNumber: EncodedSize { MemoryLayout.size } } + +extension SaturatingNumber: CompactEncodable { + public func toUInt() throws -> UInt { + guard let result = UInt(exactly: value) else { + throw CompactEncodingError.valueOutOfRange(value: "\(value)", sourceType: "\(T.self)", targetType: "UInt") + } + return result + } + + public static func fromUInt(_ value: UInt) throws -> SaturatingNumber { + guard let converted = T(exactly: value) else { + throw CompactEncodingError.valueOutOfRange(value: "\(value)", sourceType: "UInt", targetType: "\(T.self)") + } + return SaturatingNumber(converted) + } +}