integrate and pass latest tests

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "JAMTests/jamtestvectors"]
 	path = JAMTests/jamtestvectors
 	url = https://github.com/open-web3-stack/jamtestvectors.git
-[submodule "JAMTests/jamixir"]
-	path = JAMTests/jamixir
-	url = https://github.com/jamixir/jamtestnet.git
 [submodule "JAMTests/javajam"]
 	path = JAMTests/javajam
 	url = https://github.com/javajamio/javajam-trace.git

Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift

@@ -727,20 +727,20 @@ extension ProtocolConfig {
             UInt16(recentHistorySize),
             UInt16(maxWorkItems),
             UInt16(maxDepsInWorkReport),
+            UInt16(maxTicketsPerExtrinsic),
             UInt32(maxLookupAnchorAge),
+            UInt16(ticketEntriesPerValidator),
             UInt16(maxAuthorizationsPoolItems),
             UInt16(slotPeriodSeconds),
             UInt16(maxAuthorizationsQueueItems),
             UInt16(coreAssignmentRotationPeriod),
-            UInt16(maxAccumulationQueueItems),
             UInt16(maxWorkPackageExtrinsics),
             UInt16(preimageReplacementPeriod),
             UInt16(totalNumberOfValidators),
             UInt32(maxIsAuthorizedCodeSize),
             UInt32(maxEncodedWorkPackageSize),
             UInt32(maxServiceCodeSize),
             UInt32(erasureCodedPieceSize),
-            UInt32(segmentSize),
             UInt32(maxWorkPackageImports),
             UInt32(erasureCodedSegmentSize),
             UInt32(maxWorkReportBlobSize),

Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift

@@ -1,3 +1,4 @@
+import Codec
 import Foundation
 import Utils
 
@@ -9,14 +10,14 @@ public struct OperandTuple: Codable {
     public var segmentRoot: Data32
     /// a
     public var authorizerHash: Data32
-    /// o
-    public var authorizerTrace: Data
     /// y
     public var payloadHash: Data32
     /// g
-    public var gasLimit: Gas
+    @CodingAs<Compact<Gas>> public var gasLimit: Gas
     /// d
     public var workResult: WorkResult
+    /// o
+    public var authorizerTrace: Data
 }
 
 public struct DeferredTransfers: Codable {

Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift

@@ -112,7 +112,7 @@ public struct AccountChanges {
             throw .duplicatedNewService
         }
         guard altered.isDisjoint(with: other.altered) else {
-            logger.debug("altered accounts have duplicates, self: \(altered), other: \(other.altered)")
+            logger.debug("same service being altered in parallel, self: \(altered), other: \(other.altered)")
             throw .duplicatedContributionToService
         }
         guard removed.isDisjoint(with: other.removed) else {
@@ -170,10 +170,10 @@ extension Accumulation {
                     packageHash: report.packageSpecification.workPackageHash,
                     segmentRoot: report.packageSpecification.segmentRoot,
                     authorizerHash: report.authorizerHash,
-                    authorizerTrace: report.authorizerTrace,
                     payloadHash: digest.payloadHash,
                     gasLimit: digest.gasLimit,
                     workResult: digest.result,
+                    authorizerTrace: report.authorizerTrace,
                 ))
             }
         }
@@ -234,7 +234,7 @@ extension Accumulation {
         var accountsRef = ServiceAccountsMutRef(state.accounts.value)
         var servicePreimageSet = Set<ServicePreimagePair>()
 
-        for service in services {
+        for service in Set(services) {
             let singleOutput = try await singleAccumulate(
                 config: config,
                 state: AccumulateState(
@@ -527,7 +527,7 @@ extension Accumulation {
             transfersStats[service] = (count, gasUsed)
         }
 
-        self = accountsMutRef.value as! Self
+        self = accumulateOutput.state.accounts.value as! Self
 
         // update accumulation history
         let accumulated = accumulatableReports[0 ..< accumulateOutput.numAccumulated]

Blockchain/Sources/Blockchain/State/ServiceAccounts.swift

@@ -41,23 +41,17 @@ public class ServiceAccountsMutRef {
 
     public func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails) {
         ref.value.set(serviceAccount: index, account: account)
-        changes.addAlteration(index: index) { accounts in
-            accounts.set(serviceAccount: index, account: account)
-        }
+        changes.addAlteration(index: index) { $0.set(serviceAccount: index, account: account) }
     }
 
     public func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) {
         ref.value.set(serviceAccount: index, storageKey: key, value: value)
-        changes.addAlteration(index: index) { accounts in
-            accounts.set(serviceAccount: index, storageKey: key, value: value)
-        }
+        changes.addAlteration(index: index) { $0.set(serviceAccount: index, storageKey: key, value: value) }
     }
 
     public func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data?) {
         ref.value.set(serviceAccount: index, preimageHash: hash, value: value)
-        changes.addAlteration(index: index) { accounts in
-            accounts.set(serviceAccount: index, preimageHash: hash, value: value)
-        }
+        changes.addAlteration(index: index) { $0.set(serviceAccount: index, preimageHash: hash, value: value) }
     }
 
     public func set(
@@ -67,9 +61,7 @@ public class ServiceAccountsMutRef {
         value: LimitedSizeArray<TimeslotIndex, ConstInt0, ConstInt3>?
     ) {
         ref.value.set(serviceAccount: index, preimageHash: hash, length: length, value: value)
-        changes.addAlteration(index: index) { accounts in
-            accounts.set(serviceAccount: index, preimageHash: hash, length: length, value: value)
-        }
+        changes.addAlteration(index: index) { $0.set(serviceAccount: index, preimageHash: hash, length: length, value: value) }
     }
 
     public func addNew(serviceAccount index: ServiceIndex, account: ServiceAccount) {

Blockchain/Sources/Blockchain/State/State.swift

@@ -390,6 +390,28 @@ extension State: ServiceAccounts {
     }
 
     public mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) {
+        // update footprint
+        let oldValue = layer[serviceAccount: index, storageKey: key]
+        let oldAccount = layer[serviceAccount: index]
+        if let oldValue {
+            if let value {
+                // replace: update byte count difference
+                layer[serviceAccount: index]?.totalByteLength =
+                    max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count))) + (32 + UInt64(value.count))
+            } else {
+                // remove: decrease count and bytes
+                layer[serviceAccount: index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 1)
+                layer[serviceAccount: index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (32 + UInt64(oldValue.count)))
+            }
+        } else {
+            if let value {
+                // add: increase count and bytes
+                layer[serviceAccount: index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 1
+                layer[serviceAccount: index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 32 + UInt64(value.count)
+            }
+        }
+
+        // update value
         layer[serviceAccount: index, storageKey: key] = value
     }
 
@@ -403,6 +425,28 @@ extension State: ServiceAccounts {
         length: UInt32,
         value: StateKeys.ServiceAccountPreimageInfoKey.Value?
     ) {
+        // update footprint
+        let oldValue = layer[serviceAccount: index, preimageHash: hash, length: length]
+        let oldAccount = layer[serviceAccount: index]
+        if let oldValue {
+            if let value {
+                // replace: update byte count difference
+                layer[serviceAccount: index]?.totalByteLength =
+                    max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count))) + (81 + UInt64(value.count))
+            } else {
+                // remove: decrease count and bytes
+                layer[serviceAccount: index]?.itemsCount = max(0, (oldAccount?.itemsCount ?? 0) - 2)
+                layer[serviceAccount: index]?.totalByteLength = max(0, (oldAccount?.totalByteLength ?? 0) - (81 + UInt64(oldValue.count)))
+            }
+        } else {
+            if let value {
+                // add: increase count and bytes
+                layer[serviceAccount: index]?.itemsCount = (oldAccount?.itemsCount ?? 0) + 2
+                layer[serviceAccount: index]?.totalByteLength = (oldAccount?.totalByteLength ?? 0) + 81 + UInt64(value.count)
+            }
+        }
+
+        // update value
         layer[serviceAccount: index, preimageHash: hash, length: length] = value
     }
 }

Blockchain/Sources/Blockchain/Types/CodeAndMeta.swift

@@ -1,6 +1,9 @@
 import Codec
+import TracingUtils
 import Utils
 
+private let logger = Logger(label: "CodeAndMeta")
+
 /// account preimage data is: meta length + meta + code
 public struct CodeAndMeta: Sendable, Equatable {
     public enum Error: Swift.Error {
@@ -15,6 +18,7 @@ public struct CodeAndMeta: Sendable, Equatable {
         let metaLength = slice.decode()
         guard let metaLength else { throw Error.invalidMetadataLength }
         metadata = data[slice.startIndex ..< slice.startIndex + Int(metaLength)]
+        logger.debug("Metadata: \(String(data: metadata, encoding: .utf8) ?? "nil")")
         codeBlob = data[slice.startIndex + Int(metaLength) ..< slice.endIndex]
     }
 }

Blockchain/Sources/Blockchain/Types/PrivilegedServices.swift

@@ -9,12 +9,36 @@ public struct PrivilegedServices: Sendable, Equatable, Codable {
     // v
     public var designate: ServiceIndex
     // g
-    @CodingAs<SortedKeyValues<ServiceIndex, Gas>> public var basicGas: [ServiceIndex: Gas]
+    public var basicGas: [ServiceIndex: Gas]
 
     public init(blessed: ServiceIndex, assign: ServiceIndex, designate: ServiceIndex, basicGas: [ServiceIndex: Gas]) {
         self.blessed = blessed
         self.assign = assign
         self.designate = designate
         self.basicGas = basicGas
     }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        blessed = try container.decode(ServiceIndex.self, forKey: .blessed)
+        assign = try container.decode(ServiceIndex.self, forKey: .assign)
+        designate = try container.decode(ServiceIndex.self, forKey: .designate)
+
+        let compactGas = try container.decode(SortedKeyValues<ServiceIndex, Compact<Gas>>.self, forKey: .basicGas)
+        basicGas = compactGas.alias.mapValues { $0.alias }
+    }
+
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encode(blessed, forKey: .blessed)
+        try container.encode(assign, forKey: .assign)
+        try container.encode(designate, forKey: .designate)
+
+        let compactGas = SortedKeyValues(alias: basicGas.mapValues { Compact(alias: $0) })
+        try container.encode(compactGas, forKey: .basicGas)
+    }
+
+    private enum CodingKeys: String, CodingKey {
+        case blessed, assign, designate, basicGas
+    }
 }

Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift

@@ -63,15 +63,15 @@ public class Fetch: HostCall {
     }
 
     private func getWorkItemMeta(item: WorkItem) throws -> Data {
-        let encoder = JamEncoder(capacity: 4 + 4 + 8 + 8 + 2 + 2 + 2 + 4)
+        let encoder = JamEncoder(capacity: 4 + 32 + 8 + 8 + 2 + 2 + 2 + 4)
         try encoder.encode(item.serviceIndex)
         try encoder.encode(item.codeHash)
         try encoder.encode(item.refineGasLimit)
         try encoder.encode(item.accumulateGasLimit)
         try encoder.encode(item.exportsCount)
-        try encoder.encode(item.inputs.count)
-        try encoder.encode(item.outputs.count)
-        try encoder.encode(item.payloadBlob.count)
+        try encoder.encode(UInt16(item.inputs.count))
+        try encoder.encode(UInt16(item.outputs.count))
+        try encoder.encode(UInt32(item.payloadBlob.count))
         return encoder.data
     }
 
@@ -177,8 +177,12 @@ public class Fetch: HostCall {
         let first = min(Int(reg8), value?.count ?? 0)
         let len = min(Int(reg9), (value?.count ?? 0) - first)
 
+        logger.debug("writeAddr: \(writeAddr), first: \(first), len: \(len)")
+
         let isWritable = state.isMemoryWritable(address: writeAddr, length: len)
 
+        logger.debug("isWritable: \(isWritable), value: \(value?.toDebugHexString() ?? "nil")")
+
         if !isWritable {
             throw VMInvocationsError.panic
         } else if value == nil {

Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift

@@ -50,7 +50,7 @@ public func accumulate(
         )
     )
     let ctx = AccumulateContext(context: contextContent, config: config, timeslot: timeslot, operands: arguments)
-    let argument = try JamEncoder.encode(timeslot, serviceIndex, arguments.count)
+    let argument = try JamEncoder.encode(UInt(timeslot), UInt(serviceIndex), UInt(arguments.count))
 
     let (exitReason, gas, output) = await invokePVM(
         config: config,

Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift

@@ -8,7 +8,7 @@ public func isAuthorized(
     package: WorkPackage,
     coreIndex: CoreIndex
 ) async throws -> (Result<Data, WorkResultError>, Gas) {
-    let args = try JamEncoder.encode(package, coreIndex)
+    let args = try JamEncoder.encode(coreIndex)
     let codeBlob = try await package.authorizationCode(serviceAccounts: serviceAccounts)
     guard let codeBlob else {
         return (.failure(.invalidCode), Gas(0))

Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift

@@ -31,7 +31,7 @@ public func onTransfer(
     let contextContent = OnTransferContext.ContextType(serviceIndex: serviceIndex, accounts: serviceAccounts)
     let ctx = OnTransferContext(context: contextContent, config: config, entropy: entropy, transfers: transfers)
     let gasLimitSum = transfers.reduce(Balance(0)) { $0 + $1.gasLimit }
-    let argument = try JamEncoder.encode(timeslot, serviceIndex, transfers.count)
+    let argument = try JamEncoder.encode(UInt(timeslot), UInt(serviceIndex), UInt(transfers.count))
 
     let (_, gas, _) = await invokePVM(
         config: config,

Blockchain/Sources/Blockchain/VMInvocations/Invocations/RefineInvocation.swift

@@ -37,8 +37,8 @@ public func refine(
     let codeBlob = try CodeAndMeta(data: preimage).codeBlob
 
     let argumentData = try JamEncoder.encode(
-        workItemIndex,
-        service,
+        UInt(workItemIndex),
+        UInt(service),
         workItem.payloadBlob,
         workPackage.hash(),
     )