V0.6.6 rc0

.github/workflows/ci.yml

@@ -17,6 +17,7 @@ jobs:
       matrix:
         validator:
           - codec
+          - traces
           - stf/accumulate
           - stf/assurances
           - stf/authorizations

README.md

@@ -61,3 +61,16 @@ specific subsystem.
 Validation scripts are included to verify the JSON files against the expected
 ASN.1 syntax provided with the test vectors. These scripts currently rely on my
 [asn1tools](https://github.com/davxy/asn1tools) fork.
+
+## Binary To JSON
+
+The repository binary test vector files can be converted to their JSON
+equivalents for easier inspection and debugging. Use the provided conversion
+script to transform binary files into human-readable JSON format:
+
+```bash
+./scripts/convert-all.sh
+```
+
+This conversion script requires the [jam-types](https://github.com/davxy/jam-types)
+Python library to be installed.

lib/bin_to_json.py

@@ -31,13 +31,16 @@ def decode(self):
         return decoded
 
 
-def convert_to_json(filename, subsystem_type):
+def convert_to_json(filename, subsystem_type, spec_name = None):
+    if spec_name in ("tiny", "full"):
+        spec.set_spec(spec_name)
+    print("* Converting:", filename)
     with open(filename, 'rb') as file:
         blob = file.read()
         scale_bytes = ScaleBytes(blob)
         dump = subsystem_type(data=scale_bytes)
         decoded = dump.decode()
-        json_filename = filename.replace('.bin', '.json')
+        json_filename = str(filename).replace('.bin', '.json')
         with open(json_filename, 'w') as json_file:
             json.dump(decoded, json_file, indent=4)
             json_file.write('\n')
@@ -47,5 +50,4 @@ def convert_group(group_name, spec_name, subsystem_type):
         spec.set_spec(spec_name)
     print(f"\n[Converting {group_name} ({spec_name})]")
     for file in glob.glob(f"{spec_name}/*.bin"):
-        print("* Converting:", file)
         convert_to_json(file, subsystem_type)

lib/validate_asn1.py

@@ -76,24 +76,30 @@ def path_to_type_name(path):
     return name
 
 
-def validate(schema, json_file, json_tweaks_callback=None):
+def validate(schema, json_file, root_type = None, json_tweaks_callback = None):
     """Validate a JSON file against an ASN.1 schema.
 
-    The root type for decoding is determined as follows:
-    - If the schema contains "TestCase", use that type
-    - Otherwise, derive the type from the filename (e.g., "my_type.json" → "MyType")
-
     Args:
         schema: Compiled ASN.1 schema
         json_file: Path to JSON file to validate
+        root_type_name: Optional schema type name to be used to parse the json_file
         json_tweaks_callback: Optional callback to modify JSON before validation
+
+    If the `root_type_name` arg is None or is not present in the schema, then the root
+    type for decoding is determined as follows:
+    - If the schema contains "TestCase", use that type
+    - Otherwise, derive the type from the filename (e.g., "my_type.json" → "MyType")
+
     """
+    print("* Validating:", json_file)
+
     # Determine root type used for decoding
-    if "TestCase" in schema.types:
-        root_type = "TestCase"
-    else:
-        # Auto-detect root type from filename
-        root_type = path_to_type_name(json_file)
+    if root_type is None:
+        if "TestCase" in schema.types:
+            root_type = "TestCase"
+        else:
+            # Auto-detect root type from filename
+            root_type = path_to_type_name(json_file)
 
     # Read and prepare JSON
     with open(json_file, "rb") as f:
@@ -121,7 +127,7 @@ def validate_group(group_name, group_schema, spec_name, json_tweaks_callback=Non
     Args:
         group_name: Name of the validation group (for display)
         group_schema: ASN.1 schema file name (or None for base schema only)
-        spec_name: Specification name ("tiny", "full", or "data")
+        spec_name: Specification name ("tiny", "full")
         json_tweaks_callback: Optional callback to modify JSON before validation
     """
     print(f"\n[Validating {group_name} ({spec_name})]")
@@ -134,5 +140,4 @@ def validate_group(group_name, group_schema, spec_name, json_tweaks_callback=Non
     # Compile schema and validate all JSON files
     schema = asn1tools.compile_files(schema_files, codec="jer")
     for json_file in glob.glob(f"{spec_name}/*.json"):
-        print("* Validating:", json_file)
-        validate(schema, json_file, json_tweaks_callback)
+        validate(schema, json_file, None, json_tweaks_callback)

scripts/convert-all.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Get the directory containing this script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Get the parent directory
+PARENT_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Check if jam_types python library can be imported
+if ! python3 -c "import jam_types" 2>/dev/null; then
+    echo "Error: We depend on jam-types. Please install the jam_types Python library." >&2
+    exit 1
+fi
+
+echo "Searching for convert.py scripts in: $PARENT_DIR"
+
+# Find all convert.py files and execute them
+find "$PARENT_DIR" -name "convert.py" -type f | while read -r convert_script; do
+    echo "Executing: $convert_script"
+    cd "$(dirname "$convert_script")"
+    python3 "$(basename "$convert_script")"
+done
+
+echo "All convert.py scripts have been executed."

scripts/validate-all.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Get the directory containing this script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Get the parent directory
+PARENT_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Check if jam_types python library can be imported
+if ! python3 -c "import asn1tools" 2>/dev/null; then
+    echo "Error: We depend on asn1tools. Please install the asn1tools Python library." >&2
+    exit 1
+fi
+
+echo "Searching for validate.py scripts in: $PARENT_DIR"
+
+# Find all validate.py files and execute them
+find "$PARENT_DIR" -name "validate.py" -type f | while read -r validate_script; do
+    echo "Executing: $validate_script"
+    cd "$(dirname "$validate_script")"
+    python3 "$(basename "$validate_script")"
+done
+
+echo "All validate.py scripts have been executed."

stf/history/README.md

@@ -25,11 +25,11 @@ the input `accumulate_root`.
 
 ## Vectors
 
-- [progress_blocks_history-1](data/progress_blocks_history-1.json) 🟢 
+- [progress_blocks_history-1](tiny/progress_blocks_history-1.json) 🟢 
   - Empty history queue.
-- [progress_blocks_history-2](data/progress_blocks_history-2.json) 🟢 
+- [progress_blocks_history-2](tiny/progress_blocks_history-2.json) 🟢 
   - Not empty nor full history queue.
-- [progress_blocks_history-3](data/progress_blocks_history-3.json) 🟢 
+- [progress_blocks_history-3](tiny/progress_blocks_history-3.json) 🟢 
   - Fill the history queue.
-- [progress_blocks_history-4](data/progress_blocks_history-4.json) 🟢 
+- [progress_blocks_history-4](tiny/progress_blocks_history-4.json) 🟢 
   - Shift the history queue.

stf/history/convert.py

@@ -11,4 +11,5 @@
 
 os.chdir(script_dir)
 
-convert_group("history", "data", HistoryTestVector)
+for spec in ["tiny", "full"]:
+    convert_group("history", spec, HistoryTestVector)

stf/history/tiny/progress_blocks_history-1.json

@@ -0,0 +1,44 @@
+{
+    "input": {
+        "header_hash": "0x530ef4636fedd498e99c7601581271894a53e965e901e8fa49581e525f165dae",
+        "parent_state_root": "0x0e6c6cbf80b5fb00175001f7b0966bf1af83ff4406ede84f29a666a0fcbac801",
+        "accumulate_root": "0x8720b97ddd6acc0f6eb66e095524038675a4e4067adc10ec39939eaefc47d842",
+        "work_packages": [
+            {
+                "hash": "0x016cb55eb7b84e0d495d40832c7238965baeb468932c415dc2ceffe0afb039e5",
+                "exports_root": "0x935f6dfef36fa06e10a9ba820f933611c05c06a207b07141fe8d87465870c11c"
+            },
+            {
+                "hash": "0x76bcb24901299c331f0ca7342f4874f19b213ee72df613d50699e7e25edb82a6",
+                "exports_root": "0xc825d16b7325ca90287123bd149d47843c999ce686ed51eaf8592dd2759272e3"
+            }
+        ]
+    },
+    "pre_state": {
+        "beta": []
+    },
+    "output": null,
+    "post_state": {
+        "beta": [
+            {
+                "header_hash": "0x530ef4636fedd498e99c7601581271894a53e965e901e8fa49581e525f165dae",
+                "mmr": {
+                    "peaks": [
+                        "0x8720b97ddd6acc0f6eb66e095524038675a4e4067adc10ec39939eaefc47d842"
+                    ]
+                },
+                "state_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
+                "reported": [
+                    {
+                        "hash": "0x016cb55eb7b84e0d495d40832c7238965baeb468932c415dc2ceffe0afb039e5",
+                        "exports_root": "0x935f6dfef36fa06e10a9ba820f933611c05c06a207b07141fe8d87465870c11c"
+                    },
+                    {
+                        "hash": "0x76bcb24901299c331f0ca7342f4874f19b213ee72df613d50699e7e25edb82a6",
+                        "exports_root": "0xc825d16b7325ca90287123bd149d47843c999ce686ed51eaf8592dd2759272e3"
+                    }
+                ]
+            }
+        ]
+    }
+}

stf/history/tiny/progress_blocks_history-2.json

@@ -0,0 +1,76 @@
+{
+    "input": {
+        "header_hash": "0x241d129c6edc2114e6dfba7d556f7f7c66399b55ceec3078a53d44c752ba7e9a",
+        "parent_state_root": "0x1831dde64e40bfd8639c2d122e5ac00fe133c48cd16e1621ca6d5cf0b8e10d3b",
+        "accumulate_root": "0x7507515a48439dc58bc318c48a120b656136699f42bfd2bd45473becba53462d",
+        "work_packages": [
+            {
+                "hash": "0x3cc8d8c94e7b3ee01e678c63fd6b5db894fc807dff7fe10a11ab41e70194894d",
+                "exports_root": "0xc0edfe377d20b9f4ed7d9df9511ef904c87e24467364f0f7f75f20cfe90dd8fb"
+            }
+        ]
+    },
+    "pre_state": {
+        "beta": [
+            {
+                "header_hash": "0x530ef4636fedd498e99c7601581271894a53e965e901e8fa49581e525f165dae",
+                "mmr": {
+                    "peaks": [
+                        "0x8720b97ddd6acc0f6eb66e095524038675a4e4067adc10ec39939eaefc47d842"
+                    ]
+                },
+                "state_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
+                "reported": [
+                    {
+                        "hash": "0x016cb55eb7b84e0d495d40832c7238965baeb468932c415dc2ceffe0afb039e5",
+                        "exports_root": "0x935f6dfef36fa06e10a9ba820f933611c05c06a207b07141fe8d87465870c11c"
+                    },
+                    {
+                        "hash": "0x76bcb24901299c331f0ca7342f4874f19b213ee72df613d50699e7e25edb82a6",
+                        "exports_root": "0xc825d16b7325ca90287123bd149d47843c999ce686ed51eaf8592dd2759272e3"
+                    }
+                ]
+            }
+        ]
+    },
+    "output": null,
+    "post_state": {
+        "beta": [
+            {
+                "header_hash": "0x530ef4636fedd498e99c7601581271894a53e965e901e8fa49581e525f165dae",
+                "mmr": {
+                    "peaks": [
+                        "0x8720b97ddd6acc0f6eb66e095524038675a4e4067adc10ec39939eaefc47d842"
+                    ]
+                },
+                "state_root": "0x1831dde64e40bfd8639c2d122e5ac00fe133c48cd16e1621ca6d5cf0b8e10d3b",
+                "reported": [
+                    {
+                        "hash": "0x016cb55eb7b84e0d495d40832c7238965baeb468932c415dc2ceffe0afb039e5",
+                        "exports_root": "0x935f6dfef36fa06e10a9ba820f933611c05c06a207b07141fe8d87465870c11c"
+                    },
+                    {
+                        "hash": "0x76bcb24901299c331f0ca7342f4874f19b213ee72df613d50699e7e25edb82a6",
+                        "exports_root": "0xc825d16b7325ca90287123bd149d47843c999ce686ed51eaf8592dd2759272e3"
+                    }
+                ]
+            },
+            {
+                "header_hash": "0x241d129c6edc2114e6dfba7d556f7f7c66399b55ceec3078a53d44c752ba7e9a",
+                "mmr": {
+                    "peaks": [
+                        null,
+                        "0x7076c31882a5953e097aef8378969945e72807c4705e53a0c5aacc9176f0d56b"
+                    ]
+                },
+                "state_root": "0x0000000000000000000000000000000000000000000000000000000000000000",
+                "reported": [
+                    {
+                        "hash": "0x3cc8d8c94e7b3ee01e678c63fd6b5db894fc807dff7fe10a11ab41e70194894d",
+                        "exports_root": "0xc0edfe377d20b9f4ed7d9df9511ef904c87e24467364f0f7f75f20cfe90dd8fb"
+                    }
+                ]
+            }
+        ]
+    }
+}