From 007945ad92a8ba8c99a589820c248a0539783a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81kos=20Hervay?= Date: Thu, 17 Apr 2025 03:12:01 +0200 Subject: [PATCH] Add support for preserving scientific notation for floats Introduced mechanisms to handle and preserve scientific notation formats across parsing, transformation, and reconstruction processes. Added tests for various float representations to ensure functionality, including edge cases with scientific notation and negative decimals. --- hcl2/api.py | 4 + hcl2/builder.py | 14 +++ hcl2/reconstructor.py | 89 ++++++++++++++++++++ hcl2/transformer.py | 21 ++++- test/helpers/terraform-config/test_floats.tf | 27 ++++++ test/unit/test_complex_floats.py | 73 ++++++++++++++++ 6 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 test/helpers/terraform-config/test_floats.tf create mode 100644 test/unit/test_complex_floats.py diff --git a/hcl2/api.py b/hcl2/api.py index 399ba929..a7db8956 100644 --- a/hcl2/api.py +++ b/hcl2/api.py @@ -29,6 +29,10 @@ def loads(text: str, with_meta=False) -> dict: tree = parser().parse(text + "\n") return DictTransformer(with_meta=with_meta).transform(tree) +def loads_preserving_format(text: str, with_meta=False) -> dict: + """Load HCL2 from a string, preserving the format of values like scientific notation.""" + tree = parser().parse(text + "\n") + return DictTransformer(with_meta=with_meta, preserve_format=True).transform(tree) def parse(file: TextIO) -> Tree: """Load HCL2 syntax tree from a file. diff --git a/hcl2/builder.py b/hcl2/builder.py index b5b149da..ff1476b9 100644 --- a/hcl2/builder.py +++ b/hcl2/builder.py @@ -84,3 +84,17 @@ def _add_nested_blocks( block[key] = [] block[key].extend(value) return block + + def sci_float(self, value, original_format=None): + """ + Creates a special float representation that preserves scientific notation + format information. + """ + if original_format is None: + original_format = str(value) + + return { + "__sci_float__": True, + "value": value, + "format": original_format + } diff --git a/hcl2/reconstructor.py b/hcl2/reconstructor.py index bf653c05..c71adafc 100644 --- a/hcl2/reconstructor.py +++ b/hcl2/reconstructor.py @@ -253,6 +253,9 @@ def _should_add_space(self, rule, current_terminal): r"^__(tuple|arguments)_(star|plus)_.*", self._last_rule ): + if self._last_terminal == Terminal("COMMA"): + return True + # string literals, decimals, and identifiers should always be # preceded by a space if they're following a comma in a tuple or # function arg @@ -260,9 +263,16 @@ def _should_add_space(self, rule, current_terminal): Terminal("STRING_LIT"), Terminal("DECIMAL"), Terminal("NAME"), + Terminal("NEGATIVE_DECIMAL") ]: return True + if self._last_terminal == Terminal("COMMA") and ( + current_terminal == Terminal("NEGATIVE_DECIMAL") or + current_terminal == Terminal("DECIMAL") + ): + return True + # the catch-all case, we're not sure, so don't add a space return False @@ -548,6 +558,49 @@ def _transform_value_to_expr_term(self, value, level) -> Union[Token, Tree]: [Tree(Token("RULE", "identifier"), [Token("NAME", "null")])], ) + # Special handling for scientific notation metadata + if isinstance(value, dict) and value.get("__sci_float__") is True: + # Extract the format string from metadata + format_str = value.get("format", str(value.get("value"))) + + # Check if it's scientific notation + if 'e' in format_str.lower(): + # Parse the scientific notation format + base_part, exp_part = format_str.lower().split('e') + + # Handle the base part + int_part, dec_part = base_part.split('.') if '.' in base_part else (base_part, "") + + # Create tokens + tokens = [] + + # Handle negative sign if present + is_negative = int_part.startswith('-') + if is_negative: + int_part = int_part[1:] + tokens.append(Token("NEGATIVE_DECIMAL", "-" + int_part[0])) + for digit in int_part[1:]: + tokens.append(Token("DECIMAL", digit)) + else: + for digit in int_part: + tokens.append(Token("DECIMAL", digit)) + + # Add decimal part if exists + if dec_part: + tokens.append(Token("DOT", ".")) + for digit in dec_part: + tokens.append(Token("DECIMAL", digit)) + + # Use the sign from the original format + exp_sign = "+" if "+" in exp_part else "-" if "-" in exp_part else "" + exp_digits = exp_part.lstrip("+-") + tokens.append(Token("EXP_MARK", f"e{exp_sign}{exp_digits}")) + + return Tree( + Token("RULE", "expr_term"), + [Tree(Token("RULE", "float_lit"), tokens)] + ) + # for dicts, recursively turn the child k/v pairs into object elements # and store within an object if isinstance(value, dict): @@ -599,6 +652,42 @@ def _transform_value_to_expr_term(self, value, level) -> Union[Token, Tree]: ], ) + if isinstance(value, float): + str_value = str(value) + + # For regular floats (no scientific notation) + # Split into integer and decimal parts + if '.' in str_value: + int_part, dec_part = str_value.split('.') + else: + int_part, dec_part = str_value, "" + + # Check if negative + is_negative = int_part.startswith('-') + if is_negative: + int_part = int_part[1:] # Remove negative sign for processing + + tokens = [] + + # Handle integer part based on negative flag + if is_negative: + tokens.append(Token("NEGATIVE_DECIMAL", "-" + int_part[0])) + for digit in int_part[1:]: + tokens.append(Token("DECIMAL", digit)) + else: + for digit in int_part: + tokens.append(Token("DECIMAL", digit)) + + # Add decimal part + tokens.append(Token("DOT", ".")) + for digit in dec_part: + tokens.append(Token("DECIMAL", digit)) + + return Tree( + Token("RULE", "expr_term"), + [Tree(Token("RULE", "float_lit"), tokens)] + ) + # store integers as literals, digit by digit if isinstance(value, int): return Tree( diff --git a/hcl2/transformer.py b/hcl2/transformer.py index 7c7e4bd8..517caebf 100644 --- a/hcl2/transformer.py +++ b/hcl2/transformer.py @@ -3,7 +3,7 @@ import re import sys from collections import namedtuple -from typing import List, Dict, Any +from typing import List, Dict, Any, Union from lark import Token from lark.tree import Meta @@ -35,16 +35,29 @@ class DictTransformer(Transformer): def is_type_keyword(value: str) -> bool: return value in {"bool", "number", "string"} - def __init__(self, with_meta: bool = False): + def __init__(self, with_meta: bool = False, preserve_format: bool = False): """ :param with_meta: If set to true then adds `__start_line__` and `__end_line__` parameters to the output dict. Default to false. + :param preserve_format: If set to true, preserves formatting of special values + like scientific notation. Default to false. """ self.with_meta = with_meta + self.preserve_format = preserve_format super().__init__() - def float_lit(self, args: List) -> float: - return float("".join([self.to_tf_inline(arg) for arg in args])) + def float_lit(self, args: List) -> Union[float, Dict]: + original_string = "".join([self.to_tf_inline(arg) for arg in args]) + float_value = float(original_string) + + # Check if it's in scientific notation + if 'e' in original_string.lower() and self.preserve_format: + return { + "__sci_float__": True, + "value": float_value, + "format": original_string + } + return float_value def int_lit(self, args: List) -> int: return int("".join([self.to_tf_inline(arg) for arg in args])) diff --git a/test/helpers/terraform-config/test_floats.tf b/test/helpers/terraform-config/test_floats.tf new file mode 100644 index 00000000..bc14b94b --- /dev/null +++ b/test/helpers/terraform-config/test_floats.tf @@ -0,0 +1,27 @@ +resource "test_resource" "float_examples" { + simple_float = 123.456 + small_float = 0.123 + large_float = 9876543.21 + negative_float = -42.5 + negative_small = -0.001 + scientific_positive = 1.23e5 + scientific_negative = 9.87e-3 + scientific_large = 6.022e+23 + integer_as_float = 100.0 + float_calculation = 10.5 * 3.0 / 2.1 + float_comparison = 5.6 > 2.3 ? 1.0 : 0.0 + float_list = [1.1, 2.2, 3.3, -4.4, 5.5e2] + float_object = { + pi = 3.14159 + euler = 2.71828 + sqrt2 = 1.41421 + } +} + +variable "float_variable" { + default = 3.14159 +} + +output "float_output" { + value = var.float_variable * 2.0 +} diff --git a/test/unit/test_complex_floats.py b/test/unit/test_complex_floats.py new file mode 100644 index 00000000..f20b526a --- /dev/null +++ b/test/unit/test_complex_floats.py @@ -0,0 +1,73 @@ +"""Test building HCL files with complex float values""" + +from pathlib import Path +from unittest import TestCase + +import hcl2 +import hcl2.builder + + +HELPERS_DIR = Path(__file__).absolute().parent.parent / "helpers" +HCL2_DIR = HELPERS_DIR / "terraform-config" +JSON_DIR = HELPERS_DIR / "terraform-config-json" +HCL2_FILES = [str(file.relative_to(HCL2_DIR)) for file in HCL2_DIR.iterdir()] + + +class TestComplexFloats(TestCase): + """Test building hcl files with various float representations""" + + # print any differences fully to the console + maxDiff = None + + def test_builder_with_complex_floats(self): + builder = hcl2.Builder() + + builder.block( + "resource", + ["test_resource", "float_examples"], + simple_float = 123.456, + small_float = 0.123, + large_float = 9876543.21, + negative_float = -42.5, + negative_small = -0.001, + scientific_positive = builder.sci_float(1.23e5, "1.23e5"), + scientific_negative = builder.sci_float(9.87e-3, "9.87e-3"), + scientific_large = builder.sci_float(6.022e+23, "6.022e+23"), + integer_as_float= 100.0, + float_calculation = "${10.5 * 3.0 / 2.1}", + float_comparison = "${5.6 > 2.3 ? 1.0 : 0.0}", + float_list = [1.1, 2.2, 3.3, -4.4, builder.sci_float(5.5e2, "5.5e2")], + float_object = { + "pi": 3.14159, + "euler": 2.71828, + "sqrt2": 1.41421 + } + ) + + builder.block( + "variable", + ["float_variable"], + default=3.14159, + ) + + builder.block( + "output", + ["float_output"], + value="${var.float_variable * 2.0}", + ) + + self.compare_filenames(builder, "test_floats.tf") + + def compare_filenames(self, builder: hcl2.Builder, filename: str): + hcl_dict = builder.build() + hcl_ast = hcl2.reverse_transform(hcl_dict) + hcl_content_built = hcl2.writes(hcl_ast) + + hcl_path = (HCL2_DIR / filename).absolute() + with hcl_path.open("r") as hcl_file: + hcl_file_content = hcl_file.read() + self.assertMultiLineEqual( + hcl_content_built, + hcl_file_content, + f"file {filename} does not match its programmatically built version.", + )