Merge pull request #45 from hit9/dev-intx

[working in progress] Support signed integers with arbitrary bits (e.g. int24)

Author: hit9

.gitignore

@@ -12,3 +12,4 @@ dist
 .mypy_cache
 spam
 .DS_Store
+compile_flags.txt

changes.rst

@@ -1,5 +1,12 @@
 .. currentmodule:: bitproto
 
+.. _version-0.4.6:
+
+Version 0.4.6
+-------------
+
+- Support signed integers with arbitrary bits, e.g. int24 PR#45.
+
 .. _version-0.4.5:
 
 Version 0.4.5

compiler/bitproto/__init__.py

@@ -8,5 +8,5 @@
 
 """
 
-__version__ = "0.4.5"
+__version__ = "0.4.6"
 __description__ = "bit level data interchange format."

compiler/bitproto/_ast.py

@@ -42,11 +42,20 @@
 """
 
 from collections import OrderedDict as dict_
-from dataclasses import dataclass
-from dataclasses import field as dataclass_field
-from typing import Any, Callable, ClassVar, Dict, List, Optional, Tuple
-from typing import Type as T
-from typing import TypeVar, Union, cast
+from dataclasses import dataclass, field as dataclass_field
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type as T,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from bitproto.errors import (
     DuplicatedDefinition,
@@ -70,8 +79,8 @@
     PROTO_OPTTIONS,
     OptionDescriptor,
     OptionDescriptors,
+    Validator as OptionValidator,
 )
-from bitproto.options import Validator as OptionValidator
 from bitproto.utils import (
     cache,
     conditional_cache,
@@ -706,21 +715,21 @@ def __repr__(self) -> str:
 @frozen
 @dataclass
 class Int(Integer):
-    """Int is the signed Integer.
-    Different from uint, int type only supports capacity: 8, 16, 32, 64.
-    """
+    """Int is the signed Integer."""
 
     cap: int = 0
 
-    @override(Node)
-    def validate_post_freeze(self) -> None:
-        if self.cap not in (8, 16, 32, 64):
-            raise InvalidIntCap.from_token(token=self)
-
     @override(Type)
     def nbits(self) -> int:
         return self.cap
 
+    @override(Node)
+    def validate_post_freeze(self) -> None:
+        if self._is_missing:
+            return
+        if not (0 < self.cap <= 64):
+            raise InvalidIntCap.from_token(token=self)
+
     def __repr__(self) -> str:
         return "<type int{0}>".format(self.cap)
 

compiler/bitproto/errors.py

@@ -16,9 +16,7 @@
 """
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, ClassVar, Optional
-from typing import Type as T
-from typing import TypeVar
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Type as T, TypeVar
 
 from bitproto.utils import Color, colored, overridable, override, write_stderr
 
@@ -150,7 +148,7 @@ class InvalidUintCap(LexerError):
 
 @dataclass
 class InvalidIntCap(LexerError):
-    """Invalid bits capacity for a int type, should be one of 8,16,32,64."""
+    """Invalid bits capacity for a int type, should between [1, 64]."""
 
 
 @dataclass

compiler/bitproto/lexer.py

@@ -6,7 +6,6 @@
 """
 
 from contextlib import contextmanager
-from dataclasses import dataclass
 from typing import Dict, Iterator, List, Optional, Tuple
 
 from ply import lex  # type: ignore
@@ -153,9 +152,6 @@ def t_UINT_TYPE(self, t: LexToken) -> LexToken:
 
     def t_INT_TYPE(self, t: LexToken) -> LexToken:
         r"\bint[0-9]+\b"
-        # Why use the regexp `int[0-9]+` instead of `int(8|16|32|64)`?
-        # We want the unsupported cases like `int0`, `int3` to raise an error instead of
-        # lexing into identifiers.
         cap: int = int(t.value[3:])
         t.value = Int(
             cap=cap, token=t.value, lineno=t.lineno, filepath=self.current_filepath()

compiler/bitproto/linter.py

@@ -7,9 +7,7 @@
 
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Callable, Generic, List, Optional, Tuple
-from typing import Type as T
-from typing import TypeVar
+from typing import Callable, Generic, List, Optional, Tuple, Type as T, TypeVar
 
 from bitproto._ast import (
     Alias,

compiler/bitproto/parser.py

@@ -7,21 +7,15 @@
 
 import os
 from contextlib import contextmanager
-from dataclasses import dataclass
-from dataclasses import field as dataclass_field
-from typing import Iterator, List, Optional, Tuple
-from typing import Type as T
-from typing import cast
+from typing import Iterator, List, Optional, Tuple, Type as T, cast
 
 from ply import yacc  # type: ignore
 from ply.lex import LexToken  # type: ignore
-from ply.yacc import LRParser as PlyParser  # type: ignore
-from ply.yacc import YaccProduction as P  # type: ignore
+from ply.yacc import LRParser as PlyParser, YaccProduction as P  # type: ignore
 
 from bitproto._ast import (
     Alias,
     Array,
-    BooleanConstant,
     Comment,
     Constant,
     Definition,
@@ -33,7 +27,6 @@
     Option,
     Proto,
     Scope,
-    StringConstant,
     Type,
 )
 from bitproto.errors import (

compiler/bitproto/renderer/__init__.py

@@ -10,7 +10,6 @@
 from bitproto._ast import Proto
 from bitproto.errors import UnsupportedLanguageToRender
 from bitproto.renderer.impls import renderer_registry
-from bitproto.renderer.renderer import Renderer
 
 
 def render(

compiler/bitproto/renderer/block.py

@@ -22,25 +22,22 @@
 from abc import abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Generic, Iterator, List, Optional
-from typing import Type as T
-from typing import TypeVar, Union, cast
+from typing import Generic, Iterator, List, Optional, Union
 
 from bitproto._ast import (
     Alias,
     BoundDefinition,
     Comment,
     Constant,
     D,
-    Definition,
     Enum,
     EnumField,
     Message,
     MessageField,
     Proto,
 )
 from bitproto.errors import InternalError
-from bitproto.renderer.formatter import F, Formatter
+from bitproto.renderer.formatter import F
 from bitproto.utils import (
     cached_property,
     final,
@@ -503,7 +500,7 @@ def render(self) -> None:
         self.after()
 
     @abstractmethod
-    def wraps(self) -> Block[F]:
+    def wraps(self) -> Optional[Block[F]]:
         """Returns the wrapped block instance."""
         raise NotImplementedError
 

compiler/bitproto/renderer/formatter.py

@@ -6,11 +6,18 @@
 """
 import os
 from abc import abstractmethod
-from enum import Enum as Enum_
-from enum import unique
-from typing import Callable, Dict, List, Optional, Tuple
-from typing import Type as T
-from typing import TypeVar, Union, cast
+from enum import Enum as Enum_, unique
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type as T,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from bitproto._ast import (
     Alias,
@@ -259,6 +266,21 @@ def definition_name_prefix_option_name(self) -> str:
         """
         return ""
 
+    @overridable
+    def post_format_op_mode_endecode_single_type(
+        self, t: Type, chain: str, is_encode: bool
+    ) -> List[str]:
+        """
+        Overridable hook function that would be called every time after a single type's encoding or decoding
+        code is generated in the optimization mode. The lines returned by this function would be generated
+        right follow the field' encoding (or decoding) code.
+
+        :param t: The type of this value.
+        :param chain: The chained name in this message's encode/decode function.
+        :param is_encode: Is generating code for encoding or decoding now.
+        """
+        return []
+
     ###########
     # Finals
     ###########
@@ -634,6 +656,9 @@ def format_op_mode_endecode_single_type(
             # Maintains the j and i counter
             j += c
             i[0] += c
+
+        # hook function
+        l.extend(self.post_format_op_mode_endecode_single_type(t, chain, is_encode))
         return l
 
     @final

compiler/bitproto/renderer/impls/__init__.py

@@ -2,8 +2,7 @@
 Renderer implementations.
 """
 
-from typing import Dict, Tuple
-from typing import Type as T
+from typing import Dict, Tuple, Type as T
 
 from bitproto.renderer.renderer import Renderer
 

compiler/bitproto/renderer/impls/c/__init__.py

@@ -1,2 +1,4 @@
 from .renderer_c import RendererC
 from .renderer_h import RendererCHeader
+
+__all__ = ("RendererC", "RendererCHeader")

compiler/bitproto/renderer/impls/c/formatter.py

@@ -2,7 +2,7 @@
 C formatter.
 """
 
-from typing import Optional
+from typing import List, Optional
 
 from bitproto._ast import (
     Alias,
@@ -17,7 +17,6 @@
     Message,
     MessageField,
     Proto,
-    SingleType,
     Type,
     Uint,
 )
@@ -311,7 +310,7 @@ def format_op_mode_encoder_item(
     def format_op_mode_decoder_item(
         self, chain: str, t: Type, si: int, fi: int, shift: int, mask: int, r: int
     ) -> str:
-        """Implements format_op_mode_encoder_item for C.
+        """Implements format_op_mode_decoder_item for C.
         Generated C statement like:
 
             ((unsigned char *)&((*m).color))[0] = (s[0] << 3) & 7;
@@ -320,3 +319,55 @@ def format_op_mode_decoder_item(
         assign = "=" if r == 0 else "|="
         shift_s = self.format_op_mode_smart_shift(shift)
         return f"((unsigned char *)&({chain}))[{fi}] {assign} (s[{si}] {shift_s}) & {mask};"
+
+    @override(Formatter)
+    def post_format_op_mode_endecode_single_type(
+        self, t: Type, chain: str, is_encode: bool
+    ) -> List[str]:
+        """
+        Hook function called after a message field is generated.
+        """
+        if isinstance(t, Alias):
+            t = t.type
+        if isinstance(t, Int):
+            return self.post_format_op_mode_endecode_int(t, chain, is_encode)
+        return []
+
+    def post_format_op_mode_endecode_int(
+        self, t: Int, chain: str, is_encode: bool
+    ) -> List[str]:
+        """
+        Process signed integers during decoding code generation in optimization mode.
+
+        Generated C statement example:
+
+            if (((*m).pressure_sensor.pressure >> 23) & 1) (*m).pressure_sensor.pressure |= -16777216;
+        """
+        if is_encode:
+            return []
+
+        # Signed integers processing is only about decoding
+        n = t.nbits()
+
+        if n in {8, 16, 32, 64}:
+            # No need to do additional actions
+            # int8/16/32/64 signed integers' sign bit is already on the highest bit position.
+            return []
+
+        m = ~((1 << n) - 1)
+        mask = f"{m}"
+
+        if n == 63:
+            # Avoid this compiler error for mask == -9223372036854775808;
+            # warning: integer literal is too large to be represented in a signed integer type,
+            # interpreting as unsigned [-Wimplicitly-unsigned-literal]
+            mask = f"(-9223372036854775807 - 1)"
+
+        # For a signed integer e.g. 16777205, the most concise approach should be: 16777205 << 24 >> 24,
+        # this shifts the 24th bit to the highest bit position, and then shift right again.
+        # By doing an arithmetic right shifting, the leftmost sign bit got propagated, the result is -11.
+        # This way is concise, but may not be portable.
+        # Right shift behavior on negative signed integers is implementation-defined.
+        # Another safe approach is: test the 24th bit at first ((16777205 >> 23) & 1), if it’s 1,
+        # then do a OR operation with a mask,  16777205 |= ~(1<<24 -1) , the result is also -11.
+        return [f"if (({chain} >> {n-1}) & 1) {chain} |= {mask};"]