refactor(texture): texture loading is now blazingly fast

Author: danila-schelkov

.pre-commit-config.yaml

@@ -1,20 +1,18 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 6.0.1
     hooks:
       - id: isort
         args: ['--profile','black']
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-      - id: black
-        language_version: python3.11
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.261'
+    rev: v0.11.6
     hooks:
+      # Run the linter.
       - id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format

xcoder/features/place_sprites.py

@@ -79,7 +79,7 @@ def place_sprites(
             bbox = int(rect.left), int(rect.top), int(rect.right), int(rect.bottom)
 
             region_image = Image.open(
-                f'{folder}{"/overwrite" if overwrite else ""}/{filename}'
+                f"{folder}{'/overwrite' if overwrite else ''}/{filename}"
             ).convert("RGBA")
 
             sheets[region_info.texture_id].paste(

xcoder/features/sc/__init__.py

@@ -42,7 +42,13 @@ def compile_sc(
         file_size = width * height * pixel_size + 5
 
         logger.info(
-            locale.about_sc % (file_info.name, picture_index, pixel_type, width, height)
+            locale.about_sc.format(
+                filename=file_info.name,
+                index=picture_index,
+                pixel_type=pixel_type,
+                width=width,
+                height=height,
+            )
         )
 
         sc.write(struct.pack("<BIBHH", file_type, file_size, pixel_type, width, height))

xcoder/features/update/check.py

@@ -27,7 +27,7 @@ def get_run_output(command: str):
 
 def get_pip_info(outdated: bool = False) -> list:
     output = get_run_output(
-        f'pip --disable-pip-version-check list {"-o" if outdated else ""}'
+        f"pip --disable-pip-version-check list {'-o' if outdated else ''}"
     )
     output = output.splitlines()
     output = output[2:]

xcoder/images.py

@@ -11,51 +11,61 @@
 from .localization import locale
 from .math.point import Point
 from .matrices import Matrix2x3
-from .pixel_utils import (
-    get_channel_count_by_pixel_type,
-    get_pixel_encode_function,
-    get_read_function,
-)
+from .pixel_utils import get_pixel_encode_function, get_raw_mode
 
 CHUNK_SIZE = 32
 
 
-def load_image_from_buffer(pixel_type: int, width: int, height: int) -> Image.Image:
-    with open("pixel_buffer", "rb") as pixel_buffer:
-        pixel_buffer.read(1)
-
-        return Image.frombuffer(
-            get_format_by_pixel_type(pixel_type), (width, height), pixel_buffer.read()
-        )
+def load_image_from_buffer(
+    pixel_type: int, width: int, height: int, pixel_buffer: Reader
+) -> Image.Image:
+    raw_mode = get_raw_mode(pixel_type)
+    bytes_per_pixel = get_byte_count_by_pixel_type(pixel_type)
+
+    return Image.frombuffer(
+        get_format_by_pixel_type(pixel_type),
+        (width, height),
+        pixel_buffer.read(width * height * bytes_per_pixel),
+        "raw",
+        raw_mode,
+        0,
+        1,
+    )
 
 
-def join_image(pixel_type: int, width: int, height: int) -> Image.Image:
+def join_image(
+    pixel_type: int, width: int, height: int, pixel_buffer: Reader
+) -> Image.Image:
     mode = get_format_by_pixel_type(pixel_type)
+    bytes_per_pixel = get_byte_count_by_pixel_type(pixel_type)
     image = Image.new(mode, (width, height))
 
-    with open("pixel_buffer", "rb") as pixel_buffer:
-        channel_count = int.from_bytes(pixel_buffer.read(1), "little")
+    chunk_count_x = math.ceil(width / CHUNK_SIZE)
+    chunk_count_y = math.ceil(height / CHUNK_SIZE)
+    chunk_count = chunk_count_x * chunk_count_y
 
-        chunk_count_x = math.ceil(width / CHUNK_SIZE)
-        chunk_count_y = math.ceil(height / CHUNK_SIZE)
-        chunk_count = chunk_count_x * chunk_count_y
+    raw_mode = get_raw_mode(pixel_type)
 
-        for chunk_index in range(chunk_count):
-            chunk_x = chunk_index % chunk_count_x
-            chunk_y = chunk_index // chunk_count_x
+    for chunk_index in range(chunk_count):
+        chunk_x = chunk_index % chunk_count_x
+        chunk_y = chunk_index // chunk_count_x
 
-            chunk_width = min(width - chunk_x * CHUNK_SIZE, CHUNK_SIZE)
-            chunk_height = min(height - chunk_y * CHUNK_SIZE, CHUNK_SIZE)
+        chunk_width = min(width - chunk_x * CHUNK_SIZE, CHUNK_SIZE)
+        chunk_height = min(height - chunk_y * CHUNK_SIZE, CHUNK_SIZE)
 
-            sub_image = Image.frombuffer(
-                mode,
-                (chunk_width, chunk_height),
-                pixel_buffer.read(channel_count * chunk_width * chunk_height),
-            )
+        sub_image = Image.frombuffer(
+            mode,
+            (chunk_width, chunk_height),
+            pixel_buffer.read(bytes_per_pixel * chunk_width * chunk_height),
+            "raw",
+            raw_mode,
+            0,
+            1,
+        )
 
-            image.paste(sub_image, (chunk_x * CHUNK_SIZE, chunk_y * CHUNK_SIZE))
+        image.paste(sub_image, (chunk_x * CHUNK_SIZE, chunk_y * CHUNK_SIZE))
 
-            Console.progress_bar(locale.join_pic, chunk_index, chunk_count)
+        Console.progress_bar(locale.join_pic, chunk_index, chunk_count)
 
     return image
 
@@ -125,23 +135,6 @@ def get_format_by_pixel_type(pixel_type: int) -> str:
     raise Exception(locale.unknown_pixel_type % pixel_type)
 
 
-def load_texture(reader: Reader, pixel_type: int, width: int, height: int) -> None:
-    channel_count = get_channel_count_by_pixel_type(pixel_type)
-    read_pixel = get_read_function(pixel_type)
-    if read_pixel is None:
-        raise Exception(locale.unknown_pixel_type % pixel_type)
-
-    with open("pixel_buffer", "wb") as pixel_buffer:
-        pixel_buffer.write(channel_count.to_bytes(1, "little"))
-
-        for y in range(height):
-            pixel_buffer.write(
-                b"".join([bytearray(read_pixel(reader)) for _ in range(width)])
-            )
-
-            Console.progress_bar(locale.crt_pic, y, height)
-
-
 def save_texture(writer: Writer, image: Image.Image, pixel_type: int) -> None:
     encode_pixel = get_pixel_encode_function(pixel_type)
     if encode_pixel is None:

xcoder/languages/en-EU.json

@@ -54,7 +54,7 @@
 
     "not_latest": "Not the latest version installed",
     "collecting_inf": "File: %s. Collecting information...",
-    "about_sc": "About texture. Filename: %s (%d), Pixel type: %d, Size: %sx%s",
+    "about_sc": "About texture. Filename: {filename} ({index}), Pixel type: {pixel_type}, Size: {width}x{height}",
     "decompression_error": "Error while decompressing! Trying to decode as is...",
     "skip_not_installed": "%s isn't installed! Reinitialize",
     "detected_comp": "%s compression detected",

xcoder/languages/ua-UA.json

@@ -54,7 +54,7 @@
 
     "not_latest": "Встановлена не остання версія",
     "collecting_inf": "Файл: %s. Збираємо інформацію...",
-    "about_sc": "Про текстуру. Назва файлу: %s (%d), Тип пікселів: %d, Величина: %sx%s",
+    "about_sc": "Про текстуру. Назва файлу: {filename} ({index}), Тип пікселів: {pixel_type}, Величина: {width}x{height}",
     "decompression_error": "Помилка при розпакуванні! Намагаємся розпакувати як є...",
     "skip_not_installed": "%s не встановлений, повторіть налаштування",
     "detected_comp": "Виявлено компресію: %s",

xcoder/languages/zh-CN.json

@@ -54,7 +54,7 @@
 
   "not_latest": "安装的不是最新版本",
   "collecting_inf": "文件：%s。正在收集信息...",
-  "about_sc": "关于纹理。文件名：%s (%d)，像素类型：%d，尺寸：%sx%s",
+  "about_sc": "关于纹理。文件名：{filename} ({index})，像素类型：{pixel_type}，尺寸：{width}x{height}",
   "decompression_error": "解压时出错！尝试直接解码...",
   "skip_not_installed": "%s 未安装！请重新初始化",
   "detected_comp": "检测到 %s 压缩",

xcoder/main_menu.py

@@ -23,7 +23,7 @@ def check_auto_update():
 
 def check_files_updated():
     if config.has_update:
-        logger.opt(colors=True).info(f'<green>{locale.update_done % ""}</green>')
+        logger.opt(colors=True).info(f"<green>{locale.update_done % ''}</green>")
         if Console.question(locale.done_qu):
             latest_tag = get_tags(config.repo_owner, config.repo_name)[0]
             latest_tag_name = latest_tag["name"][1:]

xcoder/objects/plain_object.py

@@ -9,5 +9,4 @@
 
 class PlainObject(ABC):
     @abstractmethod
-    def load(self, swf: SupercellSWF, tag: int):
-        ...
+    def load(self, swf: SupercellSWF, tag: int): ...

xcoder/objects/renderable/display_object.py

@@ -14,12 +14,10 @@ def __init__(self):
         self._color_transform = ColorTransform()
 
     @abstractmethod
-    def calculate_bounds(self, matrix: Matrix2x3) -> Rect:
-        ...
+    def calculate_bounds(self, matrix: Matrix2x3) -> Rect: ...
 
     @abstractmethod
-    def render(self, matrix: Matrix2x3) -> Image.Image:
-        ...
+    def render(self, matrix: Matrix2x3) -> Image.Image: ...
 
     def set_matrix(self, matrix: Matrix2x3):
         self._matrix = matrix

xcoder/objects/shape/region.py

@@ -93,7 +93,7 @@ def get_image(self) -> Image.Image:
         height = max(int(rect.height), 1)
         if width + height <= 2:  # The same speed as without this return
             return Image.new(
-                "RGBA",
+                self._texture.image.mode,
                 (1, 1),
                 color=self._texture.image.getpixel((int(rect.left), int(rect.top))),
             )
@@ -102,12 +102,13 @@ def get_image(self) -> Image.Image:
             "L", self._texture.width, self._texture.height, self._uv_points, 0xFF
         )
 
-        rendered_region = Image.new("RGBA", (width, height))
+        rendered_region = Image.new(self._texture.image.mode, (width, height))
         rendered_region.paste(
             self._texture.image.crop(rect.as_tuple()),
             (0, 0),
             mask_image.crop(rect.as_tuple()),
         )
+        rendered_region = rendered_region.convert("RGBA")
 
         self._cache_image = rendered_region
 

xcoder/objects/texture.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
-import os
 from typing import TYPE_CHECKING
 
 import zstandard
 from PIL import Image
 
-from xcoder.images import join_image, load_image_from_buffer, load_texture
+from xcoder.bytestream import Reader
+from xcoder.images import join_image, load_image_from_buffer
 from xcoder.pvr_tex_tool import get_image_from_ktx_data
 
 if TYPE_CHECKING:
@@ -54,14 +54,10 @@ def load(self, swf: SupercellSWF, tag: int, has_texture: bool):
             )
             return
 
-        try:
-            load_texture(swf.reader, self.pixel_type, self.width, self.height)
+        self.image = self._load_texture(swf.reader, tag)
 
-            if tag in (27, 28, 29):
-                image = join_image(self.pixel_type, self.width, self.height)
-            else:
-                image = load_image_from_buffer(self.pixel_type, self.width, self.height)
-        finally:
-            os.remove("pixel_buffer")
+    def _load_texture(self, reader: Reader, tag: int) -> Image.Image:
+        if tag in (27, 28, 29):
+            return join_image(self.pixel_type, self.width, self.height, reader)
 
-        self.image = image
+        return load_image_from_buffer(self.pixel_type, self.width, self.height, reader)

xcoder/pixel_utils.py

@@ -1,15 +1,18 @@
 import struct
-from typing import Callable, TypeAlias
+from typing import Callable, Literal, TypeAlias
 
-from xcoder.bytestream import Reader
+from xcoder.localization import locale
 
 PixelChannels: TypeAlias = tuple[int, ...]
 EncodeFunction: TypeAlias = Callable[[PixelChannels], bytes]
-DecodeFunction: TypeAlias = Callable[[Reader], PixelChannels]
+RawMode: TypeAlias = Literal["RGBA", "RGBA;4B", "RGBA;15", "BGR;16", "LA", "L"]
 
 
-def get_read_function(pixel_type: int) -> DecodeFunction | None:
-    return _decode_functions.get(pixel_type, None)
+def get_raw_mode(pixel_type: int) -> RawMode:
+    if pixel_type in _raw_modes:
+        return _raw_modes[pixel_type]
+
+    raise Exception(locale.unknown_pixel_type % pixel_type)
 
 
 def get_pixel_encode_function(pixel_type: int) -> EncodeFunction | None:
@@ -26,43 +29,6 @@ def get_channel_count_by_pixel_type(pixel_type: int) -> int:
     return 4
 
 
-def _read_rgba8(reader: Reader) -> PixelChannels:
-    return tuple(reader.read(4))
-
-
-def _read_rgba4(reader: Reader) -> PixelChannels:
-    p = reader.read_ushort()
-    return (
-        (p >> 12 & 15) << 4,
-        (p >> 8 & 15) << 4,
-        (p >> 4 & 15) << 4,
-        (p >> 0 & 15) << 4,
-    )
-
-
-def _read_rgb5a1(reader: Reader) -> PixelChannels:
-    p = reader.read_ushort()
-    return (
-        (p >> 11 & 31) << 3,
-        (p >> 6 & 31) << 3,
-        (p >> 1 & 31) << 3,
-        (p & 255) << 7,
-    )
-
-
-def _read_rgb565(reader: Reader) -> PixelChannels:
-    p = reader.read_ushort()
-    return (p >> 11 & 31) << 3, (p >> 5 & 63) << 2, (p & 31) << 3
-
-
-def _read_luminance8_alpha8(reader: Reader) -> PixelChannels:
-    return tuple(reader.read(2))[::-1]
-
-
-def _read_luminance8(reader: Reader) -> PixelChannels:
-    return tuple(reader.read(1))
-
-
 def _write_rgba8(pixel: PixelChannels) -> bytes:
     return struct.pack("4B", *pixel)
 
@@ -109,12 +75,13 @@ def _write_luminance8(pixel: PixelChannels) -> bytes:
     10: _write_luminance8,
 }
 
-_decode_functions: dict[int, DecodeFunction] = {
-    0: _read_rgba8,
-    1: _read_rgba8,
-    2: _read_rgba4,
-    3: _read_rgb5a1,
-    4: _read_rgb565,
-    6: _read_luminance8_alpha8,
-    10: _read_luminance8,
+# here is a problem with this names https://github.com/python-pillow/Pillow/pull/8158
+_raw_modes: dict[int, RawMode] = {
+    0: "RGBA",
+    1: "RGBA",
+    2: "RGBA;4B",  # ABGR;4
+    3: "RGBA;15",  # ABGR;1555
+    4: "BGR;16",  # RGB;565
+    6: "LA",
+    10: "L",
 }