From b9ec2b8968184842925ac2d44e71a9ba9efd0ef0 Mon Sep 17 00:00:00 2001 From: Peter Harris Date: Mon, 26 May 2025 22:53:08 +0100 Subject: [PATCH 1/3] Update to Pylint 3.x --- .pylintrc | 691 ++++++++++++++++++++++++--------------- Docs/Building.md | 25 +- Test/astc_test_python.py | 33 +- 3 files changed, 469 insertions(+), 280 deletions(-) diff --git a/.pylintrc b/.pylintrc index 775f16356..87a0415f0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,4 +1,79 @@ -[MASTER] +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=Test/DocSource + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked and +# will not be imported (useful for modules/projects where namespaces are +# manipulated during runtime and thus existing member attributes cannot be +# deduced by static analysis). It supports qualified module names, as well as +# Unix pattern matching. +ignored-modules=signal + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 # Control the amount of potential inferred values when inferring a single # object. This can help the performance when dealing with large functions or @@ -12,6 +87,23 @@ load-plugins=pylint.extensions.docparams # Pickle collected data for later comparisons. persistent=yes +# Resolve imports to .pyi stubs if available. May reduce no-member messages and +# increase not-an-iterable messages. +prefer-stubs=no + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.12 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes @@ -20,139 +112,8 @@ suggestion-mode=yes # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# Ignore specific directories we don't author ourselves -ignore=Test/DocSource - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. -never-returning-functions=sys.exit +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= [BASIC] @@ -161,13 +122,15 @@ never-returning-functions=sys.exit argument-naming-style=camelCase # Regular expression matching correct argument names. Overrides argument- -# naming-style. +# naming-style. If left empty, argument names will be checked with the set +# naming style. #argument-rgx= # Naming style matching correct attribute names. attr-naming-style=camelCase # Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming # style. #attr-rgx= @@ -179,24 +142,38 @@ bad-names=foo, tutu, tata +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + # Naming style matching correct class attribute names. class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. #class-attribute-rgx= +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + # Naming style matching correct class names. class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming- -# style. +# style. If left empty, class names will be checked with the set naming style. #class-rgx= # Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming # style. #const-rgx= @@ -208,11 +185,29 @@ docstring-min-length=-1 function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- -# naming-style. +# naming-style. If left empty, function names will be checked with the set +# naming style. #function-rgx= # Good variable names which should always be accepted, separated by a comma. -good-names=i,j,k,x,y,z,w,r,g,b,a,ex,Run,_ +good-names=i, + j, + k, + x, + y, + z, + w, + r, + g, + b, + a, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= # Include a hint for the correct naming format with invalid-name. include-naming-hint=no @@ -221,21 +216,22 @@ include-naming-hint=no inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. #inlinevar-rgx= # Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- -# style. +# style. If left empty, method names will be checked with the set naming style. #method-rgx= # Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- -# style. +# style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when @@ -251,9 +247,99 @@ no-docstring-rgx=^_ # These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + # Naming style matching correct variable names. variable-naming-style=camelCase +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=16 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=16 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of positional arguments for function / method. +max-positional-arguments=5 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. @@ -284,10 +370,49 @@ single-line-class-stmt=no single-line-if-stmt=no +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + [LOGGING] -# Format style used to check logging format string. `old` means using % -# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. logging-format-style=old # Logging modules to check that the string format arguments are in logging @@ -295,23 +420,142 @@ logging-format-style=old logging-modules=logging +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[PARAMETER_DOCUMENTATION] + +# Whether to accept totally missing parameter documentation in the docstring of +# a function that has parameters. +accept-no-param-doc=yes + +# Whether to accept totally missing raises documentation in the docstring of a +# function that raises an exception. +accept-no-raise-doc=yes + +# Whether to accept totally missing return documentation in the docstring of a +# function that returns a statement. +accept-no-return-doc=yes + +# Whether to accept totally missing yields documentation in the docstring of a +# generator. +accept-no-yields-doc=yes + +# If the docstring type cannot be guessed the specified docstring type will be +# used. +default-docstring-type=default + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + +# Let 'consider-using-join' be raised when the separator to join on would be +# non-empty (resulting in expected fixes of the type: ``"- " + " - +# ".join(items)``) +suggest-join-with-non-empty-separator=yes + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: 'text', 'parseable', +# 'colorized', 'json2' (improved json format), 'json' (old json format), msvs +# (visual studio) and 'github' (GitHub actions). You can also give a reporter +# class, e.g. mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes [SIMILARITIES] -# Ignore comments when computing similarities. +# Comments are removed from the similarity computation ignore-comments=yes -# Ignore docstrings when computing similarities. +# Docstrings are removed from the similarity computation ignore-docstrings=yes -# Ignore imports when computing similarities. +# Imports are removed from the similarity computation ignore-imports=no +# Signatures are removed from the similarity computation +ignore-signatures=yes + # Minimum lines number of a similarity. min-similarity-lines=4 @@ -321,10 +565,14 @@ min-similarity-lines=4 # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. spelling-dict= +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + # List of comma separated words that should not be checked. spelling-ignore-words= @@ -338,9 +586,12 @@ spelling-store-unknown-words=no [STRING] -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. check-str-concat-over-line-jumps=no @@ -356,10 +607,6 @@ contextmanager-decorators=contextlib.contextmanager # expressions are accepted. generated-members= -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. ignore-none=yes @@ -372,22 +619,22 @@ ignore-none=yes # the rest of the inferred objects. ignore-on-opaque-inference=yes +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=signal - # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. missing-member-hint=yes -# The minimum edit distance a name should have in order to be considered a +# The maximum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 @@ -395,6 +642,9 @@ missing-member-hint-distance=1 # showing a hint for a missing member. missing-member-max-choices=1 +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + # List of decorators that change the signature of a decorated function. signature-mutators= @@ -408,16 +658,19 @@ additional-builtins= # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes +# List of names allowed to shadow builtins +allowed-redefined-builtins= + # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. -callbacks=cb_,_cb +callbacks=cb_, + _cb # A regular expression matching the name of dummy variables (i.e. expected to # not be used). dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. +# Argument names that match this expression will be ignored. ignored-argument-names=_.*|^ignored_|^unused_ # Tells whether we should check for unused import in __init__ files. @@ -426,107 +679,3 @@ init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=16 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=16 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=0 - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception diff --git a/Docs/Building.md b/Docs/Building.md index 475226e4c..0a1c01a47 100644 --- a/Docs/Building.md +++ b/Docs/Building.md @@ -283,6 +283,29 @@ cmake \ make -j16 ``` +## Configuring a new build machine + +Install native packages: + +```shell +sudo apt update +# Build essentials +sudo apt install clang cmake git +# Test essentials +sudo apt install imagemagick python3-pip +# Profile essentials +sudo apt install graphviz valgrind +``` + +Install Python modules from within venv: + +```shell +# Test essentials +pip install numpy pillow pycodestyle pylint +# Profile essentials +pip install gprof2dot +``` + ## Packaging a release bundle We support building a release bundle of all enabled binary configurations in @@ -312,4 +335,4 @@ details. - - - -_Copyright © 2019-2024, Arm Limited and contributors. All rights reserved._ +_Copyright © 2019-2025, Arm Limited and contributors. All rights reserved._ diff --git a/Test/astc_test_python.py b/Test/astc_test_python.py index 1abc8e697..557bbf9ec 100644 --- a/Test/astc_test_python.py +++ b/Test/astc_test_python.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # ----------------------------------------------------------------------------- -# Copyright 2020 Arm Limited +# Copyright 2020-2025 Arm Limited # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy @@ -20,12 +20,15 @@ test code base. """ +import io import re import sys import unittest import pycodestyle -import pylint.epylint as lint +import pylint +from pylint.lint import Run +from pylint.reporters.text import TextReporter class PythonTests(unittest.TestCase): @@ -37,22 +40,36 @@ def test_pylint(self): """ Run pylint over the codebase. """ - pylintOut, _ = lint.py_run("./Test", True) + # Run Pylint + stream = io.StringIO() + reporter = TextReporter(stream) + pylint.lint.Run(["./Test"], reporter, False) + pylintOut = stream.getvalue() + + # Write the Pylint log + with open("pylint.log", "w") as fileHandle: + fileHandle.write(pylintOut) + + # Analyze the results pattern = re.compile(r"Your code has been rated at (.*?)/10") - match = pattern.search(pylintOut.getvalue()) + match = pattern.search(pylintOut) self.assertIsNotNone(match) score = float(match.group(1)) self.assertGreaterEqual(score, 9.8) - with open("pylint.log", "w") as fileHandle: - fileHandle.write(pylintOut.getvalue()) - def test_pycodestyle(self): """ Test that we conform to PEP-8. """ style = pycodestyle.StyleGuide() - result = style.check_files(["./Test"]) + + # Write the Pycodestyle log + with open("pycodestyle.log", "w") as fileHandle: + oldStdout = sys.stdout + sys.stdout = fileHandle + result = style.check_files(["./Test"]) + sys.stdout = oldStdout + self.assertEqual(result.total_errors, 0, "Found code style errors (and warnings).") From 7110269f56a046255fce19208060776b0124e644 Mon Sep 17 00:00:00 2001 From: Peter Harris Date: Mon, 26 May 2025 22:57:59 +0100 Subject: [PATCH 2/3] Remove obsolete Python tools --- Test/DocSource/astc_image_info.rst | 8 - Test/DocSource/astc_size_binary.rst | 8 - Test/DocSource/astc_test_functional.rst | 8 - Test/DocSource/astc_test_image.rst | 8 - Test/DocSource/astc_test_image_dl.rst | 8 - Test/DocSource/conf.py | 56 ----- Test/DocSource/index.rst | 18 -- Test/DocSource/testlib-encoder.rst | 8 - Test/DocSource/testlib-image.rst | 8 - Test/DocSource/testlib-misc.rst | 8 - Test/DocSource/testlib-resultset.rst | 8 - Test/DocSource/testlib-testset.rst | 8 - Test/Makefile | 20 -- Test/astc_dump_binary.py | 125 ---------- Test/astc_image_info.py | 212 ---------------- Test/astc_image_sweep.py | 91 ------- Test/astc_profile_valgrind.py | 2 +- Test/astc_quality_test.py | 124 ---------- Test/astc_test_competitive.py | 125 ---------- Test/astc_test_competitive_plot.py | 127 ---------- Test/astc_test_result_report.py | 315 ------------------------ Test/make.bat | 35 --- 22 files changed, 1 insertion(+), 1329 deletions(-) delete mode 100644 Test/DocSource/astc_image_info.rst delete mode 100644 Test/DocSource/astc_size_binary.rst delete mode 100644 Test/DocSource/astc_test_functional.rst delete mode 100644 Test/DocSource/astc_test_image.rst delete mode 100644 Test/DocSource/astc_test_image_dl.rst delete mode 100644 Test/DocSource/conf.py delete mode 100644 Test/DocSource/index.rst delete mode 100644 Test/DocSource/testlib-encoder.rst delete mode 100644 Test/DocSource/testlib-image.rst delete mode 100644 Test/DocSource/testlib-misc.rst delete mode 100644 Test/DocSource/testlib-resultset.rst delete mode 100644 Test/DocSource/testlib-testset.rst delete mode 100644 Test/Makefile delete mode 100644 Test/astc_dump_binary.py delete mode 100644 Test/astc_image_info.py delete mode 100644 Test/astc_image_sweep.py delete mode 100644 Test/astc_quality_test.py delete mode 100644 Test/astc_test_competitive.py delete mode 100644 Test/astc_test_competitive_plot.py delete mode 100644 Test/astc_test_result_report.py delete mode 100644 Test/make.bat diff --git a/Test/DocSource/astc_image_info.rst b/Test/DocSource/astc_image_info.rst deleted file mode 100644 index 7417322ab..000000000 --- a/Test/DocSource/astc_image_info.rst +++ /dev/null @@ -1,8 +0,0 @@ -astc_image_info -=============== - -.. automodule:: astc_image_info - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/astc_size_binary.rst b/Test/DocSource/astc_size_binary.rst deleted file mode 100644 index 852dbbcff..000000000 --- a/Test/DocSource/astc_size_binary.rst +++ /dev/null @@ -1,8 +0,0 @@ -astc_size_binary -================ - -.. automodule:: astc_size_binary - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/astc_test_functional.rst b/Test/DocSource/astc_test_functional.rst deleted file mode 100644 index 1055a3185..000000000 --- a/Test/DocSource/astc_test_functional.rst +++ /dev/null @@ -1,8 +0,0 @@ -astc_test_functional -==================== - -.. automodule:: astc_test_functional - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/astc_test_image.rst b/Test/DocSource/astc_test_image.rst deleted file mode 100644 index 25ba58065..000000000 --- a/Test/DocSource/astc_test_image.rst +++ /dev/null @@ -1,8 +0,0 @@ -astc_test_image -=============== - -.. automodule:: astc_test_image - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/astc_test_image_dl.rst b/Test/DocSource/astc_test_image_dl.rst deleted file mode 100644 index 8054105ce..000000000 --- a/Test/DocSource/astc_test_image_dl.rst +++ /dev/null @@ -1,8 +0,0 @@ -astc_test_image_dl -================== - -.. automodule:: astc_test_image_dl - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/conf.py b/Test/DocSource/conf.py deleted file mode 100644 index 74d0b4e18..000000000 --- a/Test/DocSource/conf.py +++ /dev/null @@ -1,56 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('../')) -sys.path.insert(0, os.path.abspath('../testlib')) - - -# -- Project information ----------------------------------------------------- - -project = 'astcenc' -copyright = '2020, Arm Limited' -author = 'Arm Limited' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.coverage' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'classic' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] diff --git a/Test/DocSource/index.rst b/Test/DocSource/index.rst deleted file mode 100644 index 4fad3962d..000000000 --- a/Test/DocSource/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -Welcome to astcenc's documentation! -=================================== - -.. toctree:: - :maxdepth: 2 - :glob: - - - astc_* - testlib-* - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/Test/DocSource/testlib-encoder.rst b/Test/DocSource/testlib-encoder.rst deleted file mode 100644 index b033bdfa3..000000000 --- a/Test/DocSource/testlib-encoder.rst +++ /dev/null @@ -1,8 +0,0 @@ -testlib.encoder -=============== - -.. automodule:: testlib.encoder - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/testlib-image.rst b/Test/DocSource/testlib-image.rst deleted file mode 100644 index 00250bee1..000000000 --- a/Test/DocSource/testlib-image.rst +++ /dev/null @@ -1,8 +0,0 @@ -testlib.image -============= - -.. automodule:: testlib.image - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/testlib-misc.rst b/Test/DocSource/testlib-misc.rst deleted file mode 100644 index 2d1b8b6d5..000000000 --- a/Test/DocSource/testlib-misc.rst +++ /dev/null @@ -1,8 +0,0 @@ -testlib.misc -============ - -.. automodule:: testlib.misc - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/testlib-resultset.rst b/Test/DocSource/testlib-resultset.rst deleted file mode 100644 index 05cf86530..000000000 --- a/Test/DocSource/testlib-resultset.rst +++ /dev/null @@ -1,8 +0,0 @@ -testlib.resultset -================= - -.. automodule:: testlib.resultset - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/DocSource/testlib-testset.rst b/Test/DocSource/testlib-testset.rst deleted file mode 100644 index c13f6994f..000000000 --- a/Test/DocSource/testlib-testset.rst +++ /dev/null @@ -1,8 +0,0 @@ -testlib.testset -=============== - -.. automodule:: testlib.testset - :members: - :undoc-members: - :inherited-members: - :show-inheritance: diff --git a/Test/Makefile b/Test/Makefile deleted file mode 100644 index 2ffd335d3..000000000 --- a/Test/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = DocSource -BUILDDIR = DocOut - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/Test/astc_dump_binary.py b/Test/astc_dump_binary.py deleted file mode 100644 index 1e8399175..000000000 --- a/Test/astc_dump_binary.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2021-2022 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -The ``astc_dump_binary`` utility provides a wrapper around the ``objdump`` -utility to extract disassembly of specific functions. Currently only matches -the root name, for sake of command line sanity, so all overloads get dumped. - -Using __attribute__((noinline)) can be useful during profiling to stop any -functions of interest getting inlined once they get too small ... -""" - - -import argparse -import re -import shutil -import subprocess as sp -import sys - - -def run_objdump(binary, symbol): - """ - Run objdump on a single binary and extract the range for a given symbol. - - Output is printed to stdout. - - Args: - binary (str): The path of the binary file to process. - symbol (str): The symbol to match. - - Raises: - CalledProcessException: The ``objdump`` subprocess failed for any reason. - """ - args = [ - "objdump", "-C", - "-M", "intel", - "--no-show-raw", - "-d", "-S", - binary - ] - - result = sp.run(args, stdout=sp.PIPE, stderr=sp.PIPE, - check=True, universal_newlines=True) - - funcPattern = re.compile(r"^[0-9a-f]{16} <(.*?)\(.*\)>:$") - - funcLines = [] - funcActive = False - lines = result.stdout.splitlines() - - for line in lines: - match = funcPattern.match(line) - if match: - funcName = match.group(1) - if funcName == symbol: - funcActive = True - else: - funcActive = False - - if funcActive: - funcLines.append(line) - - print("\n".join(funcLines)) - -def parse_command_line(): - """ - Parse the command line. - - Returns: - Namespace: The parsed command line container. - """ - parser = argparse.ArgumentParser() - - parser.add_argument("binary", type=argparse.FileType("r"), - help="The new binary to dump") - - parser.add_argument("symbol", type=str, - help="The function name to dump") - - return parser.parse_args() - - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - args = parse_command_line() - - # Preflight - check that size exists. Note that size might still fail at - # runtime later, e.g. if the binary is not of the correct format - path = shutil.which("objdump") - if not path: - print("ERROR: The 'objdump' utility is not installed on the PATH") - return 1 - - # Collect the data - try: - run_objdump(args.binary.name, args.symbol) - except sp.CalledProcessError as ex: - print("ERROR: The 'objdump' utility failed") - print(" %s" % ex.stderr.strip()) - return 1 - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/astc_image_info.py b/Test/astc_image_info.py deleted file mode 100644 index 1fc42f9ab..000000000 --- a/Test/astc_image_info.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2020 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -The ``astc_image_info`` utility provides basic image query capabilities. It is -a modal command line utility, exposing multiple available operators. - -* ``info``: Query structural information about the image, such as image - dimensions, number of color channels, and the min/max of each channel. -* ``color``: Query the stored color value at a specific pixel coordinate, and - print the result in a variety of different formats. - -Both modes allow multiple images to be specified on the command line. -""" - -import argparse -import sys - -from PIL import Image - - -def main_color(args): - """ - Main function for the "color" mode. - - This mode prints the color at a specific pixel coordinate in each image. - The color value is printed in a variety of color formats (decimal, HTML - string, float). - - Args: - args (Namespace): The parsed command line arguments. - - Returns: - int: The process return code. - """ - retCode = 0 - - for i, image in enumerate(args.images): - if i != 0: - print("") - - img = Image.open(image.name) - x = args.location[0] - y = args.location[1] - - print(image.name) - print("=" * len(image.name)) - - if (x >= img.size[0]) or (y >= img.size[1]): - print("- ERROR: location out-of-bounds [%ux%u]" % img.size) - retCode = 1 - else: - color = img.getpixel((x, y)) - - # Print byte values - print("+ Byte: %s" % str(color)) - - # Print hex values - fmtString = "+ Hex: #" + ("%02X" * len(color)) - print(fmtString % color) - - # Print float values - parts = ["%g"] * len(color) - parts = ", ".join(parts) - fmtString = "+ Float: (" + parts + ")" - print(fmtString % tuple(float(x)/255.0 for x in color)) - - return retCode - - -def main_info(args): - """ - Main function for the "info" mode. - - This mode prints the basic metadata of an image: - - - the overall image size. - - the number of color channels. - - the min/max value in each color channel. - - Args: - args (Namespace): The parsed command line arguments. - - Returns: - int: The process return code. - """ - for i, image in enumerate(args.images): - if i != 0: - print("") - - img = Image.open(image.name) - minmax = img.getextrema() - - print(image.name) - print("=" * len(image.name)) - print("+ Size: %ux%u" % (img.size[0], img.size[1])) - print("+ Channels: %s" % ("".join(img.getbands()))) - for j, channel in enumerate(img.getbands()): - print(" + %s: %u - %u" % (channel, *minmax[j])) - - return 0 - - -def parse_loc(value): - """ - Command line argument parser for position arguments. - - Args: - value (str): The command line argument string to parse. Must be of the - form x", where both integers must be zero or positive. - - Returns: - list(int, int): The parsed location. - - Raises: - ArgumentTypeError: The value is not a valid location. - """ - error = argparse.ArgumentTypeError("%s is an invalid location" % value) - svalue = value.split("x") - - if len(svalue) != 2: - raise error - - try: - ivalue = [int(x) for x in svalue if int(x) >= 0] - except ValueError: - raise error - - if len(ivalue) != len(svalue): - raise error - - return ivalue - - -def parse_command_line(): - """ - Parse the command line. - - Returns: - Namespace: The parsed command line container. - """ - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers( - title="Operations") - - # Create the parser for the "pipette" command - parserA = subparsers.add_parser( - "color", - help="Print color at given coordinate") - - parserA.set_defaults(func=main_color) - - parserA.add_argument( - "location", metavar="loc", type=parse_loc, - help="The location spec XxY") - - parserA.add_argument( - "images", metavar="image", nargs="+", type=argparse.FileType("r"), - help="The images to query") - - # Create the parser for the "size" command - parserB = subparsers.add_parser( - "info", - help="Print image metadata info") - - parserB.set_defaults(func=main_info) - - parserB.add_argument( - "images", metavar="image", nargs="+", type=argparse.FileType("r"), - help="The images to query") - - # Cope with the user failing to specify any sub-command. Note on Python 3.8 - # we could use required=True on the add_subparsers call, but we cannot do - # this on 3.6 which is our current min-spec. - args = parser.parse_args() - if not hasattr(args, "func"): - parser.print_help() - return None - - return args - - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - args = parse_command_line() - if args: - return args.func(args) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/astc_image_sweep.py b/Test/astc_image_sweep.py deleted file mode 100644 index 575a75a21..000000000 --- a/Test/astc_image_sweep.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2021-2022 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -A benchmarking sweep helper, which can generate a performance-vs-quality sweep -for a single input images. Like other test functionality, this uses structured -image directory layouts for determining image settings to pass to the codec. -""" - -import argparse -import os -import platform -import sys - -import testlib.encoder as te -import testlib.image as ti - - -def parse_command_line(): - """ - Parse the command line. - - Returns: - Namespace: The parsed command line container. - """ - parser = argparse.ArgumentParser() - - # All reference encoders - parser.add_argument("--step", dest="step", default="10", type=int, help="step size") - parser.add_argument("--repeats", dest="repeats", type=int, default=1, help="repeats") - - parser.add_argument(dest="image", default=None, - help="select the test image to run") - - args = parser.parse_args() - return args - - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - # Parse command lines - args = parse_command_line() - - blockSizes = ["4x4", "5x5", "6x6", "8x8", "10x10"] - repeats = max(args.repeats, 1) - step = max(args.step, 1) - - image = ti.TestImage(args.image) - codec = te.Encoder2x("avx2") - - print("Block Size, Quality, PSNR (dB), Coding Time (s), Coding Rate (MT/s)") - - for blockSize in blockSizes: - for quality in range(0, 101, args.step): - localRepeats = repeats - if quality < 20: - localRepeats = localRepeats * 2 - if quality < 40: - localRepeats = localRepeats * 2 - - results = codec.run_test(image, blockSize, f"{quality}", localRepeats, False) - psnr = results[0] - codingTime = results[2] - mts = results[3] - - print(f"{blockSize}, {quality}, {psnr}, {codingTime}, {mts}") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/astc_profile_valgrind.py b/Test/astc_profile_valgrind.py index 3cd08445b..4ee42777d 100644 --- a/Test/astc_profile_valgrind.py +++ b/Test/astc_profile_valgrind.py @@ -41,7 +41,7 @@ def postprocess_cga(lines, outfile): lines ([str]): The output of callgrind_annotate. outfile (str): The output file path to write. """ - pattern = re.compile("^\s*([0-9,]+)\s+\([ 0-9.]+%\)\s+Source/(\S+):(\S+)\(.*\).*$") + pattern = re.compile(r"^\s*([0-9,]+)\s+\([ 0-9.]+%\)\s+Source/(\S+):(\S+)\(.*\).*$") totalCost = 0.0 functionTable = [] diff --git a/Test/astc_quality_test.py b/Test/astc_quality_test.py deleted file mode 100644 index 6d0641937..000000000 --- a/Test/astc_quality_test.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2021-2022 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -The ``astc_quality_test`` utility provides a tool to sweep quality settings. -""" - -import numpy as np -import re -import subprocess as sp -import sys - - -def get_psnr_pattern(): - return r"\s*PSNR \(LDR-RGB\):\s*([0-9.]*) dB" - - -def get_coding_rate_pattern(): - return r"\s*Coding rate:\s*([0-9.]*) MT/s" - - -def parse_output(output): - # Regex pattern for image quality - patternPSNR = re.compile(get_psnr_pattern()) - patternCRate = re.compile(get_coding_rate_pattern()) - - # Extract results from the log - runPSNR = None - runCRate = None - - for line in output: - match = patternPSNR.match(line) - if match: - runPSNR = float(match.group(1)) - - match = patternCRate.match(line) - if match: - runCRate = float(match.group(1)) - - assert runPSNR is not None, "No coding PSNR found" - assert runCRate is not None, "No coding rate found" - return (runPSNR, runCRate) - -def execute(command): - """ - Run a subprocess with the specified command. - - Args: - command (list(str)): The list of command line arguments. - - Returns: - list(str): The output log (stdout) split into lines. - """ - try: - result = sp.run(command, stdout=sp.PIPE, stderr=sp.PIPE, - check=True, universal_newlines=True) - except (OSError, sp.CalledProcessError): - print("ERROR: Test run failed") - print(" + %s" % " ".join(command)) - qcommand = ["\"%s\"" % x for x in command] - print(" + %s" % ", ".join(qcommand)) - sys.exit(1) - - return result.stdout.splitlines() - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - for block in ("4x4", "5x5", "6x6", "8x8", "10x10"): - - for quality in range (0, 101, 2): - - resultsQ = [] - resultsS = [] - - if (quality < 40): - repeats = 20 - elif (quality < 75): - repeats = 10 - else: - repeats = 5 - - for _ in range(0, repeats): - command = [ - "./bin/astcenc-avx2", - "-tl", - "./Test/Images/Kodak/LDR-RGB/ldr-rgb-kodak23.png", - "/dev/null", - block, - "%s" % quality, - "-silent" - ] - - stdout = execute(command) - psnr, mts = parse_output(stdout) - resultsQ.append(psnr) - resultsS.append(mts) - - print("%s, %u, %0.3f, %0.3f" % (block, quality, np.mean(resultsS), np.mean(resultsQ))) - - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/astc_test_competitive.py b/Test/astc_test_competitive.py deleted file mode 100644 index 188f4324d..000000000 --- a/Test/astc_test_competitive.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2022 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -This script is a simple test runner for sweeps on multiple compressors. -""" - -import os -import subprocess as sp -import re -import sys - -LOG_COMMANDS = False -LOG_PATTERN = re.compile(r"\s*Coding rate:\s*(.*)\s*MT/s") - -ISPC_BIN = "./Binaries/ISPC/ispc_astc.exe" -ISPC_QUALITY = ["rgba", "rgb"] - -ASTC_BIN = "./bin/astcenc-avx2" -ASTC_QUALITY = ["0", "8", "10", "20", "30", "40", "50", "60"] - -TEST_BLOCK_SIZES = ["4x4", "6x6", "8x8"] - -TEST_IMAGE = "./Test/Images/Kodak/LDR-RGB/ldr-rgb-kodak%02u.png" -TEST_RANGE = 24 -TEST_REPEATS = 5 - -OUT_CIMAGE = "out.astc" -OUT_DIMAGE = "out.png" - - -def run(command): - if LOG_COMMANDS: - print(" ".join(command)) - - return sp.run(command, capture_output=True, universal_newlines=True) - - -def run_astcenc(in_image, out_image, block_size, quality): - args = [ASTC_BIN, "-tl", in_image, out_image, block_size, quality, "-j", "1"] - result = run(args) - return float(LOG_PATTERN.search(result.stdout).group(1)) - - -def run_ispc(in_image, out_image, block_size, quality): - args = [ISPC_BIN, in_image, out_image, block_size, quality] - result = run(args) - return float(LOG_PATTERN.search(result.stdout).group(1)) - - -def decompress(in_image, out_image): - args = [ASTC_BIN, "-dl", in_image, out_image] - result = run(args) - os.remove(in_image) - - -def compare(in_image, out_image): - args = ["compare", "-metric", "PSNR", in_image, out_image, "diff.png"] - result = run(args) - os.remove("diff.png") - os.remove(out_image) - return float(result.stderr) - - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - # ISPC Tests - for block_size in TEST_BLOCK_SIZES: - for quality in ISPC_QUALITY: - print(f"ISPC {quality} {block_size}") - print(f"ISPC {quality} {block_size}", file=sys.stderr) - for index in range(1, TEST_RANGE + 1): - result_rate = 0.0 - for repeat in range(0, TEST_REPEATS): - image = TEST_IMAGE % index - result_rate += run_ispc(image, OUT_CIMAGE, block_size, quality) - decompress(OUT_CIMAGE, OUT_DIMAGE) - result_error = compare(image, OUT_DIMAGE) - result_rate /= TEST_REPEATS - - print("%s,Kodak%02u,%0.4f,%0.4f" % (block_size, index, result_rate, result_error)) - - # ASTCENC Tests - for block_size in TEST_BLOCK_SIZES: - for quality in ASTC_QUALITY: - print(f"ASTC {quality} {block_size}") - print(f"ASTC {quality} {block_size}", file=sys.stderr) - for index in range(1, TEST_RANGE + 1): - result_rate = 0.0 - for repeat in range(0, TEST_REPEATS): - image = TEST_IMAGE % index - result_rate += run_astcenc(image, OUT_DIMAGE, block_size, quality) - result_error = compare(image, OUT_DIMAGE) - result_rate /= TEST_REPEATS - - print("%s,Kodak%02u,%0.4f,%0.4f" % (block_size, index, result_rate, result_error)) - - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except sp.CalledProcessError as ex: - print(ex.stdout) - print(ex.stderr) diff --git a/Test/astc_test_competitive_plot.py b/Test/astc_test_competitive_plot.py deleted file mode 100644 index d412713e8..000000000 --- a/Test/astc_test_competitive_plot.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2022 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -This script is a simple test result plotter for sweeps on multiple compressors. -""" -import csv -import numpy as np -import matplotlib.pyplot as plt -import sys - -DATABASE = "competitive.csv" - - -class Series: - - def __init__(self, name, perf, qual): - self.name = name - self.perf = perf - self.qual = qual - - -def get_series(database, compressor, quality, block_size): - title = f"{compressor} {quality} {block_size}" - in_section = False - - perf = [] - qual = [] - - with open(database) as csvfile: - reader = csv.reader(csvfile) - for row in reader: - if len(row) == 1: - in_section = row[0] == title - continue - - if in_section: - perf.append(float(row[2])) - qual.append(float(row[3])) - - return (perf, qual) - - -def plot(block_size, series_set): - - for series in series_set: - plt.scatter(series.perf, series.qual, s=2, label=series.name) - - plt.xlabel("Speed (MT/s)") - plt.ylabel("PSNR dB") - plt.legend(loc='lower right', prop={'size': 6}) - - plt.tight_layout() - plt.savefig(f"ASTC_v_ISPC_{block_size}.png") - plt.clf() - - -def plot_diff(series_a, series_b): - - diff_perf = np.divide(series_a.perf, series_b.perf) - diff_qual = np.subtract(series_a.qual, series_b.qual) - label = f"{series_a.name} vs {series_b.name}" - - plt.scatter(diff_perf, diff_qual, s=2, c="#0091BD", label=label) - plt.scatter(np.mean(diff_perf), np.mean(diff_qual), s=10, c="#FFA500", marker="*") - - plt.axhline(y=0, color="r", linestyle="dotted", lw=0.5) - plt.axvline(x=1, color="r", linestyle="dotted", lw=0.5) - - plt.xlabel("Relative speed") - plt.ylabel("PSNR diff (dB)") - plt.legend(loc='lower right', prop={'size': 6}) - - plt.tight_layout() - file_name = label.replace(" ", "_") + ".png" - plt.savefig(file_name) - plt.clf() - - -def main(): - - block_sizes = ["4x4", "6x6", "8x8"] - - for block_size in block_sizes: - series_set = [] - - perf, qual = get_series(DATABASE, "ISPC", "rgba", block_size) - series_set.append(Series(f"{block_size} IPSC Slow", perf, qual)) - - perf, qual = get_series(DATABASE, "ISPC", "rgb", block_size) - series_set.append(Series(f"{block_size} IPSC Fast", perf, qual)) - - perf, qual = get_series(DATABASE, "ASTC", "60", block_size) - series_set.append(Series(f"{block_size} ASTC 60", perf, qual)) - - perf, qual = get_series(DATABASE, "ASTC", "50", block_size) - series_set.append(Series(f"{block_size} ASTC 50", perf, qual)) - - perf, qual = get_series(DATABASE, "ASTC", "10", block_size) - series_set.append(Series(f"{block_size} ASTC 10", perf, qual)) - - perf, qual = get_series(DATABASE, "ASTC", "8", block_size) - series_set.append(Series(f"{block_size} ASTC 8", perf, qual)) - - plot(block_size, series_set) - - plot_diff(series_set[3], series_set[0]) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/astc_test_result_report.py b/Test/astc_test_result_report.py deleted file mode 100644 index 94721d8fb..000000000 --- a/Test/astc_test_result_report.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# ----------------------------------------------------------------------------- -# Copyright 2020-2021 Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# ----------------------------------------------------------------------------- -""" -The ``astc_test_result_report.py`` script consolidates all current sets of -reference results into a single report giving PSNR diffs (absolute) and -performance diffs (relative speedup, 1 = no change). -""" - -import re -import os -import sys - - -import testlib.resultset as trs -from collections import defaultdict as ddict - - -CONFIG_FILTER = [ - re.compile(r"^.*1\.7.*$"), - re.compile(r"^.*sse.*$") -] - -TESTSET_FILTER = [ - re.compile(r"^Small$"), - re.compile(r"^Frymire$"), -] - -QUALITY_FILTER = [ -] - -BLOCKSIZE_FILTER = [ - re.compile(r"^12x12$") -] - - -def find_reference_results(): - """ - Scrape the Test/Images directory for result CSV files and return an - mapping of the result sets. - - Returns: - Returns a three deep tree of dictionaries, with the final dict - pointing at a `ResultSet` object. The hierarchy is: - - imageSet => quality => encoder => result - """ - scriptDir = os.path.dirname(__file__) - imageDir = os.path.join(scriptDir, "Images") - - # Pattern for extracting useful data from the CSV file name - filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv") - - # Build a three level dictionary we can write into - results = ddict(lambda: ddict(lambda: ddict())) - - # Final all CSVs, load them and store them in the dict tree - for root, dirs, files in os.walk(imageDir): - for name in files: - match = filePat.match(name) - if match: - - # Skip results set in the filter - skip = [1 for filt in CONFIG_FILTER if filt.match(name)] - if skip: - continue - - fullPath = os.path.join(root, name) - - encoder = match.group(1) - quality = match.group(2) - imageSet = os.path.basename(root) - - # Skip results set in the filter - skip = [1 for filt in TESTSET_FILTER if filt.match(imageSet)] - if skip: - continue - - # Skip results set in the filter - skip = [1 for filt in QUALITY_FILTER if filt.match(quality)] - if skip: - continue - - testRef = trs.ResultSet(imageSet) - testRef.load_from_file(fullPath) - - patchedRef = trs.ResultSet(imageSet) - - for result in testRef.records: - skip = [1 for filt in BLOCKSIZE_FILTER if filt.match(result.blkSz)] - if not skip: - patchedRef.add_record(result) - - results[imageSet][quality]["ref-%s" % encoder] = patchedRef - - return results - - -class DeltaRecord(): - """ - Record a single image result from N different encoders. - - Attributes: - imageSet: The image set this cme from. - quality: The compressor quality used. - encoders: The names of the encoders used. The first encoder in this - list will be used as the reference result. - records: The raw records for the encoders. The order of records in this - list matches the order of the `encoders` list. - """ - - def __init__(self, imageSet, quality, encoders, records): - self.imageSet = imageSet - self.quality = quality - - self.encoders = list(encoders) - self.records = list(records) - - assert(len(self.encoders) == len(self.records)) - - def get_delta_header(self, tag): - """ - Get the delta encoding header. - - Args: - tag: The field name to include in the tag. - - Return: - The array of strings, providing the header names. - """ - result = [] - - for encoder in self.encoders[1:]: - result.append("%s %s" % (tag, encoder)) - - return result - - def get_abs_delta(self, field): - """ - Get an absolute delta result. - - Args: - field: The Record attribute name to diff. - - Return: - The array of float delta values. - """ - result = [] - - root = self.records[0] - for record in self.records[1:]: - result.append(getattr(record, field) - getattr(root, field)) - - return result - - def get_rel_delta(self, field): - """ - Get an relative delta result (score / ref). - - Args: - field: The Record attribute name to diff. - - Return: - The array of float delta values. - """ - result = [] - - root = self.records[0] - for record in self.records[1:]: - result.append(getattr(record, field) / getattr(root, field)) - - return result - - def get_irel_delta(self, field): - """ - Get an inverse relative delta result (ref / score). - - Args: - field: The Record attribute name to diff. - - Return: - The array of float delta values. - """ - return [1.0 / x for x in self.get_rel_delta(field)] - - def get_full_row_header_csv(self): - """ - Get a CSV encoded delta record header. - - Return: - The string for the row. - """ - rows = [ - "Image Set", - "Quality", - "Size", - "Name" - ] - - rows.append("") - rows.extend(self.get_delta_header("PSNR")) - - rows.append("") - rows.extend(self.get_delta_header("Speed")) - - return ",".join(rows) - - def get_full_row_csv(self): - """ - Get a CSV encoded delta record. - - Return: - The string for the row. - """ - rows = [ - self.imageSet, - self.quality, - self.records[0].name, - self.records[0].blkSz - ] - - rows.append("") - data = ["%0.3f" % x for x in self.get_abs_delta("psnr")] - rows.extend(data) - - rows.append("") - data = ["%0.3f" % x for x in self.get_irel_delta("cTime")] - rows.extend(data) - - return ",".join(rows) - - -def print_result_set(imageSet, quality, encoders, results, printHeader): - """ - Attributes: - imageSet: The image set name. - quality: The compressor quality used. - encoders: The names of the encoders used. The first encoder in this - list will be used as the reference result. - results: The dict of results, indexed by encoder. - printHeader: True if the table header should be printed, else False. - """ - results = [results[x] for x in encoders] - recordSizes = [len(x.records) for x in results] - - # Skip result sets that are not the same size - # TODO: We can take the set intersection here to report what we can - if min(recordSizes) != max(recordSizes): - return - - # Interleave all result records - recordSets = zip(*[x.records for x in results]) - - # Iterate each image - for recordSet in recordSets: - base = recordSet[0] - - # Sanity check consistency - for record in recordSet[1:]: - assert(record.blkSz == base.blkSz) - assert(record.name == base.name) - - dr = DeltaRecord(imageSet, quality, encoders, recordSet) - - if printHeader: - print(dr.get_full_row_header_csv()) - printHeader = False - - print(dr.get_full_row_csv()) - - -def main(): - """ - The main function. - - Returns: - int: The process return code. - """ - - results = find_reference_results() - - imageSet = sorted(results.keys()) - - first = True - for image in imageSet: - qualityTree = results[image] - qualitySet = sorted(qualityTree.keys()) - - for qual in qualitySet: - encoderTree = qualityTree[qual] - encoderSet = sorted(encoderTree.keys()) - - if len(encoderSet) > 1: - print_result_set(image, qual, encoderSet, encoderTree, first) - first = False - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Test/make.bat b/Test/make.bat deleted file mode 100644 index 8634effe4..000000000 --- a/Test/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=DocSource -set BUILDDIR=DocOut - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From 5116ca39d9d7b22c1311b5f3fdee89e17192f063 Mon Sep 17 00:00:00 2001 From: Peter Harris Date: Mon, 26 May 2025 23:00:14 +0100 Subject: [PATCH 3/3] Remove obsolete native utils --- Utils/astc_blend_test.cpp | 298 -------------------------------- Utils/astcenc_u8_test_bench.cpp | 117 ------------- 2 files changed, 415 deletions(-) delete mode 100644 Utils/astc_blend_test.cpp delete mode 100644 Utils/astcenc_u8_test_bench.cpp diff --git a/Utils/astc_blend_test.cpp b/Utils/astc_blend_test.cpp deleted file mode 100644 index 1db604202..000000000 --- a/Utils/astc_blend_test.cpp +++ /dev/null @@ -1,298 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// ---------------------------------------------------------------------------- -// Copyright 2021-2024 Arm Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy -// of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations -// under the License. -// ---------------------------------------------------------------------------- - -// This is a utility tool to test blend modes. - -#include -#include -#include - -#include "astcenc_mathlib.h" - -#define STB_IMAGE_IMPLEMENTATION -#include "ThirdParty/stb_image.h" - -#define STB_IMAGE_WRITE_IMPLEMENTATION -#include "ThirdParty/stb_image_write.h" - -/** - * @brief Linearize an sRGB value. - * - * @return The linearized value. - */ -static float srgb_to_linear( - float a -) { - if (a <= 0.04045f) - { - return a * (1.0f / 12.92f); - } - - return powf((a + 0.055f) * (1.0f / 1.055f), 2.4f); -} - -/** - * @brief sRGB gamma-encode a linear value. - * - * @return The gamma encoded value. - */ -static float linear_to_srgb( - float a -) { - if (a <= 0.0031308f) - { - return a * 12.92f; - } - - return 1.055f * powf(a, 1.0f / 2.4f) - 0.055f; -} - -int main(int argc, char **argv) -{ - // Parse command line - if (argc != 6) - { - printf("Usage: astc_blend_test \n"); - exit(1); - } - - const char* src_file = argv[1]; - const char* dst_file = argv[2]; - - bool use_linear = false; - if (!strcmp(argv[3], "linear")) - { - use_linear = true; - } - else if (!strcmp(argv[3], "srgb")) - { - use_linear = false; - } - else - { - printf(" must be either 'linear' or 'srgb'\n"); - exit(1); - } - - bool use_post_blend = false; - if (!strcmp(argv[4], "post")) - { - use_post_blend = true; - } - else if (!strcmp(argv[4], "pre")) - { - use_post_blend = false; - } - else - { - printf(" must be either 'post' or 'pre'\n"); - exit(1); - } - - bool use_filter = false; - if (!strcmp(argv[5], "on")) - { - use_filter = true; - } - else if (!strcmp(argv[5], "off")) - { - use_filter = false; - } - else - { - printf(" must be either 'on' or 'off'\n"); - exit(1); - } - - // Load the input image - int dim_x; - int dim_y; - const uint8_t* data_in = stbi_load(src_file, &dim_x, &dim_y, nullptr, 4); - if (!data_in) - { - printf("ERROR: Failed to load input image.\n"); - exit(1); - } - - // Allocate the output image - uint8_t* data_out = (uint8_t*)malloc(4 * dim_y * dim_x); - if (!data_out) - { - printf("ERROR: Failed to allocate output image.\n"); - exit(1); - } - - // For each pixel blending and filtering - if (!use_filter) - { - for (int y = 0; y < dim_y; y++) - { - const uint8_t* row_in = data_in + (4 * dim_x * y); - uint8_t* row_out = data_out + (4 * dim_x * y); - - for (int x = 0; x < dim_x; x++) - { - const uint8_t* pixel_in = row_in + 4 * x; - uint8_t* pixel_out = row_out + 4 * x; - - float r_src = static_cast(pixel_in[0]) / 255.0f; - float g_src = static_cast(pixel_in[1]) / 255.0f; - float b_src = static_cast(pixel_in[2]) / 255.0f; - float a_src = static_cast(pixel_in[3]) / 255.0f; - - if (use_linear == false) - { - r_src = srgb_to_linear(r_src); - g_src = srgb_to_linear(g_src); - b_src = srgb_to_linear(b_src); - } - - float r_dst = 0.53f; - float g_dst = 0.53f; - float b_dst = 0.53f; - - float r_out; - float g_out; - float b_out; - float a_out; - - // Post-multiply blending - if (use_post_blend) - { - r_out = (r_dst * (1.0f - a_src)) + (r_src * a_src); - g_out = (g_dst * (1.0f - a_src)) + (g_src * a_src); - b_out = (b_dst * (1.0f - a_src)) + (b_src * a_src); - a_out = 1.0f; - } - // Pre-multiply blending - else - { - r_out = (r_dst * (1.0f - a_src)) + (r_src * 1.0f); - g_out = (g_dst * (1.0f - a_src)) + (g_src * 1.0f); - b_out = (b_dst * (1.0f - a_src)) + (b_src * 1.0f); - a_out = 1.0f; - } - - // Clamp color between 0 and 1.0f - r_out = astc::min(r_out, 1.0f); - g_out = astc::min(g_out, 1.0f); - b_out = astc::min(b_out, 1.0f); - - if (use_linear == false) - { - r_out = linear_to_srgb(r_out); - g_out = linear_to_srgb(g_out); - b_out = linear_to_srgb(b_out); - } - - pixel_out[0] = (uint8_t)(r_out * 255.0f); - pixel_out[1] = (uint8_t)(g_out * 255.0f); - pixel_out[2] = (uint8_t)(b_out * 255.0f); - pixel_out[3] = (uint8_t)(a_out * 255.0f); - } - } - } - else - { - for (int y = 0; y < dim_y - 1; y++) - { - const uint8_t* row_in_0 = data_in + (4 * dim_x * y); - const uint8_t* row_in_1 = data_in + (4 * dim_x * (y + 1)); - - uint8_t* row_out = data_out + (4 * (dim_x - 1) * y); - - for (int x = 0; x < dim_x - 1; x++) - { - const uint8_t* pixel_in_00 = row_in_0 + 4 * x; - const uint8_t* pixel_in_01 = row_in_0 + 4 * (x + 1); - const uint8_t* pixel_in_10 = row_in_1 + 4 * x; - const uint8_t* pixel_in_11 = row_in_1 + 4 * (x + 1); - - uint8_t* pixel_out = row_out + 4 * x; - - // Bilinear filter with a half-pixel offset - float r_src = static_cast(pixel_in_00[0] + pixel_in_01[0] + pixel_in_10[0] + pixel_in_11[0]) / (255.0f * 4.0f); - float g_src = static_cast(pixel_in_00[1] + pixel_in_01[1] + pixel_in_10[1] + pixel_in_11[1]) / (255.0f * 4.0f); - float b_src = static_cast(pixel_in_00[2] + pixel_in_01[2] + pixel_in_10[2] + pixel_in_11[2]) / (255.0f * 4.0f); - float a_src = static_cast(pixel_in_00[3] + pixel_in_01[3] + pixel_in_10[3] + pixel_in_11[3]) / (255.0f * 4.0f); - - if (use_linear == false) - { - r_src = srgb_to_linear(r_src); - g_src = srgb_to_linear(g_src); - b_src = srgb_to_linear(b_src); - } - - float r_dst = 0.8f; - float g_dst = 1.0f; - float b_dst = 0.8f; - - float r_out; - float g_out; - float b_out; - float a_out; - - // Post-multiply blending - if (use_post_blend) - { - r_out = (r_dst * (1.0f - a_src)) + (r_src * a_src); - g_out = (g_dst * (1.0f - a_src)) + (g_src * a_src); - b_out = (b_dst * (1.0f - a_src)) + (b_src * a_src); - a_out = 1.0f; - } - // Pre-multiply blending - else - { - r_out = (r_dst * (1.0f - a_src)) + (r_src * 1.0f); - g_out = (g_dst * (1.0f - a_src)) + (g_src * 1.0f); - b_out = (b_dst * (1.0f - a_src)) + (b_src * 1.0f); - a_out = 1.0f; - } - - // Clamp color between 0 and 1.0f - r_out = astc::min(r_out, 1.0f); - g_out = astc::min(g_out, 1.0f); - b_out = astc::min(b_out, 1.0f); - - if (use_linear == false) - { - r_out = linear_to_srgb(r_out); - g_out = linear_to_srgb(g_out); - b_out = linear_to_srgb(b_out); - } - - pixel_out[0] = (uint8_t)(r_out * 255.0f); - pixel_out[1] = (uint8_t)(g_out * 255.0f); - pixel_out[2] = (uint8_t)(b_out * 255.0f); - pixel_out[3] = (uint8_t)(a_out * 255.0f); - } - } - } - - // Write out the result - if (!use_filter) - { - stbi_write_png(dst_file, dim_x, dim_y, 4, data_out, 4 * dim_x); - } - else - { - stbi_write_png(dst_file, dim_x - 1, dim_y - 1, 4, data_out, 4 * (dim_x - 1)); - } - - - return 0; -} diff --git a/Utils/astcenc_u8_test_bench.cpp b/Utils/astcenc_u8_test_bench.cpp deleted file mode 100644 index b44d6954e..000000000 --- a/Utils/astcenc_u8_test_bench.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// ---------------------------------------------------------------------------- -// Copyright 2023 Arm Limited -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy -// of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations -// under the License. -// ---------------------------------------------------------------------------- - -// astcenc doesn't use the top 8 integer bits directly for sRGB RGB components -// or when using the decode_unorm8 decode mode. An alterantive is used which -// allows a common code path to be used. This test program shows that the two -// produce equivalent output once rounded to a decode_unorm8 output. - -// Compile with e.g. clang++ astcenc_u8_test_bench.cpp -o astcenc_u8_test_bench -mavx2 -mf16c - -#define ASTCENC_AVX 2 -#define ASTCENC_F16C 1 -#define ASTCENC_SSE 41 - -#include "../Source/astcenc_mathlib.cpp" -#include "../Source/astcenc_color_unquantize.cpp" -#include "../Source/astcenc_decompress_symbolic.cpp" - -int main() -{ - printf("Decode mode test bench\n"); - - for (int ep0 = 0; ep0 < 256; ep0++) - { - for (int ep1 = 0; ep1 < 256; ep1++) - { - for (int wt1 = 0; wt1 < 65; wt1++) - { - // Validate linear data with decode_unorm8 mode - { - // Expand 8 bit to 16 bit - vint4 weights(wt1); - int ep0_v0 = ep0 * 257; - int ep1_v0 = ep1 * 257; - - // Linear with decode_u8 handling - vmask4 decode_u8_v0(true, true, true, true); - vint4 ep0v0(ep0_v0, ep0_v0, ep0_v0, ep0_v0); - vint4 ep1v0(ep1_v0, ep1_v0, ep1_v0, ep1_v0); - - // Linear without decode_u8 handling - vmask4 decode_u8_v1(false, false, false, false); - vint4 ep0v1(ep0_v0, ep0_v0, ep0_v0, ep0_v0); - vint4 ep1v1(ep1_v0, ep1_v0, ep1_v0, ep1_v0); - - // Lerp both styles - vint4 colorv0 = lerp_color_int(decode_u8_v0, ep0v0, ep1v0, weights); - vint4 colorv1 = lerp_color_int(decode_u8_v1, ep0v1, ep1v1, weights); - - // Validate top 8 integer bits match in both cases - // - Shows that astcenc-style U8 doesn't differ from Khronos-style U8 - vint4 cs0 = lsr<8>(colorv0); - vint4 cs1 = lsr<8>(colorv1); - assert(cs0.lane<0>() == cs1.lane<0>()); - assert(cs0.lane<3>() == cs1.lane<3>()); - - // Validate that astcenc output matches the top 8 integer bits - vfloat4 colorv0f = decode_texel(colorv0, vmask4(false)); - vint4 colorv0_out = float_to_int_rtn(colorv0f * 255.0f); - assert(colorv0_out.lane<0>() == cs0.lane<0>()); - } - - // Validate sRGB data with decode_unorm8 mode - { - // Expand 8 bit to 16 bit - vint4 weights(wt1); - int ep0_v0s = (ep0 << 8) | 0x80; - int ep1_v0s = (ep1 << 8) | 0x80; - int ep0_v0 = ep0 * 257; - int ep1_v0 = ep1 * 257; - - // sRGB RGB and linear A with decode_u8 handling - vmask4 decode_u8_v0(true, true, true, true); - vint4 ep0v0(ep0_v0s, ep0_v0s, ep0_v0s, ep0_v0); - vint4 ep1v0(ep1_v0s, ep1_v0s, ep1_v0s, ep1_v0); - - // sRGB RGB and linear A without decode_u8 handling - vmask4 decode_u8_v1(false, false, false, false); - vint4 ep0v1(ep0_v0s, ep0_v0s, ep0_v0s, ep0_v0); - vint4 ep1v1(ep1_v0s, ep1_v0s, ep1_v0s, ep1_v0); - - // Lerp both styles - vint4 colorv0 = lerp_color_int(decode_u8_v0, ep0v0, ep1v0, weights); - vint4 colorv1 = lerp_color_int(decode_u8_v1, ep0v1, ep1v1, weights); - - // Validate top 8 integer bits match in both cases - // - Shows that astcenc-style U8 doesn't differ from Khronos-style U8 - vint4 cs0 = lsr<8>(colorv0); - vint4 cs1 = lsr<8>(colorv1); - assert(cs0.lane<0>() == cs1.lane<0>()); - assert(cs0.lane<3>() == cs1.lane<3>()); - - // Validate that astcenc output matches the top 8 integer bits - vfloat4 colorv0f = decode_texel(colorv0, vmask4(false)); - vint4 colorv0_out = float_to_int_rtn(colorv0f * 255.0f); - assert(colorv0_out.lane<0>() == cs0.lane<0>()); - } - } - } - } - - return 0; -}