diff --git a/packages/compare-transforms/CMakeLists.txt b/packages/compare-transforms/CMakeLists.txt new file mode 100644 index 000000000..8a6a92a5f --- /dev/null +++ b/packages/compare-transforms/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.16) +project(compare-transforms LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) + +find_package(ITK REQUIRED COMPONENTS + WebAssemblyInterface +) +include(${ITK_USE_FILE}) + +enable_testing() + +# Begin create-itk-wasm added pipelines. +add_subdirectory(compare-transforms) +# End create-itk-wasm added pipelines. diff --git a/packages/compare-transforms/compare-transforms/CMakeLists.txt b/packages/compare-transforms/compare-transforms/CMakeLists.txt new file mode 100644 index 000000000..90e788f73 --- /dev/null +++ b/packages/compare-transforms/compare-transforms/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable(compare-transforms compare-transforms.cxx) +target_link_libraries(compare-transforms PUBLIC ${ITK_LIBRARIES}) + +add_test(NAME compare-transforms-help COMMAND compare-transforms --help) + +add_test(NAME compare-transforms-same + COMMAND compare-transforms + ${CMAKE_CURRENT_SOURCE_DIR}/../test/data/input/translation.iwt.cbor + same-metrics.json + --baseline-transforms ${CMAKE_CURRENT_SOURCE_DIR}/../test/data/input/translation.iwt.cbor +) diff --git a/packages/compare-transforms/compare-transforms/compare-transforms.cxx b/packages/compare-transforms/compare-transforms/compare-transforms.cxx new file mode 100644 index 000000000..872b17598 --- /dev/null +++ b/packages/compare-transforms/compare-transforms/compare-transforms.cxx @@ -0,0 +1,277 @@ +/*========================================================================= + * + * Copyright NumFOCUS + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *=========================================================================*/ + +#include "itkPipeline.h" +#include "itkInputTransform.h" +#include "itkOutputTextStream.h" +#include "itkSupportInputTransformTypes.h" + +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +#include +#include + +template +std::tuple +compareTransforms(const TTransform * transform0, + const TTransform * transform1, + const double parametersTolerance, + const double fixedParametersTolerance) +{ + bool almostEqual = false; + bool parametersAlmostEqual = false; + bool fixedParametersAlmostEqual = false; + double parametersMaximumDifference = 0.0; + double fixedParametersMaximumDifference = 0.0; + + if (transform0 != nullptr && transform1 != nullptr) + { + // Check transform types + if (transform0->GetTransformTypeAsString() != transform1->GetTransformTypeAsString()) + { + return std::make_tuple(false, false, false, + itk::NumericTraits::max(), + itk::NumericTraits::max()); + } + + // Compare parameters + const auto parameters0 = transform0->GetParameters(); + const auto parameters1 = transform1->GetParameters(); + + if (parameters0.GetSize() == parameters1.GetSize()) + { + parametersAlmostEqual = true; + for (unsigned int i = 0; i < parameters0.GetSize(); ++i) + { + const double difference = std::abs(parameters0[i] - parameters1[i]); + parametersMaximumDifference = std::max(parametersMaximumDifference, difference); + if (difference > parametersTolerance) + { + parametersAlmostEqual = false; + } + } + } + + // Compare fixed parameters + const auto fixedParameters0 = transform0->GetFixedParameters(); + const auto fixedParameters1 = transform1->GetFixedParameters(); + + if (fixedParameters0.GetSize() == fixedParameters1.GetSize()) + { + fixedParametersAlmostEqual = true; + for (unsigned int i = 0; i < fixedParameters0.GetSize(); ++i) + { + const double difference = std::abs(fixedParameters0[i] - fixedParameters1[i]); + fixedParametersMaximumDifference = std::max(fixedParametersMaximumDifference, difference); + if (difference > fixedParametersTolerance) + { + fixedParametersAlmostEqual = false; + } + } + } + + almostEqual = parametersAlmostEqual && fixedParametersAlmostEqual; + } + + return std::make_tuple(almostEqual, parametersAlmostEqual, fixedParametersAlmostEqual, + parametersMaximumDifference, fixedParametersMaximumDifference); +} + +template +int +compareTransformsFunction(itk::wasm::Pipeline & pipeline) +{ + using TransformType = TTransform; + + using ParametersValueType = typename TransformType::ParametersValueType; + + itk::wasm::InputTransform testTransformInput; + pipeline.get_option("test-transform")->required()->type_name("INPUT_TRANSFORM"); + + std::vector> baselineTransformsInput; + pipeline.get_option("baseline-transforms")->required()->type_name("INPUT_TRANSFORM"); + + ParametersValueType parametersTolerance = 1e-7; + pipeline.get_option("parameters-tolerance")->type_name("FLOAT"); + + ParametersValueType fixedParametersTolerance = 1e-7; + pipeline.get_option("fixed-parameters-tolerance")->type_name("FLOAT"); + + itk::wasm::OutputTextStream metricsStream; + pipeline.get_option("metrics")->type_name("OUTPUT_JSON"); + + ITK_WASM_PARSE(pipeline); + + const TransformType * testTransform = testTransformInput.Get(); + + rapidjson::Document document; + document.SetObject(); + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + + bool almostEqual = false; + bool parametersAlmostEqual = false; + bool fixedParametersAlmostEqual = false; + double parametersMaximumDifference = itk::NumericTraits::max(); + double fixedParametersMaximumDifference = itk::NumericTraits::max(); + + // Compare with baseline transforms + for (const auto & baselineTransformInput : baselineTransformsInput) + { + const TransformType * baselineTransform = baselineTransformInput.Get(); + + auto [currentAlmostEqual, currentParametersAlmostEqual, currentFixedParametersAlmostEqual, + currentParametersMaxDiff, currentFixedParametersMaxDiff] = + compareTransforms(testTransform, baselineTransform, parametersTolerance, fixedParametersTolerance); + + if (currentAlmostEqual) + { + almostEqual = true; + parametersAlmostEqual = currentParametersAlmostEqual; + fixedParametersAlmostEqual = currentFixedParametersAlmostEqual; + parametersMaximumDifference = currentParametersMaxDiff; + fixedParametersMaximumDifference = currentFixedParametersMaxDiff; + break; // Found a match + } + else + { + // Keep track of closest match + if (currentParametersMaxDiff < parametersMaximumDifference) + { + parametersMaximumDifference = currentParametersMaxDiff; + parametersAlmostEqual = currentParametersAlmostEqual; + } + if (currentFixedParametersMaxDiff < fixedParametersMaximumDifference) + { + fixedParametersMaximumDifference = currentFixedParametersMaxDiff; + fixedParametersAlmostEqual = currentFixedParametersAlmostEqual; + } + } + } + + // Create metrics JSON + document.AddMember("almostEqual", almostEqual, allocator); + + rapidjson::Value parametersObject(rapidjson::kObjectType); + parametersObject.AddMember("almostEqual", parametersAlmostEqual, allocator); + parametersObject.AddMember("maximumDifference", parametersMaximumDifference, allocator); + document.AddMember("parameters", parametersObject, allocator); + + rapidjson::Value fixedParametersObject(rapidjson::kObjectType); + fixedParametersObject.AddMember("almostEqual", fixedParametersAlmostEqual, allocator); + fixedParametersObject.AddMember("maximumDifference", fixedParametersMaximumDifference, allocator); + document.AddMember("fixedParameters", fixedParametersObject, allocator); + + rapidjson::StringBuffer stringBuffer; + rapidjson::Writer writer(stringBuffer); + document.Accept(writer); + + metricsStream.Get() << stringBuffer.GetString(); + + return EXIT_SUCCESS; +} + +template +int +compareTransformListsFunction(itk::wasm::Pipeline & pipeline) +{ + using TransformListType = TTransformList; + using TransformType = typename TransformListType::value_type; + using ParametersValueType = typename TransformType::ParametersValueType; + + itk::wasm::InputTransform testTransformListInput; + pipeline.get_option("test-transform-list")->required()->type_name("INPUT_TRANSFORM"); + + std::vector> baselineTransformListsInput; + pipeline.get_option("baseline-transform-lists")->required()->type_name("INPUT_TRANSFORM"); + + ParametersValueType parametersTolerance = 1e-7; + pipeline.get_option("parameters-tolerance")->type_name("FLOAT"); + + ParametersValueType fixedParametersTolerance = 1e-7; + pipeline.get_option("fixed-parameters-tolerance")->type_name("FLOAT"); + + itk::wasm::OutputTextStream metricsStream; + pipeline.get_option("metrics")->type_name("OUTPUT_JSON"); + + ITK_WASM_PARSE(pipeline); + + const TransformListType * testTransformList = testTransformListInput.Get(); + + rapidjson::Document document; + document.SetObject(); + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + + bool almostEqual = false; + bool allTransformsAlmostEqual = false; + + // Compare with baseline transform lists + for (const auto & baselineTransformListInput : baselineTransformListsInput) + { + const TransformListType * baselineTransformList = baselineTransformListInput.Get(); + + if (testTransformList->size() != baselineTransformList->size()) + { + continue; // Different number of transforms + } + + bool currentListAlmostEqual = true; + for (size_t i = 0; i < testTransformList->size(); ++i) + { + auto [transformAlmostEqual, parametersAlmostEqual, fixedParametersAlmostEqual, + parametersMaxDiff, fixedParametersMaxDiff] = + compareTransforms((*testTransformList)[i].GetPointer(), + (*baselineTransformList)[i].GetPointer(), + parametersTolerance, fixedParametersTolerance); + + if (!transformAlmostEqual) + { + currentListAlmostEqual = false; + break; + } + } + + if (currentListAlmostEqual) + { + almostEqual = true; + allTransformsAlmostEqual = true; + break; + } + } + + // Create metrics JSON + document.AddMember("almostEqual", almostEqual, allocator); + document.AddMember("allTransformsAlmostEqual", allTransformsAlmostEqual, allocator); + + rapidjson::StringBuffer stringBuffer; + rapidjson::Writer writer(stringBuffer); + document.Accept(writer); + + metricsStream.Get() << stringBuffer.GetString(); + + return EXIT_SUCCESS; +} + +int +main(int argc, char * argv[]) +{ + itk::wasm::Pipeline pipeline("compare-transforms", "Compare transforms with a tolerance for regression testing.", argc, argv); + + return itk::wasm::SupportInputTransformTypes("test-transform", pipeline); +} diff --git a/packages/compare-transforms/package.json b/packages/compare-transforms/package.json new file mode 100644 index 000000000..647476d6c --- /dev/null +++ b/packages/compare-transforms/package.json @@ -0,0 +1,51 @@ +{ + "name": "@itk-wasm/compare-meshes-build", + "version": "0.6.0", + "private": true, + "description": "@itk-wasm/compare-meshes build configuration.", + "type": "module", + "itk-wasm": { + "emscripten-docker-image": "quay.io/itkwasm/emscripten:latest", + "wasi-docker-image": "quay.io/itkwasm/wasi:latest", + "test-data-hash": "bafkreibsonywg3w3gscmookip3elsyydfsn2cbubk6dukatkmjgeguhiri", + "test-data-urls": [ + "https://github.com/InsightSoftwareConsortium/ITK-Wasm/releases/download/itk-wasm-v1.0.0-b.178/compare-meshes-data.tar.gz" + ], + "package-description": "Compare meshes and polydata for regression testing.", + "typescript-package-name": "@itk-wasm/compare-meshes", + "python-package-name": "itkwasm-compare-meshes", + "repository": "https://github.com/InsightSoftwareConsortium/ITK-Wasm" + }, + "license": "Apache-2.0", + "scripts": { + "build": "pnpm build:gen:typescript && pnpm build:gen:python", + "build:emscripten": "itk-wasm pnpm-script build:emscripten", + "build:emscripten:debug": "itk-wasm pnpm-script build:emscripten:debug", + "build:wasi": "itk-wasm pnpm-script build:wasi", + "build:wasi:debug": "itk-wasm pnpm-script build:wasi:debug", + "build:python:wasi": "itk-wasm pnpm-script build:python:wasi", + "bindgen:typescript": "itk-wasm pnpm-script bindgen:typescript", + "bindgen:python": "itk-wasm pnpm-script bindgen:python", + "build:gen:typescript": "itk-wasm pnpm-script build:gen:typescript", + "build:gen:python": "pnpm build:wasi && pnpm bindgen:python", + "test": "pnpm test:data:download && pnpm build:gen:python && pnpm test:python", + "test:data:download": "dam download test/data test/data.tar.gz bafkreibsonywg3w3gscmookip3elsyydfsn2cbubk6dukatkmjgeguhiri https://github.com/InsightSoftwareConsortium/ITK-Wasm/releases/download/itk-wasm-v1.0.0-b.178/compare-meshes-data.tar.gz", + "test:data:pack": "dam pack test/data test/data.tar.gz", + "test:python:wasi": "pnpm test:data:download && pixi run --manifest-path=./pixi.toml test-wasi", + "test:python:emscripten": "pnpm test:data:download && pixi run --manifest-path=./pixi.toml test-emscripten", + "test:python:dispatch": "pnpm test:data:download && pixi run --manifest-path=./pixi.toml test-dispatch", + "test:python": "itk-wasm pnpm-script test:python", + "test:wasi": "itk-wasm pnpm-script test:wasi" + }, + "devDependencies": { + "@itk-wasm/dam": "^1.1.1", + "itk-wasm": "workspace:^", + "@itk-wasm/mesh-io-build": "workspace:^", + "@itk-wasm/compare-meshes-build": "workspace:^" + }, + "author": "Matt McCormick", + "repository": { + "type": "git", + "url": "https://github.com/InsightSoftwareConsortium/ITK-Wasm" + } +}