From 35808e4656404b26bdb5e16b856427906fed559f Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 19 Feb 2025 02:02:16 +0300 Subject: [PATCH 1/6] chore: use poetry for dependency management instead requirements.txt --- .gitignore | 3 ++- pyproject.toml | 57 +++++++++++++++++++++----------------------- requirements.dev.txt | 4 ---- 3 files changed, 29 insertions(+), 35 deletions(-) delete mode 100644 requirements.dev.txt diff --git a/.gitignore b/.gitignore index 369afbf..61bb621 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +poetry.lock # PyInstaller # Usually these files are written by a python script from a template @@ -157,7 +158,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ #vscode .vscode diff --git a/pyproject.toml b/pyproject.toml index 1992755..993dac3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,47 +1,44 @@ -# pyproject.toml - -[build-system] -requires = ["setuptools>=61.0.0", "wheel"] -build-backend = "setuptools.build_meta" - [project] name = "mpest" version = "0.1.0" description = "Parameter estimation of mixture distribution problem solver, based on EM algorithm" -readme = "README.md" -authors = [{ name = "ToxaKaz", email = "anton.a.kazancev@gmail.com" }] +keywords = ["EM algorithm", "mixture distribution", "parameter estimation"] +authors = [ + {name = "ToxaKaz", email = "anton.a.kazancev@gmail.com"}, + {name = "iraedeus", email = "dtotjmyanin@mail.ru"} +] license = { file = "LICENSE" } +readme = "README.md" +requires-python = ">=3.11" classifiers = [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", ] -keywords = ["EM algorithm", "mixture distribution", "parameter estimation"] dependencies = [ - "numpy", - "scipy", + "matplotlib (>=3.10.0,<4.0.0)", + "numpy (>=2.2.3,<3.0.0)", + "scikit-learn (>=1.6.1,<2.0.0)", + "scipy (>=1.15.2,<2.0.0)", + "seaborn (>=0.13.2,<0.14.0)" ] -requires-python = ">=3.11" -[project.optional-dependencies] -dev = [ - "matplotlib", - "multiprocess", - "pandas", - "pickleshare", - "pre-commit", - "pytest", - "scikit-learn", - "seaborn", - "tqdm", -] +[tool.poetry] + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.4" +pre-commit = "^4.1.0" +ruff = "^0.9.6" +mypy = "^1.15.0" +sphinx = "^8.2.0" -[project.urls] -Homepage = "https://github.com/toxakaz/EM-algo" -[tool.isort] -profile = "black" +[tool.poetry.group.experiments.dependencies] +tqdm = "^4.67.1" +multiprocess = "^0.70.17" +ruamel-yaml = "^0.18.10" -[tool.pylint] -disable = "too-few-public-methods, too-many-locals, line-too-long, too-many-positional-arguments, fixme" +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements.dev.txt b/requirements.dev.txt deleted file mode 100644 index 5204541..0000000 --- a/requirements.dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -black -isort -pre-commit -pytest From 1c444aa19fc65c52b8ccca197d4b67c6c3f25457 Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 19 Feb 2025 04:14:54 +0300 Subject: [PATCH 2/6] chore: replace black with ruff and format all files --- examples/big_mono_tests.py | 9 +-- examples/config.py | 6 +- examples/diff_test.py | 36 ++-------- examples/mono_test_generator.py | 17 ++--- examples/prepare_result.py | 20 ++---- examples/quick_test.py | 9 +-- examples/utils.py | 13 ++-- experimental_env/analysis/analysis.py | 26 ++++--- .../analyze_strategies/analysis_strategy.py | 14 ++-- .../analyze_strategies/density_plot.py | 2 +- .../analyze_strategies/error_convergence.py | 2 +- .../analysis/analyze_strategies/time_plot.py | 2 +- .../analysis_summarizer.py | 14 ++-- .../analyze_summarizers/error_summarizer.py | 6 +- .../analyze_summarizers/time_summarizer.py | 2 +- experimental_env/analysis/metrics.py | 24 +++---- experimental_env/experiment/estimators.py | 68 +++++++------------ .../experiment/experiment_description.py | 9 ++- .../experiment_executors/abstract_executor.py | 14 ++-- .../experiment_executors/random_executor.py | 2 +- .../experiment_executors/standart_executor.py | 2 +- .../experiment/experiment_parser.py | 16 ++--- .../experiment/experiment_saver.py | 2 +- .../mixture_generators/abstract_generator.py | 3 +- .../dataset_mixture_generator.py | 8 ++- .../random_mixture_generator.py | 3 +- .../standart_mixture_generator.py | 2 +- experimental_env/mixture_generators/utils.py | 3 +- .../preparation/dataset_description.py | 1 + .../preparation/dataset_generator.py | 7 +- .../preparation/dataset_parser.py | 13 ++-- experimental_env/utils.py | 26 ++++--- mpest/distribution.py | 4 +- .../breakpointers/step_count_breakpointer.py | 4 +- .../breakpointers/unionable_breakpointer.py | 11 +-- .../distribution_checkers/finite_checker.py | 4 +- .../prior_probability_threshold_checker.py | 10 ++- .../unionable_distribution_checker.py | 15 ++-- mpest/em/em.py | 17 ++--- mpest/em/methods/l_moments_method.py | 38 +++-------- mpest/em/methods/likelihood_method.py | 13 ++-- mpest/mixture_distribution.py | 29 ++------ mpest/models/__init__.py | 1 + mpest/models/abstract_model.py | 4 +- mpest/models/exponential.py | 28 ++++---- mpest/models/gaussian.py | 4 +- mpest/models/weibull.py | 46 ++++++------- mpest/optimizers/abstract_optimizer.py | 2 +- mpest/optimizers/scipy_cg.py | 2 +- mpest/optimizers/scipy_cobyla.py | 2 +- mpest/optimizers/scipy_nelder_mead.py | 2 +- mpest/optimizers/scipy_newton_cg.py | 2 +- mpest/optimizers/scipy_slsqp.py | 2 +- mpest/optimizers/scipy_tnc.py | 2 +- mpest/utils.py | 3 +- pyproject.toml | 18 ++++- requirements.txt | 10 --- .../test_any_distributions_complex_l.py | 11 +-- .../test_one_distribution_l.py | 4 +- .../test_two_same_distributions_complex_l.py | 8 +-- .../test_any_distributions_complex.py | 11 +-- .../tests_likelihood/test_one_distribution.py | 4 +- .../test_two_same_distributions_complex.py | 8 +-- tests/utils.py | 10 +-- 64 files changed, 266 insertions(+), 444 deletions(-) delete mode 100644 requirements.txt diff --git a/examples/big_mono_tests.py b/examples/big_mono_tests.py index 9e89597..d844dff 100644 --- a/examples/big_mono_tests.py +++ b/examples/big_mono_tests.py @@ -22,9 +22,7 @@ counter = Clicker() - def _generate_test( - model: type[AModelWithGenerator], o_borders: list[tuple[float, float]] - ) -> list[Test]: + def _generate_test(model: type[AModelWithGenerator], o_borders: list[tuple[float, float]]) -> list[Test]: return generate_mono_test( model_t=model, params_borders=o_borders, @@ -36,10 +34,7 @@ def _generate_test( tests_per_size=8, tests_per_cond=2, runs_per_test=1, - solvers=[ - init_solver(16, 0.1, 0.001, 3, optimizer) - for optimizer in TESTS_OPTIMIZERS - ], + solvers=[init_solver(16, 0.1, 0.001, 3, optimizer) for optimizer in TESTS_OPTIMIZERS], ) tests += _generate_test(WeibullModelExp, [(0.25, 25), (0.25, 25)]) diff --git a/examples/config.py b/examples/config.py index ad0d839..8d1b009 100644 --- a/examples/config.py +++ b/examples/config.py @@ -7,11 +7,7 @@ CPU_COUNT = os.cpu_count() MAX_WORKERS_PERCENT = 0.75 -MAX_WORKERS = ( - min(round(CPU_COUNT * MAX_WORKERS_PERCENT), CPU_COUNT) - if CPU_COUNT is not None - else 1 -) +MAX_WORKERS = min(round(CPU_COUNT * MAX_WORKERS_PERCENT), CPU_COUNT) if CPU_COUNT is not None else 1 # MAX_WORKERS = 4 RESULTS_FOLDER = EXAMPLES / "results" diff --git a/examples/diff_test.py b/examples/diff_test.py index 097b46a..582b777 100644 --- a/examples/diff_test.py +++ b/examples/diff_test.py @@ -28,11 +28,7 @@ for sp in gaussian_start_params: main_distr = list(gaussian.generate(np.array(sp), BASE_SIZE // 2, normalized=False)) for second_sp in np.linspace(sp[0] - 5, sp[0] + 5, num=8, endpoint=True): - x = main_distr + list( - gaussian.generate( - np.array((second_sp, 3.0)), BASE_SIZE // 2, normalized=False - ) - ) + x = main_distr + list(gaussian.generate(np.array((second_sp, 3.0)), BASE_SIZE // 2, normalized=False)) random.shuffle(x) start_params_borders = [ @@ -45,12 +41,7 @@ samples = random.sample(x, size) for _ in range(TESTS_PER_SIZE): start_params = [ - np.array( - [ - random.uniform(border[0], border[1]) - for border in start_params_borders - ] - ) + np.array([random.uniform(border[0], border[1]) for border in start_params_borders]) for _ in range(2) ] tests.append( @@ -81,10 +72,7 @@ ] ), ), - [ - init_solver(16, 0.1, 0.001, 3, optimizer) - for optimizer in TESTS_OPTIMIZERS - ], + [init_solver(16, 0.1, 0.001, 3, optimizer) for optimizer in TESTS_OPTIMIZERS], 1, ) ) @@ -92,11 +80,7 @@ for sp in weibull_start_params: main_distr = list(weibull.generate(np.array(sp), BASE_SIZE // 2, normalized=False)) for second_sp in np.linspace(max(sp[0] - 5, 0.1), sp[0] + 5, num=8, endpoint=True): - x = main_distr + list( - weibull.generate( - np.array((second_sp, 1.0)), BASE_SIZE // 2, normalized=False - ) - ) + x = main_distr + list(weibull.generate(np.array((second_sp, 1.0)), BASE_SIZE // 2, normalized=False)) random.shuffle(x) start_params_borders = [ @@ -109,12 +93,7 @@ samples = random.sample(x, size) for _ in range(TESTS_PER_SIZE): start_params = [ - np.array( - [ - random.uniform(border[0], border[1]) - for border in start_params_borders - ] - ) + np.array([random.uniform(border[0], border[1]) for border in start_params_borders]) for _ in range(2) ] tests.append( @@ -145,10 +124,7 @@ ] ), ), - [ - init_solver(16, 0.1, 0.001, 3, optimizer) - for optimizer in TESTS_OPTIMIZERS - ], + [init_solver(16, 0.1, 0.001, 3, optimizer) for optimizer in TESTS_OPTIMIZERS], 1, ) ) diff --git a/examples/mono_test_generator.py b/examples/mono_test_generator.py index f4bc18a..b474211 100644 --- a/examples/mono_test_generator.py +++ b/examples/mono_test_generator.py @@ -1,7 +1,7 @@ """Module which contains mixture distributions of single model tests generator""" import random -from typing import Iterable +from collections.abc import Iterable import numpy as np @@ -42,9 +42,7 @@ def generate_mono_test( models: list[AModel] = [] for _ in range(k): - params = np.array( - [random.uniform(border[0], border[1]) for border in params_borders] - ) + params = np.array([random.uniform(border[0], border[1]) for border in params_borders]) model = model_t() x += list(model.generate(params, per_model, normalized=False)) @@ -62,12 +60,7 @@ def generate_mono_test( samples = random.sample(x, size) for _ in range(tests_per_size): start_params = [ - np.array( - [ - random.uniform(border[0], border[1]) - for border in start_params_borders - ] - ) + np.array([random.uniform(border[0], border[1]) for border in start_params_borders]) for _ in range(k) ] tests.append( @@ -91,9 +84,7 @@ def generate_mono_test( model, params, ) - for model, params in zip( - models, start_params - ) + for model, params in zip(models, start_params) ] ), ), diff --git a/examples/prepare_result.py b/examples/prepare_result.py index cb2b823..63bc830 100644 --- a/examples/prepare_result.py +++ b/examples/prepare_result.py @@ -21,9 +21,7 @@ def nll(samples: Samples, mixture: MixtureDistribution) -> float: return occur -def identity_guessing_chance( - dx: MixtureDistribution, dy: MixtureDistribution, sample: Samples -): +def identity_guessing_chance(dx: MixtureDistribution, dy: MixtureDistribution, sample: Samples): """Identity guessing chance metric""" dxs = list(dx) @@ -64,9 +62,7 @@ def result_to_df_diff(result: SingleSolverResult): dct[clicker.click()] = (sp[0], second_sp) for sp in weibull_start_params: - for second_sp in np.linspace( - max(sp[0] - 5, 0.1), sp[0] + 5, num=8, endpoint=True - ): + for second_sp in np.linspace(max(sp[0] - 5, 0.1), sp[0] + 5, num=8, endpoint=True): for _ in sizes: for _ in range(tests_per_cond): for _ in range(tests_per_size): @@ -76,7 +72,7 @@ def result_to_df_diff(result: SingleSolverResult): [ ( d - if (d.prior_probability is not None) and (d.prior_probability > 0.001) + if (d.prior_probability is not None) and (d.prior_probability > 0.001) # noqa: PLR2004 else DistributionInMixture(d.model, d.params, None) ) for d in result.result.content @@ -100,14 +96,12 @@ def result_to_df_diff(result: SingleSolverResult): "time": result.time, "model": result.test.true_mixture[0].model.name, "size": len(result.test.problem.samples), - "success": (result.steps < 128) and not failed, + "success": (result.steps < 128) and not failed, # noqa: PLR2004 "failed": failed, "occur": nll(result.test.all_data, mixture_distribution), "start": start, "diff": diff, - "res_err": identity_guessing_chance( - result.test.true_mixture, result.result.content, result.test.all_data - ), + "res_err": identity_guessing_chance(result.test.true_mixture, result.result.content, result.test.all_data), } @@ -131,7 +125,7 @@ def result_to_df(result: SingleSolverResult): [ ( d - if (d.prior_probability is not None) and (d.prior_probability > 0.001) + if (d.prior_probability is not None) and (d.prior_probability > 0.001) # noqa: PLR2004 else DistributionInMixture(d.model, d.params, None) ) for d in result.result.content @@ -152,7 +146,7 @@ def result_to_df(result: SingleSolverResult): "time": result.time, "model": result.test.true_mixture[0].model.name, "size": len(result.test.problem.samples), - "success": (result.steps < 16) and failed, + "success": (result.steps < 16) and failed, # noqa: PLR2004 "failed": failed, "occur": nll(result.test.all_data, mixture_distribution), } diff --git a/examples/quick_test.py b/examples/quick_test.py index 3661193..434fdf6 100644 --- a/examples/quick_test.py +++ b/examples/quick_test.py @@ -26,9 +26,7 @@ def run_test(): counter = Clicker() - def _generate_test( - model: type[AModelWithGenerator], params_borders: list[tuple[float, float]] - ) -> list[Test]: + def _generate_test(model: type[AModelWithGenerator], params_borders: list[tuple[float, float]]) -> list[Test]: test = generate_mono_test( model_t=model, clicker=counter, @@ -40,10 +38,7 @@ def _generate_test( tests_per_size=1, tests_per_cond=1, runs_per_test=1, - solvers=[ - init_solver(16, 0.1, 0.001, 3, optimizer) - for optimizer in ALL_OPTIMIZERS - ], + solvers=[init_solver(16, 0.1, 0.001, 3, optimizer) for optimizer in ALL_OPTIMIZERS], ) return test diff --git a/examples/utils.py b/examples/utils.py index 5b71427..c202eb2 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -4,7 +4,7 @@ import random import time from functools import partial -from typing import Callable, NamedTuple +from typing import ClassVar, NamedTuple import numpy as np from tqdm.contrib.concurrent import process_map @@ -48,7 +48,7 @@ class SingleSolverResult(NamedTuple): steps: int time: float - log: list[EM.Log.Item] = [] + log: ClassVar[list[EM.Log.Item]] = [] class TestResult(NamedTuple): @@ -133,7 +133,10 @@ def run_tests( ) if shuffled: - key: Callable[[TestResult], int] = lambda t: t.test.index + + def key(t): + return t.test.index + results.sort(key=key) return results @@ -170,8 +173,6 @@ def init_solver( return EM( breakpointer, FiniteChecker() - + PriorProbabilityThresholdChecker( - prior_probability_threshold, prior_probability_threshold_step - ), + + PriorProbabilityThresholdChecker(prior_probability_threshold, prior_probability_threshold_step), method, ) diff --git a/experimental_env/analysis/analysis.py b/experimental_env/analysis/analysis.py index 5aa36d2..b92052d 100644 --- a/experimental_env/analysis/analysis.py +++ b/experimental_env/analysis/analysis.py @@ -1,4 +1,5 @@ -""" A module that provides a class for performing the third stage of the experiment """ +"""A module that provides a class for performing the third stage of the experiment""" + from pathlib import Path from tqdm import tqdm @@ -33,7 +34,8 @@ def analyze(self, results: ParserOutput, method: str): """ A function for analyzing the method - :param results: The result of the method on the second stage of the experiment, which was obtained using a parser. + :param results: The result of the method on the second stage of the experiment, + which was obtained using a parser. :param method: Name of analyzed method """ method_dir: Path = self._out_dir.joinpath(method) @@ -48,9 +50,7 @@ def analyze(self, results: ParserOutput, method: str): with tqdm(total=len(exp_descriptions)) as pbar: for exp_descr in exp_descriptions: pbar.update() - exp_dir: Path = mixture_dir.joinpath( - f"experiment_{exp_descr.exp_num}" - ) + exp_dir: Path = mixture_dir.joinpath(f"experiment_{exp_descr.exp_num}") for action in self._actions: action.set_path(exp_dir) @@ -66,21 +66,21 @@ def compare( """ A function for comparing the methods - :param results_1: The result of the first method on the second stage of the experiment, which was obtained using a parser. - :param results_2: The result of the second method on the second stage of the experiment, which was obtained using a parser. + :param results_1: The result of the first method on the second stage of the experiment, + which was obtained using a parser. + :param results_2: The result of the second method on the second stage of the experiment, + which was obtained using a parser. :param method_1: Name of the first method :param method_2: Name of the second method """ method_dir = self._out_dir.joinpath(f"{method_1} + {method_2}") - for mixture_name in results_1.keys(): + for mixture_name in results_1: mixture_dir = method_dir.joinpath(mixture_name) for summarizer in self._summarizers: summarizer.set_path(mixture_dir) - summarizer.compare_methods( - results_1[mixture_name], results_2[mixture_name], method_1, method_2 - ) + summarizer.compare_methods(results_1[mixture_name], results_2[mixture_name], method_1, method_2) with tqdm(total=len(results_1[mixture_name])) as pbar: for res in zip(results_1[mixture_name], results_2[mixture_name]): @@ -89,6 +89,4 @@ def compare( for action in self._actions: action.set_path(exp_dir) - action.overall_compare_methods( - res[0], res[1], method_1, method_2 - ) + action.overall_compare_methods(res[0], res[1], method_1, method_2) diff --git a/experimental_env/analysis/analyze_strategies/analysis_strategy.py b/experimental_env/analysis/analyze_strategies/analysis_strategy.py index a8bf788..f84ec31 100644 --- a/experimental_env/analysis/analyze_strategies/analysis_strategy.py +++ b/experimental_env/analysis/analyze_strategies/analysis_strategy.py @@ -1,4 +1,4 @@ -""" A module containing an abstract strategy class for analysis methods. """ +"""A module containing an abstract strategy class for analysis methods.""" # pylint: disable=duplicate-code from abc import ABC, abstractmethod @@ -9,7 +9,8 @@ class AnalysisStrategy(ABC): """ - An abstract strategy class that contains methods for analysis and comparison, as well as a method for establishing a directory + An abstract strategy class that contains methods for analysis and comparison, + as well as a method for establishing a directory """ def __init__(self): @@ -39,7 +40,8 @@ def overall_analyze_method(self, result: ExperimentDescription, method: str): """ Analyze the method result of the experiment - :param result: The result of the method on the second stage of the experiment, which was obtained using a parser. + :param result: The result of the method on the second stage of the experiment, + which was obtained using a parser. :param method: The name of the method that we are analyzing """ @@ -56,8 +58,10 @@ def overall_compare_methods( """ A function for comparing the methods by two results, with the same base mixture - :param result_1: The result of the first method on the second stage of the experiment, which was obtained using a parser. - :param result_2: The result of the second method on the second stage of the experiment, which was obtained using a parser. + :param result_1: The result of the first method on the second stage of the experiment, + which was obtained using a parser. + :param result_2: The result of the second method on the second stage of the experiment, + which was obtained using a parser. :param method_1: Name of the first method :param method_2: Name of the second method """ diff --git a/experimental_env/analysis/analyze_strategies/density_plot.py b/experimental_env/analysis/analyze_strategies/density_plot.py index 07f3a13..7278e06 100644 --- a/experimental_env/analysis/analyze_strategies/density_plot.py +++ b/experimental_env/analysis/analyze_strategies/density_plot.py @@ -1,4 +1,4 @@ -""" A module providing a class for saving a distribution density graph. """ +"""A module providing a class for saving a distribution density graph.""" import matplotlib.pyplot as plt import numpy as np diff --git a/experimental_env/analysis/analyze_strategies/error_convergence.py b/experimental_env/analysis/analyze_strategies/error_convergence.py index 927b38e..97b04a9 100644 --- a/experimental_env/analysis/analyze_strategies/error_convergence.py +++ b/experimental_env/analysis/analyze_strategies/error_convergence.py @@ -1,4 +1,4 @@ -""" A module providing a class for saving the convergence graph of the error function. """ +"""A module providing a class for saving the convergence graph of the error function.""" from matplotlib import pyplot as plt diff --git a/experimental_env/analysis/analyze_strategies/time_plot.py b/experimental_env/analysis/analyze_strategies/time_plot.py index abbd241..bcea747 100644 --- a/experimental_env/analysis/analyze_strategies/time_plot.py +++ b/experimental_env/analysis/analyze_strategies/time_plot.py @@ -1,4 +1,4 @@ -""" A module that provides a class for saving a graph showing the execution time of each step """ +"""A module that provides a class for saving a graph showing the execution time of each step""" import matplotlib.pyplot as plt diff --git a/experimental_env/analysis/analyze_summarizers/analysis_summarizer.py b/experimental_env/analysis/analyze_summarizers/analysis_summarizer.py index 2737291..1e50ca6 100644 --- a/experimental_env/analysis/analyze_summarizers/analysis_summarizer.py +++ b/experimental_env/analysis/analyze_summarizers/analysis_summarizer.py @@ -1,4 +1,4 @@ -""" A module containing an abstract summarizer class for analysis methods. """ +"""A module containing an abstract summarizer class for analysis methods.""" from abc import ABC, abstractmethod from pathlib import Path @@ -8,7 +8,8 @@ class AnalysisSummarizer(ABC): """ - A class that provides functionality for analyzing methods by examining the overall set of experimental results for a given mixture. + A class that provides functionality for analyzing methods + by examining the overall set of experimental results for a given mixture. """ def __init__(self): @@ -31,7 +32,8 @@ def analyze_method(self, results: list[ExperimentDescription], method: str): """ Analysis of the method for all experiments on this mixture. - :param results: The results of the method on the second stage of the experiment, which was obtained using a parser. + :param results: The results of the method on the second stage of the experiment, + which was obtained using a parser. :param method: The name of the method that we are analyzing """ @@ -48,8 +50,10 @@ def compare_methods( """ A function for comparing the methods - :param results_1: The results of the first method on the second stage of the experiment, which was obtained using a parser. - :param results_2: The results of the second method on the second stage of the experiment, which was obtained using a parser. + :param results_1: The results of the first method on the second stage of the experiment, + which was obtained using a parser. + :param results_2: The results of the second method on the second stage of the experiment, + which was obtained using a parser. :param method_1: Name of the first method :param method_2: Name of the second method """ diff --git a/experimental_env/analysis/analyze_summarizers/error_summarizer.py b/experimental_env/analysis/analyze_summarizers/error_summarizer.py index 09579fd..9810808 100644 --- a/experimental_env/analysis/analyze_summarizers/error_summarizer.py +++ b/experimental_env/analysis/analyze_summarizers/error_summarizer.py @@ -1,4 +1,4 @@ -""" A module that provides a class that save data about how the metric error is distributed """ +"""A module that provides a class that save data about how the metric error is distributed""" from math import isnan from pathlib import Path @@ -39,9 +39,7 @@ def calculate(self, results: list[ExperimentDescription]) -> tuple: errors.append(error) mean = np.sum(errors) / len(errors) - standart_deviation = np.sqrt( - np.sum((x - mean) ** 2 for x in errors) / len(errors) - ) + standart_deviation = np.sqrt(np.sum((x - mean) ** 2 for x in errors) / len(errors)) errors.sort() median = errors[len(errors) // 2] diff --git a/experimental_env/analysis/analyze_summarizers/time_summarizer.py b/experimental_env/analysis/analyze_summarizers/time_summarizer.py index 545b817..1c0630b 100644 --- a/experimental_env/analysis/analyze_summarizers/time_summarizer.py +++ b/experimental_env/analysis/analyze_summarizers/time_summarizer.py @@ -1,4 +1,4 @@ -""" A module that provides a class that save data about how the time of estimation is distributed """ +"""A module that provides a class that save data about how the time of estimation is distributed""" from pathlib import Path diff --git a/experimental_env/analysis/metrics.py b/experimental_env/analysis/metrics.py index bc226b5..6cf931e 100644 --- a/experimental_env/analysis/metrics.py +++ b/experimental_env/analysis/metrics.py @@ -1,4 +1,4 @@ -""" A module that provides various metrics for evaluating the quality of parameter estimation. """ +"""A module that provides various metrics for evaluating the quality of parameter estimation.""" from abc import ABC, abstractmethod from itertools import permutations @@ -27,8 +27,12 @@ def error( class SquaredError(AMetric): - """ - The class that calculates the SquaredError + r""" + The class that calculates the SquaredError. + + .. math:: + \text{err} = \int_{-\infty}^{+\infty} (f_1(x) - f_2(x))^2 \,dx + """ @property @@ -45,7 +49,8 @@ def integrand(x, base_mixture, result_mixture): class Parametric(AMetric): """ - A class that calculates the absolute difference in the parameters of mixtures. Does not use the difference of a prior probabilities + A class that calculates the absolute difference in the parameters of mixtures. + Does not use the difference of a prior probabilities. """ @property @@ -53,14 +58,9 @@ def name(self) -> str: return "ParamsError" def error(self, base_mixture, result_mixture): - base_p, res_p = ( - [d.params for d in ld] for ld in (base_mixture, result_mixture) - ) - - param_diff = min( - sum(sum(abs(x - y)) for x, y in zip(base_p, _res_p)) - for _res_p in permutations(res_p) - ) + base_p, res_p = ([d.params for d in ld] for ld in (base_mixture, result_mixture)) + + param_diff = min(sum(sum(abs(x - y)) for x, y in zip(base_p, _res_p)) for _res_p in permutations(res_p)) return param_diff diff --git a/experimental_env/experiment/estimators.py b/experimental_env/experiment/estimators.py index f5cc6b7..09d6fff 100644 --- a/experimental_env/experiment/estimators.py +++ b/experimental_env/experiment/estimators.py @@ -1,4 +1,5 @@ """Estimators for estimating parameters in second stage of experiment""" + import random from abc import abstractmethod from concurrent.futures import as_completed @@ -16,10 +17,7 @@ from mpest.utils import ANamed, Factory, ResultWithLog METHODS = { - "Likelihood": [ - [Factory(BayesEStep), Factory(LikelihoodMStep, optimizer)] - for optimizer in ALL_OPTIMIZERS - ], + "Likelihood": [[Factory(BayesEStep), Factory(LikelihoodMStep, optimizer)] for optimizer in ALL_OPTIMIZERS], "L-moments": [Factory(IndicatorEStep), Factory(LMomentsMStep)], } @@ -31,9 +29,7 @@ class AEstimator(ANamed): """ @abstractmethod - def estimate( - self, problems: list[Problem], cpu_count: int, seed: int - ) -> list[ResultWithLog]: + def estimate(self, problems: list[Problem], cpu_count: int, seed: int) -> list[ResultWithLog]: """ The process of estimating the parameters of the mixture """ @@ -45,9 +41,7 @@ class LikelihoodEstimator(AEstimator): During the estimating process, the result with the best approximation is returned. """ - def __init__( - self, brkpointer: EM.ABreakpointer, dst_checker: EM.ADistributionChecker - ): + def __init__(self, brkpointer: EM.ABreakpointer, dst_checker: EM.ADistributionChecker): """ Class constructor """ @@ -62,35 +56,26 @@ def _helper(self, problem: OrderedProblem): """ Helper function for multiprocessed estimation """ - methods = [ - Method(step[0].construct(), step[1].construct()) - for step in METHODS["Likelihood"] - ] + methods = [Method(step[0].construct(), step[1].construct()) for step in METHODS["Likelihood"]] ems = [EM(self._brkpointer, self._dst_checker, method) for method in methods] results = [em.solve_logged(problem, True, True, True) for em in ems] return choose_best_mle(problem.distributions, results), problem.number - def estimate( - self, problems: list[Problem], cpu_count: int, seed: int = 42 - ) -> list[ResultWithLog]: + def estimate(self, problems: list[Problem], cpu_count: int, seed: int = 42) -> list[ResultWithLog]: output = {} random.seed(seed) print("Starting Likelihood estimation") ordered_problem = [ - OrderedProblem(problem.samples, problem.distributions, i) - for i, problem in enumerate(problems) + OrderedProblem(problem.samples, problem.distributions, i) for i, problem in enumerate(problems) ] - with tqdm(total=len(problems)) as pbar: - with ProcessPoolExecutor(max_workers=cpu_count) as executor: - fs = [ - executor.submit(self._helper, problem) - for problem in ordered_problem - ] - - for f in as_completed(fs): - pbar.update() - res = f.result() - output[res[1]] = res[0] + with tqdm(total=len(problems)) as pbar, ProcessPoolExecutor(max_workers=cpu_count) as executor: + fs = [executor.submit(self._helper, problem) for problem in ordered_problem] + + for f in as_completed(fs): + pbar.update() + res = f.result() + output[res[1]] = res[0] + return [res for num, res in sorted(output.items())] @@ -120,25 +105,18 @@ def _helper(self, problem: OrderedProblem): problem.number, ) - def estimate( - self, problems: list[Problem], cpu_count: int, seed: int = 42 - ) -> list[ResultWithLog]: + def estimate(self, problems: list[Problem], cpu_count: int, seed: int = 42) -> list[ResultWithLog]: output = {} random.seed(seed) print("Starting L-moments estimation") ordered_problem = [ - OrderedProblem(problem.samples, problem.distributions, i) - for i, problem in enumerate(problems) + OrderedProblem(problem.samples, problem.distributions, i) for i, problem in enumerate(problems) ] - with tqdm(total=len(problems)) as pbar: - with ProcessPoolExecutor(max_workers=cpu_count) as executor: - fs = [ - executor.submit(self._helper, problem) - for problem in ordered_problem - ] - for f in as_completed(fs): - pbar.update() - res = f.result() - output[res[1]] = res[0] + with tqdm(total=len(problems)) as pbar, ProcessPoolExecutor(max_workers=cpu_count) as executor: + fs = [executor.submit(self._helper, problem) for problem in ordered_problem] + for f in as_completed(fs): + pbar.update() + res = f.result() + output[res[1]] = res[0] return [res for num, res in sorted(output.items())] diff --git a/experimental_env/experiment/experiment_description.py b/experimental_env/experiment/experiment_description.py index fe73a09..7bc0d98 100644 --- a/experimental_env/experiment/experiment_description.py +++ b/experimental_env/experiment/experiment_description.py @@ -1,5 +1,6 @@ -""" A module containing classes for storing information about the results of estimating at the second stage """ -from typing import Iterable, Iterator +"""A module containing classes for storing information about the results of estimating at the second stage""" + +from collections.abc import Iterable, Iterator import numpy as np @@ -79,9 +80,7 @@ def from_result( """ self = cls() self._init_mixture = init_mixture - self._steps = [ - StepDescription(step.result.content, step.time) for step in result.log.log - ] + self._steps = [StepDescription(step.result.content, step.time) for step in result.log.log] self._ds_descr = ds_descr self._error = bool(result.log.log[-1].result.error) return self diff --git a/experimental_env/experiment/experiment_executors/abstract_executor.py b/experimental_env/experiment/experiment_executors/abstract_executor.py index d48eed7..0f99870 100644 --- a/experimental_env/experiment/experiment_executors/abstract_executor.py +++ b/experimental_env/experiment/experiment_executors/abstract_executor.py @@ -1,4 +1,4 @@ -""" A module that provides an abstract class for performing the 2nd stage of the experiment """ +"""A module that provides an abstract class for performing the 2nd stage of the experiment""" import warnings from abc import ABC, abstractmethod @@ -34,9 +34,7 @@ def __init__(self, path: Path, cpu_count: int, seed): np.random.seed(self._seed) @abstractmethod - def init_problems( - self, ds_descriptions: list[DatasetDescrciption], models: list[type[AModel]] - ) -> list[Problem]: + def init_problems(self, ds_descriptions: list[DatasetDescrciption], models: list[type[AModel]]) -> list[Problem]: """ Function for generate problem any method user want. """ @@ -63,12 +61,8 @@ def execute(self, preparation_results: dict, estimator: AEstimator) -> None: # Saving results for i, ds_descr in enumerate(ds_descriptions): - exp_dir: Path = mixture_name_dir.joinpath( - f"experiment_{ds_descr.exp_num}" - ) + exp_dir: Path = mixture_name_dir.joinpath(f"experiment_{ds_descr.exp_num}") result = results[i] - result_descr = ExperimentDescription.from_result( - problems[i].distributions, result, ds_descr - ) + result_descr = ExperimentDescription.from_result(problems[i].distributions, result, ds_descr) ExperimentSaver(exp_dir).save(result_descr) diff --git a/experimental_env/experiment/experiment_executors/random_executor.py b/experimental_env/experiment/experiment_executors/random_executor.py index 713e9ff..c520e80 100644 --- a/experimental_env/experiment/experiment_executors/random_executor.py +++ b/experimental_env/experiment/experiment_executors/random_executor.py @@ -1,4 +1,4 @@ -""" A module that provides a class for generating initial mixtures with uniform distribution. """ +"""A module that provides a class for generating initial mixtures with uniform distribution.""" from experimental_env.experiment.experiment_executors.abstract_executor import AExecutor from experimental_env.mixture_generators.random_mixture_generator import ( diff --git a/experimental_env/experiment/experiment_executors/standart_executor.py b/experimental_env/experiment/experiment_executors/standart_executor.py index 3e0ec71..b8cdfda 100644 --- a/experimental_env/experiment/experiment_executors/standart_executor.py +++ b/experimental_env/experiment/experiment_executors/standart_executor.py @@ -1,4 +1,4 @@ -""" A module that provides a class for generating initial mixtures with standart parameters """ +"""A module that provides a class for generating initial mixtures with standart parameters""" from experimental_env.experiment.experiment_executors.abstract_executor import AExecutor from experimental_env.mixture_generators.standart_mixture_generator import ( diff --git a/experimental_env/experiment/experiment_parser.py b/experimental_env/experiment/experiment_parser.py index 1cc05e3..45317bd 100644 --- a/experimental_env/experiment/experiment_parser.py +++ b/experimental_env/experiment/experiment_parser.py @@ -1,4 +1,4 @@ -""" A module containing a class for parsing the second stage of the experiment""" +"""A module containing a class for parsing the second stage of the experiment""" import os from pathlib import Path @@ -23,13 +23,13 @@ class ExperimentParser: def _get_steps(self, exp_dir: Path): steps = [] for step in sort_human(os.listdir(exp_dir)): - if not "step" in step: + if "step" not in step: continue step_dir = exp_dir.joinpath(step) step_config_p = step_dir.joinpath("config.yaml") # Get step - with open(step_config_p, "r", encoding="utf-8") as config_file: + with open(step_config_p, encoding="utf-8") as config_file: config = yaml.safe_load(config_file) step_time = config["time"] step_mixture = create_mixture_by_key(config, "step_distributions") @@ -61,7 +61,7 @@ def parse(self, path: Path) -> ParserOutput: samples = genfromtxt(samples_p, delimiter=",") # Get base mixture - with open(config_p, "r", encoding="utf-8") as config_file: + with open(config_p, encoding="utf-8") as config_file: config = yaml.safe_load(config_file) samples_size = config["samples_size"] exp_num = config["exp_num"] @@ -74,13 +74,9 @@ def parse(self, path: Path) -> ParserOutput: steps = self._get_steps(experiment_dir) # Save results of experiment - ds_descr = DatasetDescrciption( - samples_size, samples, base_mixture, exp_num - ) + ds_descr = DatasetDescrciption(samples_size, samples, base_mixture, exp_num) - result_descr = ExperimentDescription.from_steps( - init_mixture, steps, ds_descr, error - ) + result_descr = ExperimentDescription.from_steps(init_mixture, steps, ds_descr, error) mixture_name_list.append(result_descr) diff --git a/experimental_env/experiment/experiment_saver.py b/experimental_env/experiment/experiment_saver.py index 677e0ea..8c72713 100644 --- a/experimental_env/experiment/experiment_saver.py +++ b/experimental_env/experiment/experiment_saver.py @@ -1,4 +1,4 @@ -""" A module with saver for second stage of experiment """ +"""A module with saver for second stage of experiment""" from pathlib import Path diff --git a/experimental_env/mixture_generators/abstract_generator.py b/experimental_env/mixture_generators/abstract_generator.py index cbffe98..40754c9 100644 --- a/experimental_env/mixture_generators/abstract_generator.py +++ b/experimental_env/mixture_generators/abstract_generator.py @@ -1,4 +1,5 @@ -""" A module that provides an abstract class for generating a mixture. """ +"""A module that provides an abstract class for generating a mixture.""" + import random from abc import ABC, abstractmethod diff --git a/experimental_env/mixture_generators/dataset_mixture_generator.py b/experimental_env/mixture_generators/dataset_mixture_generator.py index 0800de6..f70ba93 100644 --- a/experimental_env/mixture_generators/dataset_mixture_generator.py +++ b/experimental_env/mixture_generators/dataset_mixture_generator.py @@ -1,4 +1,8 @@ -""" A module that provides a mixture generator in which parameters are generated in accordance with the requirements for the dataset. """ +""" +A module that provides a mixture generator in which parameters are generated in accordance +with the requirements for the dataset. +""" + import numpy as np from experimental_env.mixture_generators.abstract_generator import AMixtureGenerator @@ -8,7 +12,7 @@ class DatasetMixtureGenerator(AMixtureGenerator): """ A class for generating a prior probabilities for datasets. - The alphas for the Dirichlet distribution are 5, which means that a prior probabilities will rarely be close to 0 or 1 + The alphas for the Dirichlet distribution are 5. """ def generate_dists(self, models): diff --git a/experimental_env/mixture_generators/random_mixture_generator.py b/experimental_env/mixture_generators/random_mixture_generator.py index b046ba2..184d345 100644 --- a/experimental_env/mixture_generators/random_mixture_generator.py +++ b/experimental_env/mixture_generators/random_mixture_generator.py @@ -1,4 +1,5 @@ -""" A module that provides a mixture generator class in which parameters are generated using a uniform distribution. """ +"""A module that provides a mixture generator class in which parameters are generated using a uniform distribution.""" + from experimental_env.mixture_generators.abstract_generator import AMixtureGenerator from experimental_env.mixture_generators.utils import generate_uniform_params diff --git a/experimental_env/mixture_generators/standart_mixture_generator.py b/experimental_env/mixture_generators/standart_mixture_generator.py index a622315..dc8aa04 100644 --- a/experimental_env/mixture_generators/standart_mixture_generator.py +++ b/experimental_env/mixture_generators/standart_mixture_generator.py @@ -1,4 +1,4 @@ -""" A module providing a class for generating mixtures with standard parameters. """ +"""A module providing a class for generating mixtures with standard parameters.""" from experimental_env.mixture_generators.abstract_generator import AMixtureGenerator from experimental_env.mixture_generators.utils import generate_standart_params diff --git a/experimental_env/mixture_generators/utils.py b/experimental_env/mixture_generators/utils.py index e808ad0..a8d7090 100644 --- a/experimental_env/mixture_generators/utils.py +++ b/experimental_env/mixture_generators/utils.py @@ -1,4 +1,5 @@ -""" A module containing parameter generators for mixture generators. """ +"""A module containing parameter generators for mixture generators.""" + from random import uniform from mpest import Distribution diff --git a/experimental_env/preparation/dataset_description.py b/experimental_env/preparation/dataset_description.py index 280b254..e4a816f 100644 --- a/experimental_env/preparation/dataset_description.py +++ b/experimental_env/preparation/dataset_description.py @@ -1,4 +1,5 @@ """Module which describes dataset""" + import copy from mpest.mixture_distribution import MixtureDistribution diff --git a/experimental_env/preparation/dataset_generator.py b/experimental_env/preparation/dataset_generator.py index ddc6d38..b519269 100644 --- a/experimental_env/preparation/dataset_generator.py +++ b/experimental_env/preparation/dataset_generator.py @@ -1,4 +1,5 @@ """Module from generating datasets with the given sample size, experimental counts and mixture""" + import random from pathlib import Path @@ -62,9 +63,7 @@ def __init__(self, seed: int = 42): self._dists = [] self._priors = [] - def add_distribution( - self, model: type[AModel], params: list[float], prior: float - ) -> None: + def add_distribution(self, model: type[AModel], params: list[float], prior: float) -> None: """ Add distribution with params and prior to mixture. """ @@ -88,7 +87,7 @@ def generate(self, samples_size: int, working_path: Path, exp_count: int): descr = DatasetDescrciption(samples_size, samples, mixture, i + 1) mixture_name_dir: Path = working_path.joinpath(descr.get_dataset_name()) - exp_dir: Path = mixture_name_dir.joinpath(f"experiment_{i+1}") + exp_dir: Path = mixture_name_dir.joinpath(f"experiment_{i + 1}") if exp_dir.exists(): continue diff --git a/experimental_env/preparation/dataset_parser.py b/experimental_env/preparation/dataset_parser.py index 15cb9db..e28fe57 100644 --- a/experimental_env/preparation/dataset_parser.py +++ b/experimental_env/preparation/dataset_parser.py @@ -1,4 +1,5 @@ -""" A module containing a class for parsing datasets from the first stage of the experiment """ +"""A module containing a class for parsing datasets from the first stage of the experiment""" + import os from pathlib import Path @@ -29,7 +30,9 @@ def parse(self, path: Path) -> dict: # Open each experiment dir for exp in os.listdir(mixture_name_dir): - # TODO: For macOS, there is a problem of finding files.DS_Store at which the program crashes. It is necessary to sort such files. + # TODO: For macOS, there is a problem of finding files.DS_Store at which the program crashes. + # It is necessary to sort such files. + experiment_dir: Path = mixture_name_dir.joinpath(exp) samples_p: Path = experiment_dir.joinpath("samples.csv") config_p: Path = experiment_dir.joinpath("config.yaml") @@ -38,15 +41,13 @@ def parse(self, path: Path) -> dict: samples = genfromtxt(samples_p, delimiter=",") # Get mixture - with open(config_p, "r", encoding="utf-8") as config_file: + with open(config_p, encoding="utf-8") as config_file: config = yaml.safe_load(config_file) exp_count = config["exp_num"] samples_size = config["samples_size"] base_mixture = create_mixture_by_key(config, "distributions") - mixture_name_dict.append( - DatasetDescrciption(samples_size, samples, base_mixture, exp_count) - ) + mixture_name_dict.append(DatasetDescrciption(samples_size, samples, base_mixture, exp_count)) output[mixture_name] = mixture_name_dict return output diff --git a/experimental_env/utils.py b/experimental_env/utils.py index 86d6674..a30e893 100644 --- a/experimental_env/utils.py +++ b/experimental_env/utils.py @@ -1,4 +1,5 @@ """Functions for experimental environment""" + import math import re @@ -17,17 +18,12 @@ def create_mixture_by_key(config: dict, key: str) -> MixtureDistribution: config[key] = [config[key]] priors = [d["prior"] for d in config[key]] - dists = [ - Distribution.from_params(ALL_MODELS[d["type"]], d["params"]) - for d in config[key] - ] + dists = [Distribution.from_params(ALL_MODELS[d["type"]], d["params"]) for d in config[key]] return MixtureDistribution.from_distributions(dists, priors) -def choose_best_mle( - base_mixture: MixtureDistribution, results: list[ResultWithLog] -) -> ResultWithLog: +def choose_best_mle(base_mixture: MixtureDistribution, results: list[ResultWithLog]) -> ResultWithLog: """ The method for choosing the best result in the maximum likelihood method """ @@ -74,13 +70,15 @@ def number(self): return self._number -def sort_human(l: list): +def sort_human(lst: list): """ Function for sorting list in alphanumerical order. """ - # pylint: disable = [unnecessary-lambda-assignment] - convert = lambda text: float(text) if text.isdigit() else text - alphanum = lambda key: [ - convert(c) for c in re.split(r"([-+]?[0-9]*\.?[0-9]*)", key) - ] - return sorted(l, key=alphanum) + + def convert(text): + return float(text) if text.isdigit() else text + + def alphanum(key): + return [convert(c) for c in re.split(r"([-+]?[0-9]*\.?[0-9]*)", key)] + + return sorted(lst, key=alphanum) diff --git a/mpest/distribution.py b/mpest/distribution.py index b3a04e8..53d8124 100644 --- a/mpest/distribution.py +++ b/mpest/distribution.py @@ -75,6 +75,4 @@ def generate(self, size=1): """ if not isinstance(self.model, AModelWithGenerator): raise TypeError(f"Model {self.model} hasn't got a generator") - return self.model.generate( - self.model.params_convert_to_model(self.params), size=size - ) + return self.model.generate(self.model.params_convert_to_model(self.params), size=size) diff --git a/mpest/em/breakpointers/step_count_breakpointer.py b/mpest/em/breakpointers/step_count_breakpointer.py index 307b00e..35f2485 100644 --- a/mpest/em/breakpointers/step_count_breakpointer.py +++ b/mpest/em/breakpointers/step_count_breakpointer.py @@ -25,6 +25,4 @@ def is_over( previous_step: MixtureDistribution | None, current_step: MixtureDistribution, ) -> bool: - if (self.max_step is not None) and (step >= self.max_step): - return True - return False + return (self.max_step is not None) and (step >= self.max_step) diff --git a/mpest/em/breakpointers/unionable_breakpointer.py b/mpest/em/breakpointers/unionable_breakpointer.py index 0528f92..75cf62d 100644 --- a/mpest/em/breakpointers/unionable_breakpointer.py +++ b/mpest/em/breakpointers/unionable_breakpointer.py @@ -11,9 +11,7 @@ class UnionBreakpointer(EM.ABreakpointer): def __init__(self, breakpointers: list[EM.ABreakpointer] | None) -> None: super().__init__() - self._breakpointers: list[EM.ABreakpointer] = ( - [] if breakpointers is None else breakpointers - ) + self._breakpointers: list[EM.ABreakpointer] = [] if breakpointers is None else breakpointers @property def name(self): @@ -22,7 +20,7 @@ def name(self): def __add__(self, additional: "UnionBreakpointer | EM.ABreakpointer"): if isinstance(additional, UnionBreakpointer): return UnionBreakpointer(self._breakpointers + additional._breakpointers) - return UnionBreakpointer(self._breakpointers + [additional]) + return UnionBreakpointer([*self._breakpointers, additional]) def __radd__(self, additional: "UnionBreakpointer | EM.ABreakpointer"): return self + additional @@ -45,10 +43,7 @@ def is_over( previous_step: MixtureDistribution | None, current_step: MixtureDistribution, ) -> bool: - for breakpointer in self._breakpointers: - if breakpointer.is_over(step, previous_step, current_step): - return True - return False + return any(breakpointer.is_over(step, previous_step, current_step) for breakpointer in self._breakpointers) class AUnionableBreakpointer(EM.ABreakpointer, ABC): diff --git a/mpest/em/distribution_checkers/finite_checker.py b/mpest/em/distribution_checkers/finite_checker.py index 2f9ac8f..7a90d87 100644 --- a/mpest/em/distribution_checkers/finite_checker.py +++ b/mpest/em/distribution_checkers/finite_checker.py @@ -20,8 +20,6 @@ def is_alive( step: int, distribution: DistributionInMixture, ) -> bool: - if (distribution.prior_probability is not None) and not np.isfinite( - distribution.prior_probability - ): + if (distribution.prior_probability is not None) and not np.isfinite(distribution.prior_probability): return False return bool(np.all(np.isfinite(distribution.params))) diff --git a/mpest/em/distribution_checkers/prior_probability_threshold_checker.py b/mpest/em/distribution_checkers/prior_probability_threshold_checker.py index d6206ce..bd15b60 100644 --- a/mpest/em/distribution_checkers/prior_probability_threshold_checker.py +++ b/mpest/em/distribution_checkers/prior_probability_threshold_checker.py @@ -45,9 +45,7 @@ def is_alive( return False if self.prior_probability_threshold is None: return True - if self.prior_probability_threshold_step is not None: - if step < self.prior_probability_threshold_step: - return True - if distribution.prior_probability < self.prior_probability_threshold: - return False - return True + if (self.prior_probability_threshold_step is not None) and (step < self.prior_probability_threshold_step): + return True + + return not (distribution.prior_probability < self.prior_probability_threshold) diff --git a/mpest/em/distribution_checkers/unionable_distribution_checker.py b/mpest/em/distribution_checkers/unionable_distribution_checker.py index fc296b0..2244bcf 100644 --- a/mpest/em/distribution_checkers/unionable_distribution_checker.py +++ b/mpest/em/distribution_checkers/unionable_distribution_checker.py @@ -11,26 +11,19 @@ class UnionDistributionChecker(EM.ADistributionChecker): def __init__(self, breakpointers: list[EM.ADistributionChecker] | None) -> None: super().__init__() - self._distribution_checkers: list[EM.ADistributionChecker] = ( - [] if breakpointers is None else breakpointers - ) + self._distribution_checkers: list[EM.ADistributionChecker] = [] if breakpointers is None else breakpointers @property def name(self): - return " + ".join( - distribution_checker.name - for distribution_checker in self._distribution_checkers - ) + return " + ".join(distribution_checker.name for distribution_checker in self._distribution_checkers) def __add__( self, additional: "UnionDistributionChecker | EM.ADistributionChecker", ): if isinstance(additional, UnionDistributionChecker): - return UnionDistributionChecker( - self._distribution_checkers + additional._distribution_checkers - ) - return UnionDistributionChecker(self._distribution_checkers + [additional]) + return UnionDistributionChecker(self._distribution_checkers + additional._distribution_checkers) + return UnionDistributionChecker([*self._distribution_checkers, additional]) def __radd__( self, diff --git a/mpest/em/em.py b/mpest/em/em.py index 731f33e..17f4e43 100644 --- a/mpest/em/em.py +++ b/mpest/em/em.py @@ -5,7 +5,7 @@ """ from abc import ABC, abstractmethod -from typing import Callable +from collections.abc import Callable from mpest.distribution import Distribution from mpest.em.methods.method import Method @@ -118,9 +118,7 @@ def update( self._checker = distribution_checker if len(mixture_distribution) != len(self._active): - raise ValueError( - "New mixture distribution size must be the same with previous" - ) + raise ValueError("New mixture distribution size must be the same with previous") new_active_indexes: list[int] = [] for ind, d in zip(self._active_indexes, mixture_distribution): @@ -282,11 +280,7 @@ def make_step( lambda d: self.distribution_checker.is_alive(step, d), ) - error = ( - Exception("All distributions failed") - if len(distributions) == 0 - else None - ) + error = Exception("All distributions failed") if len(distributions) == 0 else None return ResultWithError(distributions, error) @@ -303,10 +297,7 @@ def make_step( break step += 1 - history = [ - TimerResultWrapper(postprocess_result(result.content), result.runtime) - for result in history - ] + history = [TimerResultWrapper(postprocess_result(result.content), result.runtime) for result in history] return ResultWithLog( postprocess_result(ResultWithError(distributions.all_distributions)), diff --git a/mpest/em/methods/l_moments_method.py b/mpest/em/methods/l_moments_method.py index ff248fb..d50698c 100644 --- a/mpest/em/methods/l_moments_method.py +++ b/mpest/em/methods/l_moments_method.py @@ -1,4 +1,4 @@ -""" The module in which the L moments method is presented """ +"""The module in which the L moments method is presented""" import json from math import ceil @@ -88,9 +88,7 @@ def step(self, problem: Problem) -> EResult: and sorted_problem.samples[0] < 0 and not hasattr(self, "indicators") ): - self.indicators = distribute_numbers_transposed( - sorted_problem.samples, sorted_problem.distributions - ) + self.indicators = distribute_numbers_transposed(sorted_problem.samples, sorted_problem.distributions) else: self.calc_indicators(sorted_problem) @@ -104,9 +102,7 @@ def step(self, problem: Problem) -> EResult: new_priors = self.update_priors(sorted_problem) new_problem = Problem( sorted_problem.samples, - MixtureDistribution.from_distributions( - sorted_problem.distributions, new_priors - ), + MixtureDistribution.from_distributions(sorted_problem.distributions, new_priors), ) return new_problem, new_priors, self.indicators @@ -117,12 +113,10 @@ class LMomentsMStep(AMaximization[EResult]): """ def __init__(self): - with open(find_file("binoms.json", "/"), "r", encoding="utf-8") as f: + with open(find_file("binoms.json", "/"), encoding="utf-8") as f: self.binoms = json.load(f) - def calculate_mr_j( - self, r: int, j: int, samples: Samples, indicators: np.ndarray - ) -> float: + def calculate_mr_j(self, r: int, j: int, samples: Samples, indicators: np.ndarray) -> float: """ A function that calculates the list of n-th moments of each distribution. @@ -140,9 +134,7 @@ def calculate_mr_j( for k in range(r): b_num = np.sum( [ - binoms[f"{round(np.sum(indicators[j][:i+1]))} {k}"] - * samples[i] - * indicators[j][i] + binoms[f"{round(np.sum(indicators[j][: i + 1]))} {k}"] * samples[i] * indicators[j][i] for i in range(k, n) ] ) @@ -150,11 +142,7 @@ def calculate_mr_j( ind_sum = np.sum(indicators[j]) b_den = ind_sum * binoms[f"{ceil(ind_sum)} {k}"] b_k = b_num / b_den - p_rk = ( - (-1) ** (r - k - 1) - * binoms[f"{r - 1} {k}"] - * binoms[f"{r + k - 1} {k}"] - ) + p_rk = (-1) ** (r - k - 1) * binoms[f"{r - 1} {k}"] * binoms[f"{r + k - 1} {k}"] mr_j += p_rk * b_k return mr_j @@ -180,9 +168,7 @@ def step(self, e_result: EResult) -> Result: for i, d in enumerate(mixture): if d.model.name == "WeibullExp" and (l_moments[i][0] * l_moments[i][1] < 0): - error = MStepError( - "The weibul distribution degenerated in the first step." - ) + error = MStepError("The weibul distribution degenerated in the first step.") return ResultWithError(mixture.distributions, error) new_distributions = [] @@ -192,9 +178,7 @@ def step(self, e_result: EResult) -> Result: new_d = Distribution(d.model, d.model.params_convert_to_model(new_params)) new_distributions.append(new_d) - new_mixture = MixtureDistribution.from_distributions( - new_distributions, new_priors - ) + new_mixture = MixtureDistribution.from_distributions(new_distributions, new_priors) return ResultWithError(new_mixture) @@ -208,9 +192,7 @@ def distribute_numbers_transposed(numbers, mixture: MixtureDistribution): n = len(numbers) m = len(mixture) - gaussian_indices = [ - i for i, dist in enumerate(mixture) if dist.model.name == "Gaussian" - ] + gaussian_indices = [i for i, dist in enumerate(mixture) if dist.model.name == "Gaussian"] gaussian_index = gaussian_indices[0] result = np.zeros((m, n)) diff --git a/mpest/em/methods/likelihood_method.py b/mpest/em/methods/likelihood_method.py index 7cb98dd..71af68e 100644 --- a/mpest/em/methods/likelihood_method.py +++ b/mpest/em/methods/likelihood_method.py @@ -1,4 +1,4 @@ -""" The module in which the maximum likelihood method is presented """ +"""The module in which the maximum likelihood method is presented""" from functools import partial @@ -39,9 +39,7 @@ def step(self, problem: Problem) -> EResult: active_samples.append(x) if not active_samples: - error = SampleError( - "None of the elements in the sample is correct for this mixture" - ) + error = SampleError("None of the elements in the sample is correct for this mixture") return ResultWithError(mixture, error) # h[j, i] contains probability of X_i to be a part of distribution j @@ -116,8 +114,7 @@ def log_likelihood(params, ch, model: AModel): def jacobian(params, ch, model: AModelDifferentiable): return -np.sum( - ch - * np.swapaxes([model.ld_params(x, params) for x in samples], 0, 1), + ch * np.swapaxes([model.ld_params(x, params) for x in samples], 0, 1), axis=1, ) @@ -138,6 +135,4 @@ def jacobian(params, ch, model: AModelDifferentiable): ) new_distributions.append(Distribution(d.model, new_params)) - return ResultWithError( - MixtureDistribution.from_distributions(new_distributions, new_w) - ) + return ResultWithError(MixtureDistribution.from_distributions(new_distributions, new_w)) diff --git a/mpest/mixture_distribution.py b/mpest/mixture_distribution.py index a21245f..6c8a330 100644 --- a/mpest/mixture_distribution.py +++ b/mpest/mixture_distribution.py @@ -5,7 +5,7 @@ p(x | list[pdf], list[params], list[prior_probability]) = sum(prior_probability * pdf(x | params)) """ -from typing import Iterable, Iterator, Sized +from collections.abc import Iterable, Iterator, Sized import numpy as np @@ -47,9 +47,7 @@ def pdf(self, x: float): return self.prior_probability * super().pdf(x) -class MixtureDistribution( - APDFAble, AWithGenerator, Sized, Iterable[DistributionInMixture] -): +class MixtureDistribution(APDFAble, AWithGenerator, Sized, Iterable[DistributionInMixture]): """ Class which represents distributions mixture. @@ -79,16 +77,9 @@ def from_distributions( k = len(distributions) prior_probabilities = list([1 / k] * k) elif len(prior_probabilities) != len(distributions): - raise ValueError( - "Sizes of 'distributions' and 'prior_probabilities' must be the same" - ) + raise ValueError("Sizes of 'distributions' and 'prior_probabilities' must be the same") - return cls( - [ - DistributionInMixture(d.model, d.params, p) - for d, p in zip(distributions, prior_probabilities) - ] - ) + return cls([DistributionInMixture(d.model, d.params, p) for d, p in zip(distributions, prior_probabilities)]) def __iter__(self) -> Iterator[DistributionInMixture]: def iterate(instance: MixtureDistribution, index: int): @@ -111,9 +102,7 @@ def pdf(self, x: float) -> float: def has_generator(self): """Returns True if contained model has generator""" return all( - isinstance(d.model, AModelWithGenerator) - for d in self.distributions - if d.prior_probability is not None + isinstance(d.model, AModelWithGenerator) for d in self.distributions if d.prior_probability is not None ) def generate(self, size=1) -> Params: @@ -153,14 +142,10 @@ def _normalize(self): return s = sum(s) - prior_probabilities = [ - d.prior_probability / s if d.prior_probability else None - for d in self._distributions - ] + prior_probabilities = [d.prior_probability / s if d.prior_probability else None for d in self._distributions] self._distributions = [ - DistributionInMixture(d.model, d.params, p) - for d, p in zip(self._distributions, prior_probabilities) + DistributionInMixture(d.model, d.params, p) for d, p in zip(self._distributions, prior_probabilities) ] @property diff --git a/mpest/models/__init__.py b/mpest/models/__init__.py index ca2f1b2..2ddb7fd 100644 --- a/mpest/models/__init__.py +++ b/mpest/models/__init__.py @@ -1,4 +1,5 @@ """Module which represents models and abstract classes for extending""" + from mpest.models.abstract_model import ( AModel, AModelDifferentiable, diff --git a/mpest/models/abstract_model.py b/mpest/models/abstract_model.py index 765b2ab..874afe5 100644 --- a/mpest/models/abstract_model.py +++ b/mpest/models/abstract_model.py @@ -55,9 +55,7 @@ class AModelWithGenerator(AModel, ABC): """ @abstractmethod - def generate( - self, params: Params, size: int = 1, normalized: bool = False - ) -> Samples: + def generate(self, params: Params, size: int = 1, normalized: bool = False) -> Samples: """ Method which generates samples by given params :param size: sample size diff --git a/mpest/models/exponential.py b/mpest/models/exponential.py index 3efbb42..1614a2a 100644 --- a/mpest/models/exponential.py +++ b/mpest/models/exponential.py @@ -20,15 +20,13 @@ def calc_lambda(self, moments: list[float]): return 1 / moments[0] -class ExponentialModel( - AModelDifferentiable, AModelWithGenerator, LMomentsParameterMixin -): +class ExponentialModel(AModelDifferentiable, AModelWithGenerator, LMomentsParameterMixin): """ - f(x) = l * e^(-lx) + f(x) = lm * e^(-lx) - l = e^(_l) + lm = e^(_lm) - O = [_l] + O = [_lm] """ @property @@ -41,9 +39,7 @@ def params_convert_to_model(self, params): def params_convert_from_model(self, params): return np.exp(params) - def generate( - self, params: Params, size: int = 1, normalized: bool = True - ) -> Samples: + def generate(self, params: Params, size: int = 1, normalized: bool = True) -> Samples: if not normalized: return np.array(expon.rvs(scale=1 / params[0], size=size)) @@ -53,22 +49,22 @@ def generate( def pdf(self, x: float, params: Params) -> float: if x < 0: return 0 - (l,) = params - return np.exp(l - np.exp(l) * x) + (lm,) = params + return np.exp(lm - np.exp(lm) * x) def lpdf(self, x: float, params: Params) -> float: if x < 0: return -np.inf - (l,) = params - return l - np.exp(l) * x + (lm,) = params + return lm - np.exp(lm) * x def ldl(self, x: float, params: Params) -> float: - """Method which returns logarithm of derivative with respect to parameter l""" + """Method which returns logarithm of derivative with respect to parameter lm""" if x < 0: return -np.inf - (l,) = params - return 1 - np.exp(l) * x + (lm,) = params + return 1 - np.exp(lm) * x def ld_params(self, x: float, params: Params) -> np.ndarray: return np.array([self.ldl(x, params)]) diff --git a/mpest/models/gaussian.py b/mpest/models/gaussian.py index 3c89af6..dfdd908 100644 --- a/mpest/models/gaussian.py +++ b/mpest/models/gaussian.py @@ -47,9 +47,7 @@ def params_convert_to_model(self, params: Params) -> Params: def params_convert_from_model(self, params: Params) -> Params: return np.array([params[0], np.exp(params[1])]) - def generate( - self, params: Params, size: int = 1, normalized: bool = True - ) -> Samples: + def generate(self, params: Params, size: int = 1, normalized: bool = True) -> Samples: if not normalized: return np.array(norm.rvs(loc=params[0], scale=params[1], size=size)) diff --git a/mpest/models/weibull.py b/mpest/models/weibull.py index d6c9965..7df445d 100644 --- a/mpest/models/weibull.py +++ b/mpest/models/weibull.py @@ -32,17 +32,15 @@ def calc_lambda(self, moments: list[float]) -> float: return m1 / math.gamma(1 + 1 / k) -class WeibullModelExp( - AModelDifferentiable, AModelWithGenerator, LMomentsParameterMixin -): +class WeibullModelExp(AModelDifferentiable, AModelWithGenerator, LMomentsParameterMixin): """ - f(x) = (k / l) * (x / l)^(k - 1) / e^((x / l)^k) + f(x) = (k / lm) * (x / lm)^(k - 1) / e^((x / lm)^k) k = e^(_k) - l = e^(_l) + lm = e^(_lm) - O = [_k, _l] + O = [_k, _lm] """ @property @@ -55,50 +53,44 @@ def params_convert_to_model(self, params: Params) -> Params: def params_convert_from_model(self, params: Params) -> Params: return np.exp(params) - def generate( - self, params: Params, size: int = 1, normalized: bool = True - ) -> Samples: + def generate(self, params: Params, size: int = 1, normalized: bool = True) -> Samples: if not normalized: - return np.array( - weibull_min.rvs(params[0], loc=0, scale=params[1], size=size) - ) + return np.array(weibull_min.rvs(params[0], loc=0, scale=params[1], size=size)) c_params = self.params_convert_from_model(params) - return np.array( - weibull_min.rvs(c_params[0], loc=0, scale=c_params[1], size=size) - ) + return np.array(weibull_min.rvs(c_params[0], loc=0, scale=c_params[1], size=size)) def pdf(self, x: float, params: Params) -> float: if x < 0: return 0 - ek, el = np.exp(params) - xl = x / el - return (ek / el) * (xl ** (ek - 1.0)) / np.exp(xl**ek) + ek, elm = np.exp(params) + xl = x / elm + return (ek / elm) * (xl ** (ek - 1.0)) / np.exp(xl**ek) def lpdf(self, x: float, params: Params) -> float: if x < 0: return -np.inf - k, l = params - ek, el = np.exp(params) + k, lm = params + ek, elm = np.exp(params) lx = np.log(x) - return k - ((x / el) ** ek) - ek * l - lx + ek * lx + return k - ((x / elm) ** ek) - ek * lm - lx + ek * lx def ldk(self, x: float, params: Params) -> float: """Method which returns logarithm of derivative with respect to k""" if x < 0: return -np.inf - ek, el = np.exp(params) - xl = x / el - return 1.0 - ek * ((xl**ek) - 1.0) * np.log(xl) + ek, elm = np.exp(params) + xlm = x / elm + return 1.0 - ek * ((xlm**ek) - 1.0) * np.log(xlm) def ldl(self, x: float, params: Params) -> float: - """Method which returns logarithm of derivative with respect to l""" + """Method which returns logarithm of derivative with respect to lm""" if x < 0: return -np.inf - ek, el = np.exp(params) - return ek * ((x / el) ** ek - 1.0) + ek, elm = np.exp(params) + return ek * ((x / elm) ** ek - 1.0) def ld_params(self, x: float, params: Params) -> np.ndarray: return np.array([self.ldk(x, params), self.ldl(x, params)]) diff --git a/mpest/optimizers/abstract_optimizer.py b/mpest/optimizers/abstract_optimizer.py index 5b286ee..05305d6 100644 --- a/mpest/optimizers/abstract_optimizer.py +++ b/mpest/optimizers/abstract_optimizer.py @@ -1,7 +1,7 @@ """Module which contains abstract optimizer classes""" from abc import ABC, abstractmethod -from typing import Callable +from collections.abc import Callable import numpy as np diff --git a/mpest/optimizers/scipy_cg.py b/mpest/optimizers/scipy_cg.py index 7694700..611bd38 100644 --- a/mpest/optimizers/scipy_cg.py +++ b/mpest/optimizers/scipy_cg.py @@ -1,6 +1,6 @@ """Module which contains CG (Conjugate Gradient iteration) optimizer""" -from typing import Callable +from collections.abc import Callable from scipy.optimize import minimize diff --git a/mpest/optimizers/scipy_cobyla.py b/mpest/optimizers/scipy_cobyla.py index 248bfa0..ef9921d 100644 --- a/mpest/optimizers/scipy_cobyla.py +++ b/mpest/optimizers/scipy_cobyla.py @@ -1,6 +1,6 @@ """Module which contains COBYLA optimizer""" -from typing import Callable +from collections.abc import Callable from scipy.optimize import minimize diff --git a/mpest/optimizers/scipy_nelder_mead.py b/mpest/optimizers/scipy_nelder_mead.py index 3dd04d1..499e0d9 100644 --- a/mpest/optimizers/scipy_nelder_mead.py +++ b/mpest/optimizers/scipy_nelder_mead.py @@ -1,6 +1,6 @@ """Module which contains Nelder Mead optimizer""" -from typing import Callable +from collections.abc import Callable from scipy.optimize import minimize diff --git a/mpest/optimizers/scipy_newton_cg.py b/mpest/optimizers/scipy_newton_cg.py index d274463..c7dcd3b 100644 --- a/mpest/optimizers/scipy_newton_cg.py +++ b/mpest/optimizers/scipy_newton_cg.py @@ -1,6 +1,6 @@ """Module which contains Newton-CG optimizer""" -from typing import Callable +from collections.abc import Callable import numpy as np from scipy.optimize import minimize diff --git a/mpest/optimizers/scipy_slsqp.py b/mpest/optimizers/scipy_slsqp.py index 4afcbb5..aa49794 100644 --- a/mpest/optimizers/scipy_slsqp.py +++ b/mpest/optimizers/scipy_slsqp.py @@ -1,6 +1,6 @@ """Module which contains Sequential Least Squares Programming (SLSQP) optimizer""" -from typing import Callable +from collections.abc import Callable from scipy.optimize import minimize diff --git a/mpest/optimizers/scipy_tnc.py b/mpest/optimizers/scipy_tnc.py index cb6be56..f0204aa 100644 --- a/mpest/optimizers/scipy_tnc.py +++ b/mpest/optimizers/scipy_tnc.py @@ -1,6 +1,6 @@ """Module which contains truncated Newton (TNC) optimizer""" -from typing import Callable +from collections.abc import Callable from scipy.optimize import minimize diff --git a/mpest/utils.py b/mpest/utils.py index 80799a7..23ce2ba 100644 --- a/mpest/utils.py +++ b/mpest/utils.py @@ -4,7 +4,8 @@ import os import time from abc import ABC, abstractmethod -from typing import Callable, Generic, Iterator, ParamSpec, TypeVar +from collections.abc import Callable, Iterator +from typing import Generic, ParamSpec, TypeVar P = ParamSpec("P") R = TypeVar("R") diff --git a/pyproject.toml b/pyproject.toml index 993dac3..51cab4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "seaborn (>=0.13.2,<0.14.0)" ] + [tool.poetry] [tool.poetry.group.dev.dependencies] @@ -33,12 +34,27 @@ ruff = "^0.9.6" mypy = "^1.15.0" sphinx = "^8.2.0" - [tool.poetry.group.experiments.dependencies] tqdm = "^4.67.1" multiprocess = "^0.70.17" ruamel-yaml = "^0.18.10" + +[tool.ruff] +line-length = 120 +output-format = "grouped" +exclude = ["*.ipynb", "__init__.py"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +docstring-code-format = true + +[tool.ruff.lint] +select = ["A", "E", "F", "I", "PL", "RUF", "SIM", "UP", "W"] +ignore = ["PLR0913"] + + [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 52611ff..0000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -matplotlib -multiprocess -numpy -pandas -pickleshare -ruamel.yaml -scikit-learn -scipy -seaborn -tqdm diff --git a/tests/tests_l_moments/test_any_distributions_complex_l.py b/tests/tests_l_moments/test_any_distributions_complex_l.py index a735179..8e9514e 100644 --- a/tests/tests_l_moments/test_any_distributions_complex_l.py +++ b/tests/tests_l_moments/test_any_distributions_complex_l.py @@ -31,8 +31,7 @@ def idfunc(vals): @pytest.mark.parametrize( - "models, params, start_params, prior_probabilities, size, deviation, expected_params_error," - "expected_priors_error", + "models, params, start_params, prior_probabilities, size, deviation, expected_params_error,expected_priors_error", [ ( [WeibullModelExp(), GaussianModel()], @@ -109,9 +108,5 @@ def test( ) result = run_test(problem=problem, deviation=deviation) - assert check_for_params_error_tolerance( - [result], base_mixture, expected_params_error - ) - assert check_for_priors_error_tolerance( - [result], base_mixture, expected_priors_error - ) + assert check_for_params_error_tolerance([result], base_mixture, expected_params_error) + assert check_for_priors_error_tolerance([result], base_mixture, expected_priors_error) diff --git a/tests/tests_l_moments/test_one_distribution_l.py b/tests/tests_l_moments/test_one_distribution_l.py index c34b78c..ea53613 100644 --- a/tests/tests_l_moments/test_one_distribution_l.py +++ b/tests/tests_l_moments/test_one_distribution_l.py @@ -61,9 +61,7 @@ def test_one_distribution( problem = Problem( samples=x, - distributions=MixtureDistribution.from_distributions( - [Distribution(model, start_params)] - ), + distributions=MixtureDistribution.from_distributions([Distribution(model, start_params)]), ) result = run_test(problem=problem, deviation=deviation) diff --git a/tests/tests_l_moments/test_two_same_distributions_complex_l.py b/tests/tests_l_moments/test_two_same_distributions_complex_l.py index e0e4a71..9887229 100644 --- a/tests/tests_l_moments/test_two_same_distributions_complex_l.py +++ b/tests/tests_l_moments/test_two_same_distributions_complex_l.py @@ -135,9 +135,5 @@ def test_two_same_distributions_simple( ) result = run_test(problem=problem, deviation=deviation) - assert check_for_params_error_tolerance( - [result], base_mixture, expected_params_error - ) - assert check_for_priors_error_tolerance( - [result], base_mixture, expected_priors_error - ) + assert check_for_params_error_tolerance([result], base_mixture, expected_params_error) + assert check_for_priors_error_tolerance([result], base_mixture, expected_priors_error) diff --git a/tests/tests_likelihood/test_any_distributions_complex.py b/tests/tests_likelihood/test_any_distributions_complex.py index 9309f2e..94141c5 100644 --- a/tests/tests_likelihood/test_any_distributions_complex.py +++ b/tests/tests_likelihood/test_any_distributions_complex.py @@ -33,8 +33,7 @@ def idfunc(vals): @pytest.mark.parametrize( - "models, params, start_params, prior_probabilities, size, deviation, expected_params_error," - "expected_priors_error", + "models, params, start_params, prior_probabilities, size, deviation, expected_params_error,expected_priors_error", [ ( [WeibullModelExp(), GaussianModel()], @@ -111,9 +110,5 @@ def test( ) results = run_test(problem=problem, deviation=deviation) - assert check_for_params_error_tolerance( - results, base_mixture, expected_params_error - ) - assert check_for_priors_error_tolerance( - results, base_mixture, expected_priors_error - ) + assert check_for_params_error_tolerance(results, base_mixture, expected_params_error) + assert check_for_priors_error_tolerance(results, base_mixture, expected_priors_error) diff --git a/tests/tests_likelihood/test_one_distribution.py b/tests/tests_likelihood/test_one_distribution.py index 4b3c382..7112ea3 100644 --- a/tests/tests_likelihood/test_one_distribution.py +++ b/tests/tests_likelihood/test_one_distribution.py @@ -61,9 +61,7 @@ def test_one_distribution( problem = Problem( samples=x, - distributions=MixtureDistribution.from_distributions( - [Distribution(model, start_params)] - ), + distributions=MixtureDistribution.from_distributions([Distribution(model, start_params)]), ) results = run_test(problem=problem, deviation=deviation) diff --git a/tests/tests_likelihood/test_two_same_distributions_complex.py b/tests/tests_likelihood/test_two_same_distributions_complex.py index 1d1103c..18934f6 100644 --- a/tests/tests_likelihood/test_two_same_distributions_complex.py +++ b/tests/tests_likelihood/test_two_same_distributions_complex.py @@ -135,9 +135,5 @@ def test_two_same_distributions_simple( ) results = run_test(problem=problem, deviation=deviation) - assert check_for_params_error_tolerance( - results, base_mixture, expected_params_error - ) - assert check_for_priors_error_tolerance( - results, base_mixture, expected_priors_error - ) + assert check_for_params_error_tolerance(results, base_mixture, expected_params_error) + assert check_for_priors_error_tolerance(results, base_mixture, expected_priors_error) diff --git a/tests/utils.py b/tests/utils.py index d55c559..f00ebc5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -20,10 +20,7 @@ def absolute_diff_params(a: MixtureDistribution, b: MixtureDistribution): a_p, b_p = ([d.params for d in ld] for ld in (a, b)) - return min( - sum(np.sum(np.abs(x - y)) for x, y in zip(a_p, _b_p)) - for _b_p in permutations(b_p) - ) + return min(sum(np.sum(np.abs(x - y)) for x, y in zip(a_p, _b_p)) for _b_p in permutations(b_p)) for result in results: assert result.error is None @@ -48,10 +45,7 @@ def absolute_diff_priors(a: MixtureDistribution, b: MixtureDistribution): a_p, b_p = ([d.prior_probability for d in ld] for ld in (a, b)) - return min( - sum(np.sum(np.abs(x - y)) for x, y in zip(a_p, _b_p)) - for _b_p in permutations(b_p) - ) + return min(sum(np.sum(np.abs(x - y)) for x, y in zip(a_p, _b_p)) for _b_p in permutations(b_p)) for result in results: assert result.error is None From 77a989e27b05fd933c9e317b5385f23d577d6816 Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 19 Feb 2025 05:06:02 +0300 Subject: [PATCH 3/6] ci: add workflow check --- .github/workflows/{test.yml => check.yml} | 23 ++++++++---- .github/workflows/code_style.yml | 35 ------------------- .github/workflows/pylint.yml | 24 ------------- .pre-commit-config.yaml | 16 --------- examples/__init__.py | 0 examples/prepare_result.py | 6 ++-- experimental_env/__init__.py | 0 experimental_env/analysis/__init__.py | 0 .../analysis/analyze_strategies/__init__.py | 0 .../analysis/analyze_summarizers/__init__.py | 0 experimental_env/experiment/__init__.py | 0 .../experiment_executors/__init__.py | 0 .../mixture_generators/__init__.py | 0 experimental_env/preparation/__init__.py | 0 mpest/em/methods/__init__.py | 0 pyproject.toml | 11 +++++- scripts/__init__.py | 0 tests/__init__.py | 0 tests/tests_l_moments/__init__.py | 0 tests/tests_likelihood/__init__.py | 0 20 files changed, 29 insertions(+), 86 deletions(-) rename .github/workflows/{test.yml => check.yml} (52%) delete mode 100644 .github/workflows/code_style.yml delete mode 100644 .github/workflows/pylint.yml delete mode 100644 .pre-commit-config.yaml create mode 100644 examples/__init__.py create mode 100644 experimental_env/__init__.py create mode 100644 experimental_env/analysis/__init__.py create mode 100644 experimental_env/analysis/analyze_strategies/__init__.py create mode 100644 experimental_env/analysis/analyze_summarizers/__init__.py create mode 100644 experimental_env/experiment/__init__.py create mode 100644 experimental_env/experiment/experiment_executors/__init__.py create mode 100644 experimental_env/mixture_generators/__init__.py create mode 100644 experimental_env/preparation/__init__.py create mode 100644 mpest/em/methods/__init__.py create mode 100644 scripts/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/tests_l_moments/__init__.py create mode 100644 tests/tests_likelihood/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/check.yml similarity index 52% rename from .github/workflows/test.yml rename to .github/workflows/check.yml index e61f17f..2fbae3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/check.yml @@ -1,4 +1,4 @@ -name: Run tests +name: Check on: [ push, pull_request ] @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ 3.11 ] + python-version: [ "3.11", "3.12", "3.13" ] steps: - uses: actions/checkout@v3 @@ -18,10 +18,19 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install poetry + run: | + pipx install poetry + - name: Install dependencies run: | - python -m pip install --upgrade pip wheel setuptools - python -m pip install -r requirements.txt -r requirements.dev.txt - pip list - - name: Test with pytest - run: python ./scripts/run_tests.py + poetry install --with dev + + - name: Ruff + run: poetry run ruff check + + - name: Mypy + run: poetry run mypy . + + - name: Pytest + run: poetry run python -m pytest tests \ No newline at end of file diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml deleted file mode 100644 index abd4508..0000000 --- a/.github/workflows/code_style.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Check code style - -on: - [ push, pull_request ] - -jobs: - style: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: [ 3.11 ] - - steps: - - name: Set up Git repository - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install requirements - run: | - python -m pip install --upgrade pip wheel setuptools - python -m pip install -r requirements.txt -r requirements.dev.txt - python -m pip list - - - name: Install pre-commit - run: | - pre-commit install - - - name: Run pre-commit - run: | - pre-commit run --all-files --color always --verbose --show-diff-on-failure diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 473259f..0000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Pylint - -on: [ push, pull_request ] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt -r requirements.dev.txt - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index d970f2d..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 - hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace - - id: requirements-txt-fixer - - repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/prepare_result.py b/examples/prepare_result.py index 63bc830..1dc7d29 100644 --- a/examples/prepare_result.py +++ b/examples/prepare_result.py @@ -96,7 +96,7 @@ def result_to_df_diff(result: SingleSolverResult): "time": result.time, "model": result.test.true_mixture[0].model.name, "size": len(result.test.problem.samples), - "success": (result.steps < 128) and not failed, # noqa: PLR2004 + "success": (result.steps < 128) and not failed, # noqa: PLR2004 "failed": failed, "occur": nll(result.test.all_data, mixture_distribution), "start": start, @@ -125,7 +125,7 @@ def result_to_df(result: SingleSolverResult): [ ( d - if (d.prior_probability is not None) and (d.prior_probability > 0.001) # noqa: PLR2004 + if (d.prior_probability is not None) and (d.prior_probability > 0.001) # noqa: PLR2004 else DistributionInMixture(d.model, d.params, None) ) for d in result.result.content @@ -146,7 +146,7 @@ def result_to_df(result: SingleSolverResult): "time": result.time, "model": result.test.true_mixture[0].model.name, "size": len(result.test.problem.samples), - "success": (result.steps < 16) and failed, # noqa: PLR2004 + "success": (result.steps < 16) and failed, # noqa: PLR2004 "failed": failed, "occur": nll(result.test.all_data, mixture_distribution), } diff --git a/experimental_env/__init__.py b/experimental_env/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/analysis/__init__.py b/experimental_env/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/analysis/analyze_strategies/__init__.py b/experimental_env/analysis/analyze_strategies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/analysis/analyze_summarizers/__init__.py b/experimental_env/analysis/analyze_summarizers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/experiment/__init__.py b/experimental_env/experiment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/experiment/experiment_executors/__init__.py b/experimental_env/experiment/experiment_executors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/mixture_generators/__init__.py b/experimental_env/mixture_generators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/experimental_env/preparation/__init__.py b/experimental_env/preparation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mpest/em/methods/__init__.py b/mpest/em/methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 51cab4c..efb0aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ pytest = "^8.3.4" pre-commit = "^4.1.0" ruff = "^0.9.6" -mypy = "^1.15.0" +mypy = "==1.10.1" sphinx = "^8.2.0" [tool.poetry.group.experiments.dependencies] @@ -55,6 +55,15 @@ select = ["A", "E", "F", "I", "PL", "RUF", "SIM", "UP", "W"] ignore = ["PLR0913"] +[tool.mypy] +exclude = [ + "^venv/", + "^scripts/", + "^tests/", + "^examples/", +] + + [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tests_l_moments/__init__.py b/tests/tests_l_moments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tests_likelihood/__init__.py b/tests/tests_likelihood/__init__.py new file mode 100644 index 0000000..e69de29 From 65fbf7c85e29b303fe5730695ccb35a7313f9bde Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 19 Feb 2025 05:24:38 +0300 Subject: [PATCH 4/6] refactor: change repo structure by adding core module --- examples/mono_test_generator.py | 6 +++--- examples/prepare_result.py | 4 ++-- examples/utils.py | 6 +++--- experimental_env/preparation/dataset_description.py | 4 ++-- experimental_env/preparation/dataset_generator.py | 4 ++-- mpest/__init__.py | 8 ++++---- mpest/{types.py => annotations.py} | 0 mpest/core/__init__.py | 0 mpest/{ => core}/distribution.py | 2 +- mpest/{ => core}/mixture_distribution.py | 4 ++-- mpest/{ => core}/problem.py | 5 +++-- mpest/em/breakpointers/param_differ_breakpointer.py | 2 +- mpest/em/breakpointers/step_count_breakpointer.py | 2 +- mpest/em/breakpointers/unionable_breakpointer.py | 2 +- mpest/em/distribution_checkers/finite_checker.py | 2 +- .../prior_probability_threshold_checker.py | 2 +- .../unionable_distribution_checker.py | 2 +- mpest/em/em.py | 6 +++--- mpest/em/methods/abstract_steps.py | 4 ++-- mpest/em/methods/l_moments_method.py | 6 +++--- mpest/em/methods/likelihood_method.py | 6 +++--- mpest/em/methods/method.py | 4 ++-- mpest/models/abstract_model.py | 2 +- mpest/models/exponential.py | 2 +- mpest/models/gaussian.py | 2 +- mpest/models/weibull.py | 2 +- mpest/optimizers/abstract_optimizer.py | 2 +- mpest/optimizers/scipy_cg.py | 2 +- mpest/optimizers/scipy_cobyla.py | 2 +- mpest/optimizers/scipy_nelder_mead.py | 2 +- mpest/optimizers/scipy_newton_cg.py | 2 +- mpest/optimizers/scipy_slsqp.py | 2 +- mpest/optimizers/scipy_tnc.py | 2 +- tests/tests_l_moments/l_moments_utils.py | 2 +- tests/tests_l_moments/test_one_distribution_l.py | 6 +++--- .../test_two_same_distributions_complex_l.py | 6 +++--- .../test_two_same_distributions_simple_l.py | 6 +++--- tests/tests_likelihood/likelihood_utils.py | 2 +- tests/tests_likelihood/test_any_distributions_complex.py | 6 +++--- tests/tests_likelihood/test_any_distributions_simple.py | 6 +++--- tests/tests_likelihood/test_one_distribution.py | 6 +++--- .../test_two_same_distributions_complex.py | 6 +++--- .../test_two_same_distributions_simple.py | 6 +++--- tests/utils.py | 2 +- 44 files changed, 79 insertions(+), 78 deletions(-) rename mpest/{types.py => annotations.py} (100%) create mode 100644 mpest/core/__init__.py rename mpest/{ => core}/distribution.py (97%) rename mpest/{ => core}/mixture_distribution.py (97%) rename mpest/{ => core}/problem.py (91%) diff --git a/examples/mono_test_generator.py b/examples/mono_test_generator.py index b474211..f3c681b 100644 --- a/examples/mono_test_generator.py +++ b/examples/mono_test_generator.py @@ -6,11 +6,11 @@ import numpy as np from examples.utils import Clicker, Test -from mpest.distribution import Distribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.em import EM -from mpest.mixture_distribution import MixtureDistribution from mpest.models import AModel, AModelWithGenerator -from mpest.problem import Problem def generate_mono_test( diff --git a/examples/prepare_result.py b/examples/prepare_result.py index 1dc7d29..ee9e716 100644 --- a/examples/prepare_result.py +++ b/examples/prepare_result.py @@ -9,8 +9,8 @@ from examples.config import MAX_WORKERS from examples.mono_test_generator import Clicker from examples.utils import SingleSolverResult, TestResult -from mpest.mixture_distribution import DistributionInMixture, MixtureDistribution -from mpest.types import Samples +from mpest.annotations import Samples +from mpest.core.mixture_distribution import DistributionInMixture, MixtureDistribution def nll(samples: Samples, mixture: MixtureDistribution) -> float: diff --git a/examples/utils.py b/examples/utils.py index c202eb2..06984d1 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -10,6 +10,9 @@ from tqdm.contrib.concurrent import process_map from examples.config import RESULTS_FOLDER +from mpest.annotations import Samples +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem, Result from mpest.em import EM from mpest.em.breakpointers import ParamDifferBreakpointer, StepCountBreakpointer from mpest.em.distribution_checkers import ( @@ -17,10 +20,7 @@ PriorProbabilityThresholdChecker, ) from mpest.em.methods.likelihood_method import LikelihoodMethod -from mpest.mixture_distribution import MixtureDistribution from mpest.optimizers import TOptimizer -from mpest.problem import Problem, Result -from mpest.types import Samples np.seterr(all="ignore") diff --git a/experimental_env/preparation/dataset_description.py b/experimental_env/preparation/dataset_description.py index e4a816f..bd065da 100644 --- a/experimental_env/preparation/dataset_description.py +++ b/experimental_env/preparation/dataset_description.py @@ -2,8 +2,8 @@ import copy -from mpest.mixture_distribution import MixtureDistribution -from mpest.types import Samples +from mpest.annotations import Samples +from mpest.core.mixture_distribution import MixtureDistribution class DatasetDescrciption: diff --git a/experimental_env/preparation/dataset_generator.py b/experimental_env/preparation/dataset_generator.py index b519269..a560443 100644 --- a/experimental_env/preparation/dataset_generator.py +++ b/experimental_env/preparation/dataset_generator.py @@ -10,8 +10,8 @@ DatasetMixtureGenerator, ) from experimental_env.preparation.dataset_saver import DatasetDescrciption, DatasetSaver -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution from mpest.models.abstract_model import AModel diff --git a/mpest/__init__.py b/mpest/__init__.py index 505a671..786c713 100644 --- a/mpest/__init__.py +++ b/mpest/__init__.py @@ -4,7 +4,7 @@ """ from mpest import em, models, optimizers, utils -from mpest.distribution import Distribution -from mpest.mixture_distribution import DistributionInMixture, MixtureDistribution -from mpest.problem import Problem, Result -from mpest.types import Params, Samples +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import DistributionInMixture, MixtureDistribution +from mpest.core.problem import Problem, Result +from mpest.annotations import Params, Samples diff --git a/mpest/types.py b/mpest/annotations.py similarity index 100% rename from mpest/types.py rename to mpest/annotations.py diff --git a/mpest/core/__init__.py b/mpest/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mpest/distribution.py b/mpest/core/distribution.py similarity index 97% rename from mpest/distribution.py rename to mpest/core/distribution.py index 53d8124..a7b73b2 100644 --- a/mpest/distribution.py +++ b/mpest/core/distribution.py @@ -4,8 +4,8 @@ import numpy as np +from mpest.annotations import Params, Samples from mpest.models import AModel, AModelWithGenerator -from mpest.types import Params, Samples class APDFAble(ABC): diff --git a/mpest/mixture_distribution.py b/mpest/core/mixture_distribution.py similarity index 97% rename from mpest/mixture_distribution.py rename to mpest/core/mixture_distribution.py index 6c8a330..39fe4f3 100644 --- a/mpest/mixture_distribution.py +++ b/mpest/core/mixture_distribution.py @@ -9,9 +9,9 @@ import numpy as np -from mpest.distribution import APDFAble, AWithGenerator, Distribution +from mpest.annotations import Params +from mpest.core.distribution import APDFAble, AWithGenerator, Distribution from mpest.models import AModel, AModelWithGenerator -from mpest.types import Params from mpest.utils import IteratorWrapper diff --git a/mpest/problem.py b/mpest/core/problem.py similarity index 91% rename from mpest/problem.py rename to mpest/core/problem.py index 1739b31..2ee336a 100644 --- a/mpest/problem.py +++ b/mpest/core/problem.py @@ -2,8 +2,8 @@ from abc import ABC, abstractmethod -from mpest.mixture_distribution import MixtureDistribution -from mpest.types import Samples +from mpest.annotations import Samples +from mpest.core.mixture_distribution import MixtureDistribution from mpest.utils import ResultWithError @@ -13,6 +13,7 @@ class Problem: Described by samples and the initial approximation. Initial approximation is an mixture distribution. + """ def __init__( diff --git a/mpest/em/breakpointers/param_differ_breakpointer.py b/mpest/em/breakpointers/param_differ_breakpointer.py index f0d0c0a..dcc3002 100644 --- a/mpest/em/breakpointers/param_differ_breakpointer.py +++ b/mpest/em/breakpointers/param_differ_breakpointer.py @@ -2,8 +2,8 @@ import numpy as np +from mpest.core.mixture_distribution import MixtureDistribution from mpest.em.breakpointers.unionable_breakpointer import AUnionableBreakpointer -from mpest.mixture_distribution import MixtureDistribution class ParamDifferBreakpointer(AUnionableBreakpointer): diff --git a/mpest/em/breakpointers/step_count_breakpointer.py b/mpest/em/breakpointers/step_count_breakpointer.py index 35f2485..3407ebd 100644 --- a/mpest/em/breakpointers/step_count_breakpointer.py +++ b/mpest/em/breakpointers/step_count_breakpointer.py @@ -1,7 +1,7 @@ """Module which contains EM breakpointer by step count""" +from mpest.core.mixture_distribution import MixtureDistribution from mpest.em.breakpointers.unionable_breakpointer import AUnionableBreakpointer -from mpest.mixture_distribution import MixtureDistribution class StepCountBreakpointer(AUnionableBreakpointer): diff --git a/mpest/em/breakpointers/unionable_breakpointer.py b/mpest/em/breakpointers/unionable_breakpointer.py index 75cf62d..6764a79 100644 --- a/mpest/em/breakpointers/unionable_breakpointer.py +++ b/mpest/em/breakpointers/unionable_breakpointer.py @@ -2,8 +2,8 @@ from abc import ABC +from mpest.core.mixture_distribution import MixtureDistribution from mpest.em import EM -from mpest.mixture_distribution import MixtureDistribution class UnionBreakpointer(EM.ABreakpointer): diff --git a/mpest/em/distribution_checkers/finite_checker.py b/mpest/em/distribution_checkers/finite_checker.py index 7a90d87..2230004 100644 --- a/mpest/em/distribution_checkers/finite_checker.py +++ b/mpest/em/distribution_checkers/finite_checker.py @@ -2,10 +2,10 @@ import numpy as np +from mpest.core.mixture_distribution import DistributionInMixture from mpest.em.distribution_checkers.unionable_distribution_checker import ( AUnionableDistributionChecker, ) -from mpest.mixture_distribution import DistributionInMixture class FiniteChecker(AUnionableDistributionChecker): diff --git a/mpest/em/distribution_checkers/prior_probability_threshold_checker.py b/mpest/em/distribution_checkers/prior_probability_threshold_checker.py index bd15b60..f6a7921 100644 --- a/mpest/em/distribution_checkers/prior_probability_threshold_checker.py +++ b/mpest/em/distribution_checkers/prior_probability_threshold_checker.py @@ -1,9 +1,9 @@ """Module which contains distribution checker by using prior probability threshold""" +from mpest.core.mixture_distribution import DistributionInMixture from mpest.em.distribution_checkers.unionable_distribution_checker import ( AUnionableDistributionChecker, ) -from mpest.mixture_distribution import DistributionInMixture class PriorProbabilityThresholdChecker(AUnionableDistributionChecker): diff --git a/mpest/em/distribution_checkers/unionable_distribution_checker.py b/mpest/em/distribution_checkers/unionable_distribution_checker.py index 2244bcf..450b97e 100644 --- a/mpest/em/distribution_checkers/unionable_distribution_checker.py +++ b/mpest/em/distribution_checkers/unionable_distribution_checker.py @@ -2,8 +2,8 @@ from abc import ABC +from mpest.core.mixture_distribution import DistributionInMixture from mpest.em import EM -from mpest.mixture_distribution import DistributionInMixture class UnionDistributionChecker(EM.ADistributionChecker): diff --git a/mpest/em/em.py b/mpest/em/em.py index 17f4e43..d32fb8b 100644 --- a/mpest/em/em.py +++ b/mpest/em/em.py @@ -7,10 +7,10 @@ from abc import ABC, abstractmethod from collections.abc import Callable -from mpest.distribution import Distribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import DistributionInMixture, MixtureDistribution +from mpest.core.problem import ASolver, Problem, Result from mpest.em.methods.method import Method -from mpest.mixture_distribution import DistributionInMixture, MixtureDistribution -from mpest.problem import ASolver, Problem, Result from mpest.utils import ( ANamed, ObjectWrapper, diff --git a/mpest/em/methods/abstract_steps.py b/mpest/em/methods/abstract_steps.py index ac9fcfa..e1d35bc 100644 --- a/mpest/em/methods/abstract_steps.py +++ b/mpest/em/methods/abstract_steps.py @@ -3,8 +3,8 @@ from abc import ABC, abstractmethod from typing import Generic, TypeVar -from mpest.mixture_distribution import MixtureDistribution -from mpest.problem import Problem +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.utils import ResultWithError T = TypeVar("T") diff --git a/mpest/em/methods/l_moments_method.py b/mpest/em/methods/l_moments_method.py index d50698c..860ebca 100644 --- a/mpest/em/methods/l_moments_method.py +++ b/mpest/em/methods/l_moments_method.py @@ -6,11 +6,11 @@ import numpy as np from mpest import Samples -from mpest.distribution import Distribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem, Result from mpest.em.methods.abstract_steps import AExpectation, AMaximization from mpest.exceptions import EStepError, MStepError -from mpest.mixture_distribution import MixtureDistribution -from mpest.problem import Problem, Result from mpest.utils import ResultWithError, find_file EResult = tuple[Problem, list[float], np.ndarray] | ResultWithError[MixtureDistribution] diff --git a/mpest/em/methods/likelihood_method.py b/mpest/em/methods/likelihood_method.py index 71af68e..fcf2fe0 100644 --- a/mpest/em/methods/likelihood_method.py +++ b/mpest/em/methods/likelihood_method.py @@ -4,13 +4,13 @@ import numpy as np -from mpest.distribution import Distribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem, Result from mpest.em.methods.abstract_steps import AExpectation, AMaximization from mpest.exceptions import SampleError -from mpest.mixture_distribution import MixtureDistribution from mpest.models import AModel, AModelDifferentiable from mpest.optimizers import AOptimizerJacobian, TOptimizer -from mpest.problem import Problem, Result from mpest.utils import ResultWithError EResult = tuple[list[float], np.ndarray, Problem] | ResultWithError[MixtureDistribution] diff --git a/mpest/em/methods/method.py b/mpest/em/methods/method.py index f91830b..6eb876e 100644 --- a/mpest/em/methods/method.py +++ b/mpest/em/methods/method.py @@ -2,9 +2,9 @@ from typing import Generic, TypeVar +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.em.methods.abstract_steps import AExpectation, AMaximization -from mpest.mixture_distribution import MixtureDistribution -from mpest.problem import Problem from mpest.utils import ResultWithError T = TypeVar("T") diff --git a/mpest/models/abstract_model.py b/mpest/models/abstract_model.py index 874afe5..761ff6c 100644 --- a/mpest/models/abstract_model.py +++ b/mpest/models/abstract_model.py @@ -4,7 +4,7 @@ import numpy as np -from mpest.types import Params, Samples +from mpest.annotations import Params, Samples from mpest.utils import ANamed diff --git a/mpest/models/exponential.py b/mpest/models/exponential.py index 1614a2a..449fd75 100644 --- a/mpest/models/exponential.py +++ b/mpest/models/exponential.py @@ -3,8 +3,8 @@ import numpy as np from scipy.stats import expon +from mpest.annotations import Params, Samples from mpest.models.abstract_model import AModelDifferentiable, AModelWithGenerator -from mpest.types import Params, Samples class LMomentsParameterMixin: diff --git a/mpest/models/gaussian.py b/mpest/models/gaussian.py index dfdd908..cb682f9 100644 --- a/mpest/models/gaussian.py +++ b/mpest/models/gaussian.py @@ -3,8 +3,8 @@ import numpy as np from scipy.stats import norm +from mpest.annotations import Params, Samples from mpest.models.abstract_model import AModelDifferentiable, AModelWithGenerator -from mpest.types import Params, Samples class LMomentsParameterMixin: diff --git a/mpest/models/weibull.py b/mpest/models/weibull.py index 7df445d..cf8958c 100644 --- a/mpest/models/weibull.py +++ b/mpest/models/weibull.py @@ -5,8 +5,8 @@ import numpy as np from scipy.stats import weibull_min +from mpest.annotations import Params, Samples from mpest.models.abstract_model import AModelDifferentiable, AModelWithGenerator -from mpest.types import Params, Samples class LMomentsParameterMixin: diff --git a/mpest/optimizers/abstract_optimizer.py b/mpest/optimizers/abstract_optimizer.py index 05305d6..fa4b641 100644 --- a/mpest/optimizers/abstract_optimizer.py +++ b/mpest/optimizers/abstract_optimizer.py @@ -5,7 +5,7 @@ import numpy as np -from mpest.types import Params +from mpest.annotations import Params from mpest.utils import ANamed diff --git a/mpest/optimizers/scipy_cg.py b/mpest/optimizers/scipy_cg.py index 611bd38..f7c0719 100644 --- a/mpest/optimizers/scipy_cg.py +++ b/mpest/optimizers/scipy_cg.py @@ -4,8 +4,8 @@ from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizer -from mpest.types import Params class ScipyCG(AOptimizer): diff --git a/mpest/optimizers/scipy_cobyla.py b/mpest/optimizers/scipy_cobyla.py index ef9921d..b0a2856 100644 --- a/mpest/optimizers/scipy_cobyla.py +++ b/mpest/optimizers/scipy_cobyla.py @@ -4,8 +4,8 @@ from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizer -from mpest.types import Params class ScipyCOBYLA(AOptimizer): diff --git a/mpest/optimizers/scipy_nelder_mead.py b/mpest/optimizers/scipy_nelder_mead.py index 499e0d9..b965e89 100644 --- a/mpest/optimizers/scipy_nelder_mead.py +++ b/mpest/optimizers/scipy_nelder_mead.py @@ -4,8 +4,8 @@ from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizer -from mpest.types import Params class ScipyNelderMead(AOptimizer): diff --git a/mpest/optimizers/scipy_newton_cg.py b/mpest/optimizers/scipy_newton_cg.py index c7dcd3b..d870ee0 100644 --- a/mpest/optimizers/scipy_newton_cg.py +++ b/mpest/optimizers/scipy_newton_cg.py @@ -5,8 +5,8 @@ import numpy as np from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizerJacobian -from mpest.types import Params class ScipyNewtonCG(AOptimizerJacobian): diff --git a/mpest/optimizers/scipy_slsqp.py b/mpest/optimizers/scipy_slsqp.py index aa49794..e6e389e 100644 --- a/mpest/optimizers/scipy_slsqp.py +++ b/mpest/optimizers/scipy_slsqp.py @@ -4,8 +4,8 @@ from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizer -from mpest.types import Params class ScipySLSQP(AOptimizer): diff --git a/mpest/optimizers/scipy_tnc.py b/mpest/optimizers/scipy_tnc.py index f0204aa..22ae943 100644 --- a/mpest/optimizers/scipy_tnc.py +++ b/mpest/optimizers/scipy_tnc.py @@ -4,8 +4,8 @@ from scipy.optimize import minimize +from mpest.annotations import Params from mpest.optimizers.abstract_optimizer import AOptimizer -from mpest.types import Params class ScipyTNC(AOptimizer): diff --git a/tests/tests_l_moments/l_moments_utils.py b/tests/tests_l_moments/l_moments_utils.py index 8b5af1d..8f89dd4 100644 --- a/tests/tests_l_moments/l_moments_utils.py +++ b/tests/tests_l_moments/l_moments_utils.py @@ -1,5 +1,6 @@ """TODO""" +from mpest.core.problem import Problem, Result from mpest.em import EM from mpest.em.breakpointers import ParamDifferBreakpointer, StepCountBreakpointer from mpest.em.distribution_checkers import ( @@ -8,7 +9,6 @@ ) from mpest.em.methods.l_moments_method import IndicatorEStep, LMomentsMStep from mpest.em.methods.method import Method -from mpest.problem import Problem, Result def run_test(problem: Problem, deviation: float) -> Result: diff --git a/tests/tests_l_moments/test_one_distribution_l.py b/tests/tests_l_moments/test_one_distribution_l.py index ea53613..170e55a 100644 --- a/tests/tests_l_moments/test_one_distribution_l.py +++ b/tests/tests_l_moments/test_one_distribution_l.py @@ -6,15 +6,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from tests.tests_l_moments.l_moments_utils import run_test from tests.utils import check_for_params_error_tolerance diff --git a/tests/tests_l_moments/test_two_same_distributions_complex_l.py b/tests/tests_l_moments/test_two_same_distributions_complex_l.py index 9887229..e596513 100644 --- a/tests/tests_l_moments/test_two_same_distributions_complex_l.py +++ b/tests/tests_l_moments/test_two_same_distributions_complex_l.py @@ -7,15 +7,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from mpest.utils import Factory from tests.tests_l_moments.l_moments_utils import run_test from tests.utils import ( diff --git a/tests/tests_l_moments/test_two_same_distributions_simple_l.py b/tests/tests_l_moments/test_two_same_distributions_simple_l.py index 8f73578..e3a326e 100644 --- a/tests/tests_l_moments/test_two_same_distributions_simple_l.py +++ b/tests/tests_l_moments/test_two_same_distributions_simple_l.py @@ -10,15 +10,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from mpest.utils import Factory from tests.tests_l_moments.l_moments_utils import run_test from tests.utils import check_for_params_error_tolerance diff --git a/tests/tests_likelihood/likelihood_utils.py b/tests/tests_likelihood/likelihood_utils.py index 4b0c2e5..6a914a7 100644 --- a/tests/tests_likelihood/likelihood_utils.py +++ b/tests/tests_likelihood/likelihood_utils.py @@ -1,5 +1,6 @@ """TODO""" +from mpest.core.problem import Problem, Result from mpest.em import EM from mpest.em.breakpointers import ParamDifferBreakpointer, StepCountBreakpointer from mpest.em.distribution_checkers import ( @@ -9,7 +10,6 @@ from mpest.em.methods.likelihood_method import BayesEStep, LikelihoodMStep from mpest.em.methods.method import Method from mpest.optimizers import ALL_OPTIMIZERS -from mpest.problem import Problem, Result def run_test(problem: Problem, deviation: float) -> list[Result]: diff --git a/tests/tests_likelihood/test_any_distributions_complex.py b/tests/tests_likelihood/test_any_distributions_complex.py index 94141c5..2a36ad3 100644 --- a/tests/tests_likelihood/test_any_distributions_complex.py +++ b/tests/tests_likelihood/test_any_distributions_complex.py @@ -6,15 +6,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from tests.tests_likelihood.likelihood_utils import run_test from tests.utils import ( check_for_params_error_tolerance, diff --git a/tests/tests_likelihood/test_any_distributions_simple.py b/tests/tests_likelihood/test_any_distributions_simple.py index 63d91f4..dd81476 100644 --- a/tests/tests_likelihood/test_any_distributions_simple.py +++ b/tests/tests_likelihood/test_any_distributions_simple.py @@ -6,15 +6,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from tests.tests_likelihood.likelihood_utils import run_test from tests.utils import check_for_params_error_tolerance diff --git a/tests/tests_likelihood/test_one_distribution.py b/tests/tests_likelihood/test_one_distribution.py index 7112ea3..6fbf19d 100644 --- a/tests/tests_likelihood/test_one_distribution.py +++ b/tests/tests_likelihood/test_one_distribution.py @@ -6,15 +6,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from tests.tests_likelihood.likelihood_utils import run_test from tests.utils import check_for_params_error_tolerance diff --git a/tests/tests_likelihood/test_two_same_distributions_complex.py b/tests/tests_likelihood/test_two_same_distributions_complex.py index 18934f6..8c53ff8 100644 --- a/tests/tests_likelihood/test_two_same_distributions_complex.py +++ b/tests/tests_likelihood/test_two_same_distributions_complex.py @@ -7,15 +7,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from mpest.utils import Factory from tests.tests_likelihood.likelihood_utils import run_test from tests.utils import ( diff --git a/tests/tests_likelihood/test_two_same_distributions_simple.py b/tests/tests_likelihood/test_two_same_distributions_simple.py index 5eab23f..dbb86e9 100644 --- a/tests/tests_likelihood/test_two_same_distributions_simple.py +++ b/tests/tests_likelihood/test_two_same_distributions_simple.py @@ -10,15 +10,15 @@ import numpy as np import pytest -from mpest.distribution import Distribution -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.distribution import Distribution +from mpest.core.mixture_distribution import MixtureDistribution +from mpest.core.problem import Problem from mpest.models import ( AModelWithGenerator, ExponentialModel, GaussianModel, WeibullModelExp, ) -from mpest.problem import Problem from mpest.utils import Factory from tests.tests_likelihood.likelihood_utils import run_test from tests.utils import check_for_params_error_tolerance diff --git a/tests/utils.py b/tests/utils.py index f00ebc5..5565e6f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,7 @@ import numpy as np -from mpest.mixture_distribution import MixtureDistribution +from mpest.core.mixture_distribution import MixtureDistribution from mpest.utils import ResultWithError From 516c10301f5cd99c8e96b619fff88fb0c7f29345 Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 19 Feb 2025 07:26:27 +0300 Subject: [PATCH 5/6] fix: mypy --- .github/workflows/check.yml | 11 ++-- .../analyze_summarizers/error_summarizer.py | 2 +- .../analyze_summarizers/time_summarizer.py | 4 +- experimental_env/experiment/estimators.py | 2 +- .../mixture_generators/abstract_generator.py | 2 +- .../preparation/dataset_description.py | 5 +- .../preparation/dataset_generator.py | 4 +- mm.py | 66 +++++++++++++++++++ mpest/core/mixture_distribution.py | 6 +- mpest/em/em.py | 2 +- mpest/em/methods/l_moments_method.py | 4 +- mpest/em/methods/likelihood_method.py | 22 +++---- mpest/models/__init__.py | 2 +- mpest/utils.py | 6 ++ pyproject.toml | 11 ++-- 15 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 mm.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2fbae3e..ebcf60e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -19,18 +19,19 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install poetry - run: | - pipx install poetry + run: pipx install poetry - name: Install dependencies - run: | - poetry install --with dev + run: poetry install --with dev - name: Ruff run: poetry run ruff check + - name: Install stubs + run: poetry run pip install types-PyYAML types-tqdm + - name: Mypy - run: poetry run mypy . + run: poetry run mypy - name: Pytest run: poetry run python -m pytest tests \ No newline at end of file diff --git a/experimental_env/analysis/analyze_summarizers/error_summarizer.py b/experimental_env/analysis/analyze_summarizers/error_summarizer.py index 9810808..9a03575 100644 --- a/experimental_env/analysis/analyze_summarizers/error_summarizer.py +++ b/experimental_env/analysis/analyze_summarizers/error_summarizer.py @@ -39,7 +39,7 @@ def calculate(self, results: list[ExperimentDescription]) -> tuple: errors.append(error) mean = np.sum(errors) / len(errors) - standart_deviation = np.sqrt(np.sum((x - mean) ** 2 for x in errors) / len(errors)) + standart_deviation = np.sqrt(np.sum([(x - mean) ** 2 for x in errors]) / len(errors)) errors.sort() median = errors[len(errors) // 2] diff --git a/experimental_env/analysis/analyze_summarizers/time_summarizer.py b/experimental_env/analysis/analyze_summarizers/time_summarizer.py index 1c0630b..fb39684 100644 --- a/experimental_env/analysis/analyze_summarizers/time_summarizer.py +++ b/experimental_env/analysis/analyze_summarizers/time_summarizer.py @@ -23,11 +23,11 @@ def calculate(self, results: list[ExperimentDescription]) -> tuple: """ times = [] for result in results: - time = np.sum(step.time for step in result.steps) + time = np.sum([step.time for step in result.steps]) times.append(time) mean = np.sum(times) / len(times) - deviation = np.sqrt(np.sum((x - mean) ** 2 for x in times) / len(times)) + deviation = np.sqrt(np.sum([(x - mean) ** 2 for x in times]) / len(times)) return float(mean), float(deviation) def analyze_method(self, results: list[ExperimentDescription], method: str): diff --git a/experimental_env/experiment/estimators.py b/experimental_env/experiment/estimators.py index 09d6fff..f61364a 100644 --- a/experimental_env/experiment/estimators.py +++ b/experimental_env/experiment/estimators.py @@ -16,7 +16,7 @@ from mpest.optimizers import ALL_OPTIMIZERS from mpest.utils import ANamed, Factory, ResultWithLog -METHODS = { +METHODS: dict = { "Likelihood": [[Factory(BayesEStep), Factory(LikelihoodMStep, optimizer)] for optimizer in ALL_OPTIMIZERS], "L-moments": [Factory(IndicatorEStep), Factory(LMomentsMStep)], } diff --git a/experimental_env/mixture_generators/abstract_generator.py b/experimental_env/mixture_generators/abstract_generator.py index 40754c9..fa1f602 100644 --- a/experimental_env/mixture_generators/abstract_generator.py +++ b/experimental_env/mixture_generators/abstract_generator.py @@ -16,7 +16,7 @@ def __init__(self, seed: int = 42): random.seed(seed) @abstractmethod - def generate_priors(self, models: list[type[AModel]]) -> list[float]: + def generate_priors(self, models: list[type[AModel]]) -> list[float | None]: """ A method for choosing how a prior probabilities will be generated. The function is needed so that, with a uniform distribution, degenerate distributions are not generated. diff --git a/experimental_env/preparation/dataset_description.py b/experimental_env/preparation/dataset_description.py index bd065da..5696dc8 100644 --- a/experimental_env/preparation/dataset_description.py +++ b/experimental_env/preparation/dataset_description.py @@ -1,6 +1,7 @@ """Module which describes dataset""" import copy +from typing import Any from mpest.annotations import Samples from mpest.core.mixture_distribution import MixtureDistribution @@ -61,9 +62,9 @@ def to_yaml_format(self) -> dict: """ Convert info about mixture to yaml format """ - # pylint: disable=duplicate-code - output = {} + output: dict[str, Any] = {} + # Add name output["name"] = self.get_dataset_name() output["samples_size"] = self._samples_size diff --git a/experimental_env/preparation/dataset_generator.py b/experimental_env/preparation/dataset_generator.py index a560443..b171987 100644 --- a/experimental_env/preparation/dataset_generator.py +++ b/experimental_env/preparation/dataset_generator.py @@ -60,8 +60,8 @@ class ConcreteDatasetGenerator: def __init__(self, seed: int = 42): np.random.seed(seed) - self._dists = [] - self._priors = [] + self._dists: list[Distribution] = [] + self._priors: list[float | None] = [] def add_distribution(self, model: type[AModel], params: list[float], prior: float) -> None: """ diff --git a/mm.py b/mm.py new file mode 100644 index 0000000..75d1133 --- /dev/null +++ b/mm.py @@ -0,0 +1,66 @@ +"""The script implements the first step of the experiment""" + +from pathlib import Path + +import numpy as np + +from experimental_env.preparation.dataset_generator import ( + ConcreteDatasetGenerator, + RandomDatasetGenerator, +) +from mpest.models import ExponentialModel, GaussianModel, WeibullModelExp + +WORKING_DIR = Path("/home/iraedeus/Projects/stage_1") +SAMPLES_SIZE = 1000 + +np.random.seed(42) + +r_generator = RandomDatasetGenerator() +mixtures = [ + [ExponentialModel], + [GaussianModel], + [WeibullModelExp], + [WeibullModelExp, GaussianModel], + [ExponentialModel, GaussianModel], + [WeibullModelExp, WeibullModelExp], + [ExponentialModel, ExponentialModel], +] +for models in mixtures: + r_generator.generate(SAMPLES_SIZE, models, Path(WORKING_DIR), exp_count=100) + +c_generator2 = ConcreteDatasetGenerator() +models = [ExponentialModel] +c_generator2.add_distribution(models[0], [1.0], 1.0) +c_generator2.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5) + +c_generator3 = ConcreteDatasetGenerator() +models = [GaussianModel] +c_generator3.add_distribution(models[0], [0, 1.0], 1.0) +c_generator3.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5) + +c_generator4 = ConcreteDatasetGenerator() +models = [WeibullModelExp] +c_generator4.add_distribution(models[0], [1.0, 1.0], 1.0) +c_generator4.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5) + +c_generator5 = ConcreteDatasetGenerator() +models = [WeibullModelExp] +c_generator5.add_distribution(models[0], [1.0, 1.0], 1.0) +c_generator5.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5) + +c_generator6 = ConcreteDatasetGenerator() +models = [WeibullModelExp] +c_generator6.add_distribution(models[0], [1.0, 0.5], 1.0) +c_generator6.generate(SAMPLES_SIZE, Path(WORKING_DIR), 5) + +c_generator7 = ConcreteDatasetGenerator() +models = [GaussianModel, GaussianModel] +c_generator7.add_distribution(models[0], [-1.0, 2.5], 0.3) +c_generator7.add_distribution(models[1], [1.0, 0.5], 0.7) +c_generator7.generate(SAMPLES_SIZE, Path(WORKING_DIR), 10) + +c_generator8 = ConcreteDatasetGenerator() +models = [GaussianModel, GaussianModel] +c_generator8.add_distribution(models[0], [0.0, 1.5], 0.6) +c_generator8.add_distribution(models[1], [1.0, 1.0], 0.4) +c_generator8.generate(SAMPLES_SIZE, Path(WORKING_DIR), 10) diff --git a/mpest/core/mixture_distribution.py b/mpest/core/mixture_distribution.py index 39fe4f3..b89c2b2 100644 --- a/mpest/core/mixture_distribution.py +++ b/mpest/core/mixture_distribution.py @@ -122,11 +122,11 @@ def generate(self, size=1) -> Params: ) ) - x = [] + temp = [] for i, model in enumerate(models): - x += list(model[0].generate(x_models.count(i))) + temp += list(model[0].generate(x_models.count(i))) - x = np.array(x) + x = np.array(temp) np.random.shuffle(x) return x diff --git a/mpest/em/em.py b/mpest/em/em.py index d32fb8b..239c29b 100644 --- a/mpest/em/em.py +++ b/mpest/em/em.py @@ -246,7 +246,7 @@ def postprocess_result(result: ResultWithError) -> ResultWithError: return ResultWithError(new_mixture, result.error) - history = [] + history: list = [] def log_map(distributions: ResultWithError[EM._DistributionMixtureAlive]): return ResultWithError( diff --git a/mpest/em/methods/l_moments_method.py b/mpest/em/methods/l_moments_method.py index 860ebca..bd00210 100644 --- a/mpest/em/methods/l_moments_method.py +++ b/mpest/em/methods/l_moments_method.py @@ -13,7 +13,7 @@ from mpest.exceptions import EStepError, MStepError from mpest.utils import ResultWithError, find_file -EResult = tuple[Problem, list[float], np.ndarray] | ResultWithError[MixtureDistribution] +EResult = tuple[Problem, list[float | None], np.ndarray] | ResultWithError[MixtureDistribution] class IndicatorEStep(AExpectation[EResult]): @@ -57,7 +57,7 @@ def calc_indicators(self, problem: Problem) -> None: self.indicators = z return None - def update_priors(self, problem: Problem) -> list[float]: + def update_priors(self, problem: Problem) -> list[float | None]: """ A function that recalculates the list with prior probabilities. diff --git a/mpest/em/methods/likelihood_method.py b/mpest/em/methods/likelihood_method.py index fcf2fe0..d3a1413 100644 --- a/mpest/em/methods/likelihood_method.py +++ b/mpest/em/methods/likelihood_method.py @@ -59,17 +59,17 @@ def step(self, problem: Problem) -> EResult: return active_samples, h, problem -class ML(AExpectation[EResult]): - """ - Class which represents ML method for calculating matrix for M step in likelihood method - """ - - def step(self, problem: Problem) -> EResult: - """ - A function that performs E step - - :param problem: Object of class Problem, which contains samples and mixture. - """ +# class ML(AExpectation[EResult]): +# """ +# Class which represents ML method for calculating matrix for M step in likelihood method +# """ +# +# def step(self, problem: Problem) -> EResult: +# """ +# A function that performs E step +# +# :param problem: Object of class Problem, which contains samples and mixture. +# """ class LikelihoodMStep(AMaximization[EResult]): diff --git a/mpest/models/__init__.py b/mpest/models/__init__.py index 2ddb7fd..a3845e0 100644 --- a/mpest/models/__init__.py +++ b/mpest/models/__init__.py @@ -9,7 +9,7 @@ from mpest.models.gaussian import GaussianModel from mpest.models.weibull import WeibullModelExp -ALL_MODELS = { +ALL_MODELS: dict[str, type[AModel]] = { GaussianModel().name: GaussianModel, WeibullModelExp().name: WeibullModelExp, ExponentialModel().name: ExponentialModel, diff --git a/mpest/utils.py b/mpest/utils.py index 23ce2ba..f18a038 100644 --- a/mpest/utils.py +++ b/mpest/utils.py @@ -202,6 +202,9 @@ def wrapper_timer(*args: P.args, **kwargs: P.kwargs) -> TimerResultWrapper[R]: return wrapper_timer +# TODO: Fix type annotation. Here the lambda function does not match the annotation. + + def history(holder: list[T], mapper: Callable[[R], T] = lambda x: x): """ Decorator factory which allows you @@ -224,6 +227,9 @@ def wrapped_history(*args: P.args, **kwargs: P.kwargs) -> R: return current_history +# TODO: Fix type annotation. Here the lambda function does not match the annotation. + + def logged( holder: list[TimerResultWrapper[T] | ObjectWrapper[T] | float], save_results: bool = True, diff --git a/pyproject.toml b/pyproject.toml index efb0aab..5ed6049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ pytest = "^8.3.4" pre-commit = "^4.1.0" ruff = "^0.9.6" -mypy = "==1.10.1" +mypy = "==1.13.0" sphinx = "^8.2.0" [tool.poetry.group.experiments.dependencies] @@ -56,12 +56,9 @@ ignore = ["PLR0913"] [tool.mypy] -exclude = [ - "^venv/", - "^scripts/", - "^tests/", - "^examples/", -] +files = ["mpest", "experimental_env"] +ignore_missing_imports = true +exclude = ["^mpest/utils.py"] [build-system] From 7c0768e9a8ea7d2667014d71512dcd2f0f6e1dcb Mon Sep 17 00:00:00 2001 From: iraedeus Date: Wed, 26 Feb 2025 18:46:25 +0300 Subject: [PATCH 6/6] chore: update workflows in README.md --- .github/workflows/check.yml | 2 +- README.md | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ebcf60e..7a842c8 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,4 +1,4 @@ -name: Check +name: CI on: [ push, pull_request ] diff --git a/README.md b/README.md index 69077f6..18ebf61 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ -![PyLint](https://github.com/PySATL/MPEst/actions/workflows/pylint.yml/badge.svg) -![Check code style](https://github.com/PySATL/MPEst/actions/workflows/code_style.yml/badge.svg) -[![Code style](https://img.shields.io/badge/Code%20style-black-000000.svg)](https://github.com/psf/black) -![Unit Tests](https://github.com/PySATL/MPEst/actions/workflows/test.yml/badge.svg) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +![CI](https://github.com/PySATL/pysatl-mpest/actions/workflows/check.yml/badge.svg) +License: MIT ## Installation @@ -98,6 +97,6 @@ plt.show() ![plot](https://github.com/toxakaz/EM-algo/raw/main/examples/readme_example/example.png) ## Requirements -- python 3.11 +- python >= 3.11 - numpy - scipy