diff --git a/doc/rtd/MLPro-Int-scikit-learn.gif b/doc/rtd/MLPro-Int-scikit-learn.gif new file mode 100644 index 0000000..7bb5cb5 Binary files /dev/null and b/doc/rtd/MLPro-Int-scikit-learn.gif differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_008_oneclasssvm_paga_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_if_pa_1d.rst similarity index 64% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_008_oneclasssvm_paga_nd.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_if_pa_1d.rst index 27cf87a..086eb52 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_008_oneclasssvm_paga_nd.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_if_pa_1d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_008: -Howto OA-AD-008: Extended Anomaly Detection using a One Class SVM (nD) -====================================================================== +.. _Howto_OA_AD_001: +Howto OA-AD-001: Anomaly Detection using Isolation Forest (1D) +================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_008_oneclasssvm_paga_nd.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_001_if_pa_1d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-008: Extended Anomaly Detection using a One Class SVM (nD) **Results** .. image:: - images/howto_oa_ad_008_oneclasssvm_paga_nd.png + images/howto_oa_ad_001_if_pa_1d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_oneclasssvm_pa_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_oneclasssvm_pa_1d.rst deleted file mode 100644 index cd8c787..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_001_oneclasssvm_pa_1d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_001: -Howto OA-AD-001: Anomaly Detection using a One Class SVM (1D) -============================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_001_oneclasssvm_pa_1d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_001_oneclasssvm_pa_1d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_005_oneclasssvm_paga_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_if_pa_2d.rst similarity index 64% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_005_oneclasssvm_paga_1d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_if_pa_2d.rst index aa65eb2..48c7d1d 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_005_oneclasssvm_paga_1d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_if_pa_2d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_005: -Howto OA-AD-005: Extended Anomaly Detection using a One Class SVM (1D) -====================================================================== +.. _Howto_OA_AD_002: +Howto OA-AD-002: Anomaly Detection using Isolation Forest (2D) +================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_005_oneclasssvm_paga_1d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_002_if_pa_2d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-005: Extended Anomaly Detection using a One Class SVM (1D) **Results** .. image:: - images/howto_oa_ad_005_oneclasssvm_paga_1d.png + images/howto_oa_ad_002_if_pa_2d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_oneclasssvm_pa_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_oneclasssvm_pa_2d.rst deleted file mode 100644 index 78acf4d..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_002_oneclasssvm_pa_2d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_002: -Howto OA-AD-002: Anomaly Detection using a One Class SVM (2D) -============================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_002_oneclasssvm_pa_2d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_002_oneclasssvm_pa_2d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_006_oneclasssvm_paga_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_if_pa_3d.rst similarity index 64% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_006_oneclasssvm_paga_2d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_if_pa_3d.rst index a0f66bd..1b6c9f7 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_006_oneclasssvm_paga_2d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_if_pa_3d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_006: -Howto OA-AD-006: Extended Anomaly Detection using a One Class SVM (2D) -====================================================================== +.. _Howto_OA_AD_003: +Howto OA-AD-003: Anomaly Detection using Isolation Forest (3D) +================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_006_oneclasssvm_paga_2d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_003_if_pa_3d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-006: Extended Anomaly Detection using a One Class SVM (2D) **Results** .. image:: - images/howto_oa_ad_006_oneclasssvm_paga_2d.png + images/howto_oa_ad_003_if_pa_3d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_oneclasssvm_pa_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_oneclasssvm_pa_3d.rst deleted file mode 100644 index d9e1e67..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_003_oneclasssvm_pa_3d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_003: -Howto OA-AD-003: Anomaly Detection using a One Class SVM (3D) -============================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_003_oneclasssvm_pa_3d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_003_oneclasssvm_pa_3d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_007_oneclasssvm_paga_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_if_pa_nd.rst similarity index 64% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_007_oneclasssvm_paga_3d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_if_pa_nd.rst index 5e0ba6b..71b9d5a 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_007_oneclasssvm_paga_3d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_if_pa_nd.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_007: -Howto OA-AD-007: Extended Anomaly Detection using a One Class SVM (3D) -====================================================================== +.. _Howto_OA_AD_004: +Howto OA-AD-004: Anomaly Detection using Isolation Forest (nD) +================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_007_oneclasssvm_paga_3d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_004_if_pa_nd.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-007: Extended Anomaly Detection using a One Class SVM (3D) **Results** .. image:: - images/howto_oa_ad_007_oneclasssvm_paga_3d.png + images/howto_oa_ad_004_if_pa_nd.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_oneclasssvm_pa_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_oneclasssvm_pa_nd.rst deleted file mode 100644 index e2503fc..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_004_oneclasssvm_pa_nd.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_004: -Howto OA-AD-004: Anomaly Detection using a One Class SVM (nD) -============================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_004_oneclasssvm_pa_nd.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_004_oneclasssvm_pa_nd.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_lof_pa_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_011_lof_pa_1d.rst similarity index 59% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_lof_pa_1d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_011_lof_pa_1d.rst index 42abcc7..d26e025 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_lof_pa_1d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_011_lof_pa_1d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_021: -Howto OA-AD-021: Anomaly Detection using Local Outlier Factor (1D) +.. _Howto_OA_AD_011: +Howto OA-AD-011: Anomaly Detection using Local Outlier Factor (1D) ================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_021_lof_pa_1d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_011_lof_pa_1d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-021: Anomaly Detection using Local Outlier Factor (1D) **Results** .. image:: - images/howto_oa_ad_021_lof_pa_1d.png + images/howto_oa_ad_011_lof_pa_1d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_lof_pa_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_012_lof_pa_2d.rst similarity index 59% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_lof_pa_2d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_012_lof_pa_2d.rst index eef55f0..dafd0b3 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_lof_pa_2d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_012_lof_pa_2d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_022: -Howto OA-AD-022: Anomaly Detection using Local Outlier Factor (2D) +.. _Howto_OA_AD_012: +Howto OA-AD-012: Anomaly Detection using Local Outlier Factor (2D) ================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_022_lof_pa_2d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_012_lof_pa_2d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-022: Anomaly Detection using Local Outlier Factor (2D) **Results** .. image:: - images/howto_oa_ad_022_lof_pa_2d.png + images/howto_oa_ad_012_lof_pa_2d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_lof_pa_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_013_lof_pa_3d.rst similarity index 59% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_lof_pa_3d.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_013_lof_pa_3d.rst index 7f4a8a3..3647f76 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_lof_pa_3d.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_013_lof_pa_3d.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_023: -Howto OA-AD-023: Anomaly Detection using Local Outlier Factor (3D) +.. _Howto_OA_AD_013: +Howto OA-AD-013: Anomaly Detection using Local Outlier Factor (3D) ================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_023_lof_pa_3d.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_013_lof_pa_3d.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-023: Anomaly Detection using Local Outlier Factor (3D) **Results** .. image:: - images/howto_oa_ad_023_lof_pa_3d.png + images/howto_oa_ad_013_lof_pa_3d.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_lof_pa_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_014_lof_pa_nd.rst similarity index 59% rename from doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_lof_pa_nd.rst rename to doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_014_lof_pa_nd.rst index d2b85f7..876ef45 100644 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_lof_pa_nd.rst +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_014_lof_pa_nd.rst @@ -1,10 +1,10 @@ -.. _Howto_OA_AD_024: -Howto OA-AD-024: Anomaly Detection using Local Outlier Factor (nD) +.. _Howto_OA_AD_014: +Howto OA-AD-014: Anomaly Detection using Local Outlier Factor (nD) ================================================================== **Executable code** -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_024_lof_pa_nd.py +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_014_lof_pa_nd.py :language: python @@ -12,7 +12,7 @@ Howto OA-AD-024: Anomaly Detection using Local Outlier Factor (nD) **Results** .. image:: - images/howto_oa_ad_024_lof_pa_nd.png + images/howto_oa_ad_014_lof_pa_nd.png :width: 600px diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_ee_pa_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_ee_pa_1d.rst new file mode 100644 index 0000000..65e66c6 --- /dev/null +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_021_ee_pa_1d.rst @@ -0,0 +1,21 @@ +.. _Howto_OA_AD_021: +Howto OA-AD-021: Anomaly Detection using Elliptic Envelope (1D) +=============================================================== + +**Executable code** + +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_021_ee_pa_1d.py + :language: python + + + +**Results** + +.. image:: + images/howto_oa_ad_021_ee_pa_1d.png + :width: 600px + + + +**Cross reference** + - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_ee_pa_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_ee_pa_2d.rst new file mode 100644 index 0000000..e776311 --- /dev/null +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_022_ee_pa_2d.rst @@ -0,0 +1,21 @@ +.. _Howto_OA_AD_022: +Howto OA-AD-022: Anomaly Detection using Elliptic Envelope (2D) +=============================================================== + +**Executable code** + +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_022_ee_pa_2d.py + :language: python + + + +**Results** + +.. image:: + images/howto_oa_ad_022_ee_pa_2d.png + :width: 600px + + + +**Cross reference** + - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_ee_pa_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_ee_pa_3d.rst new file mode 100644 index 0000000..6f083d5 --- /dev/null +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_023_ee_pa_3d.rst @@ -0,0 +1,21 @@ +.. _Howto_OA_AD_023: +Howto OA-AD-023: Anomaly Detection using Elliptic Envelope (3D) +=============================================================== + +**Executable code** + +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_023_ee_pa_3d.py + :language: python + + + +**Results** + +.. image:: + images/howto_oa_ad_023_ee_pa_3d.png + :width: 600px + + + +**Cross reference** + - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_ee_pa_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_ee_pa_nd.rst new file mode 100644 index 0000000..675b6f9 --- /dev/null +++ b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_024_ee_pa_nd.rst @@ -0,0 +1,21 @@ +.. _Howto_OA_AD_024: +Howto OA-AD-024: Anomaly Detection using Elliptic Envelope (nD) +=============================================================== + +**Executable code** + +.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_024_ee_pa_nd.py + :language: python + + + +**Results** + +.. image:: + images/howto_oa_ad_024_ee_pa_nd.png + :width: 600px + + + +**Cross reference** + - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_025_lof_paga_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_025_lof_paga_1d.rst deleted file mode 100644 index bb90171..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_025_lof_paga_1d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_025: -Howto OA-AD-025: Extended Anomaly Detection using Local Outlier Factor (1D) -=========================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_025_lof_paga_1d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_025_lof_paga_1d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_026_lof_paga_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_026_lof_paga_2d.rst deleted file mode 100644 index 9152634..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_026_lof_paga_2d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_026: -Howto OA-AD-026: Extended Anomaly Detection using Local Outlier Factor (2D) -=========================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_026_lof_paga_2d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_026_lof_paga_2d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_027_lof_paga_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_027_lof_paga_3d.rst deleted file mode 100644 index dd60341..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_027_lof_paga_3d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_027: -Howto OA-AD-027: Extended Anomaly Detection using Local Outlier Factor (3D) -=========================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_027_lof_paga_3d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_027_lof_paga_3d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_028_lof_paga_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_028_lof_paga_nd.rst deleted file mode 100644 index 55314c5..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_028_lof_paga_nd.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_028: -Howto OA-AD-028: Extended Anomaly Detection using Local Outlier Factor (nD) -=========================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_028_lof_paga_nd.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_028_lof_paga_nd.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_041_if_pa_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_041_if_pa_1d.rst deleted file mode 100644 index 14326d4..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_041_if_pa_1d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_041: -Howto OA-AD-041: Anomaly Detection using Isolation Forest (1D) -================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_041_if_pa_1d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_041_if_pa_1d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_042_if_pa_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_042_if_pa_2d.rst deleted file mode 100644 index e77764b..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_042_if_pa_2d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_042: -Howto OA-AD-042: Anomaly Detection using Isolation Forest (2D) -================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_042_if_pa_2d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_042_if_pa_2d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_043_if_pa_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_043_if_pa_3d.rst deleted file mode 100644 index 34073e2..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_043_if_pa_3d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_043: -Howto OA-AD-043: Anomaly Detection using Isolation Forest (3D) -================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_043_if_pa_3d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_043_if_pa_3d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_044_if_pa_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_044_if_pa_nd.rst deleted file mode 100644 index 7cb49ee..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_044_if_pa_nd.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_044: -Howto OA-AD-044: Anomaly Detection using Isolation Forest (nD) -================================================================== - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_044_if_pa_nd.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_044_if_pa_nd.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_045_if_paga_1d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_045_if_paga_1d.rst deleted file mode 100644 index c071021..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_045_if_paga_1d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_045: -Howto OA-AD-045: Extended Anomaly Detection using Isolation Forest (1D) -======================================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_045_if_paga_1d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_045_if_paga_1d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_046_if_paga_2d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_046_if_paga_2d.rst deleted file mode 100644 index e829a82..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_046_if_paga_2d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_046: -Howto OA-AD-046: Extended Anomaly Detection using Isolation Forest (2D) -======================================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_046_if_paga_2d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_046_if_paga_2d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_047_if_paga_3d.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_047_if_paga_3d.rst deleted file mode 100644 index 3a2d9df..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_047_if_paga_3d.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_047: -Howto OA-AD-047: Extended Anomaly Detection using Isolation Forest (3D) -======================================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_047_if_paga_3d.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_047_if_paga_3d.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_048_if_paga_nd.rst b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_048_if_paga_nd.rst deleted file mode 100644 index a669ae8..0000000 --- a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/howto_oa_ad_048_if_paga_nd.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _Howto_OA_AD_048: -Howto OA-AD-048: Extended Anomaly Detection using Isolation Forest (nD) -======================================================================= - -**Executable code** - -.. literalinclude:: ../../../../../test/howtos/oa/howto_oa_ad_048_if_paga_nd.py - :language: python - - - -**Results** - -.. image:: - images/howto_oa_ad_048_if_paga_nd.png - :width: 600px - - - -**Cross reference** - - :ref:`API Reference: Wrapper for scikit-learn Anomaly Detectors ` diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_if_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_if_pa_1d.png new file mode 100644 index 0000000..657282e Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_if_pa_1d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_oneclasssvm_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_oneclasssvm_pa_1d.png deleted file mode 100644 index 2348a04..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_001_oneclasssvm_pa_1d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_if_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_if_pa_2d.png new file mode 100644 index 0000000..f53a239 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_if_pa_2d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_oneclasssvm_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_oneclasssvm_pa_2d.png deleted file mode 100644 index 7ea3469..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_002_oneclasssvm_pa_2d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_if_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_if_pa_3d.png new file mode 100644 index 0000000..595dd67 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_if_pa_3d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_oneclasssvm_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_oneclasssvm_pa_3d.png deleted file mode 100644 index 8f43c9d..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_003_oneclasssvm_pa_3d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_if_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_if_pa_nd.png new file mode 100644 index 0000000..3172909 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_if_pa_nd.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_oneclasssvm_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_oneclasssvm_pa_nd.png deleted file mode 100644 index 2516ca2..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_004_oneclasssvm_pa_nd.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_005_oneclasssvm_paga_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_005_oneclasssvm_paga_1d.png deleted file mode 100644 index bad3316..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_005_oneclasssvm_paga_1d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_011_lof_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_011_lof_pa_1d.png new file mode 100644 index 0000000..f30537d Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_011_lof_pa_1d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_012_lof_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_012_lof_pa_2d.png new file mode 100644 index 0000000..5e0a3ec Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_012_lof_pa_2d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_013_lof_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_013_lof_pa_3d.png new file mode 100644 index 0000000..327c4e4 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_013_lof_pa_3d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_014_lof_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_014_lof_pa_nd.png new file mode 100644 index 0000000..a005849 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_014_lof_pa_nd.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_ee_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_ee_pa_1d.png new file mode 100644 index 0000000..7cc0d10 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_ee_pa_1d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_lof_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_lof_pa_1d.png deleted file mode 100644 index d75572d..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_021_lof_pa_1d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_ee_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_ee_pa_2d.png new file mode 100644 index 0000000..c9fc8d5 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_ee_pa_2d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_lof_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_lof_pa_2d.png deleted file mode 100644 index 9692960..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_022_lof_pa_2d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_ee_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_ee_pa_3d.png new file mode 100644 index 0000000..114ad15 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_ee_pa_3d.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_lof_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_lof_pa_3d.png deleted file mode 100644 index 9773437..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_023_lof_pa_3d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_ee_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_ee_pa_nd.png new file mode 100644 index 0000000..46c04e0 Binary files /dev/null and b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_ee_pa_nd.png differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_lof_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_lof_pa_nd.png deleted file mode 100644 index e452428..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_024_lof_pa_nd.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_041_if_pa_1d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_041_if_pa_1d.png deleted file mode 100644 index 2706bc3..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_041_if_pa_1d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_042_if_pa_2d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_042_if_pa_2d.png deleted file mode 100644 index d760dd8..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_042_if_pa_2d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_043_if_pa_3d.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_043_if_pa_3d.png deleted file mode 100644 index 219303b..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_043_if_pa_3d.png and /dev/null differ diff --git a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_044_if_pa_nd.png b/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_044_if_pa_nd.png deleted file mode 100644 index 9a58dd7..0000000 Binary files a/doc/rtd/content/01_example_pool/02_howtos_anomaly_detectors/images/howto_oa_ad_044_if_pa_nd.png and /dev/null differ diff --git a/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio b/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio index 2fb982f..0246e3f 100644 --- a/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio +++ b/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio @@ -1,6 +1,6 @@ - + - + @@ -52,7 +52,7 @@ - + @@ -74,151 +74,116 @@ - - - - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - + + - + - + - - + + - - + + - - - - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio.png b/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio.png index 9c1de38..83318b2 100644 Binary files a/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio.png and b/doc/rtd/content/02_api/images/MLPro-Int-scikit-learn-Wrappers.drawio.png differ diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 7d97dff..0051c6a 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -15,6 +15,8 @@ provides numerous state-of-the-art algorithms for a vast amount of machine learn MLPro-Int-scikit-learn provides wrapper classes that enable the use of scikit-learn algorithms and data streams in your MLPro applications. The use of these wrappers is illustrated in various example programs. +.. image:: MLPro-Int-scikit-learn.gif + :width: 300 **Preparation** diff --git a/requirements.txt b/requirements.txt index caa0230..e296cfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -mlpro[full]>=1.9.7 +mlpro[full]>=2.0.2 scikit-learn>=1.6.1 diff --git a/src/mlpro_int_sklearn/wrappers/anomalydetectors/basics.py b/src/mlpro_int_sklearn/wrappers/anomalydetectors/basics.py index 496ba48..698bd3b 100644 --- a/src/mlpro_int_sklearn/wrappers/anomalydetectors/basics.py +++ b/src/mlpro_int_sklearn/wrappers/anomalydetectors/basics.py @@ -15,10 +15,14 @@ ## -- 2024-05-24 1.3.0 DA Refactoring ## -- 2024-11-27 1.4.0 DA Alignment with MLPro 1.9.2 ## -- 2025-03-05 2.0.0 DA Alignment with MLPro 1.9.5 and generalization +## -- 2025-05-07 2.1.0 DA Alignment with MLPro 2.0.0 +## -- 2025-05-30 2.2.0 DA/DS Alignment with MLPro 2.0.2 +## -- 2025-06-12 2.3.0 DA/DS - Alignment with MLPro 2.0.2 +## -- - Rework and optimization ## ------------------------------------------------------------------------------------------------- """ -Ver. 2.0.0 (2025-03-05) +Ver. 2.3.0 (2025-06-12) This module provides wrapper root classes from Scikit-learn to MLPro, specifically for anomaly detectors. @@ -31,9 +35,10 @@ from sklearn.base import OutlierMixin from mlpro.bf.various import Log -from mlpro.bf.streams import StreamTask, Instance, InstDict, InstTypeNew -from mlpro.oa.streams.tasks.anomalydetectors.instancebased import AnomalyDetectorIBPG -from mlpro.oa.streams.tasks.anomalydetectors.anomalies.instancebased import PointAnomaly +from mlpro.bf.exceptions import ParamError +from mlpro.bf.streams import StreamTask, Instance +from mlpro.oa.streams.tasks.changedetectors.anomalydetectors.instancebased import AnomalyDetectorIBPG +from mlpro.oa.streams.tasks.changedetectors.anomalydetectors.anomalies.instancebased import PointAnomaly from mlpro_int_sklearn.wrappers import WrapperSklearn @@ -43,31 +48,46 @@ ## ------------------------------------------------------------------------------------------------- class WrAnomalyDetectorSklearn2MLPro (AnomalyDetectorIBPG, WrapperSklearn): """ - MLPro's wrapper for anomaly detectors of the scikit-learn project. The wrapper raises anomalies - of type PointAnomaly and GroupAnomaly. + MLPro's wrapper for anomaly detectors of the scikit-learn project. The wrapper is limited to + detectors of type 'OutlierMixin' providing the method fit_predict() for a mixed data + training/prediction. Parameters ---------- p_algo_scikit_learn : OutlierMixin Outlier algorithm from the scikit-learn framework to be wrapped - p_delay : int - Number of instances before the detection starts. Default = 3. - p_instance_buffer_size : int - Number of instances to be buffered internally as the basis for anomaly detection. Default = 20. - p_group_anomaly_det : bool - Paramter to activate group anomaly detection. Default is True. p_range_max : int Maximum range of asynchonicity. See class Range. Default is Range.C_RANGE_PROCESS. - p_ada : bool - Boolean switch for adaptivitiy. Default = True. p_duplicate_data : bool If True, instances will be duplicated before processing. Default = False. p_visualize : bool Boolean switch for visualisation. Default = False. - p_logging + p_logging : int = Log.C_LOG_ALL Log level (see constants of class Log). Default: Log.C_LOG_ALL p_anomaly_buffer_size : int = 100 Size of the internal anomaly buffer self.anomalies. Default = 100. + p_instance_buffer_size : int = 20 + Number of instances to be buffered internally as the basis for anomaly detection. Default = 20. + p_detection_steprate : int = 1 + Detection steprate in the interval [1,p_instance_buffer_size]. + p_group_anomaly_det : bool + Paramter to activate group anomaly detection. Default is True. + + Notes + ----- + Supported algorithms + - LOF - Local Outlier Factor + - IF - Isolation Forest + - Elliptic Envelope + - Further ones inherited from 'OutlierMixin' + + Additional features + - Optional group anomaly detection + - 2D/3D anomaly visualization + + Supported types of anomalies + - PointAnomaly + - GroupAnomaly """ C_TYPE = 'Anomaly Detector (scikit-learn)' @@ -75,15 +95,15 @@ class WrAnomalyDetectorSklearn2MLPro (AnomalyDetectorIBPG, WrapperSklearn): ## ------------------------------------------------------------------------------------------------- def __init__( self, p_algo_scikit_learn : OutlierMixin, - p_delay : int = 3, - p_instance_buffer_size : int = 20, - p_group_anomaly_det : bool = True, p_range_max = StreamTask.C_RANGE_THREAD, - p_ada = True, p_duplicate_data = False, p_visualize = False, p_logging=Log.C_LOG_ALL, - p_anomaly_buffer_size = 100 ): + p_anomaly_buffer_size = 100, + p_instance_buffer_size : int = 20, + p_detection_steprate : int = 1, + p_group_anomaly_det : bool = True, + **p_kwargs ): WrapperSklearn.__init__( self, p_logging = p_logging ) @@ -91,79 +111,95 @@ def __init__( self, p_group_anomaly_det = p_group_anomaly_det, p_name = type(p_algo_scikit_learn).__name__, p_range_max = p_range_max, - p_ada = p_ada, + p_ada = True, p_duplicate_data = p_duplicate_data, p_visualize = p_visualize, p_logging = p_logging, - p_anomaly_buffer_size = p_anomaly_buffer_size ) + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_thrs_inst = 1, + **p_kwargs ) - self._algo_scikitlearn = p_algo_scikit_learn - self._inst_buffer_size = p_instance_buffer_size - self._delay = p_delay - self._inst_buffer = [] - self._ano_scores = None - - -## ------------------------------------------------------------------------------------------------- - def _run(self, p_inst: InstDict ): - - for inst_id, (inst_type, inst) in sorted(p_inst.items()): - - if inst_type != InstTypeNew: continue + if ( p_detection_steprate > p_instance_buffer_size ) or ( p_detection_steprate < 1 ): + raise ParamError('Please set the parameter "p_detection_steprate" >= 1 and <= "p_instance_buffer_size"') + + self._algo_scikitlearn = p_algo_scikit_learn + self._inst_buffer_size = p_instance_buffer_size + self._detection_steprate = p_detection_steprate + self._inst_counter = 0 - self.adapt( p_inst = { inst_id : ( inst_type, inst ) } ) + self._inst_data_buffer : np.ndarray = None + self._inst_data_buffer_full : bool = False + self._inst_ref_buffer : np.ndarray = np.empty(self._inst_buffer_size, dtype = object) - if -1 in self._ano_scores: - anomaly = PointAnomaly( p_id = self._get_next_anomaly_id(), - p_instances = [inst], - p_ano_scores = self._ano_scores, - p_visualize = self.get_visualization(), - p_raising_object = self, - p_tstamp = inst.tstamp ) - - self._raise_anomaly_event( p_anomaly = anomaly ) + self._inst_buffer_pos : int = 0 + self._block_mode = ( self._detection_steprate == self._inst_buffer_size ) + ## ------------------------------------------------------------------------------------------------- - def _adapt(self, p_inst_new: Instance) -> bool: + def _detect(self, p_instance : Instance, **p_kwargs): - # 1 Intr0 - adapted = False - feature_data = p_inst_new.get_feature_data() - feature_values = feature_data.get_values() - self._ano_scores = [] + # 1 Intro + feature_data = p_instance.get_feature_data() + feature_values = feature_data.get_values() + num_features = feature_data.get_related_set().get_num_dim() # 2 Preparation of instance data buffer - if len(self._inst_buffer) == 0: - for i in range(len(feature_values)): - self._inst_buffer.append([]) + if self._inst_data_buffer is None: + self._inst_data_buffer = np.empty((self._inst_buffer_size, num_features)) - # 3 Update of instance data buffer - if len(self._inst_buffer[0]) >= self._inst_buffer_size: - for i in range(len(feature_values)): - self._inst_buffer[i].pop(0) + # 3 Update the instance buffer + if self._block_mode: + # 3.1 Here, the buffer is used as an inplace ring buffer. Anomaly detection takes place, + # whenever the buffer is overwritten completely. + self._inst_data_buffer[self._inst_buffer_pos] = feature_values + self._inst_ref_buffer[self._inst_buffer_pos] = p_instance + self._inst_buffer_pos = ( self._inst_buffer_pos + 1 ) % self._inst_buffer_size + if self._inst_buffer_pos != 0: return - for i, value in enumerate(feature_values): - self._inst_buffer[i].append(value) + else: + # 3.2 Here, the buffer is used as a sliding window. Anomaly detection takes place, + # once the buffer is filled and the given step rate has been reached. + if self._inst_data_buffer_full: + # 3.2.1 Buffer full -> inplace shift left and new entry on the right + self._inst_data_buffer[:-1] = self._inst_data_buffer[1:] + self._inst_data_buffer[self._inst_buffer_size -1] = feature_values + self._inst_ref_buffer[:-1] = self._inst_ref_buffer[1:] + self._inst_ref_buffer[self._inst_buffer_size -1] = p_instance + self._inst_counter = ( self._inst_counter + 1 ) % self._detection_steprate + if self._inst_counter != 0: return - # 4 Anomaly detection - if len(self._inst_buffer[0]) >= self._delay: - for i in range(len(feature_values)): - try: - # Algorithms like Isolation Forest and One Class SVM provide methods fit() and predict() - self._algo_scikitlearn.fit(np.array(self._inst_buffer[i]).reshape(-1, 1)) - scores = self._algo_scikitlearn.predict(np.array(self._inst_buffer[i]).reshape(-1, 1)) - except: - # Algorithms implementing method OutlierMixin.fit_predict() - scores = self._algo_scikitlearn.fit_predict(np.array(self._inst_buffer[i]).reshape(-1, 1)) + else: + # 3.2.2 Buffer to be filled + self._inst_data_buffer[self._inst_buffer_pos] = feature_values + self._inst_ref_buffer[self._inst_buffer_pos] = p_instance + self._inst_buffer_pos = ( self._inst_buffer_pos + 1 ) % self._inst_buffer_size - self._ano_scores.append(scores[-1]) + if self._inst_buffer_pos != 0: return - adapted = True + self._inst_data_buffer_full = True - # 5 Outro - return adapted + # 4 Anomaly detection + scores = self._algo_scikitlearn.fit_predict(self._inst_data_buffer) + + # 4.1 Check for anomalies in scores + for i in np.where(scores == -1)[0]: + related_instance = self._inst_ref_buffer[i] + + if not self._block_mode: + # 4.1.1 In case of sliding window, multiple raise of anomalies for the same instance + # needs to be avoided. + if related_instance is np.nan: continue + self._inst_ref_buffer[i] = np.nan + + anomaly = PointAnomaly( p_status = True, + p_tstamp = related_instance.tstamp, + p_visualize = self.get_visualization(), + p_raising_object = self, + p_instances = [related_instance] ) + + self._raise_anomaly_event( p_anomaly = anomaly, p_instance = p_instance ) \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_041_if_pa_1d.py b/test/howtos/oa/howto_oa_ad_001_if_pa_1d.py similarity index 52% rename from test/howtos/oa/howto_oa_ad_041_if_pa_1d.py rename to test/howtos/oa/howto_oa_ad_001_if_pa_1d.py index bd1039c..81706cc 100644 --- a/test/howtos/oa/howto_oa_ad_041_if_pa_1d.py +++ b/test/howtos/oa/howto_oa_ad_001_if_pa_1d.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_041_if_pa_1d.py +## -- Module : howto_oa_ad_001_if_pa_1d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -10,10 +10,11 @@ ## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.2 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -34,9 +35,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -50,12 +53,22 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], p_outlier_rate=0.02, - p_visualize=p_visualize, + p_seed = 1, p_logging=p_logging ) # 2 Creation of a workflow @@ -66,14 +79,15 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_logging=p_logging ) # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, p_group_anomaly_det = False, p_visualize = p_visualize, p_logging = p_logging ) @@ -91,28 +105,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -125,6 +173,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_001_oneclasssvm_pa_1d.py b/test/howtos/oa/howto_oa_ad_001_oneclasssvm_pa_1d.py deleted file mode 100644 index 640b3ed..0000000 --- a/test/howtos/oa/howto_oa_ad_001_oneclasssvm_pa_1d.py +++ /dev/null @@ -1,130 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_001_oneclasssvm_pa_1d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.various import Log -from mlpro.bf.streams.streams import StreamMLProPOutliers -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], - p_outlier_rate=0.02, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -myscenario.reset() - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - input('Press ENTER to start stream processing...') - - -# 3 Run own stream scenario -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_046_if_paga_2d.py b/test/howtos/oa/howto_oa_ad_002_if_pa_2d.py similarity index 51% rename from test/howtos/oa/howto_oa_ad_046_if_paga_2d.py rename to test/howtos/oa/howto_oa_ad_002_if_pa_2d.py index 9a37263..12042ef 100644 --- a/test/howtos/oa/howto_oa_ad_046_if_paga_2d.py +++ b/test/howtos/oa/howto_oa_ad_002_if_pa_2d.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_046_if_paga_2d.py +## -- Module : howto_oa_ad_002_if_pa_2d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -11,11 +11,11 @@ ## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.3 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) - +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -36,9 +36,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -52,12 +54,22 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate=0.1, - p_visualize=p_visualize, + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], + p_outlier_rate=0.02, + p_seed = 1, p_logging=p_logging ) # 2 Creation of a workflow @@ -66,17 +78,18 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, p_visualize = p_visualize, p_logging = p_logging ) @@ -93,28 +106,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -127,6 +174,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_002_oneclasssvm_pa_2d.py b/test/howtos/oa/howto_oa_ad_002_oneclasssvm_pa_2d.py deleted file mode 100644 index c0f2cf1..0000000 --- a/test/howtos/oa/howto_oa_ad_002_oneclasssvm_pa_2d.py +++ /dev/null @@ -1,133 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_002_oneclasssvm_pa_2d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-04-05 1.0.1 DA Activated 2D visualization -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate=0.02, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_047_if_paga_3d.py b/test/howtos/oa/howto_oa_ad_003_if_pa_3d.py similarity index 51% rename from test/howtos/oa/howto_oa_ad_047_if_paga_3d.py rename to test/howtos/oa/howto_oa_ad_003_if_pa_3d.py index 2e24e9c..0769c04 100644 --- a/test/howtos/oa/howto_oa_ad_047_if_paga_3d.py +++ b/test/howtos/oa/howto_oa_ad_003_if_pa_3d.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_047_if_paga_3d.py +## -- Module : howto_oa_ad_003_if_pa_3d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -11,10 +11,11 @@ ## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.3 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -35,9 +36,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -51,12 +54,22 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, - p_visualize=p_visualize, + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, p_logging=p_logging ) # 2 Creation of a workflow @@ -65,17 +78,18 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, p_visualize = p_visualize, p_logging = p_logging ) @@ -92,28 +106,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -126,6 +174,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_003_oneclasssvm_pa_3d.py b/test/howtos/oa/howto_oa_ad_003_oneclasssvm_pa_3d.py deleted file mode 100644 index a356caf..0000000 --- a/test/howtos/oa/howto_oa_ad_003_oneclasssvm_pa_3d.py +++ /dev/null @@ -1,133 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_003_oneclasssvm_pa_3d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-08-02 0.0.0 SK Creation -## -- 2023-08-02 1.0.0 SK First version release -## -- 2024-01-30 1.1.0 DA Relocation to separate github repository -## -- 2024-02-23 1.1.1 SK Bug fix -## -- 2024-04-05 1.1.2 DA Activated 3D visualization -## -- 2024-05-07 1.1.3 SK Change in parameter p_outlier_rate -## -- 2025-03-05 1.2.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.2.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.02, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_044_if_pa_nd.py b/test/howtos/oa/howto_oa_ad_004_if_pa_nd.py similarity index 52% rename from test/howtos/oa/howto_oa_ad_044_if_pa_nd.py rename to test/howtos/oa/howto_oa_ad_004_if_pa_nd.py index 7172551..f10b1eb 100644 --- a/test/howtos/oa/howto_oa_ad_044_if_pa_nd.py +++ b/test/howtos/oa/howto_oa_ad_004_if_pa_nd.py @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_044_if_pa_nd.py +## -- Module : howto_oa_ad_004_if_pa_nd.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -10,10 +10,11 @@ ## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.3 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -34,9 +35,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -50,12 +53,22 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], p_outlier_rate=0.02, - p_visualize=p_visualize, + p_seed = 1, p_logging=p_logging ) # 2 Creation of a workflow @@ -64,16 +77,17 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, p_group_anomaly_det = False, p_visualize = p_visualize, p_logging = p_logging ) @@ -91,24 +105,58 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, @@ -125,6 +173,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_004_oneclasssvm_pa_nd.py b/test/howtos/oa/howto_oa_ad_004_oneclasssvm_pa_nd.py deleted file mode 100644 index fc66afd..0000000 --- a/test/howtos/oa/howto_oa_ad_004_oneclasssvm_pa_nd.py +++ /dev/null @@ -1,131 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_004_oneclasssvm_pa_nd.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions=['sin', 'cos', 'const'], - p_outlier_rate=0.02, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_045_if_paga_1d.py b/test/howtos/oa/howto_oa_ad_005_if_paga_1d.py.off similarity index 51% rename from test/howtos/oa/howto_oa_ad_045_if_paga_1d.py rename to test/howtos/oa/howto_oa_ad_005_if_paga_1d.py.off index ecf7b0d..e37db96 100644 --- a/test/howtos/oa/howto_oa_ad_045_if_paga_1d.py +++ b/test/howtos/oa/howto_oa_ad_005_if_paga_1d.py.off @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_045_if_paga_1d.py +## -- Module : howto_oa_ad_005_if_paga_1d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -10,14 +10,15 @@ ## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.2 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. +The workflow consists of a standard task 'Anomaly Detector'. You will learn: @@ -34,9 +35,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -50,11 +53,21 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], - p_outlier_rate=0.3, + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], + p_outlier_rate=0.02, p_visualize=p_visualize, p_logging=p_logging ) @@ -64,22 +77,23 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, p_group_anomaly_det = True, p_visualize = p_visualize, p_logging = p_logging ) workflow.add_task( p_task=anomalydetector ) - + # 5 Return stream and workflow return mystream, workflow @@ -91,28 +105,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -125,6 +173,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_005_oneclasssvm_paga_1d.py b/test/howtos/oa/howto_oa_ad_005_oneclasssvm_paga_1d.py deleted file mode 100644 index 25525ac..0000000 --- a/test/howtos/oa/howto_oa_ad_005_oneclasssvm_paga_1d.py +++ /dev/null @@ -1,132 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_005_oneclasssvm_paga_1d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.1.0 DA Review/refactoring -## -- 2025-03-05 1.2.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.2.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], - p_outlier_rate = 0.3, - p_visualize = p_visualize, - p_logging = p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name = 'wf1', - p_range_max = OAStreamWorkflow.C_RANGE_NONE, - p_ada = p_ada, - p_visualize = p_visualize, - p_logging = p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_042_if_pa_2d.py b/test/howtos/oa/howto_oa_ad_006_if_paga_2d.py.off similarity index 52% rename from test/howtos/oa/howto_oa_ad_042_if_pa_2d.py rename to test/howtos/oa/howto_oa_ad_006_if_paga_2d.py.off index c33fcee..9965489 100644 --- a/test/howtos/oa/howto_oa_ad_042_if_pa_2d.py +++ b/test/howtos/oa/howto_oa_ad_006_if_paga_2d.py.off @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_042_if_pa_2d.py +## -- Module : howto_oa_ad_006_if_paga_2d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -11,10 +11,11 @@ ## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.3 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -35,9 +36,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -51,10 +54,20 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], p_outlier_rate=0.02, p_visualize=p_visualize, p_logging=p_logging ) @@ -65,17 +78,18 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, p_visualize = p_visualize, p_logging = p_logging ) @@ -92,28 +106,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -126,6 +174,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_006_oneclasssvm_paga_2d.py b/test/howtos/oa/howto_oa_ad_006_oneclasssvm_paga_2d.py deleted file mode 100644 index 4fc5090..0000000 --- a/test/howtos/oa/howto_oa_ad_006_oneclasssvm_paga_2d.py +++ /dev/null @@ -1,133 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_006_oneclasssvm_paga_2d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 2D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate = 0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_043_if_pa_3d.py b/test/howtos/oa/howto_oa_ad_007_if_paga_3d.py.off similarity index 52% rename from test/howtos/oa/howto_oa_ad_043_if_pa_3d.py rename to test/howtos/oa/howto_oa_ad_007_if_paga_3d.py.off index fc8c58f..76084fc 100644 --- a/test/howtos/oa/howto_oa_ad_043_if_pa_3d.py +++ b/test/howtos/oa/howto_oa_ad_007_if_paga_3d.py.off @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_043_if_pa_3d.py +## -- Module : howto_oa_ad_007_if_paga_3d.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -11,10 +11,11 @@ ## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.3 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -35,14 +36,17 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + ## ------------------------------------------------------------------------------------------------- ## ------------------------------------------------------------------------------------------------- class ADScenarioIF (OAStreamScenario): @@ -50,10 +54,20 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], p_outlier_rate=0.02, p_visualize=p_visualize, p_logging=p_logging ) @@ -64,17 +78,18 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, p_visualize = p_visualize, p_logging = p_logging ) @@ -91,28 +106,62 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, p_step_rate = step_rate ) ) @@ -125,6 +174,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_007_oneclasssvm_paga_3d.py b/test/howtos/oa/howto_oa_ad_007_oneclasssvm_paga_3d.py deleted file mode 100644 index 0bba834..0000000 --- a/test/howtos/oa/howto_oa_ad_007_oneclasssvm_paga_3d.py +++ /dev/null @@ -1,131 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_007_oneclasssvm_paga_3d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 3D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_048_if_paga_nd.py b/test/howtos/oa/howto_oa_ad_008_if_paga_nd.py.off similarity index 52% rename from test/howtos/oa/howto_oa_ad_048_if_paga_nd.py rename to test/howtos/oa/howto_oa_ad_008_if_paga_nd.py.off index 3550a8f..50f69e4 100644 --- a/test/howtos/oa/howto_oa_ad_048_if_paga_nd.py +++ b/test/howtos/oa/howto_oa_ad_008_if_paga_nd.py.off @@ -1,7 +1,7 @@ ## ------------------------------------------------------------------------------------------------- ## -- Project : MLPro - The integrative middleware framework for standardized machine learning ## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_048_if_paga_nd.py +## -- Module : howto_oa_ad_008_if_paga_nd.py ## ------------------------------------------------------------------------------------------------- ## -- History : ## -- yyyy-mm-dd Ver. Auth. Description @@ -10,11 +10,11 @@ ## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate ## -- 2024-11-27 1.0.2 DA Correction for unit testing ## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-15 1.2.0 DA Refactoring and generalization by user input parameters ## ------------------------------------------------------------------------------------------------- """ -Ver. 1.1.0 (2025-03-05) - +Ver. 1.2.0 (2025-06-15) This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. @@ -35,9 +35,11 @@ from sklearn.ensemble import IsolationForest as IF -from mlpro.bf.streams.streams import * from mlpro.bf.various import Log -from mlpro.oa.streams import * +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro @@ -51,11 +53,21 @@ class ADScenarioIF (OAStreamScenario): C_NAME = 'Scikit-learn Isolation Forest' ## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_estimators: int = 50, + p_max_samples: int = 3, + p_contamination: float = 0.01, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, p_visualize=p_visualize, p_logging=p_logging ) @@ -65,16 +77,17 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): p_ada=p_ada, p_visualize=p_visualize, p_logging=p_logging ) - + # 3 Instantiation of Scikit-learn 'Isolation Forest' anomaly detector - scikit_learn_if = IF( n_estimators = 50, - max_samples = 3, - contamination = 0.01 ) + scikit_learn_if = IF( n_estimators = p_n_estimators, + max_samples = p_max_samples, + contamination = p_contamination ) # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_if, - p_delay = 5, - p_instance_buffer_size = 25, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, p_group_anomaly_det = True, p_visualize = p_visualize, p_logging = p_logging ) @@ -92,24 +105,58 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): # 1 Preparation of demo/unit test mode if __name__ == "__main__": # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_estimators = int(input(f'Algo IF: Number of estimators (press ENTER for {n_estimators}): ') or n_estimators) + max_samples = int(input(f'Algo IF: Maximum samples (press ENTER for {max_samples}): ') or max_samples) + contamination = float(input(f'Algo IF: Contamination (press ENTER for {contamination}): ') or contamination) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) else: # 1.2 Parameters for internal unit test - cycle_limit = 5 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_estimators = 50 + max_samples = 3 + contamination = 0.01 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 # 2 Instantiate the stream scenario myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, p_cycle_limit = cycle_limit, p_visualize = visualize, - p_logging = logging ) + p_logging = logging, + p_n_estimators = n_estimators, + p_max_samples = max_samples, + p_contamination = contamination, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) if visualize: myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, @@ -126,6 +173,4 @@ def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): myscenario.run() if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_008_oneclasssvm_paga_nd.py b/test/howtos/oa/howto_oa_ad_008_oneclasssvm_paga_nd.py deleted file mode 100644 index c0b8791..0000000 --- a/test/howtos/oa/howto_oa_ad_008_oneclasssvm_paga_nd.py +++ /dev/null @@ -1,132 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_008_oneclasssvm_paga_nd.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-04-01 0.0.0 SK Creation -## -- 2023-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on one class svm algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -One Class SVM - -""" - -from sklearn.svm import OneClassSVM as OCSVM - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioOCSVM (OAStreamScenario): - - C_NAME = 'Scikit-learn One Class SVM' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'One Class State Vector Machine' anomaly detector - scikit_learn_ocsvm = OCSVM( kernel = 'poly', - degree = 4, - gamma = 'scale', - coef0 = 0, - nu = 0.01 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ocsvm, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioOCSVM( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - diff --git a/test/howtos/oa/howto_oa_ad_011_lof_pa_1d.py b/test/howtos/oa/howto_oa_ad_011_lof_pa_1d.py new file mode 100644 index 0000000..860d3b3 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_011_lof_pa_1d.py @@ -0,0 +1,190 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_011_lof_pa_1d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.2 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Anomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_012_lof_pa_2d.py b/test/howtos/oa/howto_oa_ad_012_lof_pa_2d.py new file mode 100644 index 0000000..14ea911 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_012_lof_pa_2d.py @@ -0,0 +1,191 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_012_lof_pa_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-04-06 1.0.1 DA Set 2D mode +## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.3 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_013_lof_pa_3d.py b/test/howtos/oa/howto_oa_ad_013_lof_pa_3d.py new file mode 100644 index 0000000..68b8e4d --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_013_lof_pa_3d.py @@ -0,0 +1,191 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_013_lof_pa_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2023-08-02 0.0.0 SK Creation +## -- 2023-08-02 1.0.0 SK First version release +## -- 2024-01-30 1.1.0 DA Relocation to separate github repository +## -- 2024-02-23 1.1.1 SK Bug fix +## -- 2024-04-06 1.1.2 DA Set 3D mode +## -- 2024-05-07 1.1.3 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.1.4 DA Correction for unit testing +## -- 2025-03-05 1.2.0 DA Refactoring +## -- 2025-06-21 1.3.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.3.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_014_lof_pa_nd.py b/test/howtos/oa/howto_oa_ad_014_lof_pa_nd.py new file mode 100644 index 0000000..808cdec --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_014_lof_pa_nd.py @@ -0,0 +1,188 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_014_lof_pa_nd.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.2 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioIF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioIF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = False, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_015_lof_paga_1d.py.off b/test/howtos/oa/howto_oa_ad_015_lof_paga_1d.py.off new file mode 100644 index 0000000..0a3ea8a --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_015_lof_paga_1d.py.off @@ -0,0 +1,188 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_015_lof_paga_1d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.2 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Anomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_016_lof_paga_2d.py.off b/test/howtos/oa/howto_oa_ad_016_lof_paga_2d.py.off new file mode 100644 index 0000000..1452eae --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_016_lof_paga_2d.py.off @@ -0,0 +1,189 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_016_lof_paga_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-04-06 1.0.1 DA Set 2D mode +## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.3 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_017_lof_paga_3d.py.off b/test/howtos/oa/howto_oa_ad_017_lof_paga_3d.py.off new file mode 100644 index 0000000..2481ac6 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_017_lof_paga_3d.py.off @@ -0,0 +1,189 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_017_lof_paga_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-04-06 1.0.1 DA Set 3D mode +## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.3 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Local Outlier Factor' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_018_lof_paga_nd.py.off b/test/howtos/oa/howto_oa_ad_018_lof_paga_nd.py.off new file mode 100644 index 0000000..3d6d0e6 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_018_lof_paga_nd.py.off @@ -0,0 +1,189 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_028_lof_paga_nd.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-04-01 0.0.0 SK Creation +## -- 2024-04-01 1.0.0 SK First version release +## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate +## -- 2024-11-27 1.0.2 DA Correction for unit testing +## -- 2025-03-05 1.1.0 DA Refactoring +## -- 2025-06-21 1.2.0 DS Refactoring and generalization by user input parameters +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-21) + + +This module demonstrates the use of anomaly detector based on isolation forest algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.neighbors import LocalOutlierFactor as LOF + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioLOF (OAStreamScenario): + + C_NAME = 'Scikit-learn Isolation Forest' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_n_neighbors: int = 20, + p_contamination: float = 0.01, + p_novelty: bool = False, + p_metric: str = 'minkowski', + p_p : int = 2, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector + scikit_learn_lof = LOF( n_neighbors= p_n_neighbors, + contamination= p_contamination, + novelty= p_novelty, + metric= p_metric, + p = p_p) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + n_neighbours = int(input(f'Algo LOF: Number of neighbours (press ENTER for {n_neighbours}): ') or n_neighbours) + contamination = float(input(f'Algo LOF: Contamination (press ENTER for {contamination}): ') or contamination) + novelty = bool(input(f'Algo LOF: Novelty detection Y/N (press ENTER for {novelty}): ')or novelty) + metric = str(input(f'Algo LOF: Distance metric (press ENTER for {metric})') or metric) + p = int(input(f'Algo LOF: Power parameter for Minkowski distance (press ENTER for {p})') or p) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + n_neighbours = 20 + contamination = 0.01 + novelty = False + metric = 'minkowski' + p = 2 + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_n_neighbors = n_neighbours, + p_contamination = contamination, + p_novelty = novelty, + p_metric = metric, + p_p = p, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = False, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_021_ee_pa_1d.py b/test/howtos/oa/howto_oa_ad_021_ee_pa_1d.py new file mode 100644 index 0000000..1aaaddd --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_021_ee_pa_1d.py @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_021_ee_pa_1d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Anomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Elliptic Envelope' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_021_lof_pa_1d.py b/test/howtos/oa/howto_oa_ad_021_lof_pa_1d.py deleted file mode 100644 index c8c1548..0000000 --- a/test/howtos/oa/howto_oa_ad_021_lof_pa_1d.py +++ /dev/null @@ -1,128 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_021_lof_pa_1d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Anomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], - p_outlier_rate=0.01, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 1 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_022_ee_pa_2d.py b/test/howtos/oa/howto_oa_ad_022_ee_pa_2d.py new file mode 100644 index 0000000..170370a --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_022_ee_pa_2d.py @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_022_ee_pa_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Elliptic Envelope' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_022_lof_pa_2d.py b/test/howtos/oa/howto_oa_ad_022_lof_pa_2d.py deleted file mode 100644 index 002c485..0000000 --- a/test/howtos/oa/howto_oa_ad_022_lof_pa_2d.py +++ /dev/null @@ -1,129 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_022_lof_pa_2d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 2D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate=0.01, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 25, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_023_ee_pa_3d.py b/test/howtos/oa/howto_oa_ad_023_ee_pa_3d.py new file mode 100644 index 0000000..3433426 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_023_ee_pa_3d.py @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_023_ee_pa_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Elliptic Envelope' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_023_lof_pa_3d.py b/test/howtos/oa/howto_oa_ad_023_lof_pa_3d.py deleted file mode 100644 index 8bf3cb8..0000000 --- a/test/howtos/oa/howto_oa_ad_023_lof_pa_3d.py +++ /dev/null @@ -1,132 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_023_lof_pa_3d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2023-08-02 0.0.0 SK Creation -## -- 2023-08-02 1.0.0 SK First version release -## -- 2024-01-30 1.1.0 DA Relocation to separate github repository -## -- 2024-02-23 1.1.1 SK Bug fix -## -- 2024-04-06 1.1.2 DA Set 3D mode -## -- 2024-05-07 1.1.3 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.1.4 DA Correction for unit testing -## -- 2025-03-05 1.2.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.2.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.022, - p_seed=6, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_024_ee_pa_nd.py b/test/howtos/oa/howto_oa_ad_024_ee_pa_nd.py new file mode 100644 index 0000000..c361769 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_024_ee_pa_nd.py @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_024_ee_pa_nd.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Elliptic Envelope' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_seed = 1, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = False, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = False, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_024_lof_pa_nd.py b/test/howtos/oa/howto_oa_ad_024_lof_pa_nd.py deleted file mode 100644 index 08693f4..0000000 --- a/test/howtos/oa/howto_oa_ad_024_lof_pa_nd.py +++ /dev/null @@ -1,129 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_024_lof_pa_nd.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.022, - p_seed=6, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = False, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_025_ee_paga_1d.py.off b/test/howtos/oa/howto_oa_ad_025_ee_paga_1d.py.off new file mode 100644 index 0000000..00e715e --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_025_ee_paga_1d.py.off @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_025_ee_paga_1d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Anomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Elliptic Envelope' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' ], #, 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_025_lof_paga_1d.py b/test/howtos/oa/howto_oa_ad_025_lof_paga_1d.py deleted file mode 100644 index 04f5949..0000000 --- a/test/howtos/oa/howto_oa_ad_025_lof_paga_1d.py +++ /dev/null @@ -1,128 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_025_lof_paga_1d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin'], - p_outlier_rate=0.3, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 10 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 5, - p_instance_buffer_size = 20, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 1 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_026_ee_paga_2d.py.off b/test/howtos/oa/howto_oa_ad_026_ee_paga_2d.py.off new file mode 100644 index 0000000..95b91b7 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_026_ee_paga_2d.py.off @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_026_ee_paga_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Isolation Forest' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos' ], #, 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_026_lof_paga_2d.py b/test/howtos/oa/howto_oa_ad_026_lof_paga_2d.py deleted file mode 100644 index 7e0e6aa..0000000 --- a/test/howtos/oa/howto_oa_ad_026_lof_paga_2d.py +++ /dev/null @@ -1,129 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_026_lof_paga_2d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 2D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate= 0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_027_ee_paga_3d.py.off b/test/howtos/oa/howto_oa_ad_027_ee_paga_3d.py.off new file mode 100644 index 0000000..05bb535 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_027_ee_paga_3d.py.off @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_027_ee_paga_3d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 1.2.0 (2025-06-15) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Isolation Forest' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = True, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_027_lof_paga_3d.py b/test/howtos/oa/howto_oa_ad_027_lof_paga_3d.py deleted file mode 100644 index 27d256a..0000000 --- a/test/howtos/oa/howto_oa_ad_027_lof_paga_3d.py +++ /dev/null @@ -1,130 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_027_lof_paga_3d.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 3D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.3 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 2 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_3D, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_028_ee_paga_nd.py.off b/test/howtos/oa/howto_oa_ad_028_ee_paga_nd.py.off new file mode 100644 index 0000000..737e095 --- /dev/null +++ b/test/howtos/oa/howto_oa_ad_028_ee_paga_nd.py.off @@ -0,0 +1,171 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro_int_scikit_learn +## -- Module : howto_oa_ad_028_ee_paga_nd.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2025-06-21 0.1.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2025-06-21) + +This module demonstrates the use of anomaly detector based on elliptic envelope algorithm with MLPro. +To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. +The workflow consists of a standard task 'Aanomaly Detector'. + +You will learn: + +1) How to set up a stream workflow based on stream tasks. + +2) How to set up a stream scenario based on a stream and a processing stream workflow. + +3) How to add a task anomalydetector. + +4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically +Isolation Forest + +""" + +from sklearn.covariance import EllipticEnvelope as EE + +from mlpro.bf.various import Log +from mlpro.bf.ops import Mode +from mlpro.bf.plot import PlotSettings +from mlpro.bf.streams.streams import StreamMLProPOutliers +from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow + +from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class ADScenarioEE (OAStreamScenario): + + C_NAME = 'Scikit-learn Isolation Forest' + +## ------------------------------------------------------------------------------------------------- + def _setup( self, + p_mode, + p_ada: bool, + p_visualize: bool, + p_logging, + p_contamination: float = 0.01, + p_support_fraction: float = 0.5, + p_assume_centered: bool = False, + p_anomaly_buffer_size: int = 100, + p_instance_buffer_size: int = 50, + p_detection_steprate: int = 50 ): + + # 1 Get the native stream from MLPro stream provider + mystream = StreamMLProPOutliers( p_functions = ['sin' , 'cos', 'const'], + p_outlier_rate=0.02, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 2 Creation of a workflow + workflow = OAStreamWorkflow( p_name='wf1', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + # 3 Instantiation of Scikit-learn 'Elliptic Envelope' anomaly detector + scikit_learn_ee = EE( contamination = p_contamination, + support_fraction = p_support_fraction, + assume_centered = p_assume_centered ) + + # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow + anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_ee, + p_anomaly_buffer_size = p_anomaly_buffer_size, + p_instance_buffer_size = p_instance_buffer_size, + p_detection_steprate = p_detection_steprate, + p_group_anomaly_det = True, + p_visualize = p_visualize, + p_logging = p_logging ) + + workflow.add_task( p_task=anomalydetector ) + + # 5 Return stream and workflow + return mystream, workflow + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Preparation of demo/unit test mode +if __name__ == "__main__": + # 1.1 Parameters for demo mode + cycle_limit = 500 + logging = Log.C_LOG_ALL + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 50 + detection_steprate = 50 + + cycle_limit = int(input(f'\nCycle limit (press ENTER for {cycle_limit}): ') or cycle_limit) + visualize = input('Visualization Y/N (press ENTER for Y): ').upper() != 'N' + if visualize: + i = input(f'Visualization step rate (press ENTER for {step_rate}): ') + if i != '': step_rate = int(i) + + i = input('Log level: "A"=All, "W"=Warnings only, "N"=Nothing (press ENTER for "W"): ').upper() + if i == 'A': logging = Log.C_LOG_ALL + elif i == 'N': logging = Log.C_LOG_NOTHING + + contamination = float(input(f'Algo EE: Contamination (press ENTER for {contamination}): ') or contamination) + support_fraction = float(input(f'Algo EE: Support fraction (press ENTER for {support_fraction}): ') or support_fraction) + assume_centered = bool(input(f'Algo EE: Assume centered (True/False, press ENTER for {assume_centered}): ') or assume_centered) + anomaly_buffer_size = int(input(f'MLPro Wrapper: Anomaly buffer size (press ENTER for {anomaly_buffer_size}): ') or anomaly_buffer_size) + instance_buffer_size = int(input(f'MLPro Wrapper: Instance buffer size (press ENTER for {instance_buffer_size}): ') or instance_buffer_size) + detection_steprate = int(input(f'MLPro Wrapper: Detection steprate (press ENTER for {detection_steprate}): ') or detection_steprate) + +else: + # 1.2 Parameters for internal unit test + cycle_limit = 20 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + contamination = 0.01 + support_fraction = 0.5 + assume_centered = False + anomaly_buffer_size = 100 + instance_buffer_size = 10 + detection_steprate = 10 + + +# 2 Instantiate the stream scenario +myscenario = ADScenarioEE( p_mode = Mode.C_MODE_REAL, + p_cycle_limit = cycle_limit, + p_visualize = visualize, + p_logging = logging, + p_contamination = contamination, + p_support_fraction = support_fraction, + p_assume_centered = assume_centered, + p_anomaly_buffer_size = anomaly_buffer_size, + p_instance_buffer_size = instance_buffer_size, + p_detection_steprate = detection_steprate ) + +if visualize: + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, + p_view_autoselect = False, + p_step_rate = step_rate ) ) + + +# 3 Reset and run own stream scenario +myscenario.reset() + +if __name__ == '__main__': + input('Press ENTER to start stream processing...') + +myscenario.run() + +if __name__ == '__main__': + input('Press ENTER to exit...') \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_028_lof_paga_nd.py b/test/howtos/oa/howto_oa_ad_028_lof_paga_nd.py deleted file mode 100644 index c527e96..0000000 --- a/test/howtos/oa/howto_oa_ad_028_lof_paga_nd.py +++ /dev/null @@ -1,128 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_028_lof_paga_nd.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-05-07 1.0.1 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.0.2 DA Correction for unit testing -## -- 2025-03-05 1.1.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.1.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.streams.streams import * -from mlpro.bf.various import Log -from mlpro.oa.streams import * - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'AdScenario4ADlof' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos', 'const'], - p_outlier_rate=0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 4 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - workflow.add_task( p_task=anomalydetector ) - - # 5 Return stream and workflow - return mystream, workflow - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 360 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 1 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_ad_029_lof_paga_normalized_2d.py b/test/howtos/oa/howto_oa_ad_029_lof_paga_normalized_2d.py deleted file mode 100644 index 8a3d401..0000000 --- a/test/howtos/oa/howto_oa_ad_029_lof_paga_normalized_2d.py +++ /dev/null @@ -1,146 +0,0 @@ -## ------------------------------------------------------------------------------------------------- -## -- Project : MLPro - The integrative middleware framework for standardized machine learning -## -- Package : mlpro_int_scikit_learn -## -- Module : howto_oa_ad_028_lof_paga_nd.py -## ------------------------------------------------------------------------------------------------- -## -- History : -## -- yyyy-mm-dd Ver. Auth. Description -## -- 2024-04-01 0.0.0 SK Creation -## -- 2024-04-01 1.0.0 SK First version release -## -- 2024-04-06 1.0.1 DA Set 2D mode -## -- 2024-05-07 1.0.2 SK Change in parameter p_outlier_rate -## -- 2024-11-27 1.1.0 DA Alignment with MLPro 1.9.2 -## -- 2025-03-05 1.2.0 DA Refactoring -## ------------------------------------------------------------------------------------------------- - -""" -Ver. 1.2.0 (2025-03-05) - -This module demonstrates the use of anomaly detector based on local outlier factor algorithm with MLPro. -To this regard, a stream of a stream provider is combined with a stream workflow to a stream scenario. -The workflow consists of a standard task 'Aanomaly Detector'. - -You will learn: - -1) How to set up a stream workflow based on stream tasks. - -2) How to set up a stream scenario based on a stream and a processing stream workflow. - -3) How to add a task anomalydetector. - -4) How to reuse an anomaly detector algorithm from scikitlearn (https://scikit-learn.org/), specifically -Local Outlier Factor - -""" - -from sklearn.neighbors import LocalOutlierFactor as LOF - -from mlpro.bf.various import Log -from mlpro.bf.plot import PlotSettings -from mlpro.bf.ops import Mode -from mlpro.bf.streams.streams import StreamMLProPOutliers -from mlpro.oa.streams import OAStreamScenario, OAStreamWorkflow -from mlpro.oa.streams.tasks import BoundaryDetector, NormalizerMinMax - -from mlpro_int_sklearn.wrappers.anomalydetectors import WrAnomalyDetectorSklearn2MLPro - - - - -## ------------------------------------------------------------------------------------------------- -## ------------------------------------------------------------------------------------------------- -class ADScenarioLOF (OAStreamScenario): - - C_NAME = 'Scikit-learn Local Outlier Factor' - -## ------------------------------------------------------------------------------------------------- - def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): - - # 1 Get the native stream from MLPro stream provider - mystream = StreamMLProPOutliers( p_functions = ['sin', 'cos'], - p_outlier_rate= 0.1, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 2 Creation of a workflow - workflow = OAStreamWorkflow( p_name='wf1', - p_range_max=OAStreamWorkflow.C_RANGE_NONE, - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 3 Initialize a Boundarydetector task - task_bd = BoundaryDetector( p_name='Demo Boundary Detector', - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 4 Initialize a Normalizer task - task_norm = NormalizerMinMax( p_name='Demo MinMax Normalizer', - p_ada=p_ada, - p_visualize=p_visualize, - p_logging=p_logging ) - - # 5 Instantiation of Scikit-learn 'Local Outlier Factor' anomaly detector - scikit_learn_lof = LOF( n_neighbors = 3 ) - - # 6 Wrapping of the Scikit-learn algorithm and integration into the stream workflow - anomalydetector = WrAnomalyDetectorSklearn2MLPro( p_algo_scikit_learn = scikit_learn_lof, - p_delay = 3, - p_instance_buffer_size = 5, - p_group_anomaly_det = True, - p_visualize = p_visualize, - p_logging = p_logging ) - - # 6 Addition of the task to the workflow - workflow.add_task(p_task = task_bd) - workflow.add_task(p_task = task_norm, p_pred_tasks=[task_bd]) - workflow.add_task(p_task = anomalydetector, p_pred_tasks=[task_norm] ) - task_bd.register_event_handler( p_event_id = BoundaryDetector.C_EVENT_ADAPTED, p_event_handler = task_norm.adapt_on_event ) - - # 7 Return stream and workflow - return mystream, workflow - - - - -# 1 Preparation of demo/unit test mode -if __name__ == "__main__": - # 1.1 Parameters for demo mode - cycle_limit = 100 - logging = Log.C_LOG_ALL - visualize = True - step_rate = 1 - -else: - # 1.2 Parameters for internal unit test - cycle_limit = 2 - logging = Log.C_LOG_NOTHING - visualize = False - step_rate = 1 - - -# 2 Instantiate the stream scenario -myscenario = ADScenarioLOF( p_mode = Mode.C_MODE_REAL, - p_cycle_limit = cycle_limit, - p_visualize = visualize, - p_logging = logging ) - -if visualize: - myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_ND, - p_view_autoselect = False, - p_step_rate = step_rate ) ) - - -# 3 Reset and run own stream scenario -myscenario.reset() - -if __name__ == '__main__': - input('Press ENTER to start stream processing...') - -myscenario.run() - -if __name__ == '__main__': - input('Press ENTER to exit...') - - \ No newline at end of file