From a36fc3530c9b7aae9af718dd847c87ac8e3c4f36 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:01:26 +0200 Subject: [PATCH 01/79] Create a basic new Agent class --- aikido_firewall/agent/agent.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 aikido_firewall/agent/agent.py diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py new file mode 100644 index 000000000..4ab25f344 --- /dev/null +++ b/aikido_firewall/agent/agent.py @@ -0,0 +1,14 @@ +""" This file simply exports the agent class""" + + +class Agent: + """Agent class""" + + def __init__(self, block, api, token, serverless): + self.block = block + self.api = api + self.token = token + + if isinstance(serverless, str) and len(serverless) == 0: + raise ValueError("Serverless cannot be an empty string") + self.serverless = serverless From 5062dfe9d17574bc555ffbb44308fac3081408a6 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:01:55 +0200 Subject: [PATCH 02/79] Create the super class for API's --- aikido_firewall/agent/api/__init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 aikido_firewall/agent/api/__init__.py diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py new file mode 100644 index 000000000..070997a60 --- /dev/null +++ b/aikido_firewall/agent/api/__init__.py @@ -0,0 +1,21 @@ +""" +init.py file for api/ folder. Includes abstract class ReportingApi +""" +import json + +class ReportingApi: + """This is the super class for the reporting API's""" + + def to_api_response(self, status, data): + """Converts results into an Api response obj""" + if(status == 429): + return { "success": False, "error": "rate_limited"} + elif(status == 401): + return {"success": False, "error": "invalid_token"} + try: + return json.loads(data) + except Exception: + return {"success": False, "error": "unknown_error"} + + def report(self, token, event, timeout_in_ms): + """Report event to aikido server""" From 3b403fce9cd31e2b6691314f37962bc9eab8967d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:02:27 +0200 Subject: [PATCH 03/79] create an HTTP API class --- aikido_firewall/agent/api/http_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 aikido_firewall/agent/api/http_api.py diff --git a/aikido_firewall/agent/api/http_api.py b/aikido_firewall/agent/api/http_api.py new file mode 100644 index 000000000..dbd49f3db --- /dev/null +++ b/aikido_firewall/agent/api/http_api.py @@ -0,0 +1,15 @@ +""" +Exports the HTTP API class +""" + +from aikido_firewall.agent.api import ReportingApi + + +class ReportingApiHTTP(ReportingApi): + """HTTP Reporting API""" + + def __init__(self, reporting_url): + self.reporting_url = reporting_url + + def report(self, token, event, timeout_in_ms): + print("Do something here") From 4ba6703f0584973c0f4d79e287de1895d2945119 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:13:21 +0200 Subject: [PATCH 04/79] Install and import requests --- aikido_firewall/agent/api/http_api.py | 2 +- poetry.lock | 161 +++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 162 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/agent/api/http_api.py b/aikido_firewall/agent/api/http_api.py index dbd49f3db..0aaee3ee5 100644 --- a/aikido_firewall/agent/api/http_api.py +++ b/aikido_firewall/agent/api/http_api.py @@ -1,7 +1,7 @@ """ Exports the HTTP API class """ - +import requests from aikido_firewall.agent.api import ReportingApi diff --git a/poetry.lock b/poetry.lock index 652a43162..992b7b5c5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -66,6 +66,116 @@ files = [ {file = "blinker-1.8.2.tar.gz", hash = "sha256:8f77b09d3bf7c795e969e9486f39c2c5e9c39d4ee07424be2bc594ece9642d83"}, ] +[[package]] +name = "certifi" +version = "2024.7.4" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + [[package]] name = "click" version = "8.1.7" @@ -207,6 +317,17 @@ files = [ flask = "*" werkzeug = "*" +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + [[package]] name = "importhook" version = "1.0.9" @@ -596,6 +717,27 @@ files = [ {file = "regex-2024.5.15.tar.gz", hash = "sha256:d3ee02d9e5f482cc8309134a91eeaacbdd2261ba111b0fef3748eeb4913e6a2c"}, ] +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "tomlkit" version = "0.13.0" @@ -607,6 +749,23 @@ files = [ {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, ] +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "werkzeug" version = "3.0.3" @@ -627,4 +786,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "834b3f397a99c7e910e20f7f9904408c3a27141d136decaf4fb7233253984306" +content-hash = "b56496afffc10819de821835a4369beebda982e19e04f5519c669e36c81f5968" diff --git a/pyproject.toml b/pyproject.toml index 7a7caa139..307bbfceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ pytest-mock = "^3.14.0" werkzeug = "^3.0.3" flask-http-middleware = "^0.4.2" regex = "^2024.5.15" +requests = "^2.32.3" [tool.poetry.group.dev.dependencies] black = "^24.4.2" From 211421ca041b52fbfe711c33b760cef15312ada6 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:13:34 +0200 Subject: [PATCH 05/79] Linting --- aikido_firewall/agent/api/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py index 070997a60..afb8ff951 100644 --- a/aikido_firewall/agent/api/__init__.py +++ b/aikido_firewall/agent/api/__init__.py @@ -1,16 +1,18 @@ """ init.py file for api/ folder. Includes abstract class ReportingApi """ + import json + class ReportingApi: """This is the super class for the reporting API's""" def to_api_response(self, status, data): """Converts results into an Api response obj""" - if(status == 429): - return { "success": False, "error": "rate_limited"} - elif(status == 401): + if status == 429: + return {"success": False, "error": "rate_limited"} + elif status == 401: return {"success": False, "error": "invalid_token"} try: return json.loads(data) From f81f5495b37912dcfacefa49d8258af47fb4bbea Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:19:17 +0200 Subject: [PATCH 06/79] Rename to timeout_in_sec --- aikido_firewall/agent/api/__init__.py | 2 +- aikido_firewall/agent/api/http_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py index afb8ff951..06a5c592a 100644 --- a/aikido_firewall/agent/api/__init__.py +++ b/aikido_firewall/agent/api/__init__.py @@ -19,5 +19,5 @@ def to_api_response(self, status, data): except Exception: return {"success": False, "error": "unknown_error"} - def report(self, token, event, timeout_in_ms): + def report(self, token, event, timeout_in_sec): """Report event to aikido server""" diff --git a/aikido_firewall/agent/api/http_api.py b/aikido_firewall/agent/api/http_api.py index 0aaee3ee5..61c524001 100644 --- a/aikido_firewall/agent/api/http_api.py +++ b/aikido_firewall/agent/api/http_api.py @@ -11,5 +11,5 @@ class ReportingApiHTTP(ReportingApi): def __init__(self, reporting_url): self.reporting_url = reporting_url - def report(self, token, event, timeout_in_ms): + def report(self, token, event, timeout_in_sec): print("Do something here") From adbfa830062a561a8ef1faf2c033a3186593e84b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:19:28 +0200 Subject: [PATCH 07/79] Send data using requests.post --- aikido_firewall/agent/api/http_api.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/agent/api/http_api.py b/aikido_firewall/agent/api/http_api.py index 61c524001..1984b5036 100644 --- a/aikido_firewall/agent/api/http_api.py +++ b/aikido_firewall/agent/api/http_api.py @@ -1,6 +1,7 @@ """ Exports the HTTP API class """ + import requests from aikido_firewall.agent.api import ReportingApi @@ -12,4 +13,14 @@ def __init__(self, reporting_url): self.reporting_url = reporting_url def report(self, token, event, timeout_in_sec): - print("Do something here") + res = requests.post( + self.reporting_url + "api/runtime/events", + data=event, + timeout=timeout_in_sec, + headers=get_headers(token), + ) + + +def get_headers(token): + """Returns headers""" + return {"Content-Type": "application/json", "Authorization": str(token)} From 1e88fcc7ba65e078525a1c30bbccabbed65e063a Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:21:33 +0200 Subject: [PATCH 08/79] Create a class that encapsulates the token --- aikido_firewall/agent/api/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py index 06a5c592a..31d1e79f0 100644 --- a/aikido_firewall/agent/api/__init__.py +++ b/aikido_firewall/agent/api/__init__.py @@ -21,3 +21,17 @@ def to_api_response(self, status, data): def report(self, token, event, timeout_in_sec): """Report event to aikido server""" + + +class Token: + """Class that encapsulates the token""" + + def __init__(self, token): + if not isinstance(token, str): + raise ValueError("Token should be an instance of string") + if len(token) == 0: + raise ValueError("Token cannot be an empty string") + self.token = token + + def __str__(self): + return self.token From 465c7465f514b7b37eefef232dfa8aa7532ca749 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:29:44 +0200 Subject: [PATCH 09/79] Give back api response in http_api file --- aikido_firewall/agent/api/http_api.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/aikido_firewall/agent/api/http_api.py b/aikido_firewall/agent/api/http_api.py index 1984b5036..0fd6e94d5 100644 --- a/aikido_firewall/agent/api/http_api.py +++ b/aikido_firewall/agent/api/http_api.py @@ -13,12 +13,18 @@ def __init__(self, reporting_url): self.reporting_url = reporting_url def report(self, token, event, timeout_in_sec): - res = requests.post( - self.reporting_url + "api/runtime/events", - data=event, - timeout=timeout_in_sec, - headers=get_headers(token), - ) + try: + res = requests.post( + self.reporting_url + "api/runtime/events", + data=event, + timeout=timeout_in_sec, + headers=get_headers(token), + ) + except requests.exceptions.Timeout: + return {"success": False, "error": "timeout"} + except Exception as e: + raise e + return self.to_api_response(res) def get_headers(token): From 87a71aec0593af24fa296fbbfcd29ec0952fb6a8 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 11:29:56 +0200 Subject: [PATCH 10/79] update the way to_api_response() function works --- aikido_firewall/agent/api/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py index 31d1e79f0..52b7b89b5 100644 --- a/aikido_firewall/agent/api/__init__.py +++ b/aikido_firewall/agent/api/__init__.py @@ -2,20 +2,19 @@ init.py file for api/ folder. Includes abstract class ReportingApi """ -import json - class ReportingApi: """This is the super class for the reporting API's""" - def to_api_response(self, status, data): + def to_api_response(self, res): """Converts results into an Api response obj""" + status = res.status_code if status == 429: return {"success": False, "error": "rate_limited"} elif status == 401: return {"success": False, "error": "invalid_token"} try: - return json.loads(data) + return res.json() except Exception: return {"success": False, "error": "unknown_error"} From 9689a29486d04e411426c7edee03605a883e4ed0 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:04:10 +0200 Subject: [PATCH 11/79] Add tests for token class --- aikido_firewall/agent/api/init_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 aikido_firewall/agent/api/init_test.py diff --git a/aikido_firewall/agent/api/init_test.py b/aikido_firewall/agent/api/init_test.py new file mode 100644 index 000000000..aabcca088 --- /dev/null +++ b/aikido_firewall/agent/api/init_test.py @@ -0,0 +1,24 @@ +import pytest +from aikido_firewall.agent.api import Token + + +def test_token_valid_string(): + token_str = "my_token" + token = Token(token_str) + assert str(token) == token_str + + +def test_token_empty_string(): + with pytest.raises(ValueError): + Token("") + + +def test_token_invalid_type(): + with pytest.raises(ValueError): + Token(123) + + +def test_token_instance(): + token_str = "my_token" + token = Token(token_str) + assert isinstance(token, Token) From 9eecfc08070237c275375c79e4db0013d2f15080 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:09:41 +0200 Subject: [PATCH 12/79] Add tests for ReportingApi class --- aikido_firewall/agent/api/init_test.py | 45 +++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/agent/api/init_test.py b/aikido_firewall/agent/api/init_test.py index aabcca088..21f86a287 100644 --- a/aikido_firewall/agent/api/init_test.py +++ b/aikido_firewall/agent/api/init_test.py @@ -1,7 +1,50 @@ import pytest -from aikido_firewall.agent.api import Token +from aikido_firewall.agent.api import Token, ReportingApi +# Test ReportingApi Class : +from requests.models import Response + +@pytest.fixture +def reporting_api(): + return ReportingApi() + + +def test_to_api_response_rate_limited(reporting_api): + res = Response() + res.status_code = 429 + assert reporting_api.to_api_response(res) == { + "success": False, + "error": "rate_limited", + } + + +def test_to_api_response_invalid_token(reporting_api): + res = Response() + res.status_code = 401 + assert reporting_api.to_api_response(res) == { + "success": False, + "error": "invalid_token", + } + + +def test_to_api_response_unknown_error(reporting_api): + res = Response() + res.status_code = 500 # Simulating an unknown error status code + assert reporting_api.to_api_response(res) == { + "success": False, + "error": "unknown_error", + } + + +def test_to_api_response_valid_json(reporting_api): + res = Response() + res.status_code = 200 + res._content = b'{"key": "value"}' # Simulating valid JSON response + assert reporting_api.to_api_response(res) == {"key": "value"} + + +# Test Token Class : def test_token_valid_string(): token_str = "my_token" token = Token(token_str) From 434d83031096bf4b1fcbc5642166eb9e9e41c93d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:22:18 +0200 Subject: [PATCH 13/79] Move token file into a helper module with new function get_token_from_env --- aikido_firewall/agent/api/__init__.py | 14 -------------- aikido_firewall/agent/api/init_test.py | 25 +------------------------ aikido_firewall/helpers/token.py | 22 ++++++++++++++++++++++ aikido_firewall/helpers/token_test.py | 24 ++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 38 deletions(-) create mode 100644 aikido_firewall/helpers/token.py create mode 100644 aikido_firewall/helpers/token_test.py diff --git a/aikido_firewall/agent/api/__init__.py b/aikido_firewall/agent/api/__init__.py index 52b7b89b5..5e29f4d11 100644 --- a/aikido_firewall/agent/api/__init__.py +++ b/aikido_firewall/agent/api/__init__.py @@ -20,17 +20,3 @@ def to_api_response(self, res): def report(self, token, event, timeout_in_sec): """Report event to aikido server""" - - -class Token: - """Class that encapsulates the token""" - - def __init__(self, token): - if not isinstance(token, str): - raise ValueError("Token should be an instance of string") - if len(token) == 0: - raise ValueError("Token cannot be an empty string") - self.token = token - - def __str__(self): - return self.token diff --git a/aikido_firewall/agent/api/init_test.py b/aikido_firewall/agent/api/init_test.py index 21f86a287..8e6398e38 100644 --- a/aikido_firewall/agent/api/init_test.py +++ b/aikido_firewall/agent/api/init_test.py @@ -1,5 +1,5 @@ import pytest -from aikido_firewall.agent.api import Token, ReportingApi +from aikido_firewall.agent.api import ReportingApi # Test ReportingApi Class : from requests.models import Response @@ -42,26 +42,3 @@ def test_to_api_response_valid_json(reporting_api): res.status_code = 200 res._content = b'{"key": "value"}' # Simulating valid JSON response assert reporting_api.to_api_response(res) == {"key": "value"} - - -# Test Token Class : -def test_token_valid_string(): - token_str = "my_token" - token = Token(token_str) - assert str(token) == token_str - - -def test_token_empty_string(): - with pytest.raises(ValueError): - Token("") - - -def test_token_invalid_type(): - with pytest.raises(ValueError): - Token(123) - - -def test_token_instance(): - token_str = "my_token" - token = Token(token_str) - assert isinstance(token, Token) diff --git a/aikido_firewall/helpers/token.py b/aikido_firewall/helpers/token.py new file mode 100644 index 000000000..992928cee --- /dev/null +++ b/aikido_firewall/helpers/token.py @@ -0,0 +1,22 @@ +""" +Helper module for token +""" + +class Token: + """Class that encapsulates the token""" + + def __init__(self, token): + if not isinstance(token, str): + raise ValueError("Token should be an instance of string") + if len(token) == 0: + raise ValueError("Token cannot be an empty string") + self.token = token + + def __str__(self): + return self.token + +def get_token_from_env(): + """ + Fetches the token from the env variable "" + """ + return Token("xyz") diff --git a/aikido_firewall/helpers/token_test.py b/aikido_firewall/helpers/token_test.py new file mode 100644 index 000000000..8b068d958 --- /dev/null +++ b/aikido_firewall/helpers/token_test.py @@ -0,0 +1,24 @@ +import pytest +from aikido_firewall.helpers.token import Token + +# Test Token Class : +def test_token_valid_string(): + token_str = "my_token" + token = Token(token_str) + assert str(token) == token_str + + +def test_token_empty_string(): + with pytest.raises(ValueError): + Token("") + + +def test_token_invalid_type(): + with pytest.raises(ValueError): + Token(123) + + +def test_token_instance(): + token_str = "my_token" + token = Token(token_str) + assert isinstance(token, Token) From d25f1c17c0d93ab10bfeddbca87ba20262c3ad65 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:22:40 +0200 Subject: [PATCH 14/79] Create a should_block helper function --- aikido_firewall/helpers/should_block.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 aikido_firewall/helpers/should_block.py diff --git a/aikido_firewall/helpers/should_block.py b/aikido_firewall/helpers/should_block.py new file mode 100644 index 000000000..3fd0fb9d1 --- /dev/null +++ b/aikido_firewall/helpers/should_block.py @@ -0,0 +1,14 @@ +""" +Helper function file, see function docstring +""" +import os + +def should_block(): + """ + Checks the environment variable "AIKIDO_BLOCKING" + """ + # Set log level + aikido_blocking_env = os.getenv("AIKIDO_BLOCKING") + if aikido_blocking_env is not None: + return aikido_blocking_env.lower() in ["true", "1"] + return False From 58e02018a68674d934b9b8a7bc0e7bfab702e5e4 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:22:58 +0200 Subject: [PATCH 15/79] Linting --- aikido_firewall/helpers/should_block.py | 2 ++ aikido_firewall/helpers/token.py | 2 ++ aikido_firewall/helpers/token_test.py | 1 + 3 files changed, 5 insertions(+) diff --git a/aikido_firewall/helpers/should_block.py b/aikido_firewall/helpers/should_block.py index 3fd0fb9d1..e044283f1 100644 --- a/aikido_firewall/helpers/should_block.py +++ b/aikido_firewall/helpers/should_block.py @@ -1,8 +1,10 @@ """ Helper function file, see function docstring """ + import os + def should_block(): """ Checks the environment variable "AIKIDO_BLOCKING" diff --git a/aikido_firewall/helpers/token.py b/aikido_firewall/helpers/token.py index 992928cee..2bed7fe42 100644 --- a/aikido_firewall/helpers/token.py +++ b/aikido_firewall/helpers/token.py @@ -2,6 +2,7 @@ Helper module for token """ + class Token: """Class that encapsulates the token""" @@ -15,6 +16,7 @@ def __init__(self, token): def __str__(self): return self.token + def get_token_from_env(): """ Fetches the token from the env variable "" diff --git a/aikido_firewall/helpers/token_test.py b/aikido_firewall/helpers/token_test.py index 8b068d958..f710f8f26 100644 --- a/aikido_firewall/helpers/token_test.py +++ b/aikido_firewall/helpers/token_test.py @@ -1,6 +1,7 @@ import pytest from aikido_firewall.helpers.token import Token + # Test Token Class : def test_token_valid_string(): token_str = "my_token" From 3ff2bf3b6358082af0ecf724d19b411cedb77075 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:28:43 +0200 Subject: [PATCH 16/79] Create agent and add get_token_from_env func --- aikido_firewall/agent/__init__.py | 6 ++++++ aikido_firewall/helpers/token.py | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/agent/__init__.py b/aikido_firewall/agent/__init__.py index 170d3aea2..88b94370e 100644 --- a/aikido_firewall/agent/__init__.py +++ b/aikido_firewall/agent/__init__.py @@ -9,6 +9,9 @@ from threading import Thread from queue import Queue from aikido_firewall.helpers.logging import logger +from aikido_firewall.agent.agent import Agent +from aikido_firewall.helpers.should_block import should_block +from aikido_firewall.helpers.token import get_token_from_env AGENT_SEC_INTERVAL = 5 IPC_ADDRESS = ("localhost", 9898) # Specify the IP address and port @@ -23,6 +26,7 @@ def __init__(self, address, key): logger.debug("Agent thread started") listener = Listener(address, authkey=key) self.queue = Queue() + self.agent = None # Start reporting thread : Thread(target=self.reporting_thread).start() @@ -41,6 +45,8 @@ def __init__(self, address, key): def reporting_thread(self): """Reporting thread""" logger.debug("Started reporting thread") + self.agent = Agent(should_block(), {}, get_token_from_env(), None) + logger.debug("Created agent") while True: self.report_to_agent() time.sleep(AGENT_SEC_INTERVAL) diff --git a/aikido_firewall/helpers/token.py b/aikido_firewall/helpers/token.py index 2bed7fe42..043d2dcba 100644 --- a/aikido_firewall/helpers/token.py +++ b/aikido_firewall/helpers/token.py @@ -2,6 +2,8 @@ Helper module for token """ +import os + class Token: """Class that encapsulates the token""" @@ -19,6 +21,9 @@ def __str__(self): def get_token_from_env(): """ - Fetches the token from the env variable "" + Fetches the token from the env variable "AIKIDO_TOKEN" """ - return Token("xyz") + aikido_token_env = os.getenv("AIKIDO_TOKEN") + if not aikido_token_env is None: + return Token(aikido_token_env) + return None From 24f1869c15fba2d2cafacedbd5fb2069e6ff4ed0 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:51:04 +0200 Subject: [PATCH 17/79] add send_heartbeat and some auxiliary funcs --- aikido_firewall/agent/agent.py | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index 4ab25f344..3ac827a2f 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -1,5 +1,8 @@ """ This file simply exports the agent class""" +from datetime import datetime +from aikido_firewall.helpers.logging import logger + class Agent: """Agent class""" @@ -12,3 +15,45 @@ def __init__(self, block, api, token, serverless): if isinstance(serverless, str) and len(serverless) == 0: raise ValueError("Serverless cannot be an empty string") self.serverless = serverless + + def on_detected_attack(self): + """ + This will send something to the API when an attack is detected + """ + pass + + def send_heartbeat(self): + """ + This will send a heartbeat to the server + """ + if not self.token: + return + logger.debug("Aikido Agent : Sending out heartbeat") + res = self.api.report( + self.token, + { + "type": "heartbeat", + "time": datetime.now(), + "agent": self.get_agent_info(), + "stats": {"sinks": [], "startedAt": 0, "endedAt": 0, "requests": []}, + "hostnames": [], + "routes": [], + "users": [], + }, + ) + self.update_service_config(res) + + def on_start(self): + """ + This will send out an Event signalling the start to the server + """ + pass + + def get_agent_info(self): + """ + This returns info about the agent + """ + return {} + + def update_service_config(self, res): + pass From 20bf490765f3813a952eb700a0952a6d6153f9bc Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:56:16 +0200 Subject: [PATCH 18/79] self.token needs to be defined for these 2 funcs --- aikido_firewall/agent/agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index 3ac827a2f..957c3a97e 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -20,7 +20,8 @@ def on_detected_attack(self): """ This will send something to the API when an attack is detected """ - pass + if not self.token: + return def send_heartbeat(self): """ @@ -47,7 +48,8 @@ def on_start(self): """ This will send out an Event signalling the start to the server """ - pass + if not self.token: + return def get_agent_info(self): """ From c6705e57b45b538c6309783d9d7ab971325519bf Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:58:37 +0200 Subject: [PATCH 19/79] Use self.timeout_in_sec --- aikido_firewall/agent/agent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index 957c3a97e..c7d1fca7f 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -7,6 +7,8 @@ class Agent: """Agent class""" + timeout_in_sec = 5 + def __init__(self, block, api, token, serverless): self.block = block self.api = api @@ -41,6 +43,7 @@ def send_heartbeat(self): "routes": [], "users": [], }, + self.timeout_in_sec, ) self.update_service_config(res) From 41c5b053c95a30b51a7fbb4a0d536966a0f78cc7 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 12:59:19 +0200 Subject: [PATCH 20/79] add on_start function --- aikido_firewall/agent/agent.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index c7d1fca7f..f82e7ac4d 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -53,6 +53,12 @@ def on_start(self): """ if not self.token: return + res = self.api.report( + self.token, + {"type": "started", "time": datetime.now(), "agent": self.get_agent_info()}, + self.timeout_in_sec, + ) + self.update_service_config(res) def get_agent_info(self): """ From db277ed657d0ddcd7beebe66111f7cc3df4435bd Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 13:04:02 +0200 Subject: [PATCH 21/79] Update get_agent_info function --- aikido_firewall/agent/agent.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index f82e7ac4d..e6aa5c81e 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -1,6 +1,8 @@ """ This file simply exports the agent class""" from datetime import datetime +import socket +import platform from aikido_firewall.helpers.logging import logger @@ -64,7 +66,20 @@ def get_agent_info(self): """ This returns info about the agent """ - return {} + return { + "dryMode": not self.block, + "hostname": socket.gethostname(), + "version": "x.x.x", + "library": "firewall_python", + "ipAddress": socket.gethostbyname(socket.gethostname()), + "packages": [], + "serverless": bool(self.serverless), + "stack": [], + "os": { + "name": platform.system(), + "version": platform.release() + } + } def update_service_config(self, res): pass From cd6a24f549e78f955e91ae49f4a149370035de7d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 13:05:58 +0200 Subject: [PATCH 22/79] Linting --- aikido_firewall/agent/agent.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index e6aa5c81e..596f4fcba 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -75,11 +75,10 @@ def get_agent_info(self): "packages": [], "serverless": bool(self.serverless), "stack": [], - "os": { - "name": platform.system(), - "version": platform.release() - } + "os": {"name": platform.system(), "version": platform.release()}, } def update_service_config(self, res): - pass + """ + Update configuration based on the server's response + """ From 164eae3801354d08d0aa878350de728c3dc731f2 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 13:17:39 +0200 Subject: [PATCH 23/79] Add some stuff already to on_detected_attack --- aikido_firewall/agent/agent.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index 596f4fcba..460d9ff2a 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -3,6 +3,8 @@ from datetime import datetime import socket import platform +import json +from copy import deepcopy from aikido_firewall.helpers.logging import logger @@ -20,12 +22,41 @@ def __init__(self, block, api, token, serverless): raise ValueError("Serverless cannot be an empty string") self.serverless = serverless - def on_detected_attack(self): + def on_detected_attack(self, attack): """ This will send something to the API when an attack is detected """ if not self.token: return + # Modify attack so we can send it out : + try: + req = deepcopy(attack["request"]) + del attack["request"] + attack["user"] = req["user"] + attack["payload"] = json.dumps(attack["payload"])[:4096] + + self.api.report( + self.token, + { + "type": "detected_attack", + "time": datetime.now(), + "agent": self.get_agent_info(), + "attack": attack, + "request": { + "method": req["method"], + "url": req["url"], + "ipAddress": req["remoteAddress"], + "userAgent": "WIP", + "body": {}, + "headers": {}, + "source": req["source"], + "route": req["route"], + }, + }, + self.timeout_in_sec, + ) + except Exception: + logger.info("Failed to report attack") def send_heartbeat(self): """ From e0e5fefb7a52d77baee5044a6ca0e5054945463e Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 13:24:27 +0200 Subject: [PATCH 24/79] Add new helper function limit_length_metadata --- .../helpers/limit_length_metadata.py | 15 +++++++++++++ .../helpers/limit_length_metadata_test.py | 21 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 aikido_firewall/helpers/limit_length_metadata.py create mode 100644 aikido_firewall/helpers/limit_length_metadata_test.py diff --git a/aikido_firewall/helpers/limit_length_metadata.py b/aikido_firewall/helpers/limit_length_metadata.py new file mode 100644 index 000000000..4df689544 --- /dev/null +++ b/aikido_firewall/helpers/limit_length_metadata.py @@ -0,0 +1,15 @@ +""" +Helper function file, see function docstring +""" + + +def limit_length_metadata(metadata, max_length): + """ + Limits the length of the metadata obj so it can be sent out + """ + for key in metadata: + print(len(metadata[key])) + if len(metadata[key]) > max_length: + metadata[key] = metadata[key][:max_length] + + return metadata diff --git a/aikido_firewall/helpers/limit_length_metadata_test.py b/aikido_firewall/helpers/limit_length_metadata_test.py new file mode 100644 index 000000000..524c71b68 --- /dev/null +++ b/aikido_firewall/helpers/limit_length_metadata_test.py @@ -0,0 +1,21 @@ +import pytest +from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata + + +def test_limit_length_metadata(): + # Test case 1: Check if values are truncated correctly + metadata = {"key1": "value1", "key2": "value2longvalue", "key3": "value3"} + max_length = 6 + expected_result = {"key1": "value1", "key2": "value2", "key3": "value3"} + assert limit_length_metadata(metadata, max_length) == expected_result + + # Test case 2: Check if values are not truncated if within max length + metadata = {"key1": "value1", "key2": "value2", "key3": "value3"} + max_length = 10 + expected_result = {"key1": "value1", "key2": "value2", "key3": "value3"} + assert limit_length_metadata(metadata, max_length) == expected_result + + metadata = {} + max_length = 5 + expected_result = {} + assert limit_length_metadata(metadata, max_length) == expected_result From b04031bd311fb3117d28fe28c447dac5dfe8330b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Wed, 24 Jul 2024 13:30:33 +0200 Subject: [PATCH 25/79] Use new helper function in agent.py to limit metadata --- aikido_firewall/agent/agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aikido_firewall/agent/agent.py b/aikido_firewall/agent/agent.py index 460d9ff2a..40f3fffee 100644 --- a/aikido_firewall/agent/agent.py +++ b/aikido_firewall/agent/agent.py @@ -6,6 +6,7 @@ import json from copy import deepcopy from aikido_firewall.helpers.logging import logger +from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata class Agent: @@ -34,6 +35,7 @@ def on_detected_attack(self, attack): del attack["request"] attack["user"] = req["user"] attack["payload"] = json.dumps(attack["payload"])[:4096] + attack["metadata"] = limit_length_metadata(attack["metadata"], 4096) self.api.report( self.token, From 78f659990165118def1f06f2968b8e30e3ea28de Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Thu, 25 Jul 2024 14:55:40 +0200 Subject: [PATCH 26/79] Change interval to 10 minutes --- aikido_firewall/agent/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/agent/__init__.py b/aikido_firewall/agent/__init__.py index 88b94370e..8a17bced9 100644 --- a/aikido_firewall/agent/__init__.py +++ b/aikido_firewall/agent/__init__.py @@ -13,7 +13,7 @@ from aikido_firewall.helpers.should_block import should_block from aikido_firewall.helpers.token import get_token_from_env -AGENT_SEC_INTERVAL = 5 +AGENT_SEC_INTERVAL = 600 # 10 minutes IPC_ADDRESS = ("localhost", 9898) # Specify the IP address and port From d3914f4eace5ead651dca70e7886797a85eea467 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Thu, 25 Jul 2024 17:33:27 +0200 Subject: [PATCH 27/79] Renaming agent to reporter and fixing left scars of merge --- aikido_firewall/background_process/__init__.py | 8 ++++---- .../background_process/api/http_api.py | 2 +- .../background_process/api/init_test.py | 2 +- .../{agent.py => reporter.py} | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) rename aikido_firewall/background_process/{agent.py => reporter.py} (90%) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index c0fa3178f..ca744bb09 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -11,7 +11,7 @@ from threading import Thread from queue import Queue from aikido_firewall.helpers.logging import logger -from aikido_firewall.agent.agent import Agent +from aikido_firewall.background_process.reporter import Reporter from aikido_firewall.helpers.should_block import should_block from aikido_firewall.helpers.token import get_token_from_env @@ -30,7 +30,7 @@ def __init__(self, address, key): logger.debug("Background process started") listener = con.Listener(address, authkey=key) self.queue = Queue() - self.agent = None + self.reporter = None # Start reporting thread : Thread(target=self.reporting_thread).start() @@ -49,8 +49,8 @@ def __init__(self, address, key): def reporting_thread(self): """Reporting thread""" logger.debug("Started reporting thread") - self.agent = Agent(should_block(), {}, get_token_from_env(), None) - logger.debug("Created agent") + self.reporter = Reporter(should_block(), {}, get_token_from_env(), None) + logger.debug("Created Reporter") while True: self.send_to_reporter() time.sleep(REPORT_SEC_INTERVAL) diff --git a/aikido_firewall/background_process/api/http_api.py b/aikido_firewall/background_process/api/http_api.py index 0fd6e94d5..4c7d5b5c2 100644 --- a/aikido_firewall/background_process/api/http_api.py +++ b/aikido_firewall/background_process/api/http_api.py @@ -3,7 +3,7 @@ """ import requests -from aikido_firewall.agent.api import ReportingApi +from aikido_firewall.background_process.api import ReportingApi class ReportingApiHTTP(ReportingApi): diff --git a/aikido_firewall/background_process/api/init_test.py b/aikido_firewall/background_process/api/init_test.py index 8e6398e38..18b5f39ea 100644 --- a/aikido_firewall/background_process/api/init_test.py +++ b/aikido_firewall/background_process/api/init_test.py @@ -1,5 +1,5 @@ import pytest -from aikido_firewall.agent.api import ReportingApi +from aikido_firewall.background_process.api import ReportingApi # Test ReportingApi Class : from requests.models import Response diff --git a/aikido_firewall/background_process/agent.py b/aikido_firewall/background_process/reporter.py similarity index 90% rename from aikido_firewall/background_process/agent.py rename to aikido_firewall/background_process/reporter.py index 40f3fffee..648c3bede 100644 --- a/aikido_firewall/background_process/agent.py +++ b/aikido_firewall/background_process/reporter.py @@ -1,4 +1,4 @@ -""" This file simply exports the agent class""" +""" This file simply exports the Reporter class""" from datetime import datetime import socket @@ -9,8 +9,8 @@ from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata -class Agent: - """Agent class""" +class Reporter: + """Reporter class""" timeout_in_sec = 5 @@ -42,7 +42,7 @@ def on_detected_attack(self, attack): { "type": "detected_attack", "time": datetime.now(), - "agent": self.get_agent_info(), + "agent": self.get_reporter_info(), "attack": attack, "request": { "method": req["method"], @@ -66,13 +66,13 @@ def send_heartbeat(self): """ if not self.token: return - logger.debug("Aikido Agent : Sending out heartbeat") + logger.debug("Aikido Reporter : Sending out heartbeat") res = self.api.report( self.token, { "type": "heartbeat", "time": datetime.now(), - "agent": self.get_agent_info(), + "agent": self.get_reporter_info(), "stats": {"sinks": [], "startedAt": 0, "endedAt": 0, "requests": []}, "hostnames": [], "routes": [], @@ -90,14 +90,14 @@ def on_start(self): return res = self.api.report( self.token, - {"type": "started", "time": datetime.now(), "agent": self.get_agent_info()}, + {"type": "started", "time": datetime.now(), "agent": self.get_reporter_info()}, self.timeout_in_sec, ) self.update_service_config(res) - def get_agent_info(self): + def get_reporter_info(self): """ - This returns info about the agent + This returns info about the reporter """ return { "dryMode": not self.block, From 747d825300728f6e64782d21b8fc817055589806 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:02:05 +0200 Subject: [PATCH 28/79] Validate token using Token class in Reporter --- aikido_firewall/background_process/reporter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 648c3bede..b6dea5e32 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -7,6 +7,7 @@ from copy import deepcopy from aikido_firewall.helpers.logging import logger from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata +from aikido_firewall.helpers.token import Token class Reporter: @@ -17,7 +18,7 @@ class Reporter: def __init__(self, block, api, token, serverless): self.block = block self.api = api - self.token = token + self.token = Token(token) if isinstance(serverless, str) and len(serverless) == 0: raise ValueError("Serverless cannot be an empty string") From 1bceaba6df943dc6d6bb2392b7165a6dada65d0b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:02:38 +0200 Subject: [PATCH 29/79] Use .timestamp() on datetime --- aikido_firewall/background_process/reporter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index b6dea5e32..b2db425da 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -42,7 +42,7 @@ def on_detected_attack(self, attack): self.token, { "type": "detected_attack", - "time": datetime.now(), + "time": datetime.now().timestamp(), "agent": self.get_reporter_info(), "attack": attack, "request": { @@ -72,7 +72,7 @@ def send_heartbeat(self): self.token, { "type": "heartbeat", - "time": datetime.now(), + "time": datetime.now().timestamp(), "agent": self.get_reporter_info(), "stats": {"sinks": [], "startedAt": 0, "endedAt": 0, "requests": []}, "hostnames": [], @@ -91,7 +91,11 @@ def on_start(self): return res = self.api.report( self.token, - {"type": "started", "time": datetime.now(), "agent": self.get_reporter_info()}, + { + "type": "started", + "time": datetime.now().timestamp(), + "agent": self.get_reporter_info(), + }, self.timeout_in_sec, ) self.update_service_config(res) From 0d3b1d37a7cd28c1e543c1f4dcd0a9119d551306 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:03:00 +0200 Subject: [PATCH 30/79] Bugfix, should be json= instead of data= --- aikido_firewall/background_process/api/http_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/api/http_api.py b/aikido_firewall/background_process/api/http_api.py index 4c7d5b5c2..0f7b169e6 100644 --- a/aikido_firewall/background_process/api/http_api.py +++ b/aikido_firewall/background_process/api/http_api.py @@ -16,7 +16,7 @@ def report(self, token, event, timeout_in_sec): try: res = requests.post( self.reporting_url + "api/runtime/events", - data=event, + json=event, timeout=timeout_in_sec, headers=get_headers(token), ) From ca25686c3dc7067679f82933044bd15aeeafd4fe Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:03:49 +0200 Subject: [PATCH 31/79] Only parse if status code is 200, use json.loads and debug if error --- aikido_firewall/background_process/api/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/aikido_firewall/background_process/api/__init__.py b/aikido_firewall/background_process/api/__init__.py index 5e29f4d11..3f8324b18 100644 --- a/aikido_firewall/background_process/api/__init__.py +++ b/aikido_firewall/background_process/api/__init__.py @@ -2,6 +2,9 @@ init.py file for api/ folder. Includes abstract class ReportingApi """ +import json +from aikido_firewall.helpers.logging import logger + class ReportingApi: """This is the super class for the reporting API's""" @@ -13,10 +16,13 @@ def to_api_response(self, res): return {"success": False, "error": "rate_limited"} elif status == 401: return {"success": False, "error": "invalid_token"} - try: - return res.json() - except Exception: - return {"success": False, "error": "unknown_error"} + elif status == 200: + try: + return json.loads(res.text) + except Exception as e: + logger.debug(e) + logger.debug(res.text) + return {"success": False, "error": "unknown_error"} def report(self, token, event, timeout_in_sec): """Report event to aikido server""" From 0a47b8f0bfdf5f98bc2bd9720d7409ba476c0d71 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:23:06 +0200 Subject: [PATCH 32/79] Fix bug, unixtime needs to be sent in ms --- aikido_firewall/background_process/reporter.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index b2db425da..26325098e 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -1,6 +1,6 @@ """ This file simply exports the Reporter class""" -from datetime import datetime +import time import socket import platform import json @@ -42,7 +42,7 @@ def on_detected_attack(self, attack): self.token, { "type": "detected_attack", - "time": datetime.now().timestamp(), + "time": get_unixtime_ms(), "agent": self.get_reporter_info(), "attack": attack, "request": { @@ -72,7 +72,7 @@ def send_heartbeat(self): self.token, { "type": "heartbeat", - "time": datetime.now().timestamp(), + "time": get_unixtime_ms(), "agent": self.get_reporter_info(), "stats": {"sinks": [], "startedAt": 0, "endedAt": 0, "requests": []}, "hostnames": [], @@ -93,7 +93,7 @@ def on_start(self): self.token, { "type": "started", - "time": datetime.now().timestamp(), + "time": get_unixtime_ms(), "agent": self.get_reporter_info(), }, self.timeout_in_sec, @@ -109,14 +109,21 @@ def get_reporter_info(self): "hostname": socket.gethostname(), "version": "x.x.x", "library": "firewall_python", - "ipAddress": socket.gethostbyname(socket.gethostname()), + "ipAddress": "127.0.0.1", "packages": [], "serverless": bool(self.serverless), "stack": [], "os": {"name": platform.system(), "version": platform.release()}, + "preventedPrototypePollution": False, # Get this out of the API maybe? } def update_service_config(self, res): """ Update configuration based on the server's response """ + print(res) + + +def get_unixtime_ms(): + """Get the current unix time but in ms""" + return int(time.time() * 1000) From dbfa324b84f3b9fd06d7374094057a763014897a Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:46:06 +0200 Subject: [PATCH 33/79] Use a PKG_VERSION const --- aikido_firewall/__init__.py | 2 ++ aikido_firewall/background_process/reporter.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/__init__.py b/aikido_firewall/__init__.py index 18eafc779..21a6aee2a 100644 --- a/aikido_firewall/__init__.py +++ b/aikido_firewall/__init__.py @@ -4,6 +4,8 @@ from dotenv import load_dotenv +# Constants +PKG_VERSION = "0.0.1" # Import logger from aikido_firewall.helpers.logging import logger diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 26325098e..0127b55d1 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -8,6 +8,7 @@ from aikido_firewall.helpers.logging import logger from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata from aikido_firewall.helpers.token import Token +from aikido_firewall import PKG_VERSION class Reporter: @@ -107,7 +108,7 @@ def get_reporter_info(self): return { "dryMode": not self.block, "hostname": socket.gethostname(), - "version": "x.x.x", + "version": PKG_VERSION, "library": "firewall_python", "ipAddress": "127.0.0.1", "packages": [], From 2bce5f6d103130464e98f14af0845146439018b8 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 11:46:24 +0200 Subject: [PATCH 34/79] Create a get_ip() function --- aikido_firewall/background_process/reporter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 0127b55d1..ff343ee67 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -110,7 +110,7 @@ def get_reporter_info(self): "hostname": socket.gethostname(), "version": PKG_VERSION, "library": "firewall_python", - "ipAddress": "127.0.0.1", + "ipAddress": get_ip(), "packages": [], "serverless": bool(self.serverless), "stack": [], @@ -128,3 +128,11 @@ def update_service_config(self, res): def get_unixtime_ms(): """Get the current unix time but in ms""" return int(time.time() * 1000) + + +def get_ip(): + """Tries to fetch the IP and returns 0.0.0.0 on failure""" + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "0.0.0.0" From 66fcb28b57b191ab699b1874c41a239f1b2f6e46 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 12:06:59 +0200 Subject: [PATCH 35/79] Update blocking in the updateConfig function --- aikido_firewall/background_process/reporter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index ff343ee67..7ac79b3ae 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -122,6 +122,9 @@ def update_service_config(self, res): """ Update configuration based on the server's response """ + if(res["block"] and res["block"] != self.block): + logger.debug("Updating blocking, setting blocking to : %s", res["block"]) + self.block = bool(res["block"]) print(res) From 224a864245ef203aedd0c684b036fdb14ad1fa81 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 12:14:47 +0200 Subject: [PATCH 36/79] Linting --- aikido_firewall/background_process/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 7ac79b3ae..ccade2d66 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -122,7 +122,7 @@ def update_service_config(self, res): """ Update configuration based on the server's response """ - if(res["block"] and res["block"] != self.block): + if res["block"] and res["block"] != self.block: logger.debug("Updating blocking, setting blocking to : %s", res["block"]) self.block = bool(res["block"]) print(res) From 177c9fc2710a2111120496a6273f7a7906497c4b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 12:15:04 +0200 Subject: [PATCH 37/79] Add a heartbeats.py file with heartbeat logic (interval, event sched) --- .../background_process/heartbeats.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 aikido_firewall/background_process/heartbeats.py diff --git a/aikido_firewall/background_process/heartbeats.py b/aikido_firewall/background_process/heartbeats.py new file mode 100644 index 000000000..38133633a --- /dev/null +++ b/aikido_firewall/background_process/heartbeats.py @@ -0,0 +1,33 @@ +""" +The code to send out a heartbeat is in here +""" + +import sched, time +from aikido_firewall.helpers.logging import logger + +# Create an event scheduler +s = sched.scheduler(time.time, time.sleep) + + +def send_heartbeats_every_x_secs(reporter, interval_in_secs): + """ + Start sending out heartbeats every x seconds + """ + if reporter.serverless: + logger.debug("Running in serverless environment, not starting heartbeats") + return + if not reporter.token: + logger.debug("No token provided, not starting heartbeats") + + # Interval already started code ? + + s.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs)) + + +def send_heartbeat_wrapper(rep, interval_in_secs): + """ + Wrapper function for send_heartbeat so we get an interval + """ + s.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs)) + logger.debug("Heartbeat...") + rep.send_heartbeat() From f5e6a3b5b3ef5313bd6bacabdb263a1ff374a41a Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 12:15:16 +0200 Subject: [PATCH 38/79] Add missing data (empty) to the request --- aikido_firewall/background_process/reporter.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index ccade2d66..060a0d883 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -75,7 +75,19 @@ def send_heartbeat(self): "type": "heartbeat", "time": get_unixtime_ms(), "agent": self.get_reporter_info(), - "stats": {"sinks": [], "startedAt": 0, "endedAt": 0, "requests": []}, + "stats": { + "sinks": {}, + "startedAt": 0, + "endedAt": 0, + "requests": { + "total": 0, + "aborted": 0, + "attacksDetected": { + "total": 0, + "blocked": 0, + }, + }, + }, "hostnames": [], "routes": [], "users": [], @@ -116,6 +128,7 @@ def get_reporter_info(self): "stack": [], "os": {"name": platform.system(), "version": platform.release()}, "preventedPrototypePollution": False, # Get this out of the API maybe? + "nodeEnv": "", } def update_service_config(self, res): From 19cda6b18bab703b5bd314a9e3c0ed66d4b12b1c Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 14:31:24 +0200 Subject: [PATCH 39/79] linting for if statement in token.py --- aikido_firewall/helpers/token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/helpers/token.py b/aikido_firewall/helpers/token.py index 043d2dcba..ab39c1c14 100644 --- a/aikido_firewall/helpers/token.py +++ b/aikido_firewall/helpers/token.py @@ -24,6 +24,6 @@ def get_token_from_env(): Fetches the token from the env variable "AIKIDO_TOKEN" """ aikido_token_env = os.getenv("AIKIDO_TOKEN") - if not aikido_token_env is None: + if aikido_token_env is not None: return Token(aikido_token_env) return None From 1c8a00669576a0708a45c669f5f27c05f60e4442 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 14:31:43 +0200 Subject: [PATCH 40/79] Mistake with 2x token wrappin --- aikido_firewall/background_process/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 060a0d883..d7d382493 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -19,7 +19,7 @@ class Reporter: def __init__(self, block, api, token, serverless): self.block = block self.api = api - self.token = Token(token) + self.token = token # Should be instance of the Token class! if isinstance(serverless, str) and len(serverless) == 0: raise ValueError("Serverless cannot be an empty string") From 1ac32d029e0834e0588b2fcaf4cfd77f0e9c6c71 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 15:36:53 +0200 Subject: [PATCH 41/79] Flask shouldn't reload, messes up our bg job --- sample-apps/flask-mysql/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample-apps/flask-mysql/docker-compose.yml b/sample-apps/flask-mysql/docker-compose.yml index 1f2478edd..24dbb87ff 100644 --- a/sample-apps/flask-mysql/docker-compose.yml +++ b/sample-apps/flask-mysql/docker-compose.yml @@ -25,7 +25,7 @@ services: context: ./../../ dockerfile: ./sample-apps/flask-mysql/Dockerfile container_name: flask_mysql_backend - command: sh -c "flask --app app run --debug --host=0.0.0.0" + command: sh -c "flask --app app run --debug --host=0.0.0.0 --no-reload" restart: always volumes: - .:/app @@ -35,4 +35,4 @@ services: - db volumes: - db_data: \ No newline at end of file + db_data: From 739f18a0f7c1d4382c426b78e05250049f886f28 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 15:37:18 +0200 Subject: [PATCH 42/79] Create a reporter with a local api url --- aikido_firewall/background_process/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 2cf3e020a..2aa510791 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -39,7 +39,8 @@ def __init__(self, address, key): pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess self.queue = Queue() - self.reporter = None + api = ReportingApiHTTP("http://app.local.aikido.io/") + self.reporter = Reporter(should_block(), api, get_token_from_env(), False) # Start reporting thread : Thread(target=self.reporting_thread).start() From caece69a9a65cb24fc2af2d625dad231777c1290 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 15:40:37 +0200 Subject: [PATCH 43/79] Using an array as data for send_data function is pointless --- aikido_firewall/background_process/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 2aa510791..2b2f6ad37 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -140,12 +140,11 @@ def send_data(self, action, obj): # We want to make sure that sending out this data affects the process as little as possible # So we run it inside a seperate thread with a timeout of 3 seconds - def target(address, key, data_array): + def target(address, key, data): try: conn = con.Client(address, authkey=key) logger.debug("Created connection %s", conn) - for data in data_array: - conn.send(data) + conn.send(data) conn.send(("CLOSE", {})) conn.close() logger.debug("Connection closed") @@ -153,7 +152,7 @@ def target(address, key, data_array): logger.info("Failed to send data to bg process : %s", e) t = Thread( - target=target, args=(self.address, self.key, [(action, obj)]), daemon=True + target=target, args=(self.address, self.key, (action, obj)), daemon=True ) t.start() t.join(timeout=3) From bbc0b1ec7745090a2159bf3ee087d8d3b5c720d8 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:13:23 +0200 Subject: [PATCH 44/79] Add aikido as host in docker --- sample-apps/flask-mysql/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sample-apps/flask-mysql/docker-compose.yml b/sample-apps/flask-mysql/docker-compose.yml index 24dbb87ff..86f34326c 100644 --- a/sample-apps/flask-mysql/docker-compose.yml +++ b/sample-apps/flask-mysql/docker-compose.yml @@ -33,6 +33,8 @@ services: - "8080:5000" depends_on: - db + extra_hosts: + - "app.local.aikido.io:host-gateway" volumes: db_data: From 1b9fd0f06c6c06154937da04cf94d86a309fc74b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:13:48 +0200 Subject: [PATCH 45/79] Run on_start when initiating a Reporter --- aikido_firewall/background_process/reporter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index d7d382493..26872a35a 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -25,6 +25,8 @@ def __init__(self, block, api, token, serverless): raise ValueError("Serverless cannot be an empty string") self.serverless = serverless + self.on_start() + def on_detected_attack(self, attack): """ This will send something to the API when an attack is detected From 0ffe0685e7359825e55cc36b7004f265b87d5299 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:14:10 +0200 Subject: [PATCH 46/79] Check if result is successfull of API --- aikido_firewall/background_process/reporter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 26872a35a..ef12204a2 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -137,10 +137,12 @@ def update_service_config(self, res): """ Update configuration based on the server's response """ + if res["success"] is False: + logger.debug(res) + return if res["block"] and res["block"] != self.block: logger.debug("Updating blocking, setting blocking to : %s", res["block"]) self.block = bool(res["block"]) - print(res) def get_unixtime_ms(): From 1e869ea733928a522d7f19e18b2839c029d13d29 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:14:24 +0200 Subject: [PATCH 47/79] Fix bug with not detecting the timeout error --- aikido_firewall/background_process/api/http_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/api/http_api.py b/aikido_firewall/background_process/api/http_api.py index 0f7b169e6..3374eabdd 100644 --- a/aikido_firewall/background_process/api/http_api.py +++ b/aikido_firewall/background_process/api/http_api.py @@ -20,7 +20,7 @@ def report(self, token, event, timeout_in_sec): timeout=timeout_in_sec, headers=get_headers(token), ) - except requests.exceptions.Timeout: + except requests.exceptions.ConnectionError: return {"success": False, "error": "timeout"} except Exception as e: raise e From 058ab89bd7bb28227079b3df33aec9be4408901f Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:15:08 +0200 Subject: [PATCH 48/79] Forgotten import + report attacks when emptying queue --- aikido_firewall/background_process/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 2b2f6ad37..31e259b68 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -16,6 +16,7 @@ from aikido_firewall.background_process.reporter import Reporter from aikido_firewall.helpers.should_block import should_block from aikido_firewall.helpers.token import get_token_from_env +from aikido_firewall.background_process.api.http_api import ReportingApiHTTP REPORT_SEC_INTERVAL = 600 # 10 minutes IPC_ADDRESS = ("localhost", 9898) # Specify the IP address and port @@ -74,12 +75,11 @@ def send_to_reporter(self): """ Reports the found data to an Aikido server """ - items_to_report = [] + logger.debug("Checking queue") while not self.queue.empty(): - items_to_report.append(self.queue.get()) - logger.debug("Reporting to aikido server") - logger.critical("Items to report : %s", items_to_report) - # Currently not making API calls + attack = self.queue.get() + logger.debug("Reporting attack : %s", attack) + self.reporter.on_detected_attack(attack) # pylint: disable=invalid-name # This variable does change From 3d2143e48d085873d2ff6d33bddf96fc5cf3efe9 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:47:36 +0200 Subject: [PATCH 49/79] Make context object picklable --- aikido_firewall/context/__init__.py | 26 +++++++++------- aikido_firewall/context/init_test.py | 31 ++++++++++++++++--- aikido_firewall/middleware/django.py | 2 +- aikido_firewall/sources/flask.py | 2 +- .../django-mysql-gunicorn/gunicorn_config.py | 2 +- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index c0d9dde2c..fe859e03e 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -51,7 +51,11 @@ class Context: for vulnerability detection """ - def __init__(self, req, source): + def __init__(self, context_obj=None, req=None, source=None): + if context_obj: + self.__dict__.update(context_obj) + return + if not source in SUPPORTED_SOURCES: raise ValueError(f"Source {source} not supported") self.source = source @@ -92,16 +96,16 @@ def set_flask_attrs(self, req): def __reduce__(self): return ( self.__class__, - ( - self.method, - self.remote_address, - self.url, - self.body, - self.headers, - self.query, - self.cookies, - self.source, - ), + ({ + "method": self.method, + "remote_address": self.remote_address, + "url": self.url, + "body": self.body, + "headers": self.headers, + "query": self.query, + "cookies": self.cookies, + "source": self.source, + }, None, None), ) def set_as_current_context(self): diff --git a/aikido_firewall/context/init_test.py b/aikido_firewall/context/init_test.py index f73c06e33..27e9fadb1 100644 --- a/aikido_firewall/context/init_test.py +++ b/aikido_firewall/context/init_test.py @@ -1,4 +1,5 @@ import pytest +import pickle from aikido_firewall.context import Context, get_current_context @@ -10,7 +11,7 @@ def test_get_current_context_no_context(): def test_set_as_current_context(mocker): # Test set_as_current_context() method sample_request = mocker.MagicMock() - context = Context(sample_request, "flask") + context = Context(req=sample_request, source="flask") context.set_as_current_context() assert get_current_context() == context @@ -18,7 +19,7 @@ def test_set_as_current_context(mocker): def test_get_current_context_with_context(mocker): # Test get_current_context() when a context is set sample_request = mocker.MagicMock() - context = Context(sample_request, "flask") + context = Context(req=sample_request, source="flask") context.set_as_current_context() assert get_current_context() == context @@ -33,7 +34,7 @@ def test_context_init_flask(mocker): req.args.to_dict.return_value = {"key": "value"} req.cookies.to_dict.return_value = {"cookie": "value"} - context = Context(req, "flask") + context = Context(req=req, source="flask") assert context.source == "flask" assert context.method == "GET" assert context.remote_address == "127.0.0.1" @@ -54,7 +55,7 @@ def test_context_init_django(mocker): req.GET = {"key": "value"} req.COOKIES = {"cookie": "value"} - context = Context(req, "django") + context = Context(req=req, source="django") assert context.source == "django" assert context.method == "POST" assert context.remote_address == "127.0.0.1" @@ -63,3 +64,25 @@ def test_context_init_django(mocker): assert context.headers == {"Content-Type": "application/json"} assert context.query == {"key": "value"} assert context.cookies == {"cookie": "value"} + +def test_context_is_picklable(mocker): + req = mocker.MagicMock() + req.method = "POST" + req.META.get.return_value = "127.0.0.1" + req.build_absolute_uri.return_value = "http://example.com" + req.POST = {"key": "value"} + req.headers = {"Content-Type": "application/json"} + req.GET = {"key": "value"} + req.COOKIES = {"cookie": "value"} + context = Context(req=req, source="django") + + pickled_obj = pickle.dumps(context) + unpickled_obj = pickle.loads(pickled_obj) + assert unpickled_obj.source == "django" + assert unpickled_obj.method == "POST" + assert unpickled_obj.remote_address == "127.0.0.1" + assert unpickled_obj.url == "http://example.com" + assert unpickled_obj.body == {"key": "value"} + assert unpickled_obj.headers == {"Content-Type": "application/json"} + assert unpickled_obj.query == {"key": "value"} + assert unpickled_obj.cookies == {"cookie": "value"} diff --git a/aikido_firewall/middleware/django.py b/aikido_firewall/middleware/django.py index 276387b36..2fcb2ce7c 100644 --- a/aikido_firewall/middleware/django.py +++ b/aikido_firewall/middleware/django.py @@ -17,7 +17,7 @@ def __init__(self, get_response): def __call__(self, request, *args, **kwargs): logger.debug("Aikido middleware for `django` was called : __call__") - context = Context(request, "django") + context = Context(req=request, source="django") context.set_as_current_context() return self.get_response(request) diff --git a/aikido_firewall/sources/flask.py b/aikido_firewall/sources/flask.py index d5d92cced..bc0de0eb0 100644 --- a/aikido_firewall/sources/flask.py +++ b/aikido_firewall/sources/flask.py @@ -21,7 +21,7 @@ def __init__(self): def dispatch(self, request, call_next): """Dispatch function""" logger.debug("Aikido middleware for `flask` was called") - context = Context(request, "flask") + context = Context(req=request, source="flask") context.set_as_current_context() response = call_next(request) diff --git a/sample-apps/django-mysql-gunicorn/gunicorn_config.py b/sample-apps/django-mysql-gunicorn/gunicorn_config.py index c722b0f04..ced815ec9 100644 --- a/sample-apps/django-mysql-gunicorn/gunicorn_config.py +++ b/sample-apps/django-mysql-gunicorn/gunicorn_config.py @@ -19,7 +19,7 @@ def post_fork(server, worker): def pre_request(worker, req): req.body, req.body_copy = clone_body(req.body) - django_context = Context(req, "django-gunicorn") + django_context = Context(req=req, source="django-gunicorn") django_context.set_as_current_context() worker.log.debug("%s %s", req.method, req.path) From 0121d6a717d7394bf90749a84f0f21322688ebbc Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 16:51:55 +0200 Subject: [PATCH 50/79] Update the on_detected_attack function --- .../background_process/__init__.py | 4 +-- .../background_process/reporter.py | 30 ++++++++++--------- aikido_firewall/sinks/pymysql.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 31e259b68..d31d6c78d 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -65,8 +65,6 @@ def __init__(self, address, key): def reporting_thread(self): """Reporting thread""" logger.debug("Started reporting thread") - self.reporter = Reporter(should_block(), {}, get_token_from_env(), None) - logger.debug("Created Reporter") while True: self.send_to_reporter() time.sleep(REPORT_SEC_INTERVAL) @@ -79,7 +77,7 @@ def send_to_reporter(self): while not self.queue.empty(): attack = self.queue.get() logger.debug("Reporting attack : %s", attack) - self.reporter.on_detected_attack(attack) + self.reporter.on_detected_attack(attack[0], attack[1]) # pylint: disable=invalid-name # This variable does change diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index ef12204a2..870fc150f 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -27,7 +27,7 @@ def __init__(self, block, api, token, serverless): self.on_start() - def on_detected_attack(self, attack): + def on_detected_attack(self, attack, context): """ This will send something to the API when an attack is detected """ @@ -35,33 +35,35 @@ def on_detected_attack(self, attack): return # Modify attack so we can send it out : try: - req = deepcopy(attack["request"]) - del attack["request"] - attack["user"] = req["user"] + attack["user"] = None attack["payload"] = json.dumps(attack["payload"])[:4096] attack["metadata"] = limit_length_metadata(attack["metadata"], 4096) - self.api.report( - self.token, - { + payload = { "type": "detected_attack", "time": get_unixtime_ms(), "agent": self.get_reporter_info(), "attack": attack, "request": { - "method": req["method"], - "url": req["url"], - "ipAddress": req["remoteAddress"], + "method": context.method, + "url": context.url, + "ipAddress": context.remote_address, "userAgent": "WIP", "body": {}, "headers": {}, - "source": req["source"], - "route": req["route"], + "source": context.source, + "route": "?", }, - }, + } + logger.debug(json.dumps(payload)) + result = self.api.report( + self.token, + payload, self.timeout_in_sec, ) - except Exception: + logger.debug("Result : %s", result) + except Exception as e: + logger.debug(e) logger.info("Failed to report attack") def send_heartbeat(self): diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index a48e86420..557fa7040 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -37,7 +37,7 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.info("sql_injection results : %s", json.dumps(result)) if result: - get_comms().send_data("ATTACK", result) + get_comms().send_data("ATTACK", (result, context)) raise Exception("SQL Injection [aikido_firewall]") return prev_query_function(_self, sql, unbuffered=False) From bd98fe7e505af721ba1c9e3ff232aa433e3fd3a0 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 17:02:18 +0200 Subject: [PATCH 51/79] Linting --- aikido_firewall/context/__init__.py | 24 ++++++++++++++---------- aikido_firewall/context/init_test.py | 1 + 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index fe859e03e..fa2eb5314 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -96,16 +96,20 @@ def set_flask_attrs(self, req): def __reduce__(self): return ( self.__class__, - ({ - "method": self.method, - "remote_address": self.remote_address, - "url": self.url, - "body": self.body, - "headers": self.headers, - "query": self.query, - "cookies": self.cookies, - "source": self.source, - }, None, None), + ( + { + "method": self.method, + "remote_address": self.remote_address, + "url": self.url, + "body": self.body, + "headers": self.headers, + "query": self.query, + "cookies": self.cookies, + "source": self.source, + }, + None, + None, + ), ) def set_as_current_context(self): diff --git a/aikido_firewall/context/init_test.py b/aikido_firewall/context/init_test.py index 27e9fadb1..829e3e237 100644 --- a/aikido_firewall/context/init_test.py +++ b/aikido_firewall/context/init_test.py @@ -65,6 +65,7 @@ def test_context_init_django(mocker): assert context.query == {"key": "value"} assert context.cookies == {"cookie": "value"} + def test_context_is_picklable(mocker): req = mocker.MagicMock() req.method = "POST" From c5bce7d65d132f0827d1c17091fcbe61c4a271f4 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Mon, 29 Jul 2024 17:02:31 +0200 Subject: [PATCH 52/79] Extract extra nformation and user-agent from headers --- .../background_process/reporter.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 870fc150f..346b6dd3d 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -40,21 +40,21 @@ def on_detected_attack(self, attack, context): attack["metadata"] = limit_length_metadata(attack["metadata"], 4096) payload = { - "type": "detected_attack", - "time": get_unixtime_ms(), - "agent": self.get_reporter_info(), - "attack": attack, - "request": { - "method": context.method, - "url": context.url, - "ipAddress": context.remote_address, - "userAgent": "WIP", - "body": {}, - "headers": {}, - "source": context.source, - "route": "?", - }, - } + "type": "detected_attack", + "time": get_unixtime_ms(), + "agent": self.get_reporter_info(), + "attack": attack, + "request": { + "method": context.method, + "url": context.url, + "ipAddress": context.remote_address, + "userAgent": get_ua_from_context(context), + "body": context.body, + "headers": context.headers, + "source": context.source, + "route": "?", + }, + } logger.debug(json.dumps(payload)) result = self.api.report( self.token, @@ -158,3 +158,11 @@ def get_ip(): return socket.gethostbyname(socket.gethostname()) except Exception: return "0.0.0.0" + + +def get_ua_from_context(context): + """Tries to retrieve the user agent from context""" + for k, v in context.headers.items(): + if k.lower() == "user-agent": + return v + return None From 49e41f08135a15689ed041fb09f955ca2dbce0b3 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 09:16:17 +0200 Subject: [PATCH 53/79] Allow polling for config, and do it to see if we should block --- aikido_firewall/background_process/__init__.py | 13 +++++++++++++ aikido_firewall/background_process/reporter.py | 1 + aikido_firewall/sinks/pymysql.py | 4 +++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index d31d6c78d..4a1c4d732 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -61,6 +61,9 @@ def __init__(self, address, key): conn.close() pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess + elif data[0] == "READ_PROPERTY": + if hasattr(self.reporter, data[1]): + conn.send(self.reporter.__dict__[data[1]]) def reporting_thread(self): """Reporting thread""" @@ -154,3 +157,13 @@ def target(address, key, data): ) t.start() t.join(timeout=3) + + def poll_config(self, prop): + """ + This will poll the config from the Background Process + """ + conn = con.Client(self.address, authkey=self.key) + conn.send(("READ_PROPERTY", prop)) + prop_value = conn.recv() + logger.debug("Received property %s as %s", prop, prop_value) + return prop_value diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 346b6dd3d..96fb4e151 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -38,6 +38,7 @@ def on_detected_attack(self, attack, context): attack["user"] = None attack["payload"] = json.dumps(attack["payload"])[:4096] attack["metadata"] = limit_length_metadata(attack["metadata"], 4096) + attack["blocked"] = self.block payload = { "type": "detected_attack", diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index 557fa7040..e5127e944 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -38,7 +38,9 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.info("sql_injection results : %s", json.dumps(result)) if result: get_comms().send_data("ATTACK", (result, context)) - raise Exception("SQL Injection [aikido_firewall]") + should_block = get_comms().poll_config("block") + if should_block: + raise Exception("SQL Injection [aikido_firewall]") return prev_query_function(_self, sql, unbuffered=False) # pylint: disable=no-member From ae4c5859efc2236105638a21ed6c314097122f86 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 11:26:39 +0200 Subject: [PATCH 54/79] Report route to aikido server --- aikido_firewall/background_process/reporter.py | 2 +- aikido_firewall/context/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 96fb4e151..48db2e979 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -53,7 +53,7 @@ def on_detected_attack(self, attack, context): "body": context.body, "headers": context.headers, "source": context.source, - "route": "?", + "route": context.route, }, } logger.debug(json.dumps(payload)) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index fa2eb5314..718345bf3 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -5,7 +5,7 @@ import threading from urllib.parse import parse_qs from http.cookies import SimpleCookie - +from aikido_firewall.helpers.build_route_from_url import build_route_from_url SUPPORTED_SOURCES = ["django", "flask", "django-gunicorn"] UINPUT_SOURCES = ["body", "cookies", "query", "headers"] @@ -67,6 +67,7 @@ def __init__(self, context_obj=None, req=None, source=None): self.set_django_attrs(req) elif source == "django-gunicorn": self.set_django_gunicorn_attrs(req) + self.route = build_route_from_url(self.url) def set_django_gunicorn_attrs(self, req): """Set properties that are specific to django-gunicorn""" @@ -106,6 +107,7 @@ def __reduce__(self): "query": self.query, "cookies": self.cookies, "source": self.source, + "route": self.route, }, None, None, From cd3bfab71ad185948d8da9753ffe7aeaf51c057d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 11:26:48 +0200 Subject: [PATCH 55/79] Fix bug with hung connection --- aikido_firewall/background_process/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 4a1c4d732..3e8660b31 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -165,5 +165,7 @@ def poll_config(self, prop): conn = con.Client(self.address, authkey=self.key) conn.send(("READ_PROPERTY", prop)) prop_value = conn.recv() + conn.send(("CLOSE", {})) + conn.close() logger.debug("Received property %s as %s", prop, prop_value) return prop_value From 771c949c6ad5d69b0d95e84432fb9adbb1fee6fb Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 11:40:22 +0200 Subject: [PATCH 56/79] Create get_subdomains_from_url helper function --- .../helpers/get_subdomains_from_url.py | 16 +++++++++++ .../helpers/get_subdomains_from_url_test.py | 27 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 aikido_firewall/helpers/get_subdomains_from_url.py create mode 100644 aikido_firewall/helpers/get_subdomains_from_url_test.py diff --git a/aikido_firewall/helpers/get_subdomains_from_url.py b/aikido_firewall/helpers/get_subdomains_from_url.py new file mode 100644 index 000000000..319e540ee --- /dev/null +++ b/aikido_firewall/helpers/get_subdomains_from_url.py @@ -0,0 +1,16 @@ +""" +Helper function file, see function docstring +""" + +from urllib.parse import urlparse + + +def get_subdomains_from_url(url): + """ + Returns a list with subdomains from url + """ + host = urlparse(url).hostname + if not host: + return [] + parts = host.split(".") + return parts[:-2] diff --git a/aikido_firewall/helpers/get_subdomains_from_url_test.py b/aikido_firewall/helpers/get_subdomains_from_url_test.py new file mode 100644 index 000000000..f0c67918d --- /dev/null +++ b/aikido_firewall/helpers/get_subdomains_from_url_test.py @@ -0,0 +1,27 @@ +import pytest +from aikido_firewall.helpers.get_subdomains_from_url import get_subdomains_from_url + + +def test_get_subdomains_from_url(): + # Test cases with expected results + test_cases = [ + # Test with a standard URL + ("http://tobi.ferrets.example.com", ["tobi", "ferrets"]), + # Test with a URL that has no subdomains + ("http://example.com", []), + # Test with a URL that has multiple subdomains + ("http://a.b.c.example.com", ["a", "b", "c"]), + # Test with a URL that has a port + ("http://tobi.ferrets.example.com:8080", ["tobi", "ferrets"]), + # Test with a URL that has only the main domain + ("http://localhost", []), + # Test with an invalid URL + ("http://.com", []), + # Test with an empty string + ("", []), + # Test with a URL with subdomains and a path + ("http://tobi.ferrets.example.com/path/to/resource", ["tobi", "ferrets"]), + ] + + for url, expected in test_cases: + assert get_subdomains_from_url(url) == expected From f42108d4f8602981013ade6f0b7620be935bc4cb Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 11:41:59 +0200 Subject: [PATCH 57/79] Add subdomains to context --- aikido_firewall/context/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aikido_firewall/context/__init__.py b/aikido_firewall/context/__init__.py index 718345bf3..ce6862fbe 100644 --- a/aikido_firewall/context/__init__.py +++ b/aikido_firewall/context/__init__.py @@ -6,6 +6,7 @@ from urllib.parse import parse_qs from http.cookies import SimpleCookie from aikido_firewall.helpers.build_route_from_url import build_route_from_url +from aikido_firewall.helpers.get_subdomains_from_url import get_subdomains_from_url SUPPORTED_SOURCES = ["django", "flask", "django-gunicorn"] UINPUT_SOURCES = ["body", "cookies", "query", "headers"] @@ -68,6 +69,7 @@ def __init__(self, context_obj=None, req=None, source=None): elif source == "django-gunicorn": self.set_django_gunicorn_attrs(req) self.route = build_route_from_url(self.url) + self.subdomains = get_subdomains_from_url(self.url) def set_django_gunicorn_attrs(self, req): """Set properties that are specific to django-gunicorn""" @@ -108,6 +110,7 @@ def __reduce__(self): "cookies": self.cookies, "source": self.source, "route": self.route, + "subdomains": self.subdomains, }, None, None, From c55eaaa1d7c697ac6268089ecf86be3d465e8d30 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 12:12:25 +0200 Subject: [PATCH 58/79] Create a scheduler in reporting thread, send out heartbeats using it --- .../background_process/__init__.py | 21 +++++++++++++------ .../background_process/heartbeats.py | 15 ++++++------- .../background_process/reporter.py | 5 ++++- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/aikido_firewall/background_process/__init__.py b/aikido_firewall/background_process/__init__.py index 3e8660b31..cb00a112b 100644 --- a/aikido_firewall/background_process/__init__.py +++ b/aikido_firewall/background_process/__init__.py @@ -8,6 +8,7 @@ import secrets import signal import socket +import sched import multiprocessing.connection as con from multiprocessing import Process from threading import Thread @@ -40,8 +41,7 @@ def __init__(self, address, key): pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess self.queue = Queue() - api = ReportingApiHTTP("http://app.local.aikido.io/") - self.reporter = Reporter(should_block(), api, get_token_from_env(), False) + self.reporter = None # Start reporting thread : Thread(target=self.reporting_thread).start() @@ -68,14 +68,23 @@ def __init__(self, address, key): def reporting_thread(self): """Reporting thread""" logger.debug("Started reporting thread") - while True: - self.send_to_reporter() - time.sleep(REPORT_SEC_INTERVAL) + s = sched.scheduler(time.time, time.sleep) # Create an event scheduler + self.send_to_reporter(s) + + api = ReportingApiHTTP("http://app.local.aikido.io/") + # We need to pass along the scheduler so that the heartbeat also gets sent + self.reporter = Reporter(should_block(), api, get_token_from_env(), False, s) + + s.run() - def send_to_reporter(self): + def send_to_reporter(self, event_scheduler): """ Reports the found data to an Aikido server """ + # Add back to event scheduler in REPORT_SEC_INTERVAL secs : + event_scheduler.enter( + REPORT_SEC_INTERVAL, 1, self.send_to_reporter, (event_scheduler,) + ) logger.debug("Checking queue") while not self.queue.empty(): attack = self.queue.get() diff --git a/aikido_firewall/background_process/heartbeats.py b/aikido_firewall/background_process/heartbeats.py index 38133633a..b0c999c5b 100644 --- a/aikido_firewall/background_process/heartbeats.py +++ b/aikido_firewall/background_process/heartbeats.py @@ -2,14 +2,10 @@ The code to send out a heartbeat is in here """ -import sched, time from aikido_firewall.helpers.logging import logger -# Create an event scheduler -s = sched.scheduler(time.time, time.sleep) - -def send_heartbeats_every_x_secs(reporter, interval_in_secs): +def send_heartbeats_every_x_secs(reporter, interval_in_secs, s): """ Start sending out heartbeats every x seconds """ @@ -18,16 +14,17 @@ def send_heartbeats_every_x_secs(reporter, interval_in_secs): return if not reporter.token: logger.debug("No token provided, not starting heartbeats") + return - # Interval already started code ? + logger.debug("Starting heartbeats") - s.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs)) + s.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs, s)) -def send_heartbeat_wrapper(rep, interval_in_secs): +def send_heartbeat_wrapper(rep, interval_in_secs, s): """ Wrapper function for send_heartbeat so we get an interval """ - s.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs)) + s.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs, s)) logger.debug("Heartbeat...") rep.send_heartbeat() diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 48db2e979..f69dd5636 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -9,14 +9,16 @@ from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata from aikido_firewall.helpers.token import Token from aikido_firewall import PKG_VERSION +from aikido_firewall.background_process.heartbeats import send_heartbeats_every_x_secs class Reporter: """Reporter class""" timeout_in_sec = 5 + heartbeat_secs = 10 - def __init__(self, block, api, token, serverless): + def __init__(self, block, api, token, serverless, event_scheduler): self.block = block self.api = api self.token = token # Should be instance of the Token class! @@ -26,6 +28,7 @@ def __init__(self, block, api, token, serverless): self.serverless = serverless self.on_start() + send_heartbeats_every_x_secs(self, self.heartbeat_secs, event_scheduler) def on_detected_attack(self, attack, context): """ From f5b5c3ec91ab92bd40f7fb64ed2377849c969ac0 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 12:20:13 +0200 Subject: [PATCH 59/79] Fix bug with blocking not being updated by using hasattr --- aikido_firewall/background_process/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index f69dd5636..bb0789bfe 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -146,7 +146,7 @@ def update_service_config(self, res): if res["success"] is False: logger.debug(res) return - if res["block"] and res["block"] != self.block: + if "block" in res.keys() and res["block"] != self.block: logger.debug("Updating blocking, setting blocking to : %s", res["block"]) self.block = bool(res["block"]) From 1bfa13af38215ae37841b7aa517ea18fdde1b055 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 13:33:02 +0200 Subject: [PATCH 60/79] Make sure the url is a string --- aikido_firewall/helpers/get_subdomains_from_url.py | 2 ++ aikido_firewall/helpers/get_subdomains_from_url_test.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/aikido_firewall/helpers/get_subdomains_from_url.py b/aikido_firewall/helpers/get_subdomains_from_url.py index 319e540ee..c43e02765 100644 --- a/aikido_firewall/helpers/get_subdomains_from_url.py +++ b/aikido_firewall/helpers/get_subdomains_from_url.py @@ -9,6 +9,8 @@ def get_subdomains_from_url(url): """ Returns a list with subdomains from url """ + if not isinstance(url, str): + return [] host = urlparse(url).hostname if not host: return [] diff --git a/aikido_firewall/helpers/get_subdomains_from_url_test.py b/aikido_firewall/helpers/get_subdomains_from_url_test.py index f0c67918d..06d7bc681 100644 --- a/aikido_firewall/helpers/get_subdomains_from_url_test.py +++ b/aikido_firewall/helpers/get_subdomains_from_url_test.py @@ -21,6 +21,8 @@ def test_get_subdomains_from_url(): ("", []), # Test with a URL with subdomains and a path ("http://tobi.ferrets.example.com/path/to/resource", ["tobi", "ferrets"]), + ({}, []), + (None, []), ] for url, expected in test_cases: From 9877d0068a7b7eba6a01c49e24531b7592fffc01 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:25:46 +0200 Subject: [PATCH 61/79] Don't raise an exception at HTTPApi, just logger.error it --- aikido_firewall/background_process/api/http_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/api/http_api.py b/aikido_firewall/background_process/api/http_api.py index 3374eabdd..f2a707da9 100644 --- a/aikido_firewall/background_process/api/http_api.py +++ b/aikido_firewall/background_process/api/http_api.py @@ -4,6 +4,7 @@ import requests from aikido_firewall.background_process.api import ReportingApi +from aikido_firewall.helpers.logging import logger class ReportingApiHTTP(ReportingApi): @@ -23,7 +24,7 @@ def report(self, token, event, timeout_in_sec): except requests.exceptions.ConnectionError: return {"success": False, "error": "timeout"} except Exception as e: - raise e + logger.error(e) return self.to_api_response(res) From 310f5387f6ae97e88caa6ad41db4e1081e7c9e52 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:35:39 +0200 Subject: [PATCH 62/79] Rename to context_contains_sql_injection --- aikido_firewall/sinks/mysqlclient.py | 6 +++--- aikido_firewall/sinks/pymysql.py | 6 +++--- ...r_sql_injection.py => context_contains_sql_injection.py} | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename aikido_firewall/vulnerabilities/sql_injection/{check_context_for_sql_injection.py => context_contains_sql_injection.py} (94%) diff --git a/aikido_firewall/sinks/mysqlclient.py b/aikido_firewall/sinks/mysqlclient.py index 801df40fe..9cc95fa79 100644 --- a/aikido_firewall/sinks/mysqlclient.py +++ b/aikido_firewall/sinks/mysqlclient.py @@ -7,8 +7,8 @@ from importlib.metadata import version import importhook from aikido_firewall.context import get_current_context -from aikido_firewall.vulnerabilities.sql_injection.check_context_for_sql_injection import ( - check_context_for_sql_injection, +from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import ( + context_contains_sql_injection, ) from aikido_firewall.vulnerabilities.sql_injection.dialects import MySQL from aikido_firewall.helpers.logging import logger @@ -30,7 +30,7 @@ def aikido_new_query(_self, sql): logger.debug("Wrapper - `mysqlclient` version : %s", version("mysqlclient")) context = get_current_context() - result = check_context_for_sql_injection( + result = context_contains_sql_injection( sql.decode("utf-8"), "MySQLdb.connections.query", context, MySQL() ) diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index c1a63c1f0..7ae075e6d 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -8,8 +8,8 @@ from importlib.metadata import version import importhook from aikido_firewall.context import get_current_context -from aikido_firewall.vulnerabilities.sql_injection.check_context_for_sql_injection import ( - check_context_for_sql_injection, +from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import ( + context_contains_sql_injection, ) from aikido_firewall.vulnerabilities.sql_injection.dialects import MySQL from aikido_firewall.background_process import get_comms @@ -33,7 +33,7 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.debug("Wrapper - `pymysql` version : %s", version("pymysql")) context = get_current_context() - result = check_context_for_sql_injection( + result = context_contains_sql_injection( sql, "pymysql.connections.query", context, MySQL() ) diff --git a/aikido_firewall/vulnerabilities/sql_injection/check_context_for_sql_injection.py b/aikido_firewall/vulnerabilities/sql_injection/context_contains_sql_injection.py similarity index 94% rename from aikido_firewall/vulnerabilities/sql_injection/check_context_for_sql_injection.py rename to aikido_firewall/vulnerabilities/sql_injection/context_contains_sql_injection.py index 77c205ae2..41d761156 100644 --- a/aikido_firewall/vulnerabilities/sql_injection/check_context_for_sql_injection.py +++ b/aikido_firewall/vulnerabilities/sql_injection/context_contains_sql_injection.py @@ -11,7 +11,7 @@ from aikido_firewall.context import UINPUT_SOURCES as SOURCES -def check_context_for_sql_injection(sql, operation, context, dialect): +def context_contains_sql_injection(sql, operation, context, dialect): """ This will check the context of the request for SQL Injections """ From 5c50abf1b5650d9be6686396a746be9601f9d88d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:35:55 +0200 Subject: [PATCH 63/79] Rename s to event_scheduler in heartbeats.py --- aikido_firewall/background_process/heartbeats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aikido_firewall/background_process/heartbeats.py b/aikido_firewall/background_process/heartbeats.py index b0c999c5b..96810d843 100644 --- a/aikido_firewall/background_process/heartbeats.py +++ b/aikido_firewall/background_process/heartbeats.py @@ -5,7 +5,7 @@ from aikido_firewall.helpers.logging import logger -def send_heartbeats_every_x_secs(reporter, interval_in_secs, s): +def send_heartbeats_every_x_secs(reporter, interval_in_secs, event_scheduler): """ Start sending out heartbeats every x seconds """ @@ -18,13 +18,13 @@ def send_heartbeats_every_x_secs(reporter, interval_in_secs, s): logger.debug("Starting heartbeats") - s.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs, s)) + event_scheduler.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs, event_scheduler)) -def send_heartbeat_wrapper(rep, interval_in_secs, s): +def send_heartbeat_wrapper(rep, interval_in_secs, event_scheduler): """ Wrapper function for send_heartbeat so we get an interval """ - s.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs, s)) + event_scheduler.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs, event_scheduler)) logger.debug("Heartbeat...") rep.send_heartbeat() From a83ab9157bd35b3868abdec5a12f12d10c6f246d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:40:18 +0200 Subject: [PATCH 64/79] Fix merge issues, add back lost code --- .../aikido_background_process.py | 40 ++++++++++++++----- aikido_firewall/background_process/comms.py | 12 ++++++ .../background_process/heartbeats.py | 11 ++++- aikido_firewall/sinks/pymysql.py | 2 +- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index 1e760af9e..c7fba0445 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -6,9 +6,15 @@ import os import time import signal +import sched from threading import Thread from queue import Queue from aikido_firewall.helpers.logging import logger +from aikido_firewall.background_process.reporter import Reporter +from aikido_firewall.helpers.should_block import should_block +from aikido_firewall.helpers.token import get_token_from_env +from aikido_firewall.background_process.api.http_api import ReportingApiHTTP + REPORT_SEC_INTERVAL = 600 # 10 minutes @@ -31,6 +37,7 @@ def __init__(self, address, key): pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess self.queue = Queue() + self.reporter = None # Start reporting thread : Thread(target=self.reporting_thread).start() @@ -52,21 +59,36 @@ def __init__(self, address, key): conn.close() pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess + elif data[0] == "READ_PROPERTY": + if hasattr(self.reporter, data[1]): + conn.send(self.reporter.__dict__[data[1]]) def reporting_thread(self): """Reporting thread""" logger.debug("Started reporting thread") - while True: - self.send_to_reporter() - time.sleep(REPORT_SEC_INTERVAL) + event_scheduler = sched.scheduler( + time.time, time.sleep + ) # Create an event scheduler + self.send_to_reporter(event_scheduler) + + api = ReportingApiHTTP("http://app.local.aikido.io/") + # We need to pass along the scheduler so that the heartbeat also gets sent + self.reporter = Reporter( + should_block(), api, get_token_from_env(), False, event_scheduler + ) + + event_scheduler.run() - def send_to_reporter(self): + def send_to_reporter(self, event_scheduler): """ Reports the found data to an Aikido server """ - items_to_report = [] + # Add back to event scheduler in REPORT_SEC_INTERVAL secs : + event_scheduler.enter( + REPORT_SEC_INTERVAL, 1, self.send_to_reporter, (event_scheduler,) + ) + logger.debug("Checking queue") while not self.queue.empty(): - items_to_report.append(self.queue.get()) - logger.debug("Reporting to aikido server") - logger.critical("Items to report : %s", items_to_report) - # Currently not making API calls + attack = self.queue.get() + logger.debug("Reporting attack : %s", attack) + self.reporter.on_detected_attack(attack[0], attack[1]) diff --git a/aikido_firewall/background_process/comms.py b/aikido_firewall/background_process/comms.py index ec951c6df..2c08d4be4 100644 --- a/aikido_firewall/background_process/comms.py +++ b/aikido_firewall/background_process/comms.py @@ -81,3 +81,15 @@ def target(address, key, data_array): t.start() # This joins the thread for 3 seconds, afterwards the thread is forced to close (daemon=True) t.join(timeout=3) + + def poll_config(self, prop): + """ + This will poll the config from the Background Process + """ + conn = con.Client(self.address, authkey=self.key) + conn.send(("READ_PROPERTY", prop)) + prop_value = conn.recv() + conn.send(("CLOSE", {})) + conn.close() + logger.debug("Received property %s as %s", prop, prop_value) + return prop_value diff --git a/aikido_firewall/background_process/heartbeats.py b/aikido_firewall/background_process/heartbeats.py index 96810d843..310022c18 100644 --- a/aikido_firewall/background_process/heartbeats.py +++ b/aikido_firewall/background_process/heartbeats.py @@ -18,13 +18,20 @@ def send_heartbeats_every_x_secs(reporter, interval_in_secs, event_scheduler): logger.debug("Starting heartbeats") - event_scheduler.enter(0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs, event_scheduler)) + event_scheduler.enter( + 0, 1, send_heartbeat_wrapper, (reporter, interval_in_secs, event_scheduler) + ) def send_heartbeat_wrapper(rep, interval_in_secs, event_scheduler): """ Wrapper function for send_heartbeat so we get an interval """ - event_scheduler.enter(interval_in_secs, 1, send_heartbeat_wrapper, (rep, interval_in_secs, event_scheduler)) + event_scheduler.enter( + interval_in_secs, + 1, + send_heartbeat_wrapper, + (rep, interval_in_secs, event_scheduler), + ) logger.debug("Heartbeat...") rep.send_heartbeat() diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index 7ae075e6d..d30da8ebb 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -39,7 +39,7 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.info("sql_injection results : %s", json.dumps(result)) if result: - get_comms().send_data("ATTACK", (result, context)) + get_comms().send_data_to_bg_process("ATTACK", (result, context)) should_block = get_comms().poll_config("block") if should_block: raise Exception("SQL Injection [aikido_firewall]") From f76ea568f060ffc556060d6fa5a06e58e778197b Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:43:12 +0200 Subject: [PATCH 65/79] Replac with contains_injection and add blocking code to mysqlclient --- aikido_firewall/sinks/mysqlclient.py | 17 ++++++++++------- aikido_firewall/sinks/pymysql.py | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/aikido_firewall/sinks/mysqlclient.py b/aikido_firewall/sinks/mysqlclient.py index 9cc95fa79..044bf8a92 100644 --- a/aikido_firewall/sinks/mysqlclient.py +++ b/aikido_firewall/sinks/mysqlclient.py @@ -7,11 +7,10 @@ from importlib.metadata import version import importhook from aikido_firewall.context import get_current_context -from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import ( - context_contains_sql_injection, -) +from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import context_contains_sql_injection from aikido_firewall.vulnerabilities.sql_injection.dialects import MySQL from aikido_firewall.helpers.logging import logger +from aikido_firewall.background_process import get_comms @importhook.on_import("MySQLdb.connections") @@ -30,13 +29,17 @@ def aikido_new_query(_self, sql): logger.debug("Wrapper - `mysqlclient` version : %s", version("mysqlclient")) context = get_current_context() - result = context_contains_sql_injection( + contains_injection = context_contains_sql_injection( sql.decode("utf-8"), "MySQLdb.connections.query", context, MySQL() ) - logger.debug("sql_injection results : %s", json.dumps(result)) - if result: - raise Exception("SQL Injection [aikido_firewall]") + logger.debug("sql_injection results : %s", json.dumps(contains_injection)) + if contains_injection: + get_comms().send_data_to_bg_process("ATTACK", (contains_injection, context)) + should_block = get_comms().poll_config("block") + if should_block: + raise Exception("SQL Injection [aikido_firewall]") + return prev_query_function(_self, sql) # pylint: disable=no-member diff --git a/aikido_firewall/sinks/pymysql.py b/aikido_firewall/sinks/pymysql.py index d30da8ebb..88f5e1353 100644 --- a/aikido_firewall/sinks/pymysql.py +++ b/aikido_firewall/sinks/pymysql.py @@ -33,13 +33,13 @@ def aikido_new_query(_self, sql, unbuffered=False): logger.debug("Wrapper - `pymysql` version : %s", version("pymysql")) context = get_current_context() - result = context_contains_sql_injection( + contains_injection = context_contains_sql_injection( sql, "pymysql.connections.query", context, MySQL() ) - logger.info("sql_injection results : %s", json.dumps(result)) - if result: - get_comms().send_data_to_bg_process("ATTACK", (result, context)) + logger.info("sql_injection results : %s", json.dumps(contains_injection)) + if contains_injection: + get_comms().send_data_to_bg_process("ATTACK", (contains_injection, context)) should_block = get_comms().poll_config("block") if should_block: raise Exception("SQL Injection [aikido_firewall]") From 6a9288215004ec5002d09c1050e8c99c8595b839 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:44:33 +0200 Subject: [PATCH 66/79] Linting --- aikido_firewall/sinks/mysqlclient.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/sinks/mysqlclient.py b/aikido_firewall/sinks/mysqlclient.py index 044bf8a92..a8d999c30 100644 --- a/aikido_firewall/sinks/mysqlclient.py +++ b/aikido_firewall/sinks/mysqlclient.py @@ -7,7 +7,9 @@ from importlib.metadata import version import importhook from aikido_firewall.context import get_current_context -from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import context_contains_sql_injection +from aikido_firewall.vulnerabilities.sql_injection.context_contains_sql_injection import ( + context_contains_sql_injection, +) from aikido_firewall.vulnerabilities.sql_injection.dialects import MySQL from aikido_firewall.helpers.logging import logger from aikido_firewall.background_process import get_comms From 87eb9f91f536da6202a6c2acc84010f0b4cc4bd9 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:44:42 +0200 Subject: [PATCH 67/79] Heartbeat every 10 minutes --- aikido_firewall/background_process/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index bb0789bfe..2f435beba 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -16,7 +16,7 @@ class Reporter: """Reporter class""" timeout_in_sec = 5 - heartbeat_secs = 10 + heartbeat_secs = 600 # Heartbeat every 10 minutes def __init__(self, block, api, token, serverless, event_scheduler): self.block = block From 9523f51bc0dfc38e1a0d74b2415b44d9dbdb0a08 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 15:57:22 +0200 Subject: [PATCH 68/79] Report sec interval needs to be near instant (5 seconds) --- aikido_firewall/background_process/aikido_background_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index c7fba0445..800e44bde 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -16,7 +16,7 @@ from aikido_firewall.background_process.api.http_api import ReportingApiHTTP -REPORT_SEC_INTERVAL = 600 # 10 minutes +REPORT_SEC_INTERVAL = 5 # 5 seconds class AikidoBackgroundProcess: From bde23b830774679b19783eacecee07278878dcde Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 17:45:51 +0200 Subject: [PATCH 69/79] Split all helper functions for reporter up into files --- .../background_process/reporter.py | 24 +++---------------- .../helpers/get_current_unixtime_ms.py | 10 ++++++++ aikido_firewall/helpers/get_machine_ip.py | 13 ++++++++++ .../helpers/get_ua_from_context.py | 6 +++++ 4 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 aikido_firewall/helpers/get_current_unixtime_ms.py create mode 100644 aikido_firewall/helpers/get_machine_ip.py create mode 100644 aikido_firewall/helpers/get_ua_from_context.py diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 2f435beba..011bf3e54 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -8,6 +8,9 @@ from aikido_firewall.helpers.logging import logger from aikido_firewall.helpers.limit_length_metadata import limit_length_metadata from aikido_firewall.helpers.token import Token +from aikido_firewall.helpers.get_machine_ip import get_ip +from aikido_firewall.helpers.get_ua_from_context import get_ua_from_context +from aikido_firewall.helpers.get_current_unixtime_ms import get_unixtime_ms from aikido_firewall import PKG_VERSION from aikido_firewall.background_process.heartbeats import send_heartbeats_every_x_secs @@ -149,24 +152,3 @@ def update_service_config(self, res): if "block" in res.keys() and res["block"] != self.block: logger.debug("Updating blocking, setting blocking to : %s", res["block"]) self.block = bool(res["block"]) - - -def get_unixtime_ms(): - """Get the current unix time but in ms""" - return int(time.time() * 1000) - - -def get_ip(): - """Tries to fetch the IP and returns 0.0.0.0 on failure""" - try: - return socket.gethostbyname(socket.gethostname()) - except Exception: - return "0.0.0.0" - - -def get_ua_from_context(context): - """Tries to retrieve the user agent from context""" - for k, v in context.headers.items(): - if k.lower() == "user-agent": - return v - return None diff --git a/aikido_firewall/helpers/get_current_unixtime_ms.py b/aikido_firewall/helpers/get_current_unixtime_ms.py new file mode 100644 index 000000000..a2cb48c11 --- /dev/null +++ b/aikido_firewall/helpers/get_current_unixtime_ms.py @@ -0,0 +1,10 @@ +""" +Helper function file, see function docstring +""" + +import time + + +def get_unixtime_ms(): + """Get the current unix time but in ms""" + return int(time.time() * 1000) diff --git a/aikido_firewall/helpers/get_machine_ip.py b/aikido_firewall/helpers/get_machine_ip.py new file mode 100644 index 000000000..d36eef729 --- /dev/null +++ b/aikido_firewall/helpers/get_machine_ip.py @@ -0,0 +1,13 @@ +""" +Helper function file, see function docstring +""" + +import socket + + +def get_ip(): + """Tries to fetch the IP and returns x.x.x.x on failure""" + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "x.x.x.x" diff --git a/aikido_firewall/helpers/get_ua_from_context.py b/aikido_firewall/helpers/get_ua_from_context.py new file mode 100644 index 000000000..5c182f172 --- /dev/null +++ b/aikido_firewall/helpers/get_ua_from_context.py @@ -0,0 +1,6 @@ +def get_ua_from_context(context): + """Tries to retrieve the user agent from context""" + for k, v in context.headers.items(): + if k.lower() == "user-agent": + return v + return None From d6a5d324b25ec848e09f712d856f6de5ea8ba70d Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 17:49:04 +0200 Subject: [PATCH 70/79] Add tests for get_ua_from_context --- .../helpers/get_ua_from_context_test.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 aikido_firewall/helpers/get_ua_from_context_test.py diff --git a/aikido_firewall/helpers/get_ua_from_context_test.py b/aikido_firewall/helpers/get_ua_from_context_test.py new file mode 100644 index 000000000..9a99d4239 --- /dev/null +++ b/aikido_firewall/helpers/get_ua_from_context_test.py @@ -0,0 +1,54 @@ +import pytest +from aikido_firewall.helpers.get_ua_from_context import get_ua_from_context + + +class Context: + def __init__(self, headers): + self.headers = headers + + +def test_user_agent_present(): + context = Context( + { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" + } + ) + assert ( + get_ua_from_context(context) + == "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" + ) + + +def test_user_agent_present_case_insensitive(): + context = Context( + { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" + } + ) + assert ( + get_ua_from_context(context) + == "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" + ) + + +def test_user_agent_not_present(): + context = Context({"Accept": "text/html", "Content-Type": "application/json"}) + assert get_ua_from_context(context) is None + + +def test_user_agent_empty_value(): + context = Context({"User-Agent": ""}) + assert get_ua_from_context(context) == "" + + +def test_user_agent_with_other_headers(): + context = Context( + { + "Accept": "text/html", + "User-Agent": "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Mobile Safari/537.36", + } + ) + assert ( + get_ua_from_context(context) + == "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Mobile Safari/537.36" + ) From 89dd55cb6365f1e44389e57e950a38b196445f24 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 17:52:33 +0200 Subject: [PATCH 71/79] Add tests for get_unixtime_ms --- .../helpers/get_current_unixtime_ms_test.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 aikido_firewall/helpers/get_current_unixtime_ms_test.py diff --git a/aikido_firewall/helpers/get_current_unixtime_ms_test.py b/aikido_firewall/helpers/get_current_unixtime_ms_test.py new file mode 100644 index 000000000..6aa08678c --- /dev/null +++ b/aikido_firewall/helpers/get_current_unixtime_ms_test.py @@ -0,0 +1,23 @@ +import pytest +from aikido_firewall.helpers.get_current_unixtime_ms import get_unixtime_ms +import time + + +def test_get_unixtime_ms(monkeypatch): + # Mock time.time to return a specific timestamp + monkeypatch.setattr( + time, "time", lambda: 1633072800.123 + ) # Example timestamp in seconds + + # Calculate the expected result in milliseconds + expected_result = int(1633072800.123 * 1000) + + assert get_unixtime_ms() == expected_result + + +def test_get_unixtime_ms_zero(monkeypatch): + # Mock time.time to return zero + monkeypatch.setattr(time, "time", lambda: 0.0) + + # The expected result should be 0 milliseconds + assert get_unixtime_ms() == 0 From 72025714f38afda3606602ec2c0489e906d2a47e Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 17:52:39 +0200 Subject: [PATCH 72/79] Add tests for get_ip --- .../helpers/get_machine_ip_test.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 aikido_firewall/helpers/get_machine_ip_test.py diff --git a/aikido_firewall/helpers/get_machine_ip_test.py b/aikido_firewall/helpers/get_machine_ip_test.py new file mode 100644 index 000000000..2fe2b99ec --- /dev/null +++ b/aikido_firewall/helpers/get_machine_ip_test.py @@ -0,0 +1,25 @@ +import pytest +import socket +from aikido_firewall.helpers.get_machine_ip import get_ip + + +def test_get_ip_success(monkeypatch): + # Mock the socket.gethostname to return a specific hostname + monkeypatch.setattr(socket, "gethostname", lambda: "mocked_hostname") + # Mock the socket.gethostbyname to return a specific IP address + monkeypatch.setattr(socket, "gethostbyname", lambda hostname: "192.168.1.1") + + assert get_ip() == "192.168.1.1" + + +def test_get_ip_failure(monkeypatch): + # Mock the socket.gethostname to return a specific hostname + monkeypatch.setattr(socket, "gethostname", lambda: "mocked_hostname") + # Mock the socket.gethostbyname to raise an exception + monkeypatch.setattr( + socket, + "gethostbyname", + lambda hostname: (_ for _ in ()).throw(Exception("Mocked exception")), + ) + + assert get_ip() == "x.x.x.x" From cdf840ded49ff168cbc02c8cee3a5d5cf7855e70 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:05:44 +0200 Subject: [PATCH 73/79] Bugfix where http_api would still try and parse if an exception occured --- aikido_firewall/background_process/api/http_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aikido_firewall/background_process/api/http_api.py b/aikido_firewall/background_process/api/http_api.py index f2a707da9..3a1a74fb2 100644 --- a/aikido_firewall/background_process/api/http_api.py +++ b/aikido_firewall/background_process/api/http_api.py @@ -25,6 +25,7 @@ def report(self, token, event, timeout_in_sec): return {"success": False, "error": "timeout"} except Exception as e: logger.error(e) + return {"success": False, "error": "unknown"} return self.to_api_response(res) From 6d8bae6d3eb447e9e0ac9b513ae858f5293ed983 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:07:47 +0200 Subject: [PATCH 74/79] Add testing for http_api --- .../background_process/api/http_api_test.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 aikido_firewall/background_process/api/http_api_test.py diff --git a/aikido_firewall/background_process/api/http_api_test.py b/aikido_firewall/background_process/api/http_api_test.py new file mode 100644 index 000000000..b7eeba980 --- /dev/null +++ b/aikido_firewall/background_process/api/http_api_test.py @@ -0,0 +1,72 @@ +import pytest +import requests +from unittest.mock import patch +from aikido_firewall.background_process.api.http_api import ( + ReportingApiHTTP, +) # Replace with the actual module name + +# Sample event data for testing +sample_event = {"event_type": "test_event", "data": {"key": "value"}} + + +def test_report_data_401_code(monkeypatch): + # Create an instance of ReportingApiHTTP + api = ReportingApiHTTP("http://mocked-url.com/") + + # Mock the requests.post method to return a successful response + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def mock_post(url, json, timeout, headers): + return MockResponse({"success": False}, 401) + + monkeypatch.setattr(requests, "post", mock_post) + + # Call the report method + response = api.report("mocked_token", sample_event, 5) + + # Assert the response + assert response == {"success": False, "error": "invalid_token"} + + +def test_report_connection_error(monkeypatch): + # Create an instance of ReportingApiHTTP + api = ReportingApiHTTP("http://mocked-url.com/") + + # Mock the requests.post method to raise a ConnectionError + monkeypatch.setattr( + requests, + "post", + lambda *args, **kwargs: (_ for _ in ()).throw( + requests.exceptions.ConnectionError + ), + ) + + # Call the report method + response = api.report("mocked_token", sample_event, 5) + + # Assert the response + assert response == {"success": False, "error": "timeout"} + + +def test_report_other_exception(monkeypatch): + # Create an instance of ReportingApiHTTP + api = ReportingApiHTTP("http://mocked-url.com/") + + # Mock the requests.post method to raise a generic exception + def mock_post(url, json, timeout, headers): + raise Exception("Some error occurred") + + monkeypatch.setattr(requests, "post", mock_post) + + # Call the report method + response = api.report("mocked_token", sample_event, 5) + + # Assert that the response is None (or however you want to handle it) + assert response["error"] is "unknown" + assert not response["success"] From fce720b2413e42d7d2fae5d71ad7f96b3b6acf08 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:09:03 +0200 Subject: [PATCH 75/79] Add comments clarifying timeout_in_sec variable --- aikido_firewall/background_process/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/reporter.py b/aikido_firewall/background_process/reporter.py index 011bf3e54..5f2abed51 100644 --- a/aikido_firewall/background_process/reporter.py +++ b/aikido_firewall/background_process/reporter.py @@ -18,7 +18,7 @@ class Reporter: """Reporter class""" - timeout_in_sec = 5 + timeout_in_sec = 5 # Timeout of API calls to Aikido Server heartbeat_secs = 600 # Heartbeat every 10 minutes def __init__(self, block, api, token, serverless, event_scheduler): From a359dc3bce0db1e2b4f91163477753dcfac6b163 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:12:53 +0200 Subject: [PATCH 76/79] Add tests for heartbeats.py --- .../background_process/heartbeats_test.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 aikido_firewall/background_process/heartbeats_test.py diff --git a/aikido_firewall/background_process/heartbeats_test.py b/aikido_firewall/background_process/heartbeats_test.py new file mode 100644 index 000000000..34eca8f4c --- /dev/null +++ b/aikido_firewall/background_process/heartbeats_test.py @@ -0,0 +1,62 @@ +import pytest +from unittest.mock import Mock, patch +from aikido_firewall.background_process.heartbeats import ( + send_heartbeats_every_x_secs, + send_heartbeat_wrapper, +) + + +def test_send_heartbeats_serverless(): + reporter = Mock() + reporter.serverless = True + reporter.token = "mocked_token" + event_scheduler = Mock() + + with patch("aikido_firewall.helpers.logging.logger.debug") as mock_debug: + send_heartbeats_every_x_secs(reporter, 5, event_scheduler) + + mock_debug.assert_called_once_with( + "Running in serverless environment, not starting heartbeats" + ) + event_scheduler.enter.assert_not_called() + + +def test_send_heartbeats_no_token(): + reporter = Mock() + reporter.serverless = False + reporter.token = None + event_scheduler = Mock() + + with patch("aikido_firewall.helpers.logging.logger.debug") as mock_debug: + send_heartbeats_every_x_secs(reporter, 5, event_scheduler) + + mock_debug.assert_called_once_with("No token provided, not starting heartbeats") + event_scheduler.enter.assert_not_called() + + +def test_send_heartbeats_success(): + reporter = Mock() + reporter.serverless = False + reporter.token = "mocked_token" + event_scheduler = Mock() + + with patch("aikido_firewall.helpers.logging.logger.debug") as mock_debug: + send_heartbeats_every_x_secs(reporter, 5, event_scheduler) + + mock_debug.assert_called_with("Starting heartbeats") + event_scheduler.enter.assert_called_once_with( + 0, 1, send_heartbeat_wrapper, (reporter, 5, event_scheduler) + ) + + +def test_send_heartbeat_wrapper(): + reporter = Mock() + reporter.send_heartbeat = Mock() + event_scheduler = Mock() + + send_heartbeat_wrapper(reporter, 5, event_scheduler) + + reporter.send_heartbeat.assert_called_once() + event_scheduler.enter.assert_called_once_with( + 5, 1, send_heartbeat_wrapper, (reporter, 5, event_scheduler) + ) From 4e6dbbaa55eb9a2caf8a72d8c098731e422ab86f Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:30:23 +0200 Subject: [PATCH 77/79] REPORT_SEC_INTERVAL to EMPTY_QUEUE_INTERVAL --- .../background_process/aikido_background_process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index 800e44bde..7574d1283 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -16,7 +16,7 @@ from aikido_firewall.background_process.api.http_api import ReportingApiHTTP -REPORT_SEC_INTERVAL = 5 # 5 seconds +EMPTY_QUEUE_INTERVAL = 5 # 5 seconds class AikidoBackgroundProcess: @@ -83,9 +83,9 @@ def send_to_reporter(self, event_scheduler): """ Reports the found data to an Aikido server """ - # Add back to event scheduler in REPORT_SEC_INTERVAL secs : + # Add back to event scheduler in EMPTY_QUEUE_INTERVAL secs : event_scheduler.enter( - REPORT_SEC_INTERVAL, 1, self.send_to_reporter, (event_scheduler,) + EMPTY_QUEUE_INTERVAL, 1, self.send_to_reporter, (event_scheduler,) ) logger.debug("Checking queue") while not self.queue.empty(): From b2203d2c14628578035e68b5f249253fdb3a79ea Mon Sep 17 00:00:00 2001 From: willem-delbare <20814660+willem-delbare@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:31:27 +0200 Subject: [PATCH 78/79] Update aikido_firewall/background_process/aikido_background_process.py --- aikido_firewall/background_process/aikido_background_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index 7574d1283..88f04f9c9 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -59,7 +59,7 @@ def __init__(self, address, key): conn.close() pid = os.getpid() os.kill(pid, signal.SIGTERM) # Kill this subprocess - elif data[0] == "READ_PROPERTY": + elif data[0] == "READ_PROPERTY": # meant to get config props if hasattr(self.reporter, data[1]): conn.send(self.reporter.__dict__[data[1]]) From 34ef86f5a6bbaf84df967ae30875652b634bf624 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 30 Jul 2024 18:33:39 +0200 Subject: [PATCH 79/79] Create a new Reporter using named arguments for clarity --- .../background_process/aikido_background_process.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aikido_firewall/background_process/aikido_background_process.py b/aikido_firewall/background_process/aikido_background_process.py index 88f04f9c9..a9a2a553b 100644 --- a/aikido_firewall/background_process/aikido_background_process.py +++ b/aikido_firewall/background_process/aikido_background_process.py @@ -74,7 +74,11 @@ def reporting_thread(self): api = ReportingApiHTTP("http://app.local.aikido.io/") # We need to pass along the scheduler so that the heartbeat also gets sent self.reporter = Reporter( - should_block(), api, get_token_from_env(), False, event_scheduler + block=should_block(), + api=api, + token=get_token_from_env(), + serverless=False, + event_scheduler=event_scheduler, ) event_scheduler.run()