diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 20e22e054..acd09fbc3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -62,6 +62,36 @@ jobs: run: | k6 run -q ./benchmarks/flask-mysql-benchmarks.js + benchmark_with_starlette_k6: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Start databases + working-directory: ./sample-apps/databases + run: docker compose up --build -d + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies & build + run: | + python -m pip install --upgrade pip + make install && make build + - name: Start starlette + working-directory: ./sample-apps/starlette-postgres-uvicorn + run: nohup make runBenchmark & nohup make runZenDisabled & + - name: Install K6 + uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2 # v1 + - name: Run flask-mysql k6 Benchmark + run: | + k6 run -q ./benchmarks/starlette-benchmarks.js + benchmark_with_starlette_app: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/aikido_zen/api_discovery/update_route_info.py b/aikido_zen/api_discovery/update_route_info.py index 5ddcd97f1..6065c14dd 100644 --- a/aikido_zen/api_discovery/update_route_info.py +++ b/aikido_zen/api_discovery/update_route_info.py @@ -1,12 +1,24 @@ """Exports update_route_info function""" from aikido_zen.helpers.logging import logger +from .get_api_info import get_api_info from .merge_data_schemas import merge_data_schemas from .merge_auth_types import merge_auth_types ANALYSIS_ON_FIRST_X_ROUTES = 20 +def update_route_info_from_context(context, route): + """ + Checks if a route still needs to be updated (only analyzes first x routes), + and if so, generates a new api spec and updates the route. + """ + if route["hits"] <= ANALYSIS_ON_FIRST_X_ROUTES: + # Only analyze the first x routes for api discovery + new_apispec = get_api_info(context) + route["apispec"] = update_api_info(new_apispec, route["apispec"]) + + def update_route_info(new_apispec, route): """ Checks if a route still needs to be updated (only analyzes first x routes), diff --git a/aikido_zen/sources/functions/request_handler.py b/aikido_zen/sources/functions/request_handler.py index 16b58788b..54595a982 100644 --- a/aikido_zen/sources/functions/request_handler.py +++ b/aikido_zen/sources/functions/request_handler.py @@ -1,8 +1,7 @@ """Exports request_handler function""" import aikido_zen.context as ctx -from aikido_zen.api_discovery.get_api_info import get_api_info -from aikido_zen.api_discovery.update_route_info import update_route_info +from aikido_zen.api_discovery.update_route_info import update_route_info_from_context from aikido_zen.helpers.is_useful_route import is_useful_route from aikido_zen.helpers.logging import logger from aikido_zen.thread.thread_cache import get_cache @@ -88,7 +87,5 @@ def post_response(status_code): if cache: cache.routes.increment_route(route_metadata) - # Run API Discovery : - update_route_info( - new_apispec=get_api_info(context), route=cache.routes.get(route_metadata) - ) + # Run API Discovery + update_route_info_from_context(context, route=cache.routes.get(route_metadata)) diff --git a/benchmarks/starlette-benchmarks.js b/benchmarks/starlette-benchmarks.js new file mode 100644 index 000000000..8bbe3a857 --- /dev/null +++ b/benchmarks/starlette-benchmarks.js @@ -0,0 +1,101 @@ +import http from 'k6/http'; +import { check, sleep, fail } from 'k6'; +import exec from 'k6/execution'; +import { Trend } from 'k6/metrics'; + +const BASE_URL_8086 = 'http://localhost:8102'; +const BASE_URL_8087 = 'http://localhost:8103'; + +export const options = { + vus: 1, // Number of virtual users + thresholds: { + test_40mb_payload: [{ + threshold: "avg<15", // This is a higher threshold due to the data being processed + abortOnFail: true, + delayAbortEval: '10s', + }], + test_create_with_big_body: [{ + threshold: "avg<5", + abortOnFail: true, + delayAbortEval: '10s', + }], + test_normal_route: [{ + threshold: "avg<5", + abortOnFail: true, + delayAbortEval: '10s', + }], + test_id_route: [{ + threshold: "avg<5", + abortOnFail: true, + delayAbortEval: '10s', + }], + }, +}; +const default_headers = { + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", +}; + +const default_payload = { + dog_name: "Pops", + other_dogs: Array(2000).fill("Lorem Ipsum"), + other_dogs2: Array.from({length: 5000}, () => Math.floor(Math.random() * 99999999)), + text_message: "Lorem ipsum dolor sit amet".repeat(3000) + +}; +function generateLargeJson(sizeInMB) { + const sizeInBytes = sizeInMB * 1024; // Convert MB to Kilobytes + let long_text = "b".repeat(sizeInBytes) + return { + dog_name: "test", + long_texts: new Array(1024).fill(long_text) + } +} + +function measureRequest(url, method = 'GET', payload, status_code=200, headers=default_headers) { + let res; + if (method === 'POST') { + res = http.post(url, payload, { + headers: headers + } + ); + } else { + res = http.get(url, { + headers: headers + }); + } + check(res, { + 'status is correct': (r) => r.status === status_code, + }); + return res.timings.duration; // Return the duration of the request +} + +function route_test(trend, amount, route, method="GET", data=default_payload, status=200) { + for (let i = 0; i < amount; i++) { + let time_with_fw = measureRequest(BASE_URL_8086 + route, method, data, status) + let time_without_fw = measureRequest(BASE_URL_8087 + route, method, data, status) + trend.add(time_with_fw - time_without_fw) + } +} + +export function handleSummary(data) { + for (const [metricName, metricValue] of Object.entries(data.metrics)) { + if(!metricName.startsWith('test_') || metricValue.values.avg == 0) { + continue + } + let values = metricValue.values + console.log(`🚅 ${metricName}: ΔAverage is ${values.avg.toFixed(2)}ms | ΔMedian is ${values.med.toFixed(2)}ms.`); + } + return {stdout: ""}; +} + +let test_40mb_payload = new Trend('test_40mb_payload') +let test_create_with_big_body = new Trend("test_create_with_big_body") +let test_normal_route = new Trend("test_normal_route") +let test_id_route = new Trend("test_id_route") +export default function () { + route_test(test_40mb_payload, 30, "/create", "POST", generateLargeJson(40)) // 40 Megabytes + route_test(test_create_with_big_body, 500, "/create", "POST") + route_test(test_normal_route, 500, "/") + route_test(test_id_route, 500, "/dogpage/1") +} diff --git a/benchmarks/starlette_benchmark.py b/benchmarks/starlette_benchmark.py index 920112e5e..6e4dc77f1 100644 --- a/benchmarks/starlette_benchmark.py +++ b/benchmarks/starlette_benchmark.py @@ -5,7 +5,7 @@ def generate_wrk_command_for_url(url): # Define the command with awk included - return "wrk -t12 -c400 -d15s " + url + return "wrk -t5 -c200 -d15s " + url def extract_requests_and_latency_tuple(output): if output.returncode == 0: