DRAFT: Only do api spec analysis on first 20 hits

.github/workflows/benchmark.yml

@@ -62,6 +62,36 @@ jobs:
         run: |
           k6 run -q ./benchmarks/flask-mysql-benchmarks.js
 
+  benchmark_with_starlette_k6:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Start databases
+        working-directory:  ./sample-apps/databases
+        run: docker compose up --build -d
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies & build
+        run: |
+          python -m pip install --upgrade pip
+          make install && make build
+      - name: Start starlette
+        working-directory: ./sample-apps/starlette-postgres-uvicorn
+        run: nohup make runBenchmark & nohup make runZenDisabled &
+      - name: Install K6
+        uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2 # v1
+      - name: Run flask-mysql k6 Benchmark
+        run: |
+          k6 run -q ./benchmarks/starlette-benchmarks.js
+
   benchmark_with_starlette_app:
     runs-on: ubuntu-latest
     timeout-minutes: 10

aikido_zen/api_discovery/update_route_info.py

@@ -1,12 +1,24 @@
 """Exports update_route_info function"""
 
 from aikido_zen.helpers.logging import logger
+from .get_api_info import get_api_info
 from .merge_data_schemas import merge_data_schemas
 from .merge_auth_types import merge_auth_types
 
 ANALYSIS_ON_FIRST_X_ROUTES = 20
 
 
+def update_route_info_from_context(context, route):
+    """
+    Checks if a route still needs to be updated (only analyzes first x routes),
+    and if so, generates a new api spec and updates the route.
+    """
+    if route["hits"] <= ANALYSIS_ON_FIRST_X_ROUTES:
+        # Only analyze the first x routes for api discovery
+        new_apispec = get_api_info(context)
+        route["apispec"] = update_api_info(new_apispec, route["apispec"])
+
+
 def update_route_info(new_apispec, route):
     """
     Checks if a route still needs to be updated (only analyzes first x routes),

aikido_zen/sources/functions/request_handler.py

@@ -1,8 +1,7 @@
 """Exports request_handler function"""
 
 import aikido_zen.context as ctx
-from aikido_zen.api_discovery.get_api_info import get_api_info
-from aikido_zen.api_discovery.update_route_info import update_route_info
+from aikido_zen.api_discovery.update_route_info import update_route_info_from_context
 from aikido_zen.helpers.is_useful_route import is_useful_route
 from aikido_zen.helpers.logging import logger
 from aikido_zen.thread.thread_cache import get_cache
@@ -88,7 +87,5 @@ def post_response(status_code):
     if cache:
         cache.routes.increment_route(route_metadata)
 
-        # Run API Discovery :
-        update_route_info(
-            new_apispec=get_api_info(context), route=cache.routes.get(route_metadata)
-        )
+        # Run API Discovery
+        update_route_info_from_context(context, route=cache.routes.get(route_metadata))

benchmarks/starlette-benchmarks.js

@@ -0,0 +1,101 @@
+import http from 'k6/http';
+import { check, sleep, fail } from 'k6';
+import exec from 'k6/execution';
+import { Trend } from 'k6/metrics';
+
+const BASE_URL_8086 = 'http://localhost:8102';
+const BASE_URL_8087 = 'http://localhost:8103';
+
+export const options = {
+    vus: 1, // Number of virtual users
+    thresholds: {
+        test_40mb_payload: [{
+            threshold: "avg<15", // This is a higher threshold due to the data being processed
+            abortOnFail: true,
+            delayAbortEval: '10s',
+        }],
+        test_create_with_big_body: [{
+            threshold: "avg<5", 
+            abortOnFail: true,
+            delayAbortEval: '10s',
+        }],
+        test_normal_route: [{
+            threshold: "avg<5", 
+            abortOnFail: true,
+            delayAbortEval: '10s',
+        }],
+        test_id_route: [{
+            threshold: "avg<5", 
+            abortOnFail: true,
+            delayAbortEval: '10s',
+        }],
+    },
+};
+const default_headers = {
+    "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+};
+
+const default_payload = {
+    dog_name: "Pops",
+    other_dogs: Array(2000).fill("Lorem Ipsum"),
+    other_dogs2: Array.from({length: 5000}, () => Math.floor(Math.random() * 99999999)),
+    text_message: "Lorem ipsum dolor sit amet".repeat(3000)
+
+};
+function generateLargeJson(sizeInMB) {
+    const sizeInBytes = sizeInMB * 1024; // Convert MB to Kilobytes
+    let long_text = "b".repeat(sizeInBytes)
+    return {
+        dog_name: "test",
+        long_texts: new Array(1024).fill(long_text)
+    }
+}
+
+function measureRequest(url, method = 'GET', payload, status_code=200, headers=default_headers) {
+    let res;
+    if (method === 'POST') {
+        res = http.post(url, payload, {
+            headers: headers
+        }
+        );
+    } else {
+        res = http.get(url, {
+            headers: headers
+        });
+    }
+    check(res, {
+        'status is correct': (r) => r.status === status_code,
+    });
+    return res.timings.duration; // Return the duration of the request
+}
+
+function route_test(trend, amount, route, method="GET", data=default_payload, status=200) {
+    for (let i = 0; i < amount; i++) {
+        let time_with_fw = measureRequest(BASE_URL_8086 + route, method, data, status)
+        let time_without_fw = measureRequest(BASE_URL_8087 + route, method, data, status)
+        trend.add(time_with_fw - time_without_fw)
+    }
+}
+
+export function handleSummary(data) {
+    for (const [metricName, metricValue] of Object.entries(data.metrics)) {
+        if(!metricName.startsWith('test_') || metricValue.values.avg == 0) {
+            continue
+        }
+        let values = metricValue.values
+        console.log(`🚅 ${metricName}: ΔAverage is ${values.avg.toFixed(2)}ms | ΔMedian is ${values.med.toFixed(2)}ms.`);
+    }
+    return {stdout: ""};
+}
+
+let test_40mb_payload = new Trend('test_40mb_payload')
+let test_create_with_big_body = new Trend("test_create_with_big_body")
+let test_normal_route = new Trend("test_normal_route")
+let test_id_route = new Trend("test_id_route")
+export default function () {
+    route_test(test_40mb_payload, 30, "/create", "POST", generateLargeJson(40)) // 40 Megabytes
+    route_test(test_create_with_big_body, 500, "/create", "POST")
+    route_test(test_normal_route, 500, "/")
+    route_test(test_id_route, 500, "/dogpage/1")
+}

benchmarks/starlette_benchmark.py

@@ -5,7 +5,7 @@
 
 def generate_wrk_command_for_url(url):
     # Define the command with awk included
-    return "wrk -t12 -c400 -d15s " + url
+    return "wrk -t5 -c200 -d15s " + url
 
 def extract_requests_and_latency_tuple(output):
     if output.returncode == 0: