Create autoscaling_script.ipynb

Added an example Junyper Notebook.

Author: wierdvanderhaar

topic/autoscaling/autoscaling_script.ipynb

@@ -0,0 +1,126 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "4c9711ff-fe70-45d1-a79f-d71df86e64cd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from requests.auth import HTTPBasicAuth\n",
+    "from datetime import datetime\n",
+    "\n",
+    "auth_data = HTTPBasicAuth('API-KEY', 'API-SECRET')\n",
+    "\n",
+    "date_format = '%Y-%m-%dT%H:%M:%S.%f'\n",
+    "\n",
+    "def get_metrics(organization_id):\n",
+    "    api_url = f'https://console.cratedb.cloud/api/v2/organizations/{organization_id}/metrics/prometheus/'\n",
+    "    headers = {'accept': 'plain/text'}\n",
+    "    response = requests.get(api_url, auth=auth_data)\n",
+    "    return response.text\n",
+    "\n",
+    "def num_shards(metrics):\n",
+    "    for row in metrics.split('\\n'):\n",
+    "        line = row.split(' ')\n",
+    "        if line[0].startswith('crate_node{name=\"shard_stats\"') and 'property=\"total\"' in line[0] and 'crate-data-hot-97ebb337-b44e-446f-adca-a092fd1df0aa' in line[0]:\n",
+    "            return float(line[1])\n",
+    "\n",
+    "def get_cluster_status(cluster_id):\n",
+    "    api_url = f'https://console.cratedb.cloud/api/v2/clusters/{cluster_id}/'\n",
+    "    return requests.get(api_url, auth=auth_data).json()\n",
+    "\n",
+    "def get_cluster_running_op(cluster_status, cluster_id):\n",
+    "    if (datetime.now() - datetime.strptime(cluster_status['health']['last_seen'], date_format)).seconds > 30:\n",
+    "        cluster_status = get_cluster_status(cluster_id)\n",
+    "    return cluster_status['health']['running_operation']\n",
+    "\n",
+    "def get_cluster_num_nodes(cluster_status, cluster_id):\n",
+    "    if (datetime.now() - datetime.strptime(cluster_status['health']['last_seen'], date_format)).seconds > 30:\n",
+    "        cluster_status = get_cluster_status(cluster_id)\n",
+    "    return cluster_status['num_nodes'] - 1\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c14c779c-1809-4e19-b995-e850cfa667ba",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Current number of shards: 25.0\n",
+      "Start scaling out from 3 to 4\n"
+     ]
+    }
+   ],
+   "source": [
+    "import time\n",
+    "\n",
+    "delay_seconds = 5\n",
+    "max_num_shard = 30\n",
+    "\n",
+    "organization_id = 'Use your organisation ID here'\n",
+    "cluster_id = 'Use your Cluster ID here'\n",
+    "\n",
+    "api_url = f'https://console.cratedb.cloud/api/v2/clusters/{cluster_id}/scale/'\n",
+    "\n",
+    "while True:\n",
+    "    metrics = get_metrics(organization_id)  # Pass organization_id to get_metrics function\n",
+    "    cluster_status = get_cluster_status(cluster_id)\n",
+    "    num = num_shards(metrics)\n",
+    "    if num is not None:  # Check if num is not None\n",
+    "        print(f'Current number of shards: {num}')\n",
+    "        if num > (0.8 * max_num_shard):\n",
+    "            num_nodes = get_cluster_num_nodes(cluster_status, cluster_id)  # Pass cluster_id to get_cluster_num_nodes function\n",
+    "            print(f'Start scaling out from {num_nodes + 1} to {num_nodes + 2}')\n",
+    "            requests.put(api_url, json={'product_unit':num_nodes + 1}, auth=auth_data)\n",
+    "            time.sleep(delay_seconds)\n",
+    "            while get_cluster_running_op(cluster_status, cluster_id) != '':\n",
+    "                cluster_status = get_cluster_status(cluster_id)\n",
+    "                continue\n",
+    "            print('Scaled up successfully!')\n",
+    "                \n",
+    "        elif num < (0.5 * max_num_shard):\n",
+    "            num_nodes = get_cluster_num_nodes(cluster_status, cluster_id)  # Pass cluster_id to get_cluster_num_nodes function\n",
+    "            print(f'Start scaling down from {num_nodes + 1} to {num_nodes}')\n",
+    "            requests.put(api_url, json={'product_unit':num_nodes - 1}, auth=auth_data)\n",
+    "            time.sleep(delay_seconds)\n",
+    "            while get_cluster_running_op(cluster_status, cluster_id) != '':\n",
+    "                cluster_status = get_cluster_status(cluster_id)\n",
+    "                continue\n",
+    "            print('Scaled down successfully!')\n",
+    "        else:\n",
+    "            print('Nothing to do!')\n",
+    "    else:\n",
+    "        print('Failed to retrieve shard metrics.')\n",
+    "    time.sleep(delay_seconds)\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}