updating code with claude's latest changes

Author: jeremymanning

docs/notebooks/azure_cloud_tutorial.ipynb

@@ -642,87 +642,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "id": "azure-security-setup",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "def setup_azure_security():\n",
-    "    \"\"\"\n",
-    "    Security configuration templates for Azure resources.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    security_commands = \"\"\"\n",
-    "# Create Network Security Group with restrictive rules\n",
-    "az network nsg create \\\n",
-    "  --name clustrix-nsg \\\n",
-    "  --resource-group clustrix-tutorial-rg\n",
-    "\n",
-    "# Allow SSH only from your IP\n",
-    "az network nsg rule create \\\n",
-    "  --name SSH \\\n",
-    "  --nsg-name clustrix-nsg \\\n",
-    "  --resource-group clustrix-tutorial-rg \\\n",
-    "  --priority 1000 \\\n",
-    "  --source-address-prefixes YOUR_IP/32 \\\n",
-    "  --source-port-ranges '*' \\\n",
-    "  --destination-address-prefixes '*' \\\n",
-    "  --destination-port-ranges 22 \\\n",
-    "  --access Allow \\\n",
-    "  --protocol Tcp\n",
-    "\n",
-    "# Create Key Vault for secrets\n",
-    "az keyvault create \\\n",
-    "  --name clustrix-keyvault-$(date +%s) \\\n",
-    "  --resource-group clustrix-tutorial-rg \\\n",
-    "  --location eastus \\\n",
-    "  --enable-disk-encryption\n",
-    "\n",
-    "# Enable managed identity for VMs\n",
-    "az vm identity assign \\\n",
-    "  --name clustrix-vm-01 \\\n",
-    "  --resource-group clustrix-tutorial-rg\n",
-    "\n",
-    "# Setup private endpoint for storage\n",
-    "az storage account update \\\n",
-    "  --name clustrixstorage \\\n",
-    "  --resource-group clustrix-tutorial-rg \\\n",
-    "  --default-action Deny\n",
-    "\"\"\"\n",
-    "    \n",
-    "    security_checklist = \"\"\"\n",
-    "Azure Security Checklist for Clustrix:\n",
-    "\n",
-    "✓ Use Azure Active Directory for authentication\n",
-    "✓ Enable managed identities instead of service principals when possible\n",
-    "✓ Restrict Network Security Groups to your IP address only\n",
-    "✓ Use private endpoints for storage accounts\n",
-    "✓ Enable disk encryption for all VMs\n",
-    "✓ Use Azure Key Vault for secrets and certificates\n",
-    "✓ Enable Azure Security Center recommendations\n",
-    "✓ Use Azure Private Link for service connectivity\n",
-    "✓ Enable diagnostic logging and monitoring\n",
-    "✓ Implement Azure Policy for compliance\n",
-    "✓ Use Azure Defender for cloud workload protection\n",
-    "✓ Regularly rotate access keys and certificates\n",
-    "✓ Set up cost alerts and spending limits\n",
-    "✓ Tag all resources for governance and cost tracking\n",
-    "\"\"\"\n",
-    "    \n",
-    "    print(\"Azure Security Setup Commands:\")\n",
-    "    print(security_commands)\n",
-    "    print(\"\\nSecurity Checklist:\")\n",
-    "    print(security_checklist)\n",
-    "    \n",
-    "    return {\n",
-    "        'nsg_name': 'clustrix-nsg',\n",
-    "        'keyvault_name': 'clustrix-keyvault',\n",
-    "        'security_commands': security_commands\n",
-    "    }\n",
-    "\n",
-    "security_config = setup_azure_security()\n",
-    "print(\"Security configuration templates generated.\")"
-   ]
+   "source": "def setup_azure_security_for_clustrix(project_id):\n    \"\"\"\n    Security configuration for Azure + Clustrix deployment.\n    \"\"\"\n    \n    security_commands = f\"\"\"\n# Create VPC with private subnets\ngcloud compute networks create clustrix-vpc \\\n  --project {project_id} \\\n  --subnet-mode custom\n\ngcloud compute networks subnets create clustrix-subnet \\\n  --project {project_id} \\\n  --network clustrix-vpc \\\n  --range 10.1.0.0/24 \\\n  --region us-central1 \\\n  --enable-private-ip-google-access\n\n# Create firewall rules (restrictive)\ngcloud compute firewall-rules create clustrix-allow-ssh \\\n  --project {project_id} \\\n  --network clustrix-vpc \\\n  --allow tcp:22 \\\n  --source-ranges YOUR_IP/32 \\\n  --target-tags clustrix\n\ngcloud compute firewall-rules create clustrix-internal \\\n  --project {project_id} \\\n  --network clustrix-vpc \\\n  --allow tcp,udp,icmp \\\n  --source-ranges 10.1.0.0/24 \\\n  --target-tags clustrix\n\n# Create service account with minimal permissions\ngcloud iam service-accounts create clustrix-compute \\\n  --project {project_id} \\\n  --description=\"Service account for Clustrix compute instances\" \\\n  --display-name=\"Clustrix Compute Service Account\"\n\n# Grant only necessary permissions\ngcloud projects add-iam-policy-binding {project_id} \\\n  --member=\"serviceAccount:clustrix-compute@{project_id}.iam.gserviceaccount.com\" \\\n  --role=\"roles/storage.objectAdmin\"\n\ngcloud projects add-iam-policy-binding {project_id} \\\n  --member=\"serviceAccount:clustrix-compute@{project_id}.iam.gserviceaccount.com\" \\\n  --role=\"roles/logging.logWriter\"\n\n# Enable OS Login for better SSH key management\ngcloud compute project-info add-metadata \\\n  --project {project_id} \\\n  --metadata enable-oslogin=TRUE\n\n# Create Cloud KMS key for encryption\ngcloud kms keyrings create clustrix-keyring \\\n  --project {project_id} \\\n  --location global\n\ngcloud kms keys create clustrix-key \\\n  --project {project_id} \\\n  --keyring clustrix-keyring \\\n  --location global \\\n  --purpose encryption\n\"\"\"\n    \n    print(\"Azure Security Setup Commands:\")\n    print(security_commands)\n    \n    return {\n        'project_id': project_id,\n        'vpc_name': 'clustrix-vpc',\n        'subnet_name': 'clustrix-subnet',\n        'service_account': f'clustrix-compute@{project_id}.iam.gserviceaccount.com',\n        'security_commands': security_commands\n    }\n\nsecurity_config = setup_azure_security_for_clustrix('your-project-id')\nprint(\"Security configuration templates generated.\")"
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8jwgahv9skt",
+   "source": "### Azure Security Checklist for Clustrix\n\n✓ **Authentication and Access**\n- Use Azure Active Directory for authentication\n- Enable managed identities instead of service principals when possible\n- Restrict Network Security Groups to your IP address only\n- Use private endpoints for storage accounts\n\n✓ **Infrastructure Security**\n- Enable disk encryption for all VMs\n- Use Azure Key Vault for secrets and certificates\n- Enable Azure Security Center recommendations\n- Use Azure Private Link for service connectivity\n\n✓ **Monitoring and Compliance**\n- Enable diagnostic logging and monitoring\n- Implement Azure Policy for compliance\n- Use Azure Defender for cloud workload protection\n- Regularly rotate access keys and certificates\n\n✓ **Cost and Resource Management**\n- Set up cost alerts and spending limits\n- Tag all resources for governance and cost tracking",
+   "metadata": {}
   },
   {
    "cell_type": "markdown",
@@ -734,103 +663,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "id": "azure-cost-optimization",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "def azure_cost_optimization_guide():\n",
-    "    \"\"\"\n",
-    "    Cost optimization strategies for Azure + Clustrix.\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    cost_tips = \"\"\"\n",
-    "Azure Cost Optimization for Clustrix:\n",
-    "\n",
-    "1. Compute Optimization:\n",
-    "   - Use Azure Spot VMs for non-critical workloads (up to 90% savings)\n",
-    "   - Choose B-series burstable VMs for variable workloads\n",
-    "   - Use reserved instances for predictable workloads (1-3 year terms)\n",
-    "   - Enable auto-shutdown for dev/test VMs\n",
-    "   - Right-size VMs based on actual usage\n",
-    "\n",
-    "2. Storage Optimization:\n",
-    "   - Use appropriate storage tiers (Hot, Cool, Archive)\n",
-    "   - Enable lifecycle management for blob storage\n",
-    "   - Use managed disks with appropriate performance tiers\n",
-    "   - Implement data deduplication and compression\n",
-    "\n",
-    "3. Network Optimization:\n",
-    "   - Minimize data transfer between regions\n",
-    "   - Use Azure CDN for static content\n",
-    "   - Optimize data transfer patterns\n",
-    "\n",
-    "4. Monitoring and Management:\n",
-    "   - Set up budget alerts and spending limits\n",
-    "   - Use Azure Cost Management + Billing\n",
-    "   - Implement proper resource tagging\n",
-    "   - Regular cost reviews and optimizations\n",
-    "\n",
-    "5. Service-Specific:\n",
-    "   - Use Azure Functions for small, event-driven tasks\n",
-    "   - Consider Azure Container Instances for short-running jobs\n",
-    "   - Use Azure Batch for large-scale parallel processing\n",
-    "\"\"\"\n",
-    "    \n",
-    "    cost_monitoring_commands = \"\"\"\n",
-    "# Set up budget alerts\n",
-    "az consumption budget create \\\n",
-    "  --budget-name clustrix-monthly-budget \\\n",
-    "  --amount 100 \\\n",
-    "  --time-grain Monthly \\\n",
-    "  --time-period-start 2025-01-01 \\\n",
-    "  --time-period-end 2025-12-31\n",
-    "\n",
-    "# Get current costs\n",
-    "az consumption usage list \\\n",
-    "  --start-date 2025-01-01 \\\n",
-    "  --end-date 2025-01-31\n",
-    "\n",
-    "# List resource costs\n",
-    "az costmanagement query \\\n",
-    "  --type Usage \\\n",
-    "  --dataset-aggregation '{\"totalCost\":{\"name\":\"PreTaxCost\",\"function\":\"Sum\"}}' \\\n",
-    "  --dataset-grouping name=ResourceGroup type=Dimension\n",
-    "\"\"\"\n",
-    "    \n",
-    "    print(cost_tips)\n",
-    "    print(\"\\nCost Monitoring Commands:\")\n",
-    "    print(cost_monitoring_commands)\n",
-    "    \n",
-    "    return {\n",
-    "        'recommendations': [\n",
-    "            'Use Spot VMs for batch processing',\n",
-    "            'Enable auto-shutdown for dev resources',\n",
-    "            'Implement lifecycle policies for storage',\n",
-    "            'Set up budget alerts',\n",
-    "            'Regular cost reviews'\n",
-    "        ]\n",
-    "    }\n",
-    "\n",
-    "# Example Spot VM configuration for cost savings\n",
-    "def configure_spot_vm():\n",
-    "    \"\"\"Example configuration for using Azure Spot VMs.\"\"\"\n",
-    "    configure(\n",
-    "        cluster_type=\"ssh\",\n",
-    "        cluster_host=\"your-spot-vm-ip\",\n",
-    "        username=\"azureuser\",\n",
-    "        key_file=\"~/.ssh/id_rsa\",\n",
-    "        remote_work_dir=\"/tmp/clustrix\",\n",
-    "        # Spot VMs can be evicted, so use shorter timeouts\n",
-    "        default_time=\"00:30:00\",\n",
-    "        job_poll_interval=60,  # Check more frequently\n",
-    "        cleanup_on_success=True  # Clean up quickly\n",
-    "    )\n",
-    "    print(\"Configured for Azure Spot VMs with appropriate timeouts.\")\n",
-    "\n",
-    "cost_guide = azure_cost_optimization_guide()\n",
-    "print(\"\\nCost optimization guide generated.\")"
-   ]
+   "source": "def azure_cost_optimization_guide():\n    \"\"\"\n    Cost optimization strategies for Azure + Clustrix.\n    \"\"\"\n    \n    cost_monitoring_commands = \"\"\"\n# Set up budget alerts\naz consumption budget create \\\n  --budget-name clustrix-monthly-budget \\\n  --amount 100 \\\n  --time-grain Monthly \\\n  --time-period-start 2025-01-01 \\\n  --time-period-end 2025-12-31\n\n# Get current costs\naz consumption usage list \\\n  --start-date 2025-01-01 \\\n  --end-date 2025-01-31\n\n# List resource costs\naz costmanagement query \\\n  --type Usage \\\n  --dataset-aggregation '{\"totalCost\":{\"name\":\"PreTaxCost\",\"function\":\"Sum\"}}' \\\n  --dataset-grouping name=ResourceGroup type=Dimension\n\"\"\"\n    \n    print(\"Cost Monitoring Commands:\")\n    print(cost_monitoring_commands)\n    \n    return {\n        'recommendations': [\n            'Use Spot VMs for batch processing',\n            'Enable auto-shutdown for dev resources',\n            'Implement lifecycle policies for storage',\n            'Set up budget alerts',\n            'Regular cost reviews'\n        ]\n    }\n\n# Example Spot VM configuration for cost savings\ndef configure_spot_vm():\n    \"\"\"Example configuration for using Azure Spot VMs.\"\"\"\n    configure(\n        cluster_type=\"ssh\",\n        cluster_host=\"your-spot-vm-ip\",\n        username=\"azureuser\",\n        key_file=\"~/.ssh/id_rsa\",\n        remote_work_dir=\"/tmp/clustrix\",\n        # Spot VMs can be evicted, so use shorter timeouts\n        default_time=\"00:30:00\",\n        job_poll_interval=60,  # Check more frequently\n        cleanup_on_success=True  # Clean up quickly\n    )\n    print(\"Configured for Azure Spot VMs with appropriate timeouts.\")\n\ncost_guide = azure_cost_optimization_guide()\nprint(\"\\nCost optimization guide generated.\")"
+  },
+  {
+   "cell_type": "markdown",
+   "id": "scseti9hu",
+   "source": "### Azure Cost Optimization for Clustrix\n\n#### 1. Compute Optimization\n- **Use Azure Spot VMs** for non-critical workloads (up to 90% savings)\n- **Choose B-series burstable VMs** for variable workloads\n- **Use reserved instances** for predictable workloads (1-3 year terms)\n- **Enable auto-shutdown** for dev/test VMs\n- **Right-size VMs** based on actual usage\n\n#### 2. Storage Optimization\n- **Use appropriate storage tiers** (Hot, Cool, Archive)\n- **Enable lifecycle management** for blob storage\n- **Use managed disks** with appropriate performance tiers\n- **Implement data deduplication** and compression\n\n#### 3. Network Optimization\n- **Minimize data transfer** between regions\n- **Use Azure CDN** for static content\n- **Optimize data transfer** patterns\n\n#### 4. Monitoring and Management\n- **Set up budget alerts** and spending limits\n- **Use Azure Cost Management + Billing**\n- **Implement proper resource tagging**\n- **Regular cost reviews** and optimizations\n\n#### 5. Service-Specific\n- **Use Azure Functions** for small, event-driven tasks\n- **Consider Azure Container Instances** for short-running jobs\n- **Use Azure Batch** for large-scale parallel processing",
+   "metadata": {}
   },
   {
    "cell_type": "markdown",