|
708 | 708 | "metadata": {}, |
709 | 709 | "outputs": [], |
710 | 710 | "source": [ |
711 | | - "# Example spot instance configuration\n", |
712 | | - "def configure_spot_instances():\n", |
713 | | - " \"\"\"Example of using spot instances for cost savings.\"\"\"\n", |
714 | | - " configure(\n", |
715 | | - " cluster_type=\"ssh\",\n", |
716 | | - " cluster_host=\"your-spot-instance-ip\",\n", |
717 | | - " username=\"ec2-user\",\n", |
718 | | - " key_file=\"~/.ssh/aws-clustrix-key\",\n", |
719 | | - " remote_work_dir=\"/tmp/clustrix\",\n", |
720 | | - " # Spot instances can be terminated, so use short timeouts\n", |
721 | | - " default_time=\"00:30:00\",\n", |
722 | | - " job_poll_interval=60, # Check more frequently\n", |
723 | | - " cleanup_on_success=True # Clean up quickly\n", |
724 | | - " )\n", |
725 | | - " print(\"✓ Configured for spot instances with appropriate timeouts.\")\n", |
726 | | - " return \"Spot instance configuration completed.\"\n", |
727 | | - "\n", |
728 | | - "# Example function to check current spot prices\n", |
729 | | - "def check_spot_prices(instance_types=['t3.large', 't3.xlarge', 'c5.large'], region='us-east-1'):\n", |
730 | | - " \"\"\"Check current spot prices for different instance types.\"\"\"\n", |
731 | | - " ec2 = boto3.client('ec2', region_name=region)\n", |
| 711 | + "# Import Clustrix cost monitoring for AWS\n", |
| 712 | + "from clustrix import cost_tracking_decorator, get_cost_monitor, generate_cost_report, get_pricing_info\n", |
| 713 | + "\n", |
| 714 | + "# Example 1: Cost tracking with AWS instances\n", |
| 715 | + "@cost_tracking_decorator('aws', 'p3.2xlarge')\n", |
| 716 | + "@cluster(cores=8, memory=\"60GB\")\n", |
| 717 | + "def aws_training_with_cost_tracking():\n", |
| 718 | + " \"\"\"Example training function with AWS cost tracking.\"\"\"\n", |
| 719 | + " import time\n", |
| 720 | + " import numpy as np\n", |
732 | 721 | " \n", |
733 | | - " try:\n", |
734 | | - " response = ec2.describe_spot_price_history(\n", |
735 | | - " InstanceTypes=instance_types,\n", |
736 | | - " MaxResults=len(instance_types),\n", |
737 | | - " ProductDescriptions=['Linux/UNIX']\n", |
738 | | - " )\n", |
| 722 | + " print(\"Starting AWS training with cost monitoring...\")\n", |
| 723 | + " time.sleep(3) # Simulate training\n", |
| 724 | + " \n", |
| 725 | + " # Simulate GPU workload\n", |
| 726 | + " data = np.random.randn(2000, 2000)\n", |
| 727 | + " result = np.linalg.svd(data)\n", |
| 728 | + " \n", |
| 729 | + " print(\"Training completed!\")\n", |
| 730 | + " return {'accuracy': 0.92, 'epochs': 50}\n", |
| 731 | + "\n", |
| 732 | + "# Example 2: Compare AWS pricing\n", |
| 733 | + "def compare_aws_pricing():\n", |
| 734 | + " \"\"\"Compare AWS EC2 pricing for different instance types.\"\"\"\n", |
| 735 | + " pricing = get_pricing_info('aws')\n", |
| 736 | + " if pricing:\n", |
| 737 | + " print(\"AWS EC2 On-Demand Pricing (USD/hour):\")\n", |
| 738 | + " \n", |
| 739 | + " # Group by category\n", |
| 740 | + " gpu_instances = {k: v for k, v in pricing.items() if k.startswith(('p3', 'p4d', 'g4dn'))}\n", |
| 741 | + " compute_instances = {k: v for k, v in pricing.items() if k.startswith('c5')}\n", |
| 742 | + " memory_instances = {k: v for k, v in pricing.items() if k.startswith('r5')}\n", |
| 743 | + " \n", |
| 744 | + " print(\"\\nGPU Instances:\")\n", |
| 745 | + " for instance, price in sorted(gpu_instances.items(), key=lambda x: x[1]):\n", |
| 746 | + " print(f\" {instance:<20}: ${price:.3f}/hour\")\n", |
| 747 | + " \n", |
| 748 | + " print(\"\\nCompute Optimized:\")\n", |
| 749 | + " for instance, price in sorted(compute_instances.items(), key=lambda x: x[1]):\n", |
| 750 | + " print(f\" {instance:<20}: ${price:.3f}/hour\")\n", |
739 | 751 | " \n", |
740 | | - " for price in response['SpotPriceHistory']:\n", |
741 | | - " print(f\"{price['InstanceType']}: ${price['SpotPrice']}/hour in {price['AvailabilityZone']}\")\n", |
| 752 | + " print(\"\\nMemory Optimized:\")\n", |
| 753 | + " for instance, price in sorted(memory_instances.items(), key=lambda x: x[1]):\n", |
| 754 | + " print(f\" {instance:<20}: ${price:.3f}/hour\")\n", |
| 755 | + "\n", |
| 756 | + "# Example 3: AWS Spot vs On-Demand cost analysis\n", |
| 757 | + "def aws_spot_cost_analysis():\n", |
| 758 | + " \"\"\"Analyze potential savings with AWS Spot instances.\"\"\"\n", |
| 759 | + " monitor = get_cost_monitor('aws')\n", |
| 760 | + " if monitor:\n", |
| 761 | + " print(\"AWS Spot Instance Savings Analysis:\")\n", |
| 762 | + " print(\"-\" * 40)\n", |
| 763 | + " \n", |
| 764 | + " instance_types = ['p3.2xlarge', 'p3.8xlarge', 'g4dn.xlarge', 'c5.large']\n", |
| 765 | + " \n", |
| 766 | + " for instance in instance_types:\n", |
| 767 | + " on_demand = monitor.estimate_cost(instance, 1.0, use_spot=False)\n", |
| 768 | + " spot = monitor.estimate_cost(instance, 1.0, use_spot=True)\n", |
| 769 | + " savings = ((on_demand.hourly_rate - spot.hourly_rate) / on_demand.hourly_rate) * 100\n", |
742 | 770 | " \n", |
743 | | - " except Exception as e:\n", |
744 | | - " print(f\"✗ Error checking spot prices: {e}\")\n", |
| 771 | + " print(f\"{instance}:\")\n", |
| 772 | + " print(f\" On-Demand: ${on_demand.hourly_rate:.3f}/hour\")\n", |
| 773 | + " print(f\" Spot: ${spot.hourly_rate:.3f}/hour\")\n", |
| 774 | + " print(f\" Savings: {savings:.1f}%\")\n", |
| 775 | + " print()\n", |
| 776 | + "\n", |
| 777 | + "# Example 4: AWS Batch cost estimation\n", |
| 778 | + "def estimate_aws_batch_costs():\n", |
| 779 | + " \"\"\"Estimate costs for AWS Batch workloads.\"\"\"\n", |
| 780 | + " monitor = get_cost_monitor('aws')\n", |
| 781 | + " if monitor:\n", |
| 782 | + " batch_estimate = monitor.estimate_batch_cost(\n", |
| 783 | + " job_queue=\"clustrix-batch-queue\",\n", |
| 784 | + " compute_environment=\"clustrix-compute-env\",\n", |
| 785 | + " estimated_jobs=100,\n", |
| 786 | + " avg_job_duration_hours=0.25\n", |
| 787 | + " )\n", |
| 788 | + " \n", |
| 789 | + " print(\"AWS Batch Cost Estimation:\")\n", |
| 790 | + " print(f\" Job Queue: {batch_estimate['job_queue']}\")\n", |
| 791 | + " print(f\" Total Jobs: {batch_estimate['estimated_jobs']}\")\n", |
| 792 | + " print(f\" Avg Duration: {batch_estimate['avg_job_duration_hours']} hours\")\n", |
| 793 | + " print(f\" Total Compute Hours: {batch_estimate['total_compute_hours']}\")\n", |
| 794 | + " print(f\" Estimated Cost: ${batch_estimate['estimated_cost']:.2f}\")\n", |
| 795 | + " print(f\" Cost per Job: ${batch_estimate['cost_per_job']:.4f}\")\n", |
| 796 | + "\n", |
| 797 | + "# Example 5: Regional pricing comparison\n", |
| 798 | + "def compare_aws_regions():\n", |
| 799 | + " \"\"\"Compare AWS pricing across different regions.\"\"\"\n", |
| 800 | + " monitor = get_cost_monitor('aws')\n", |
| 801 | + " if monitor:\n", |
| 802 | + " print(\"AWS Regional Pricing Comparison for p3.2xlarge:\")\n", |
| 803 | + " print(\"-\" * 50)\n", |
| 804 | + " \n", |
| 805 | + " regional_pricing = monitor.get_region_pricing_comparison('p3.2xlarge')\n", |
| 806 | + " for region, pricing_info in regional_pricing.items():\n", |
| 807 | + " print(f\"{region}:\")\n", |
| 808 | + " print(f\" On-Demand: ${pricing_info['on_demand_hourly']:.3f}/hour\")\n", |
| 809 | + " print(f\" Est. Spot: ${pricing_info['estimated_spot_hourly']:.3f}/hour\")\n", |
| 810 | + " print()\n", |
| 811 | + "\n", |
| 812 | + "# Example 6: Real-time AWS cost monitoring\n", |
| 813 | + "def monitor_aws_costs():\n", |
| 814 | + " \"\"\"Monitor current AWS resource usage and costs.\"\"\"\n", |
| 815 | + " report = generate_cost_report('aws', 'p3.2xlarge')\n", |
| 816 | + " if report:\n", |
| 817 | + " print(\"Current AWS Resource Status:\")\n", |
| 818 | + " print(f\" CPU Usage: {report['resource_usage']['cpu_percent']:.1f}%\")\n", |
| 819 | + " print(f\" Memory Usage: {report['resource_usage']['memory_percent']:.1f}%\")\n", |
| 820 | + " if report['resource_usage']['gpu_stats']:\n", |
| 821 | + " print(f\" GPU Count: {len(report['resource_usage']['gpu_stats'])}\")\n", |
| 822 | + " print(f\" Hourly Rate: ${report['cost_estimate']['hourly_rate']:.3f}\")\n", |
| 823 | + " \n", |
| 824 | + " if report['recommendations']:\n", |
| 825 | + " print(\"\\nCost Optimization Recommendations:\")\n", |
| 826 | + " for rec in report['recommendations']:\n", |
| 827 | + " print(f\" • {rec}\")\n", |
| 828 | + "\n", |
| 829 | + "# Run examples\n", |
| 830 | + "print(\"AWS Cost Monitoring Examples:\")\n", |
| 831 | + "print(\"=\" * 40)\n", |
| 832 | + "\n", |
| 833 | + "print(\"\\n1. AWS Pricing Comparison:\")\n", |
| 834 | + "compare_aws_pricing()\n", |
| 835 | + "\n", |
| 836 | + "print(\"\\n2. Spot Instance Savings Analysis:\")\n", |
| 837 | + "aws_spot_cost_analysis()\n", |
| 838 | + "\n", |
| 839 | + "print(\"\\n3. AWS Batch Cost Estimation:\")\n", |
| 840 | + "estimate_aws_batch_costs()\n", |
| 841 | + "\n", |
| 842 | + "print(\"\\n4. Regional Pricing Comparison:\")\n", |
| 843 | + "compare_aws_regions()\n", |
| 844 | + "\n", |
| 845 | + "print(\"\\n5. Current AWS Status:\")\n", |
| 846 | + "monitor_aws_costs()\n", |
745 | 847 | "\n", |
746 | | - "# Uncomment to check current spot prices:\n", |
747 | | - "# check_spot_prices()" |
| 848 | + "print(\"\\n✅ AWS cost monitoring examples ready!\")\n", |
| 849 | + "print(\"💡 Use @cost_tracking_decorator('aws', 'instance_type') for automatic cost tracking\")" |
748 | 850 | ] |
749 | 851 | }, |
750 | 852 | { |
|
0 commit comments