Skip to content

Commit 47efc06

Browse files
git-user-9git-user-9
committed
Integrated AWS cloudwatch and AWS SNS
1 parent 29f8bf3 commit 47efc06

File tree

12 files changed

+260
-20
lines changed

12 files changed

+260
-20
lines changed

‎.github/workflows/deploy.yml

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: EC2 Deploy
33
on:
44
push:
55
branches:
6-
- devops/a4
6+
- devops/a5
77
tags:
88
- deploy-dev
99
- deploy-prod
@@ -107,11 +107,11 @@ jobs:
107107
- name: Validate App Health
108108
run: |
109109
echo -e "\n📦 Full Response from App:\n"
110-
curl -s http://${{ env.INSTANCE_IP }}:80 || echo "❌ Failed to get response"
110+
curl -s http://${{ env.INSTANCE_IP }}:80/hello || echo "❌ Failed to get response"
111111
echo -e "\n"
112-
echo "Checking app health at http://${{ env.INSTANCE_IP }}:80"
112+
echo "Checking app health at http://${{ env.INSTANCE_IP }}:80/hello"
113113
for i in {1..10}; do
114-
STATUS=$(curl -o /dev/null -s -w "%{http_code}" http://${{ env.INSTANCE_IP }}:80)
114+
STATUS=$(curl -o /dev/null -s -w "%{http_code}" http://${{ env.INSTANCE_IP }}:80/hello)
115115
if [[ "$STATUS" == "200" ]]; then
116116
echo "✅ App is healthy (HTTP 200)"
117117
exit 0
@@ -123,6 +123,26 @@ jobs:
123123
echo "❌ App failed health check"
124124
exit 1
125125
126+
127+
# Inject ERROR/EXCEPTION into App Log
128+
- name: Inject ERROR/EXCEPTION into App Log via SSH
129+
run: |
130+
131+
echo "🔐 Injecting fake error/exception logs into /var/log/my-app.log on App EC2"
132+
133+
for attempt in {1..5}; do
134+
ssh -o StrictHostKeyChecking=no ubuntu@${INSTANCE_IP} "echo '✅ SSH to App EC2 successful'" && break
135+
echo "⏳ App EC2 not ready for SSH (attempt $attempt)..."
136+
sleep 15
137+
done
138+
139+
ssh -o StrictHostKeyChecking=no ubuntu@${INSTANCE_IP} <<EOF
140+
echo "Error: Simulated failure on \$(date)" >> /var/log/my-app.log
141+
echo "Exception: Simulated exception on \$(date)" >> /var/log/my-app.log
142+
EOF
143+
144+
echo "✅ Injected fake error/exception logs into /var/log/my-app.log"
145+
126146
# Provision Verifier EC2
127147
- name: Terraform Apply Verifier EC2
128148
working-directory: ${{ env.TF_WORKING_DIR }}

‎scripts/deploy.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ sleep 200
3131
RAW_INSTANCE_IP=$(terraform output -raw instance_public_ip)
3232

3333
echo -e "\n"
34-
echo "[+] Testing app on http://$RAW_INSTANCE_IP:80"
34+
echo "[+] Testing app on http://$RAW_INSTANCE_IP:80/hello"
3535
echo -e "\n"
3636

3737
echo -e "\n"
38-
curl "http://$RAW_INSTANCE_IP:80"
38+
curl "http://$RAW_INSTANCE_IP:80/hello"
3939
echo -e "\n"
4040
echo -e "\n"
4141

@@ -46,7 +46,7 @@ terraform apply -var-file="$CONFIG_FILE" -target=aws_instance.log_verifier -auto
4646
VERIFIER_IP=$(terraform output -raw verifier_instance_public_ip)
4747

4848

49-
echo "Verified Public IP: $VERIFIER_IP"
49+
echo "Verifier instance Public IP: $VERIFIER_IP"
5050

5151

5252
# #To verify and pull logs from ec2 to local.
@@ -65,8 +65,8 @@ curl "http://$RAW_INSTANCE_IP:80"
6565
echo -e "\n"
6666
echo -e "\n"
6767

68-
echo "Terraform destroy will run after 2 minutes..."
68+
echo "Terraform destroy will run after 10 minutes..."
6969
echo "You can press ctrl+c and do it earlier as well"
70-
sleep 120
70+
sleep 600
7171

7272
terraform destroy -var-file="$CONFIG_FILE" -auto-approve

‎scripts/dev_script.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,56 @@ git clone ${repo_url} app
2323
cd app
2424
mvn clean package
2525

26+
# To fix permission issues
27+
sudo touch /var/log/my-app.log
28+
sudo chown ubuntu:ubuntu /var/log/my-app.log
29+
30+
2631
# Run the Java app
2732
nohup java -jar target/*.jar --server.port=80 > /var/log/my-app.log 2>&1 &
2833

2934
# Wait for app to run
3035
sleep 5
3136

37+
38+
39+
# ------------------------------------------------------------------------------
40+
# Install and Configure CloudWatch Agent (Ubuntu)
41+
# ------------------------------------------------------------------------------
42+
# sudo apt-get update
43+
# sudo apt-get install collectd -y
44+
45+
# echo "Downloading CloudWatch Agent .deb..."
46+
# wget https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -O /tmp/amazon-cloudwatch-agent.deb
47+
48+
# echo "Installing CloudWatch Agent..."
49+
# sudo dpkg -i /tmp/amazon-cloudwatch-agent.deb
50+
51+
echo 'export PATH=$PATH:/opt/aws/amazon-cloudwatch-agent/bin' >> ~/.bashrc
52+
source ~/.bashrc
53+
54+
55+
echo "Writing CloudWatch Agent config..."
56+
cat << 'EOF' > /opt/aws/amazon-cloudwatch-agent/bin/config.json
57+
${cw_agent_config_json}
58+
EOF
59+
60+
echo "Starting CloudWatch Agent..."
61+
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
62+
-a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json
63+
64+
# Wait a bit to ensure CW agent starts
65+
sleep 10
66+
67+
# Inject test log entry for monitoring validation
68+
echo "Error: Simulated a fake error failure on $(date)" >> /var/log/my-app.log
69+
echo "Exception: Simulated a fake exception for testing on $(date)" >> /var/log/my-app.log
70+
echo "Injected one fake Error and Exception into /var/log/my-app.log"
71+
72+
echo "Something went wrong - ERROR" >> /var/log/my-app.log
73+
echo "NullPointerException occurred" >> /var/log/my-app.log
74+
75+
3276
# Upload Logs to S3
3377
sudo aws s3 cp /var/log/cloud-init.log s3://${s3_bucket_name}/dev/system/
3478
sudo aws s3 cp /var/log/my-app.log s3://${s3_bucket_name}/dev/app/

‎scripts/prod_script.sh

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/bin/bash
22

3+
34
# Update system and install dependencies
45
# apt-get update -y
56
# apt-get install -y unzip git openjdk-21-jdk maven
@@ -22,15 +23,59 @@ git clone ${repo_url} app
2223
cd app
2324
mvn clean package
2425

26+
# To fix permission issues
27+
sudo touch /var/log/my-app.log
28+
sudo chown ubuntu:ubuntu /var/log/my-app.log
29+
30+
2531
# Run the Java app
2632
nohup java -jar target/*.jar --server.port=80 > /var/log/my-app.log 2>&1 &
2733

28-
# Wait for the app to start
34+
# Wait for app to run
2935
sleep 5
3036

37+
38+
39+
# ------------------------------------------------------------------------------
40+
# Install and Configure CloudWatch Agent (Ubuntu)
41+
# ------------------------------------------------------------------------------
42+
# sudo apt-get update
43+
# sudo apt-get install collectd -y
44+
45+
# echo "Downloading CloudWatch Agent .deb..."
46+
# wget https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -O /tmp/amazon-cloudwatch-agent.deb
47+
48+
# echo "Installing CloudWatch Agent..."
49+
# sudo dpkg -i /tmp/amazon-cloudwatch-agent.deb
50+
51+
echo 'export PATH=$PATH:/opt/aws/amazon-cloudwatch-agent/bin' >> ~/.bashrc
52+
source ~/.bashrc
53+
54+
55+
echo "Writing CloudWatch Agent config..."
56+
cat << 'EOF' > /opt/aws/amazon-cloudwatch-agent/bin/config.json
57+
${cw_agent_config_json}
58+
EOF
59+
60+
echo "Starting CloudWatch Agent..."
61+
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
62+
-a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json
63+
64+
# Wait a bit to ensure CW agent starts
65+
sleep 10
66+
67+
# Inject test log entry for monitoring validation
68+
echo "Error: Simulated a fake error failure on $(date)" >> /var/log/my-app.log
69+
echo "Exception: Simulated a fake exception for testing on $(date)" >> /var/log/my-app.log
70+
echo "Injected one fake Error and Exception into /var/log/my-app.log"
71+
72+
echo "Something went wrong - ERROR" >> /var/log/my-app.log
73+
echo "NullPointerException occurred" >> /var/log/my-app.log
74+
75+
3176
# Upload Logs to S3
32-
aws s3 cp /var/log/cloud-init.log s3://${s3_bucket_name}/prod/system/
33-
aws s3 cp /var/log/my-app.log s3://${s3_bucket_name}/prod/app/
77+
sudo aws s3 cp /var/log/cloud-init.log s3://${s3_bucket_name}/prod/system/
78+
sudo aws s3 cp /var/log/my-app.log s3://${s3_bucket_name}/prod/app/
3479

3580
# Shutdown after timeout
3681
sudo shutdown -h +${shutdown_minutes}

‎terraform/cloudwatch-configs/dev_cw_agent_config.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"logs": {
3+
"logs_collected": {
4+
"files": {
5+
"collect_list": [
6+
{
7+
"file_path": "/var/log/my-app.log",
8+
"log_group_name": "/aws/ec2/dev-app-logs",
9+
"log_stream_name": "{instance_id}",
10+
"timestamp_format": "%Y-%m-%d %H:%M:%S",
11+
"timezone": "Local"
12+
}
13+
]
14+
}
15+
}
16+
}
17+
}

‎terraform/cloudwatch-configs/prod_cw_agent_config.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"logs": {
3+
"logs_collected": {
4+
"files": {
5+
"collect_list": [
6+
{
7+
"file_path": "/var/log/my-app.log",
8+
"log_group_name": "/aws/ec2/prod-app-logs",
9+
"log_stream_name": "{instance_id}",
10+
"timestamp_format": "%Y-%m-%d %H:%M:%S",
11+
"timezone": "Local"
12+
}
13+
]
14+
}
15+
}
16+
}
17+
}

‎terraform/cloudwatch.tf

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# ----------------------------------
2+
# 1. CloudWatch Log Group
3+
# ----------------------------------
4+
resource "aws_cloudwatch_log_group" "app_log_group" {
5+
name = "/aws/ec2/${var.stage}-app-logs"
6+
retention_in_days = 7
7+
}
8+
9+
# ----------------------------------
10+
# 2. SNS Topic and Email Subscription
11+
# ----------------------------------
12+
resource "aws_sns_topic" "log_alert_topic" {
13+
name = "log-alert-topic"
14+
}
15+
16+
resource "aws_sns_topic_subscription" "log_alert_email_subscription" {
17+
topic_arn = aws_sns_topic.log_alert_topic.arn
18+
protocol = "email"
19+
endpoint = var.alert_email
20+
}
21+
22+
# ----------------------------------
23+
# 3. Log Metric Filter for ERROR
24+
# ----------------------------------
25+
resource "aws_cloudwatch_log_metric_filter" "error_filter" {
26+
name = "app-log-error-filter"
27+
log_group_name = aws_cloudwatch_log_group.app_log_group.name
28+
pattern = "Error"
29+
30+
metric_transformation {
31+
name = "AppLogErrorCount"
32+
namespace = "AppLogMetrics"
33+
value = "1"
34+
}
35+
}
36+
37+
# ----------------------------------
38+
# 4. Log Metric Filter for Exception
39+
# ----------------------------------
40+
resource "aws_cloudwatch_log_metric_filter" "exception_filter" {
41+
name = "app-log-exception-filter"
42+
log_group_name = aws_cloudwatch_log_group.app_log_group.name
43+
pattern = "Exception"
44+
45+
metric_transformation {
46+
name = "AppLogExceptionCount"
47+
namespace = "AppLogMetrics"
48+
value = "1"
49+
}
50+
}
51+
52+
# ----------------------------------
53+
# 5. CloudWatch Alarm for ERROR
54+
# ----------------------------------
55+
resource "aws_cloudwatch_metric_alarm" "error_alarm" {
56+
alarm_name = "${var.stage}-app-log-error-alarm"
57+
comparison_operator = "GreaterThanOrEqualToThreshold"
58+
evaluation_periods = 1
59+
period = 60
60+
threshold = 1
61+
statistic = "Sum"
62+
metric_name = aws_cloudwatch_log_metric_filter.error_filter.metric_transformation[0].name
63+
namespace = aws_cloudwatch_log_metric_filter.error_filter.metric_transformation[0].namespace
64+
alarm_description = "Triggered when ERROR appears in application logs"
65+
alarm_actions = [aws_sns_topic.log_alert_topic.arn]
66+
}
67+
68+
# ----------------------------------
69+
# 6. CloudWatch Alarm for Exception
70+
# ----------------------------------
71+
resource "aws_cloudwatch_metric_alarm" "exception_alarm" {
72+
alarm_name = "${var.stage}-app-log-exception-alarm"
73+
comparison_operator = "GreaterThanOrEqualToThreshold"
74+
evaluation_periods = 1
75+
period = 60
76+
threshold = 1
77+
statistic = "Sum"
78+
metric_name = aws_cloudwatch_log_metric_filter.exception_filter.metric_transformation[0].name
79+
namespace = aws_cloudwatch_log_metric_filter.exception_filter.metric_transformation[0].namespace
80+
alarm_description = "Triggered when Exception is logged in application logs"
81+
alarm_actions = [aws_sns_topic.log_alert_topic.arn]
82+
}

‎terraform/dev_config.tfvars

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ instance_type = "t2.micro"
22
key_name = "ssh-key-ec2" #change this to your key-pair name
33
# ami_id = "ami-0f918f7e67a3323f0"
44
stage = "dev"
5-
shutdown_minutes = 30
5+
shutdown_minutes = 40
66
s3_bucket_name = "techeazy-logs-unique123ss" # Change this!
77
aws_region = "ap-south-1"
8-
repo_url = "https://github.com/techeazy-consulting/techeazy-devops"
9-
verifier_lifetime = 25
8+
# repo_url = "https://github.com/techeazy-consulting/techeazy-devops"
9+
verifier_lifetime = 40
10+
alert_email = "akhilc0101@outlook.com"

‎terraform/ec2.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ resource "aws_instance" "app" {
4444
aws_region = var.aws_region
4545
repo_url = var.repo_url
4646
shutdown_minutes = var.shutdown_minutes
47+
cw_agent_config_json = file("${path.module}/cloudwatch-configs/${var.stage}_cw_agent_config.json")
4748
})
4849

4950
tags = {

‎terraform/iam.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,9 @@ resource "aws_iam_instance_profile" "ec2_instance_profile_b" {
9494
name = "${var.stage}_writeonly_instance_profile"
9595
role = aws_iam_role.role_b_uploader.name
9696
}
97+
98+
resource "aws_iam_role_policy_attachment" "cloudwatch_logs_attach" {
99+
role = aws_iam_role.role_b_uploader.name
100+
policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
101+
}
102+

0 commit comments

Comments
 (0)