Merge pull request #346 from code-dot-org/ben/fix-throttling-alarms

Fix throttling alarms

Author: bencodeorg

cicd/3-app/javabuilder/template.yml.erb

@@ -963,14 +963,14 @@ Resources:
     Type: AWS::CloudWatch::Alarm
     Properties:
         AlarmName: !Sub "${SubDomainName}_high_users_blocked"
-        AlarmDescription: Unusually high number of users being blocked by our throttling
+        AlarmDescription: Unusually high number of users being newly blocked by our throttling
             thresholds.
         ActionsEnabled: true
         OKActions: []
         AlarmActions:
           - !If [SilenceAlertsCondition, !Ref AWS::NoValue, !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:javabuilder-high-throttling"]
         InsufficientDataActions: []
-        MetricName: UserBlocked
+        MetricName: NewUserBlocked
         Namespace: Javabuilder
         Statistic: Sum
         Dimensions:
@@ -987,14 +987,14 @@ Resources:
     Type: AWS::CloudWatch::Alarm
     Properties:
         AlarmName: !Sub "${SubDomainName}_high_classrooms_blocked"
-        AlarmDescription: Unusually high number of classrooms being blocked by our throttling
+        AlarmDescription: Unusually high number of classrooms being newly blocked by our throttling
             thresholds.
         ActionsEnabled: true
         OKActions: []
         AlarmActions:
           - !If [SilenceAlertsCondition, !Ref AWS::NoValue, !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:javabuilder-high-throttling"]
         InsufficientDataActions: []
-        MetricName: ClassroomBlocked
+        MetricName: NewClassroomBlocked
         Namespace: Javabuilder
         Statistic: Sum
         Dimensions:

javabuilder-authorizer/metrics_reporter.rb

@@ -11,8 +11,13 @@ def initialize(context)
 
   def log_token_error(status, error_message)
     puts error_message
+    metric_name = TOKEN_STATUS_METRIC_NAMES[status]
+    log(metric_name)
+  end
+
+  def log(metric_name)
     metric_data = {
-      metric_name: TOKEN_STATUS_METRIC_NAMES[status],
+      metric_name: metric_name,
       dimensions: [
         {
           name: "functionName",

javabuilder-authorizer/token_status.rb

@@ -40,4 +40,7 @@ module TokenStatus
     NOT_VETTED => 'TokenNotVetted',
     TOKEN_USED => 'TokenUsed'
   }.freeze
+
+  NEW_USER_BLOCKED = 'NewUserBlocked'.freeze
+  NEW_CLASSROOM_BLOCKED = 'NewClassroomBlocked'.freeze
 end

javabuilder-authorizer/token_validator.rb

@@ -136,6 +136,7 @@ def teachers_over_hourly_limit?
             },
             condition_expression: 'attribute_not_exists(user_id)'
           )
+          @metrics_reporter.log(NEW_CLASSROOM_BLOCKED)
         rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
           # Do nothing if this teacher has already been added to blocked table
           # (possible if throttling limit was reached by a user on another lambda simultaneously)
@@ -244,6 +245,7 @@ def user_over_limit?(query_response, limit, logging_message)
           },
           condition_expression: 'attribute_not_exists(user_id)'
         )
+        @metrics_reporter.log(NEW_USER_BLOCKED)
       rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
         # Do nothing if this user has already been added to blocked table
         # (possible if throttling limit was reached by the same user executing code on another lambda simultaneously)