hyperdxio · wrn14897 · Oct 25, 2025 · Oct 25, 2025 · Oct 25, 2025 · Oct 25, 2025
diff --git a/.changeset/brave-meals-judge.md b/.changeset/brave-meals-judge.md
@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: add support for alert auto-resolve
diff --git a/.changeset/friendly-apricots-sell.md b/.changeset/friendly-apricots-sell.md
@@ -0,0 +1,7 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+---
+
+feat: support incident.io integration
diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts
@@ -18,6 +18,7 @@ import logger from '@/utils/logger';
 import { alertSchema } from '@/utils/zod';
 
 export type AlertInput = {
+  id?: string;
   source?: AlertSource;
   channel: AlertChannel;
   interval: AlertInterval;

diff --git a/packages/api/src/models/webhook.ts b/packages/api/src/models/webhook.ts
@@ -1,10 +1,8 @@
+import { WebhookService } from '@hyperdx/common-utils/dist/types';
 import { ObjectId } from 'mongodb';
 import mongoose, { Schema } from 'mongoose';
 
-export enum WebhookService {
-  Slack = 'slack',
-  Generic = 'generic',
-}
+export { WebhookService };
 
 interface MongooseMap extends Map<string, string> {
   // https://mongoosejs.com/docs/api/map.html#MongooseMap.prototype.toJSON()

diff --git a/packages/api/src/tasks/__tests__/checkAlerts.test.ts b/packages/api/src/tasks/__tests__/checkAlerts.test.ts
@@ -312,6 +312,7 @@ describe('checkAlerts', () => {
         alertProvider,
         clickhouseClient: {} as any,
         metadata: {} as any,
+        state: 'ALERT',
         template: 'Custom body @webhook-My_Web', // partial name should work
         view: {
           ...defaultSearchView,
@@ -350,6 +351,7 @@ describe('checkAlerts', () => {
         alertProvider,
         clickhouseClient: {} as any,
         metadata: {} as any,
+        state: 'ALERT',
         template: 'Custom body @webhook-My_Web', // partial name should work
         view: {
           ...defaultSearchView,
@@ -410,6 +412,7 @@ describe('checkAlerts', () => {
         alertProvider,
         clickhouseClient: {} as any,
         metadata: {} as any,
+        state: 'ALERT',
         template: 'Custom body @webhook-{{attributes.webhookName}}', // partial name should work
         view: {
           ...defaultSearchView,
@@ -481,6 +484,7 @@ describe('checkAlerts', () => {
         alertProvider,
         clickhouseClient: {} as any,
         metadata: {} as any,
+        state: 'ALERT',
         template: `
 {{#is_match "attributes.k8s.pod.name" "otel-collector-123"}}
   Runbook URL: {{attributes.runbook.url}}
@@ -518,6 +522,7 @@ describe('checkAlerts', () => {
         alertProvider,
         clickhouseClient: {} as any,
         metadata: {} as any,
+        state: 'ALERT',
         template:
           '{{#is_match "attributes.host" "web"}} @webhook-My_Web {{/is_match}}', // partial name should work
         view: {

diff --git a/packages/api/src/tasks/checkAlerts.ts b/packages/api/src/tasks/checkAlerts.ts
@@ -67,6 +67,7 @@ const fireChannelEvent = async ({
   savedSearch,
   source,
   startTime,
+  state,
   totalCount,
   windowSizeInMins,
   teamWebhooksById,
@@ -82,6 +83,7 @@ const fireChannelEvent = async ({
   savedSearch?: ISavedSearch | null;
   source?: ISource | null;
   startTime: Date;
+  state: AlertState;
   totalCount: number;
   windowSizeInMins: number;
   teamWebhooksById: Map<string, IWebhook>;
@@ -105,6 +107,7 @@ const fireChannelEvent = async ({
   const attributesNested = unflattenObject(attributes);
   const templateView: AlertMessageTemplateDefaultView = {
     alert: {
+      id: alert.id,
       channel: alert.channel,
       dashboardId: dashboard?.id,
       groupBy: alert.groupBy,
@@ -133,6 +136,7 @@ const fireChannelEvent = async ({
     alertProvider,
     clickhouseClient,
     metadata,
+    state,
     title: buildAlertMessageTemplateTitle({
       template: alert.name,
       view: templateView,
@@ -350,6 +354,7 @@ export const processAlert = async (
               savedSearch: (details as any).savedSearch,
               source,
               startTime: bucketStart,
+              state: AlertState.ALERT,
               totalCount: _value,
               windowSizeInMins,
               teamWebhooksById,
@@ -368,6 +373,47 @@ export const processAlert = async (
       }
     }
 
+    // Check if alert transitioned from ALERT to OK (resolved)
+    if (
+      previous?.state === AlertState.ALERT &&
+      history.state === AlertState.OK
+    ) {
+      logger.info({
+        message: `Alert resolved, triggering ${alert.channel.type} notification`,
+        alertId: alert.id,
+      });
+
+      try {
+        const lastValue = history.lastValues[history.lastValues.length - 1];
+        await fireChannelEvent({
+          alert,
+          alertProvider,
+          attributes: {}, // FIXME: support attributes (logs + resources ?)
+          clickhouseClient,
+          dashboard: (details as any).dashboard,
+          endTime: fns.addMinutes(
+            lastValue?.startTime || nowInMinsRoundDown,
+            windowSizeInMins,
+          ),
+          group: '',
+          metadata,
+          savedSearch: (details as any).savedSearch,
+          source,
+          startTime: lastValue?.startTime || nowInMinsRoundDown,
+          state: AlertState.OK,
+          totalCount: lastValue?.count || 0,
+          windowSizeInMins,
+          teamWebhooksById,
+        });
+      } catch (e) {
+        logger.error({
+          message: 'Failed to fire resolved channel event',
+          alertId: alert.id,
+          error: serializeError(e),
+        });
+      }
+    }
+
     await alertProvider.updateAlertState(history);
   } catch (e) {
     // Uncomment this for better error messages locally
@@ -386,6 +432,7 @@ export { handleSendGenericWebhook };
 export interface AggregatedAlertHistory {
   _id: ObjectId;
   createdAt: Date;
+  state: AlertState;
 }
 
 /**
@@ -416,11 +463,16 @@ export const getPreviousAlertHistories = async (
             createdAt: { $lte: now },
           },
         },
-        // Group by alert ID, taking the latest createdAt value for each group
+        // Sort by createdAt descending to get the latest first
+        {
+          $sort: { createdAt: -1 },
+        },
+        // Group by alert ID, taking the first (latest) values for each group
         {
           $group: {
             _id: '$alert',
-            createdAt: { $max: '$createdAt' },
+            createdAt: { $first: '$createdAt' },
+            state: { $first: '$state' },
           },
         },
       ]),

diff --git a/packages/api/src/tasks/template.ts b/packages/api/src/tasks/template.ts
@@ -8,7 +8,11 @@ import {
   WebhookService,
   zAlertChannelType,
 } from '@hyperdx/common-utils/dist/types';
-import { _useTry, formatDate } from '@hyperdx/common-utils/dist/utils';
+import {
+  _useTry,
+  formatDate,
+  objectHash,
+} from '@hyperdx/common-utils/dist/utils';
 import { isValidSlackUrl } from '@hyperdx/common-utils/dist/validation';
 import Handlebars, { HelperOptions } from 'handlebars';
 import _ from 'lodash';
@@ -59,6 +63,10 @@ interface Message {
   hdxLink: string;
   title: string;
   body: string;
+  state: string;
+  startTime: number;
+  endTime: number;
+  eventId: string;
 }
 
 export const notifyChannel = async ({
@@ -73,7 +81,10 @@ export const notifyChannel = async ({
       const webhook = channel.channel;
       if (webhook.service === WebhookService.Slack) {
         await handleSendSlackWebhook(webhook, message);
-      } else if (webhook.service === 'generic') {
+      } else if (
+        webhook.service === WebhookService.Generic ||
+        webhook.service === WebhookService.IncidentIO
+      ) {
         await handleSendGenericWebhook(webhook, message);
       }
       break;
@@ -196,11 +207,18 @@ export const handleSendGenericWebhook = async (
   let body = '';
   try {
     const handlebars = Handlebars.create();
+    // Register eq helper for conditional checks
+    handlebars.registerHelper('eq', (a, b) => a === b);
+
     body = handlebars.compile(webhook.body, {
       noEscape: true,
     })({
       body: escapeJsonString(message.body),
+      endTime: message.endTime,
+      eventId: message.eventId,
       link: escapeJsonString(message.hdxLink),
+      startTime: message.startTime,
+      state: message.state,
       title: escapeJsonString(message.title),
     });
   } catch (e) {
@@ -377,6 +395,7 @@ export const renderAlertTemplate = async ({
   alertProvider,
   clickhouseClient,
   metadata,
+  state,
   template,
   title,
   view,
@@ -385,6 +404,7 @@ export const renderAlertTemplate = async ({
   alertProvider: AlertProvider;
   clickhouseClient: ClickhouseClient;
   metadata: Metadata;
+  state: string;
   template?: string | null;
   title: string;
   view: AlertMessageTemplateDefaultView;
@@ -451,12 +471,26 @@ export const renderAlertTemplate = async ({
       );
 
       if (channel) {
+        const startTime = view.startTime.getTime();
+        const endTime = view.endTime.getTime();
+        const eventId = objectHash({
+          alertId: alert.id,
+          channel: {
+            type: channel.type,
+            id: channel.channel._id.toString(),
+          },
+        });
+
         await notifyChannel({
           channel,
           message: {
             hdxLink: buildAlertMessageTemplateHdxLink(alertProvider, view),
             title,
             body: renderedBody,
+            state,
+            startTime,
+            endTime,
+            eventId,
           },
         });
       }

diff --git a/packages/app/src/TeamPage.tsx b/packages/app/src/TeamPage.tsx
@@ -54,7 +54,15 @@ import { useSources } from './source';
 import { useConfirm } from './useConfirm';
 import { capitalizeFirstLetter } from './utils';
 
-const DEFAULT_GENERIC_WEBHOOK_BODY = ['{{title}}', '{{body}}', '{{link}}'];
+const DEFAULT_GENERIC_WEBHOOK_BODY = [
+  '{{title}}',
+  '{{body}}',
+  '{{link}}',
+  '{{state}}',
+  '{{startTime}}',
+  '{{endTime}}',
+  '{{eventId}}',
+];
 const DEFAULT_GENERIC_WEBHOOK_BODY_TEMPLATE =
   DEFAULT_GENERIC_WEBHOOK_BODY.join(' | ');
 
@@ -733,15 +741,27 @@ export function CreateWebhookForm({
         }
       }
 
+      let defaultBody = body;
+      if (!body) {
+        if (service === WebhookService.Generic) {
+          defaultBody = `{"text": "${DEFAULT_GENERIC_WEBHOOK_BODY_TEMPLATE}"}`;
+        } else if (service === WebhookService.IncidentIO) {
+          defaultBody = `{
+  "title": "{{title}}",
+  "description": "{{body}}",
+  "deduplication_key": "{{eventId}}",
+  "status": "{{#if (eq state "ALERT")}}firing{{else}}resolved{{/if}}",
+  "source_url": "{{link}}"
+}`;
+        }
+      }
+
       const response = await saveWebhook.mutateAsync({
         service,
         name,
         url,
         description: description || '',
-        body:
-          service === WebhookService.Generic && !body
-            ? `{"text": "${DEFAULT_GENERIC_WEBHOOK_BODY_TEMPLATE}"}`
-            : body,
+        body: defaultBody,
         headers: parsedHeaders,
       });
       notifications.show({
@@ -815,6 +835,11 @@ export function CreateWebhookForm({
               label="Generic"
               {...form.register('service', { required: true })}
             />
+            <Radio
+              value={WebhookService.IncidentIO}
+              label="Incident.io"
+              {...form.register('service', { required: true })}
+            />
           </Group>
         </Radio.Group>
         <TextInput
@@ -976,6 +1001,7 @@ function IntegrationsSection() {
   const { data: webhookData, refetch: refetchWebhooks } = api.useWebhooks([
     WebhookService.Slack,
     WebhookService.Generic,
+    WebhookService.IncidentIO,
   ]);
 
   const allWebhooks = useMemo(() => {

diff --git a/packages/app/src/components/Alerts.tsx b/packages/app/src/components/Alerts.tsx
@@ -2,7 +2,10 @@ import { useMemo } from 'react';
 import { Control, useController } from 'react-hook-form';
 import { Select, SelectProps } from 'react-hook-form-mantine';
 import { Label, ReferenceArea, ReferenceLine } from 'recharts';
-import type { AlertChannelType } from '@hyperdx/common-utils/dist/types';
+import {
+  type AlertChannelType,
+  WebhookService,
+} from '@hyperdx/common-utils/dist/types';
 import { Button, ComboboxData, Group, Modal } from '@mantine/core';
 import { useDisclosure } from '@mantine/hooks';
 
@@ -19,8 +22,9 @@ const WebhookChannelForm = <T extends object>(
   props: Partial<SelectProps<T>>,
 ) => {
   const { data: webhooks, refetch: refetchWebhooks } = api.useWebhooks([
-    'slack',
-    'generic',
+    WebhookService.Slack,
+    WebhookService.Generic,
+    WebhookService.IncidentIO,
   ]);
   const [opened, { open, close }] = useDisclosure(false);
 

diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts
@@ -213,6 +213,7 @@ export type StacktraceBreadcrumb = {
 export enum WebhookService {
   Slack = 'slack',
   Generic = 'generic',
+  IncidentIO = 'incidentio',
 }
 
 // -------------------------

diff --git a/packages/common-utils/src/utils.ts b/packages/common-utils/src/utils.ts
@@ -3,6 +3,8 @@ import { add as fnsAdd, format as fnsFormat } from 'date-fns';
 import { formatInTimeZone } from 'date-fns-tz';
 import { z } from 'zod';
 
+export { default as objectHash } from 'object-hash';
+
 import {
   ChartConfigWithDateRange,
   ChartConfigWithOptTimestamp,