feat(ourlogs): Add auto-refresh graph updates (#95489)

### Summary
This updates the graph when autorefresh is on to also stream in log
data. This is behind the feature flag alongside the auto-refresh.

This adds a hook which takes in the table data (which is streaming with
auto-refresh) and the timeseries and synthesizes new buckets based on
the existing timeseries structure as more logs appear in the table.

Explanation for how the timeseries streaming works is in the comments
for hook function.

Author: k-fish

static/app/utils/timeSeries/markDelayedData.tsx

@@ -2,6 +2,7 @@
  * Given a timeseries and a delay in seconds, goes through the timeseries data, and marks each point as either delayed (data bucket ended before the delay threshold) or not
  */
 
+import {defined} from 'sentry/utils';
 import type {TimeSeries} from 'sentry/views/dashboards/widgets/common/types';
 
 export function markDelayedData(timeSeries: TimeSeries, delay: number): TimeSeries {
@@ -22,6 +23,10 @@ export function markDelayedData(timeSeries: TimeSeries, delay: number): TimeSeri
       const bucketEndTimestamp = new Date(datum.timestamp).getTime() + bucketSize;
       const delayed = bucketEndTimestamp >= ingestionDelayTimestamp;
 
+      if (defined(datum.incomplete)) {
+        return datum;
+      }
+
       if (!delayed) {
         return datum;
       }

static/app/views/explore/contexts/logs/logsPageParams.tsx

@@ -547,6 +547,8 @@ export function useSetLogsAutoRefresh() {
     (autoRefresh: boolean) => {
       if (autoRefresh) {
         queryClient.removeQueries({queryKey});
+        // Until we change our timeseries hooks to be build their query keys separately, we need to remove the query via the route.
+        queryClient.removeQueries({queryKey: ['events-stats']});
       }
       setPageParams({autoRefresh});
     },

static/app/views/explore/logs/logsAutoRefresh.tsx

@@ -10,6 +10,7 @@ import usePrevious from 'sentry/utils/usePrevious';
 import {useLogsPageData} from 'sentry/views/explore/contexts/logs/logsPageData';
 import {
   useLogsAutoRefresh,
+  useLogsGroupBy,
   useLogsRefreshInterval,
   useLogsSortBys,
   useSetLogsAutoRefresh,
@@ -27,12 +28,20 @@ type DisableReason = 'sort' | 'timeout' | 'rateLimit' | 'error';
 
 const SWITCH_DISABLE_REASONS: DisableReason[] = ['sort'];
 
-export function AutorefreshToggle() {
-  const {selection} = usePageFilters();
-  const previousProjects = usePrevious(selection.projects);
+interface AutorefreshToggleProps {
+  disabled?: boolean;
+}
+
+export function AutorefreshToggle({
+  disabled: externallyDisabled,
+}: AutorefreshToggleProps) {
   const checked = useLogsAutoRefresh();
   const setChecked = useSetLogsAutoRefresh();
   const sortBys = useLogsSortBys();
+  const groupBy = useLogsGroupBy();
+  const {selection} = usePageFilters();
+  const previousSelection = usePrevious(selection);
+  const previousGroupBy = usePrevious(groupBy);
   const refreshInterval = useLogsRefreshInterval();
   const {infiniteLogsQueryResult} = useLogsPageData();
   const {fetchPreviousPage, isError} = infiniteLogsQueryResult;
@@ -50,16 +59,25 @@ export function AutorefreshToggle() {
   const isRefreshRunningRef = useRef(false);
   const consecutivePagesWithMoreDataRef = useRef(0);
 
-  const statsPeriod = usePageFilters().selection.datetime.period;
+  const statsPeriod = selection.datetime.period;
   const isDescendingTimeBasedSort = checkSortIsTimeBasedDescending(sortBys);
-  const enabled = isDescendingTimeBasedSort && checked && defined(statsPeriod);
+  const enabled =
+    isDescendingTimeBasedSort && checked && defined(statsPeriod) && !externallyDisabled;
 
-  // Disable auto-refresh if the project changes
+  // Disable auto-refresh if anything in the selection changes
   useEffect(() => {
-    if (!isEqual(selection.projects, previousProjects)) {
+    const selectionChanged = !isEqual(previousSelection, selection);
+    if (selectionChanged) {
       setChecked(false);
     }
-  }, [selection.projects, previousProjects, setChecked]);
+  }, [selection, previousSelection, setChecked]);
+
+  // Disable auto-refresh if the group-by changes
+  useEffect(() => {
+    if (groupBy && groupBy !== previousGroupBy) {
+      setChecked(false);
+    }
+  }, [groupBy, previousGroupBy, setChecked]);
 
   // Reset disableReason when sort bys change
   useEffect(() => {
@@ -68,6 +86,13 @@ export function AutorefreshToggle() {
     }
   }, [sortBysString, previousSortBysString, isDescendingTimeBasedSort]);
 
+  // Disable auto-refresh when externally disabled
+  useEffect(() => {
+    if (externallyDisabled && checked) {
+      setChecked(false);
+    }
+  }, [externallyDisabled, checked, setChecked]);
+
   useEffect(() => {
     if (isError && enabled) {
       setDisableReason('error');
@@ -209,6 +234,9 @@ export function AutorefreshToggle() {
   }, [enabled, refreshInterval]);
 
   const getTooltipMessage = (): string => {
+    if (externallyDisabled) {
+      return t('Auto-refresh is not available in the aggregates view.');
+    }
     switch (disableReason) {
       case 'rateLimit':
         return t(
@@ -235,11 +263,14 @@ export function AutorefreshToggle() {
       <AutoRefreshLabel>
         <Tooltip
           title={getTooltipMessage()}
-          disabled={disableReason === undefined}
+          disabled={disableReason === undefined && !externallyDisabled}
           skipWrapper
         >
           <Switch
-            disabled={disableReason && SWITCH_DISABLE_REASONS.includes(disableReason)}
+            disabled={
+              externallyDisabled ||
+              (disableReason && SWITCH_DISABLE_REASONS.includes(disableReason))
+            }
             checked={checked}
             onChange={() => {
               if (!checked) {

static/app/views/explore/logs/logsGraph.tsx

@@ -21,7 +21,6 @@ import {
   ChartIntervalUnspecifiedStrategy,
   useChartInterval,
 } from 'sentry/views/explore/hooks/useChartInterval';
-import {INGESTION_DELAY} from 'sentry/views/explore/settings';
 import {ChartType} from 'sentry/views/insights/common/components/chart';
 import type {useSortedTimeSeries} from 'sentry/views/insights/common/queries/useSortedTimeSeries';
 
@@ -138,7 +137,7 @@ export function LogsGraph({timeseriesResult}: LogsGraphProps) {
         <TimeSeriesWidgetVisualization
           plottables={data.map(
             timeSeries =>
-              new DataPlottableConstructor(markDelayedData(timeSeries, INGESTION_DELAY), {
+              new DataPlottableConstructor(markDelayedData(timeSeries, 60), {
                 stack: 'all',
               })
           )}

static/app/views/explore/logs/logsTab.tsx

@@ -64,6 +64,7 @@ import {LogsAggregateTable} from 'sentry/views/explore/logs/tables/logsAggregate
 import {LogsInfiniteTable as LogsInfiniteTable} from 'sentry/views/explore/logs/tables/logsInfiniteTable';
 import {LogsTable} from 'sentry/views/explore/logs/tables/logsTable';
 import {usePersistentLogsPageParameters} from 'sentry/views/explore/logs/usePersistentLogsPageParameters';
+import {useStreamingTimeseriesResult} from 'sentry/views/explore/logs/useStreamingTimeseriesResult';
 import {ColumnEditorModal} from 'sentry/views/explore/tables/columnEditorModal';
 import {TraceItemDataset} from 'sentry/views/explore/types';
 import type {PickableDays} from 'sentry/views/explore/utils';
@@ -100,7 +101,7 @@ export function LogsTabContent({
   const [sidebarOpen, setSidebarOpen] = useState(
     !!((aggregateFunction && aggregateFunction !== 'count') || groupBy)
   );
-  const timeseriesResult = useSortedTimeSeries(
+  const _timeseriesResult = useSortedTimeSeries(
     {
       search: logsSearch,
       yAxis: [aggregate],
@@ -111,6 +112,7 @@ export function LogsTabContent({
     'explore.ourlogs.main-chart',
     DiscoverDatasets.OURLOGS
   );
+  const timeseriesResult = useStreamingTimeseriesResult(tableData, _timeseriesResult);
 
   const {attributes: stringAttributes, isLoading: stringAttributesLoading} =
     useTraceItemAttributes('string');
@@ -254,7 +256,7 @@ export function LogsTabContent({
               </Feature>
               <TableActionsContainer>
                 <Feature features="organizations:ourlogs-live-refresh">
-                  <AutorefreshToggle />
+                  <AutorefreshToggle disabled={tableTab === 'aggregates'} />
                 </Feature>
                 <Button onClick={openColumnEditor} icon={<IconTable />} size="sm">
                   {t('Edit Table')}

static/app/views/explore/logs/useLogsQuery.tsx

@@ -197,12 +197,15 @@ function useLogsQueryKey({limit, referrer}: {referrer: string; limit?: number})
   const {selection, isReady: pageFiltersReady} = usePageFilters();
   const location = useLocation();
   const projectIds = useLogsProjectIds();
+  const groupBy = useLogsGroupBy();
 
   const search = baseSearch ? _search.copy() : _search;
   if (baseSearch) {
     search.tokens.push(...baseSearch.tokens);
   }
-  const fields = Array.from(new Set([...AlwaysPresentLogFields, ..._fields]));
+  const fields = Array.from(
+    new Set([...AlwaysPresentLogFields, ..._fields, ...(groupBy ? [groupBy] : [])])
+  );
   const sorts = sortBys ?? [];
   const pageFilters = selection;
   const dataset = DiscoverDatasets.OURLOGS;
@@ -512,7 +515,7 @@ export function useInfiniteLogsQuery({
     initialPageParam,
     enabled: !disabled,
     staleTime: getStaleTimeForEventView(other.eventView),
-    maxPages: 15,
+    maxPages: 30, // This number * the refresh interval must be more seconds than 2 * the smallest time interval in the chart for streaming to work.
   });
 
   const {