From 05e77893102030f6502d97bd62f9de893ec4a586 Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Thu, 5 Jun 2025 17:58:50 -0400 Subject: [PATCH 1/4] Create new waterfall data format --- .../src/apiClient/visualizationService.ts | 63 ++++-- .../components/waterfall/ScanDetailsTable.tsx | 37 +-- .../components/waterfall/WaterfallPlot.tsx | 4 +- .../waterfall/WaterfallVizContainer.tsx | 27 ++- frontend/src/components/waterfall/index.tsx | 211 ++++++++---------- frontend/src/components/waterfall/types.ts | 91 ++------ .../spx_vis/api/views.py | 101 ++++++++- .../spx_vis/capture_utils/base.py | 12 + .../spx_vis/capture_utils/radiohound.py | 102 +++++++++ 9 files changed, 417 insertions(+), 231 deletions(-) diff --git a/frontend/src/apiClient/visualizationService.ts b/frontend/src/apiClient/visualizationService.ts index 12b0253..477b3f2 100644 --- a/frontend/src/apiClient/visualizationService.ts +++ b/frontend/src/apiClient/visualizationService.ts @@ -10,7 +10,10 @@ import { CaptureSourceSchema, CaptureSchema, } from './captureService'; -import { RadioHoundFileSchema } from '../components/waterfall/types'; +import { + WaterfallFileSchema, + WaterfallFile, +} from '../components/waterfall/types'; import { FilesWithContent } from '../components/types'; import JSZip from 'jszip'; @@ -211,20 +214,6 @@ export const useVisualizationFiles = (vizRecord: VisualizationRecordDetail) => { let parsedContent: unknown = content; let isValid: boolean | undefined; - // Validate RadioHound files - if (vizRecord.capture_type === 'rh') { - parsedContent = JSON.parse(await content.text()); - const validationResult = - RadioHoundFileSchema.safeParse(parsedContent); - isValid = validationResult.success; - - if (!isValid) { - console.warn( - `Invalid RadioHound file content for ${file.name}: ${validationResult.error}`, - ); - } - } - // Add the file to our files object files[file.uuid] = { uuid: file.uuid, @@ -248,3 +237,47 @@ export const useVisualizationFiles = (vizRecord: VisualizationRecordDetail) => { return { files, isLoading, error }; }; + +/** + * Fetches waterfall data for a visualization from the backend. + * @param id - The ID of the visualization + * @returns An array of WaterfallFile objects containing the waterfall data + * @throws Error if the request fails or the response data is invalid + */ +export const getWaterfallData = async ( + id: string, +): Promise => { + try { + const response = await apiClient.get( + `/api/visualizations/${id}/get_waterfall_data/`, + ); + return zod.array(WaterfallFileSchema).parse(response.data); + } catch (error) { + console.error('Error fetching waterfall data:', error); + throw error; + } +}; + +export const useWaterfallData = (id: string) => { + const [waterfallData, setWaterfallData] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + const fetchWaterfallData = async () => { + setIsLoading(true); + try { + const data = await getWaterfallData(id); + setWaterfallData(data); + } catch (error) { + console.error('Error fetching waterfall data:', error); + setError('Failed to fetch waterfall data'); + } finally { + setIsLoading(false); + } + }; + fetchWaterfallData(); + }, [id]); + + return { waterfallData, isLoading, error }; +}; diff --git a/frontend/src/components/waterfall/ScanDetailsTable.tsx b/frontend/src/components/waterfall/ScanDetailsTable.tsx index dfa56f1..baa5f46 100644 --- a/frontend/src/components/waterfall/ScanDetailsTable.tsx +++ b/frontend/src/components/waterfall/ScanDetailsTable.tsx @@ -2,7 +2,7 @@ import { Table } from 'react-bootstrap'; import _ from 'lodash'; import { formatHertz } from '../../utils/utils'; -import { RadioHoundFile } from './types'; +import { WaterfallFile } from './types'; interface DetailRowProps { label: string; @@ -19,10 +19,10 @@ function DetailRow({ label, value }: DetailRowProps): JSX.Element { } interface ScanDetailsProps { - rhFile: RadioHoundFile; + waterfallFile: WaterfallFile; } -export function ScanDetails({ rhFile }: ScanDetailsProps): JSX.Element { +export function ScanDetails({ waterfallFile }: ScanDetailsProps): JSX.Element { // const downloadUrl = useMemo(() => { // const blob = new Blob([JSON.stringify(capture, null, 4)], { // type: 'application/json', @@ -47,7 +47,7 @@ export function ScanDetails({ rhFile }: ScanDetailsProps): JSX.Element { // Helper function to safely get nested values const getScanValue = (path: string, defaultValue?: T): T | undefined => { - return _.get(rhFile, path, defaultValue) as T; + return _.get(waterfallFile, path, defaultValue) as T; }; return ( @@ -57,7 +57,7 @@ export function ScanDetails({ rhFile }: ScanDetailsProps): JSX.Element { @@ -112,17 +112,20 @@ export function ScanDetails({ rhFile }: ScanDetailsProps): JSX.Element { - + diff --git a/frontend/src/components/waterfall/WaterfallPlot.tsx b/frontend/src/components/waterfall/WaterfallPlot.tsx index 50d3901..4275e04 100644 --- a/frontend/src/components/waterfall/WaterfallPlot.tsx +++ b/frontend/src/components/waterfall/WaterfallPlot.tsx @@ -2,7 +2,7 @@ import { useRef, useEffect, useState } from 'react'; import _ from 'lodash'; import { scaleLinear, interpolateHslLong, rgb } from 'd3'; -import { ScanState, WaterfallType, Display } from './types'; +import { ScanState, ScanWaterfallType, Display } from './types'; import { WATERFALL_MAX_ROWS } from './index'; const SCROLL_INDICATOR_SIZE = 15; @@ -31,7 +31,7 @@ const downIndicatorStyle: React.CSSProperties = { interface WaterfallPlotProps { scan: ScanState; display: Display; - setWaterfall: (waterfall: WaterfallType) => void; + setWaterfall: (waterfall: ScanWaterfallType) => void; setScaleChanged: (scaleChanged: boolean) => void; setResetScale: (resetScale: boolean) => void; currentFileIndex: number; diff --git a/frontend/src/components/waterfall/WaterfallVizContainer.tsx b/frontend/src/components/waterfall/WaterfallVizContainer.tsx index 7d96c80..e9127cd 100644 --- a/frontend/src/components/waterfall/WaterfallVizContainer.tsx +++ b/frontend/src/components/waterfall/WaterfallVizContainer.tsx @@ -2,11 +2,10 @@ import { useState } from 'react'; import { Alert, Row, Col } from 'react-bootstrap'; import { WaterfallVisualization } from '.'; -import { RadioHoundFile } from './types'; import WaterfallControls from './WaterfallControls'; import ScanDetailsTable from './ScanDetailsTable'; import { VizContainerProps } from '../types'; -import { useVisualizationFiles } from '../../apiClient/visualizationService'; +import { useWaterfallData } from '../../apiClient/visualizationService'; import LoadingBlock from '../LoadingBlock'; export interface WaterfallSettings { @@ -18,8 +17,9 @@ export interface WaterfallSettings { export const WaterfallVizContainer = ({ visualizationRecord, }: VizContainerProps) => { - const { files, isLoading, error } = - useVisualizationFiles(visualizationRecord); + const { waterfallData, isLoading, error } = useWaterfallData( + visualizationRecord.uuid, + ); const [settings, setSettings] = useState({ fileIndex: 0, isPlaying: false, @@ -36,7 +36,7 @@ export const WaterfallVizContainer = ({ ); } - if (Object.keys(files).length === 0) { + if (waterfallData.length === 0) { return ( No Data Found @@ -47,12 +47,9 @@ export const WaterfallVizContainer = ({ // We currently only support one capture per visualization, so grab the first // capture and use its files - const rhFiles = visualizationRecord.captures[0].files - .map((file) => files[file.uuid].fileContent as RadioHoundFile) - .sort( - (a, b) => - new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), - ); + const waterfallFiles = waterfallData.sort( + (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), + ); return (
@@ -62,20 +59,22 @@ export const WaterfallVizContainer = ({
- + diff --git a/frontend/src/components/waterfall/index.tsx b/frontend/src/components/waterfall/index.tsx index 54f7670..2a159af 100644 --- a/frontend/src/components/waterfall/index.tsx +++ b/frontend/src/components/waterfall/index.tsx @@ -10,10 +10,9 @@ import { DataPoint, FloatArray, ScanState, - WaterfallType, - RadioHoundFile, + ScanWaterfallType, + WaterfallFile, Display, - ApplicationType, } from './types'; import { formatHertz } from '../../utils/utils'; @@ -135,17 +134,16 @@ const initialScan: ScanState = { export const WATERFALL_MAX_ROWS = 80; interface WaterfallVisualizationProps { - rhFiles: RadioHoundFile[]; + waterfallFiles: WaterfallFile[]; settings: WaterfallSettings; setSettings: React.Dispatch>; } const WaterfallVisualization: React.FC = ({ - rhFiles, + waterfallFiles, settings, setSettings, }: WaterfallVisualizationProps) => { - const currentApplication = ['PERIODOGRAM', 'WATERFALL'] as ApplicationType[]; const [displayedFileIndex, setDisplayedFileIndex] = useState( settings.fileIndex, ); @@ -169,7 +167,7 @@ const WaterfallVisualization: React.FC = ({ resetScale, })); }; - const setWaterfall = (waterfall: WaterfallType) => { + const setScanWaterfall = (waterfall: ScanWaterfallType) => { const localScaleMin = waterfall.scaleMin ?? scan.scaleMin; const localScaleMax = waterfall.scaleMax ?? scan.scaleMax; const tmpData = _.cloneDeep(waterfall.allData ?? scan.allData); @@ -192,19 +190,22 @@ const WaterfallVisualization: React.FC = ({ } }; - // Process all RadioHound files once and store the results + // Process all files once and store the results const processedData = useMemo(() => { - return rhFiles.map((rhFile) => { + return waterfallFiles.map((waterfallFile) => { let floatArray: FloatArray | number[] | undefined; - if (typeof rhFile.data === 'string') { - if (rhFile.data.slice(0, 8) === 'AAAAAAAA') { + if (typeof waterfallFile.data === 'string') { + if (waterfallFile.data.slice(0, 8) === 'AAAAAAAA') { return { floatArray: undefined, dbValues: undefined }; } - floatArray = binaryStringToFloatArray(rhFile.data, rhFile.type); + floatArray = binaryStringToFloatArray( + waterfallFile.data, + waterfallFile.data_type, + ); } else { // Old RH data - floatArray = rhFile.data; + floatArray = waterfallFile.data; } if (!floatArray) { @@ -217,7 +218,7 @@ const WaterfallVisualization: React.FC = ({ return { floatArray, dbValues }; }); - }, [rhFiles]); + }, [waterfallFiles]); const globalYAxisRange = useMemo(() => { let globalMin = Infinity; @@ -243,28 +244,24 @@ const WaterfallVisualization: React.FC = ({ * Processes a single file for the periodogram display */ const processPeriodogramData = ( - rhFile: RadioHoundFile, + waterfallFile: WaterfallFile, processedValues: (typeof processedData)[number], ) => { let fMin: number; let fMax: number; - const requested = - rhFile.custom_fields?.requested ?? rhFile.requested ?? undefined; + const requested = waterfallFile.custom_fields?.requested; // Multiple branches to handle both new and old RH data - if (requested && requested.fmin && requested.fmax) { - fMin = requested.fmin; - fMax = requested.fmax; - } else if (rhFile.metadata.fmin && rhFile.metadata.fmax) { - fMin = rhFile.metadata.fmin; - fMax = rhFile.metadata.fmax; - } else if (rhFile.metadata.xstart && rhFile.metadata.xstop) { - fMin = rhFile.metadata.xstart; - fMax = rhFile.metadata.xstop; - } else if (rhFile.center_frequency) { - fMin = rhFile.center_frequency - rhFile.sample_rate / 2; - fMax = rhFile.center_frequency + rhFile.sample_rate / 2; + if (requested && requested.min_frequency && requested.max_frequency) { + fMin = requested.min_frequency; + fMax = requested.max_frequency; + } else if (waterfallFile.min_frequency && waterfallFile.max_frequency) { + fMin = waterfallFile.min_frequency; + fMax = waterfallFile.max_frequency; + } else if (waterfallFile.center_frequency) { + fMin = waterfallFile.center_frequency - waterfallFile.sample_rate / 2; + fMax = waterfallFile.center_frequency + waterfallFile.sample_rate / 2; } else { throw new Error('No frequency range found'); } @@ -273,14 +270,11 @@ const WaterfallVisualization: React.FC = ({ let centerFreq: number; // Multiple branches to handle both new and old RH data - if (rhFile.center_frequency) { - freqStep = rhFile.sample_rate / rhFile.metadata.nfft; - centerFreq = rhFile.center_frequency; - } else if (rhFile.metadata.xcount) { - freqStep = (fMax - fMin) / rhFile.metadata.xcount; - centerFreq = (fMax + fMin) / 2; + if (waterfallFile.center_frequency) { + freqStep = waterfallFile.sample_rate / waterfallFile.nfft; + centerFreq = waterfallFile.center_frequency; } else { - freqStep = rhFile.sample_rate / rhFile.metadata.nfft; + freqStep = waterfallFile.sample_rate / waterfallFile.nfft; centerFreq = (fMax + fMin) / 2; } @@ -289,32 +283,21 @@ const WaterfallVisualization: React.FC = ({ let m4sMean: FloatArray | undefined; let m4sMedian: FloatArray | undefined; - if ( - rhFile.m4s_min && - rhFile.m4s_max && - rhFile.m4s_mean && - rhFile.m4s_median - ) { - m4sMin = binaryStringToFloatArray(rhFile.m4s_min, rhFile.type); - m4sMax = binaryStringToFloatArray(rhFile.m4s_max, rhFile.type); - m4sMean = binaryStringToFloatArray(rhFile.m4s_mean, rhFile.type); - m4sMedian = binaryStringToFloatArray(rhFile.m4s_median, rhFile.type); - } - const tmpDisplay = _.cloneDeep(display); // Use pre-processed data const dataArray = processedValues.floatArray; if ( - display.maxHoldValues[rhFile.mac_address] === undefined || - dataArray?.length !== display.maxHoldValues[rhFile.mac_address].length + display.maxHoldValues[waterfallFile.mac_address] === undefined || + dataArray?.length !== + display.maxHoldValues[waterfallFile.mac_address].length ) { - tmpDisplay.maxHoldValues[rhFile.mac_address] = []; + tmpDisplay.maxHoldValues[waterfallFile.mac_address] = []; } const yValues = processedValues.dbValues; - const arrayLength = dataArray?.length ?? rhFile.metadata.xcount; + const arrayLength = dataArray?.length; const pointArr: DataPoint[] = []; const minArray: DataPoint[] = []; const maxArray: DataPoint[] = []; @@ -331,17 +314,19 @@ const WaterfallVisualization: React.FC = ({ pointArr.push({ x: xValue, y: yValue }); if (display.max_hold) { - if (tmpDisplay.maxHoldValues[rhFile.mac_address].length <= i) { - tmpDisplay.maxHoldValues[rhFile.mac_address].push({ + if ( + tmpDisplay.maxHoldValues[waterfallFile.mac_address].length <= i + ) { + tmpDisplay.maxHoldValues[waterfallFile.mac_address].push({ x: xValue, y: yValue, }); } else { const maxHoldValuesY = - tmpDisplay.maxHoldValues[rhFile.mac_address][i].y; + tmpDisplay.maxHoldValues[waterfallFile.mac_address][i].y; if (maxHoldValuesY && yValue > maxHoldValuesY) { - tmpDisplay.maxHoldValues[rhFile.mac_address][i] = { + tmpDisplay.maxHoldValues[waterfallFile.mac_address][i] = { x: xValue, y: yValue, }; @@ -349,7 +334,7 @@ const WaterfallVisualization: React.FC = ({ } } - if ('m4s_min' in rhFile) { + if ('m4s_min' in waterfallFile) { minArray.push({ x: xValue, y: m4sMin?.[i] }); maxArray.push({ x: xValue, y: m4sMax?.[i] }); meanArray.push({ x: xValue, y: m4sMean?.[i] }); @@ -369,7 +354,7 @@ const WaterfallVisualization: React.FC = ({ } else { //Find index for this node nextIndex = tmpChart.data.findIndex( - (element) => element._id === rhFile.mac_address, + (element) => element._id === waterfallFile.mac_address, ); if (nextIndex === -1) { nextIndex = tmpChart.data.length; @@ -383,12 +368,14 @@ const WaterfallVisualization: React.FC = ({ axisXType: 'secondary', showInLegend: true, name: - rhFile.short_name + + waterfallFile.device_name + ' (' + - rhFile.mac_address.substring(rhFile.mac_address.length - 4) + + waterfallFile.mac_address.substring( + waterfallFile.mac_address.length - 4, + ) + ')', - toolTipContent: rhFile.short_name + ': {x}, {y}', - _id: rhFile.mac_address, + toolTipContent: waterfallFile.device_name + ': {x}, {y}', + _id: waterfallFile.mac_address, }; // Ensure axisX exists and isn't an array. @@ -400,14 +387,6 @@ const WaterfallVisualization: React.FC = ({ tmpChart.axisX2!.title = 'Frequency ' + (centerFreq ? formatHertz(centerFreq) : ''); - if (rhFile.requested) { - tmpChart.axisX2!.title += rhFile.requested.rbw - ? ', RBW ' + formatHertz(rhFile.requested.rbw) - : ''; - tmpChart.axisX2!.title += rhFile.requested.span - ? ', Span ' + formatHertz(rhFile.requested.span) - : ''; - } // if (CurrentApplication === DEFS.APPLICATION_PERIODOGRAM_MULTI) // { @@ -420,18 +399,12 @@ const WaterfallVisualization: React.FC = ({ // + " (" + input.mac_address.substring(input.mac_address.length - 4) + ')'; // } - if (_.isEqual(currentApplication, ['WATERFALL'])) { - tmpChart.axisX2!.title = ''; - tmpChart.data[nextIndex].showInLegend = false; - tmpChart.data[nextIndex].name = rhFile.short_name; - } - if ( display.max_hold && - display.maxHoldValues[rhFile.mac_address].length > 0 + display.maxHoldValues[waterfallFile.mac_address].length > 0 ) { nextIndex = tmpChart.data.findIndex( - (element) => element._id === 'maxhold_' + rhFile.mac_address, + (element) => element._id === 'maxhold_' + waterfallFile.mac_address, ); if (nextIndex === -1) { nextIndex = tmpChart.data.length; @@ -441,10 +414,12 @@ const WaterfallVisualization: React.FC = ({ ..._.cloneDeep(tmpChart.data[nextIndex - 1]), name: 'Max Hold (' + - rhFile.mac_address.substring(rhFile.mac_address.length - 4) + + waterfallFile.mac_address.substring( + waterfallFile.mac_address.length - 4, + ) + ')', - _id: 'maxhold_' + rhFile.mac_address, - dataPoints: tmpDisplay.maxHoldValues[rhFile.mac_address], + _id: 'maxhold_' + waterfallFile.mac_address, + dataPoints: tmpDisplay.maxHoldValues[waterfallFile.mac_address], toolTipContent: 'Max : {x}, {y}', }; @@ -453,7 +428,7 @@ const WaterfallVisualization: React.FC = ({ } } - if ('m4s_min' in rhFile) { + if ('m4s_min' in waterfallFile) { nextIndex += 1; tmpChart.data[nextIndex] = _.cloneDeep(tmpChart.data[nextIndex - 1]); tmpChart.data[nextIndex].name = 'M4S Min'; @@ -524,13 +499,8 @@ const WaterfallVisualization: React.FC = ({ tmpChart.axisY!.interval = display.ref_interval; } - if (currentApplication.includes('PERIODOGRAM')) { - tmpChart.axisX2!.minimum = fMin / 1e6; - tmpChart.axisX2!.maximum = fMax / 1e6; - } else { - delete tmpChart.axisX2!.minimum; - delete tmpChart.axisX2!.maximum; - } + tmpChart.axisX2!.minimum = fMin / 1e6; + tmpChart.axisX2!.maximum = fMax / 1e6; // Move dummy series to the end of the data array to maintain correct data // series colors @@ -549,7 +519,7 @@ const WaterfallVisualization: React.FC = ({ * Processes multiple files for the waterfall display */ const processWaterfallData = ( - rhFiles: RadioHoundFile[], + waterfallFiles: WaterfallFile[], processedValues: typeof processedData, ) => { const processedWaterfallData: number[][] = []; @@ -558,7 +528,7 @@ const WaterfallVisualization: React.FC = ({ let xMin = Infinity; let xMax = -Infinity; - rhFiles.forEach((rhFile, index) => { + waterfallFiles.forEach((waterfallFile, index) => { const processedFile = processedValues[index]; const yValues = processedFile.dbValues; @@ -579,12 +549,11 @@ const WaterfallVisualization: React.FC = ({ // Update x range let currentXMin = Infinity; let currentXMax = -Infinity; - if (rhFile.metadata.xstart && rhFile.metadata.xstop) { - currentXMin = rhFile.metadata.xstart; - currentXMax = rhFile.metadata.xstop; - } else if (rhFile.center_frequency && rhFile.sample_rate) { - currentXMin = rhFile.center_frequency - rhFile.sample_rate / 2; - currentXMax = rhFile.center_frequency + rhFile.sample_rate / 2; + if (waterfallFile.center_frequency && waterfallFile.sample_rate) { + currentXMin = + waterfallFile.center_frequency - waterfallFile.sample_rate / 2; + currentXMax = + waterfallFile.center_frequency + waterfallFile.sample_rate / 2; } xMin = Math.min(xMin, currentXMin); xMax = Math.max(xMax, currentXMax); @@ -600,7 +569,7 @@ const WaterfallVisualization: React.FC = ({ })); // Update waterfall state - const newWaterfall: WaterfallType = { + const newWaterfall: ScanWaterfallType = { allData: processedWaterfallData, xMin: xMin / 1e6, xMax: xMax / 1e6, @@ -610,16 +579,16 @@ const WaterfallVisualization: React.FC = ({ scaleMax: globalMaxValue, }; - setWaterfall(newWaterfall); + setScanWaterfall(newWaterfall); }; useEffect(() => { // Process single file for periodogram processPeriodogramData( - rhFiles[settings.fileIndex], + waterfallFiles[settings.fileIndex], processedData[settings.fileIndex], ); - }, [rhFiles, processedData, settings.fileIndex]); + }, [waterfallFiles, processedData, settings.fileIndex]); useEffect(() => { const pageSize = WATERFALL_MAX_ROWS; @@ -633,16 +602,19 @@ const WaterfallVisualization: React.FC = ({ // Calculate new start index only when moving outside current window const idealStartIndex = Math.floor(settings.fileIndex / pageSize) * pageSize; - const lastPossibleStartIndex = Math.max(0, rhFiles.length - pageSize); + const lastPossibleStartIndex = Math.max( + 0, + waterfallFiles.length - pageSize, + ); const startIndex = Math.min(idealStartIndex, lastPossibleStartIndex); - const endIndex = Math.min(rhFiles.length, startIndex + pageSize); + const endIndex = Math.min(waterfallFiles.length, startIndex + pageSize); // Only reprocess waterfall if the range has changed if ( startIndex !== waterfallRange.startIndex || endIndex !== waterfallRange.endIndex ) { - const relevantFiles = rhFiles.slice(startIndex, endIndex); + const relevantFiles = waterfallFiles.slice(startIndex, endIndex); const relevantProcessedValues = processedData.slice( startIndex, endIndex, @@ -651,15 +623,15 @@ const WaterfallVisualization: React.FC = ({ setWaterfallRange({ startIndex, endIndex }); } } - }, [rhFiles, processedData, settings.fileIndex, waterfallRange]); + }, [waterfallFiles, processedData, settings.fileIndex, waterfallRange]); // Handle realtime playback useEffect(() => { if (!settings.isPlaying || settings.playbackSpeed !== 'realtime') return; // Pre-compute timestamps for all files - const timestamps = rhFiles.map((rhFile) => - rhFile.timestamp ? Date.parse(rhFile.timestamp) : 0, + const timestamps = waterfallFiles.map((waterfallFile) => + waterfallFile.timestamp ? Date.parse(waterfallFile.timestamp) : 0, ); // Store the start time and reference points @@ -683,7 +655,7 @@ const WaterfallVisualization: React.FC = ({ return { ...prev, fileIndex: targetIndex, - isPlaying: targetIndex < rhFiles.length - 1, + isPlaying: targetIndex < waterfallFiles.length - 1, }; } return prev; @@ -691,7 +663,12 @@ const WaterfallVisualization: React.FC = ({ }, 20); return () => clearInterval(realtimeInterval); - }, [settings.isPlaying, settings.playbackSpeed, rhFiles.length, setSettings]); + }, [ + settings.isPlaying, + settings.playbackSpeed, + waterfallFiles.length, + setSettings, + ]); // Handle constant FPS playback useEffect(() => { @@ -706,7 +683,7 @@ const WaterfallVisualization: React.FC = ({ setSettings((prev) => { const nextIndex = prev.fileIndex + 1; // Stop playback at the end - if (nextIndex >= rhFiles.length) { + if (nextIndex >= waterfallFiles.length) { return { ...prev, isPlaying: false }; } return { ...prev, fileIndex: nextIndex }; @@ -714,7 +691,12 @@ const WaterfallVisualization: React.FC = ({ }, intervalTime); return () => clearInterval(playbackInterval); - }, [settings.isPlaying, settings.playbackSpeed, rhFiles.length, setSettings]); + }, [ + settings.isPlaying, + settings.playbackSpeed, + waterfallFiles.length, + setSettings, + ]); const handleRowSelect = (index: number) => { // Update the settings with the new file index @@ -728,7 +710,8 @@ const WaterfallVisualization: React.FC = ({ return (
- Scan {displayedFileIndex + 1} ({rhFiles[displayedFileIndex].timestamp}) + Scan {displayedFileIndex + 1} ( + {waterfallFiles[displayedFileIndex].timestamp})
= ({ diff --git a/frontend/src/components/waterfall/types.ts b/frontend/src/components/waterfall/types.ts index 2045788..20b26f9 100644 --- a/frontend/src/components/waterfall/types.ts +++ b/frontend/src/components/waterfall/types.ts @@ -109,73 +109,36 @@ export interface Display { errors?: ScanOptionsType['errors']; } -/** - * RadioHound format (.rh/.rh.json) capture for periodograms - * - * Schema definition: - * https://github.com/spectrumx/schema-definitions/tree/master/definitions/sds/metadata-formats/radiohound - */ -const RequestedSchema = zod.object({ - fmin: zod.number().optional(), - fmax: zod.number().optional(), - span: zod.number().optional(), - rbw: zod.number().optional(), - samples: zod.number().optional(), - gain: zod.number().optional(), -}); - -const RadioHoundMetadataSchema = zod.object({ - data_type: zod.string(), - fmax: zod.number(), - fmin: zod.number(), - gps_lock: zod.boolean(), - nfft: zod.number(), - scan_time: zod.number(), - archive_result: zod.boolean().optional(), - // Deprecated fields - xcount: zod.number().optional(), - xstart: zod.number().optional(), - xstop: zod.number().optional(), - suggested_gain: zod.number().optional(), - uncertainty: zod.number().optional(), - archiveResult: zod.boolean().optional(), -}); - -const RadioHoundCustomFieldsSchema = zod +const WaterfallCustomFieldsSchema = zod .object({ - requested: RequestedSchema, + requested: zod + .object({ + min_frequency: zod.number().optional(), + max_frequency: zod.number().optional(), + }) + .optional(), + scan_time: zod.number().optional(), + gain: zod.number().optional(), + gps_lock: zod.boolean().optional(), + job_name: zod.string().optional(), + comments: zod.string().optional(), }) .catchall(zod.unknown()); -export const RadioHoundFileSchema = zod.object({ +export const WaterfallFileSchema = zod.object({ data: zod.string(), - gain: zod.number(), - latitude: zod.number(), - longitude: zod.number(), - mac_address: zod.string(), - metadata: RadioHoundMetadataSchema, - sample_rate: zod.number(), - short_name: zod.string(), + data_type: zod.string(), timestamp: zod.string(), - type: zod.string(), - version: zod.string(), - altitude: zod.number().optional(), + min_frequency: zod.number(), + max_frequency: zod.number(), + nfft: zod.number(), + sample_rate: zod.number(), + mac_address: zod.string(), + device_name: zod.string().optional(), center_frequency: zod.number().optional(), - custom_fields: RadioHoundCustomFieldsSchema.optional(), - hardware_board_id: zod.string().optional(), - hardware_version: zod.string().optional(), - scan_group: zod.string().optional(), - software_version: zod.string().optional(), - // Deprecated fields - batch: zod.number().optional(), - m4s_min: zod.string().optional(), - m4s_max: zod.string().optional(), - m4s_mean: zod.string().optional(), - m4s_median: zod.string().optional(), - requested: RequestedSchema.optional(), + custom_fields: WaterfallCustomFieldsSchema.optional(), }); - -export type RadioHoundFile = zod.infer; +export type WaterfallFile = zod.infer; export type FloatArray = Float32Array | Float64Array; @@ -190,20 +153,14 @@ export interface ScanState { xMin?: number; xMax?: number; spinner: boolean; - periodogram?: RadioHoundFile | number[]; + periodogram?: WaterfallFile | number[]; heatmapData: Data[]; scaleMin: number | undefined; scaleMax: number | undefined; } -export interface WaterfallType +export interface ScanWaterfallType extends Pick, Partial< Pick > {} - -export type ApplicationType = - | 'WATERFALL' - | 'PERIODOGRAM' - | 'PERIODOGRAM_SINGLE' - | 'PERIODOGRAM_MULTI'; diff --git a/spectrumx_visualization_platform/spx_vis/api/views.py b/spectrumx_visualization_platform/spx_vis/api/views.py index 37b81ab..5c456e8 100644 --- a/spectrumx_visualization_platform/spx_vis/api/views.py +++ b/spectrumx_visualization_platform/spx_vis/api/views.py @@ -1,4 +1,5 @@ import io +import json import logging import os import shutil @@ -35,6 +36,9 @@ from spectrumx_visualization_platform.spx_vis.capture_utils.digital_rf import ( DigitalRFUtility, ) +from spectrumx_visualization_platform.spx_vis.capture_utils.radiohound import ( + RadioHoundUtility, +) from spectrumx_visualization_platform.spx_vis.capture_utils.sigmf import SigMFUtility from spectrumx_visualization_platform.spx_vis.models import Capture from spectrumx_visualization_platform.spx_vis.models import CaptureType @@ -486,7 +490,7 @@ def _handle_sds_captures( """ logging.info("Getting SDS captures") sds_captures = get_sds_captures(request) - logging.info(f"Got {len(sds_captures)} SDS captures") + logging.info(f"Found {len(sds_captures)} SDS captures") token = request.user.sds_token for capture_id in visualization.capture_ids: @@ -497,9 +501,14 @@ def _handle_sds_captures( if capture is None: raise ValueError(f"Capture ID {capture_id} not found in SDS") + files = capture.get("files", []) + if not files: + raise ValueError(f"No files found for capture ID {capture_id}") + seen_filenames: set[str] = set() - for file in capture.get("files", []): + for i, file in enumerate(files): + logging.info(f"Downloading file {i + 1} of {len(files)}") try: self._process_sds_file( file, capture_id, token, seen_filenames, zip_file @@ -624,3 +633,91 @@ def save(self, request: Request, uuid=None) -> Response: serializer = self.get_serializer(visualization) return Response(serializer.data) + + @action(detail=True, methods=["get"]) + def get_waterfall_data(self, request: Request, uuid=None) -> Response: + """Get waterfall data for a visualization. + + This endpoint retrieves files from the visualization's SDS captures and converts them + to the WaterfallFile format expected by the frontend. Currently supports RadioHound + captures from SDS sources. + + Args: + request: The HTTP request + uuid: The UUID of the visualization + + Returns: + Response: A list of WaterfallFile objects + + Raises: + Response: 400 if the visualization type is not supported + Response: 400 if there's an error processing the files + """ + visualization: Visualization = self.get_object() + + # Currently only support RadioHound captures + if visualization.capture_type != CaptureType.RadioHound: + return Response( + { + "status": "error", + "message": "Only RadioHound captures are currently supported for waterfall visualization", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Only support SDS captures + if visualization.capture_source != "sds": + return Response( + { + "status": "error", + "message": "Only SDS captures are currently supported", + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + waterfall_files = [] + + # Create a BytesIO object to store the ZIP file + zip_buffer = io.BytesIO() + + with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: + self._handle_sds_captures(visualization, request, zip_file) + + # Reset buffer position to start + zip_buffer.seek(0) + + # Process the ZIP file + with zipfile.ZipFile(zip_buffer, "r") as zip_file: + for capture_id in visualization.capture_ids: + capture_dir = f"{capture_id}/" + for file_info in zip_file.infolist(): + if file_info.filename.startswith( + capture_dir + ) and file_info.filename.endswith(".json"): + with zip_file.open(file_info) as f: + try: + rh_data = json.load(f) + waterfall_file = ( + RadioHoundUtility.to_waterfall_file(rh_data) + ) + waterfall_files.append(waterfall_file) + except json.JSONDecodeError as e: + logging.error( + f"Failed to parse JSON from file {file_info.filename}: {e}" + ) + continue + except ValueError as e: + logging.error( + f"Failed to convert file {file_info.filename} to waterfall format: {e}" + ) + continue + + return Response(waterfall_files) + + except Exception as e: + logging.exception("Error processing waterfall data") + return Response( + {"error": f"Failed to process waterfall data: {e}"}, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/spectrumx_visualization_platform/spx_vis/capture_utils/base.py b/spectrumx_visualization_platform/spx_vis/capture_utils/base.py index 655849f..3794ccf 100644 --- a/spectrumx_visualization_platform/spx_vis/capture_utils/base.py +++ b/spectrumx_visualization_platform/spx_vis/capture_utils/base.py @@ -72,3 +72,15 @@ def submit_spectrogram_job( Returns: The submitted job """ + + @staticmethod + @abstractmethod + def to_waterfall_file(file: UploadedFile) -> dict: + """Convert a capture file to the WaterfallFile format. + + Args: + file: The uploaded capture file + + Returns: + dict: The converted WaterfallFile + """ diff --git a/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py b/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py index 2166e11..f4ecee1 100644 --- a/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py +++ b/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py @@ -136,3 +136,105 @@ def get_capture_name(files: list[UploadedFile], name: str | None = None) -> str: # Use the file name (without extension) as the capture name return ".".join(files[0].name.split(".")[:-1]) + + @staticmethod + def to_waterfall_file(rh_data: dict) -> dict: + """Convert RadioHound data to WaterfallFile format. + + Args: + rh_data: Dictionary containing RadioHound file data + + Returns: + dict: Data in WaterfallFile format + + Raises: + ValueError: If required fields are missing or invalid + """ + try: + metadata = rh_data.get("metadata", {}) + + waterfall_file = RadioHoundUtility._get_required_waterfall_fields( + rh_data, metadata + ) + waterfall_file.update( + RadioHoundUtility._get_extra_waterfall_fields(rh_data, metadata) + ) + + return waterfall_file + + except Exception as e: + error_message = ( + f"Error converting RadioHound data to WaterfallFile format: {e}" + ) + logger.error(error_message) + raise ValueError(error_message) + + @staticmethod + def _get_required_waterfall_fields(rh_data: dict, metadata: dict) -> dict: + """Extract required fields from RadioHound data. + + Args: + rh_data: Dictionary containing RadioHound file data + metadata: Dictionary containing RadioHound metadata + + Returns: + dict: Required fields in WaterfallFile format + """ + required_fields = { + "data": rh_data.get("data", ""), + "data_type": rh_data.get("type", ""), + "timestamp": rh_data.get("timestamp", ""), + "min_frequency": metadata.get("fmin"), + "max_frequency": metadata.get("fmax"), + "nfft": metadata.get("nfft"), + "sample_rate": rh_data.get("sample_rate"), + "mac_address": rh_data.get("mac_address", ""), + } + missing_fields = [] + for field, value in required_fields.items(): + if value is None or value == "": + missing_fields.append(field) + + if missing_fields: + raise ValueError(f"Missing required fields: {', '.join(missing_fields)}") + + return required_fields + + @staticmethod + def _get_extra_waterfall_fields(rh_data: dict, metadata: dict) -> dict: + """Extract optional and custom fields from RadioHound data. + + Args: + rh_data: Dictionary containing RadioHound file data + metadata: Dictionary containing RadioHound metadata + + Returns: + dict: Additional fields in WaterfallFile format + """ + additional_fields = {} + requested = rh_data.get("requested", {}) + + # Optional fields + if "center_frequency" in rh_data: + additional_fields["center_frequency"] = rh_data["center_frequency"] + if "short_name" in rh_data: + additional_fields["device_name"] = rh_data["short_name"] + + # Custom fields + custom_fields = {} + if "fmin" in requested or "fmax" in requested: + custom_fields["requested"] = {} + if "fmin" in requested: + custom_fields["requested"]["min_frequency"] = requested["fmin"] + if "fmax" in requested: + custom_fields["requested"]["max_frequency"] = requested["fmax"] + + for field in ["scan_time", "gain", "gps_lock", "name", "comments"]: + if field in metadata: + key = "job_name" if field == "name" else field + custom_fields[key] = metadata[field] + + if custom_fields: + additional_fields["custom_fields"] = custom_fields + + return additional_fields From 524c4fdf4cd48488b4a82f3552b46b20f685f4ca Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Fri, 6 Jun 2025 17:52:53 -0400 Subject: [PATCH 2/4] Remove unused waterfall logic --- frontend/src/components/waterfall/index.tsx | 42 --------------------- 1 file changed, 42 deletions(-) diff --git a/frontend/src/components/waterfall/index.tsx b/frontend/src/components/waterfall/index.tsx index 2a159af..c32f95d 100644 --- a/frontend/src/components/waterfall/index.tsx +++ b/frontend/src/components/waterfall/index.tsx @@ -278,11 +278,6 @@ const WaterfallVisualization: React.FC = ({ centerFreq = (fMax + fMin) / 2; } - let m4sMin: FloatArray | undefined; - let m4sMax: FloatArray | undefined; - let m4sMean: FloatArray | undefined; - let m4sMedian: FloatArray | undefined; - const tmpDisplay = _.cloneDeep(display); // Use pre-processed data @@ -299,10 +294,6 @@ const WaterfallVisualization: React.FC = ({ const yValues = processedValues.dbValues; const arrayLength = dataArray?.length; const pointArr: DataPoint[] = []; - const minArray: DataPoint[] = []; - const maxArray: DataPoint[] = []; - const meanArray: DataPoint[] = []; - const medianArray: DataPoint[] = []; let yValue: number | undefined; let xValue: number; @@ -333,13 +324,6 @@ const WaterfallVisualization: React.FC = ({ } } } - - if ('m4s_min' in waterfallFile) { - minArray.push({ x: xValue, y: m4sMin?.[i] }); - maxArray.push({ x: xValue, y: m4sMax?.[i] }); - meanArray.push({ x: xValue, y: m4sMean?.[i] }); - medianArray.push({ x: xValue, y: m4sMedian?.[i] }); - } } } } @@ -428,32 +412,6 @@ const WaterfallVisualization: React.FC = ({ } } - if ('m4s_min' in waterfallFile) { - nextIndex += 1; - tmpChart.data[nextIndex] = _.cloneDeep(tmpChart.data[nextIndex - 1]); - tmpChart.data[nextIndex].name = 'M4S Min'; - tmpChart.data[nextIndex].dataPoints = minArray; - tmpChart.data[nextIndex].toolTipContent = 'Min : {x}, {y}'; - - nextIndex += 1; - tmpChart.data[nextIndex] = _.cloneDeep(tmpChart.data[nextIndex - 1]); - tmpChart.data[nextIndex].name = 'M4S Max'; - tmpChart.data[nextIndex].dataPoints = maxArray; - tmpChart.data[nextIndex].toolTipContent = 'Max : {x}, {y}'; - - nextIndex += 1; - tmpChart.data[nextIndex] = _.cloneDeep(tmpChart.data[nextIndex - 1]); - tmpChart.data[nextIndex].name = 'M4S Mean'; - tmpChart.data[nextIndex].dataPoints = meanArray; - tmpChart.data[nextIndex].toolTipContent = 'Mean : {x}, {y}'; - - nextIndex += 1; - tmpChart.data[nextIndex] = _.cloneDeep(tmpChart.data[nextIndex - 1]); - tmpChart.data[nextIndex].name = 'M4S Median'; - tmpChart.data[nextIndex].dataPoints = medianArray; - tmpChart.data[nextIndex].toolTipContent = 'Median : {x}, {y}'; - } - // Hide legend if there is only one data series to show in the legend let seriesWithLegendIndex = -1; let showLegend = false; From 4691abd094202bb310fd1fef40900a87741307c7 Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Fri, 6 Jun 2025 18:03:00 -0400 Subject: [PATCH 3/4] Rename nfft to num_samples --- frontend/src/components/waterfall/ScanDetailsTable.tsx | 4 ++-- frontend/src/components/waterfall/index.tsx | 4 ++-- frontend/src/components/waterfall/types.ts | 2 +- .../spx_vis/capture_utils/radiohound.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/waterfall/ScanDetailsTable.tsx b/frontend/src/components/waterfall/ScanDetailsTable.tsx index baa5f46..cf6fe64 100644 --- a/frontend/src/components/waterfall/ScanDetailsTable.tsx +++ b/frontend/src/components/waterfall/ScanDetailsTable.tsx @@ -96,8 +96,8 @@ export function ScanDetails({ waterfallFile }: ScanDetailsProps): JSX.Element { diff --git a/frontend/src/components/waterfall/index.tsx b/frontend/src/components/waterfall/index.tsx index c32f95d..b9f246c 100644 --- a/frontend/src/components/waterfall/index.tsx +++ b/frontend/src/components/waterfall/index.tsx @@ -271,10 +271,10 @@ const WaterfallVisualization: React.FC = ({ // Multiple branches to handle both new and old RH data if (waterfallFile.center_frequency) { - freqStep = waterfallFile.sample_rate / waterfallFile.nfft; + freqStep = waterfallFile.sample_rate / waterfallFile.num_samples; centerFreq = waterfallFile.center_frequency; } else { - freqStep = waterfallFile.sample_rate / waterfallFile.nfft; + freqStep = waterfallFile.sample_rate / waterfallFile.num_samples; centerFreq = (fMax + fMin) / 2; } diff --git a/frontend/src/components/waterfall/types.ts b/frontend/src/components/waterfall/types.ts index 20b26f9..2a66eaa 100644 --- a/frontend/src/components/waterfall/types.ts +++ b/frontend/src/components/waterfall/types.ts @@ -131,7 +131,7 @@ export const WaterfallFileSchema = zod.object({ timestamp: zod.string(), min_frequency: zod.number(), max_frequency: zod.number(), - nfft: zod.number(), + num_samples: zod.number(), sample_rate: zod.number(), mac_address: zod.string(), device_name: zod.string().optional(), diff --git a/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py b/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py index f4ecee1..05388f0 100644 --- a/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py +++ b/spectrumx_visualization_platform/spx_vis/capture_utils/radiohound.py @@ -186,7 +186,7 @@ def _get_required_waterfall_fields(rh_data: dict, metadata: dict) -> dict: "timestamp": rh_data.get("timestamp", ""), "min_frequency": metadata.get("fmin"), "max_frequency": metadata.get("fmax"), - "nfft": metadata.get("nfft"), + "num_samples": metadata.get("nfft"), "sample_rate": rh_data.get("sample_rate"), "mac_address": rh_data.get("mac_address", ""), } From f9ccbd7b5e6ed3d89a726499de53716e58377d1b Mon Sep 17 00:00:00 2001 From: Connor Howington Date: Fri, 6 Jun 2025 18:14:33 -0400 Subject: [PATCH 4/4] Remove a bit of logging --- spectrumx_visualization_platform/spx_vis/api/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spectrumx_visualization_platform/spx_vis/api/views.py b/spectrumx_visualization_platform/spx_vis/api/views.py index 674ad06..9b60af2 100644 --- a/spectrumx_visualization_platform/spx_vis/api/views.py +++ b/spectrumx_visualization_platform/spx_vis/api/views.py @@ -448,7 +448,6 @@ def _process_sds_file( ValueError: If duplicate filename is found or file download fails """ file_uuid = file["uuid"] - logging.info(f"Downloading file with ID {file_uuid}") response = requests.get( f"https://{settings.SDS_CLIENT_URL}/api/latest/assets/files/{file_uuid}/download", @@ -457,7 +456,6 @@ def _process_sds_file( stream=True, ) response.raise_for_status() - logging.info(f"File with ID {file_uuid} downloaded") # Get filename from Content-Disposition header or use file ID content_disposition = response.headers.get("Content-Disposition", "")