Merge pull request #35 from timson/html-upload

add html ingestion pipeline

Author: mlim19

src/gprofiler-dev/gprofiler_dev/s3_profile_dal.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
+import gzip
 from datetime import datetime
 from io import BytesIO
 from typing import Callable, Optional
@@ -72,6 +72,14 @@ def download_file(self, src_file_path: str, dest_file_path: str) -> None:
                 raise FileNotFoundError("Requested file does not exist", src_file_path) from error
             raise
 
+    def get_object(self, s3_path: str, is_gzip=False) -> str:
+        s3_response = self._s3_client.get_object(Bucket=self.bucket_name, Key=s3_path)
+        response_body = s3_response["Body"].read()
+        if is_gzip:
+            with gzip.GzipFile(fileobj=BytesIO(response_body)) as gz:
+                response_body = gz.read()
+        return response_body.decode("utf-8")
+
     def write_file(self, file_path: str, content: bytes) -> None:
         io_content = BytesIO(content)
         self._s3_client.upload_fileobj(Bucket=self.bucket_name, Fileobj=io_content, Key=file_path)

src/gprofiler/backend/models/metrics_models.py

@@ -83,3 +83,7 @@ class MetricK8s(CamelModel):
     cores: Optional[int] = None
     cpu: Optional[float] = None
     memory: Optional[float] = None
+
+
+class HTMLMetadata(CamelModel):
+    content: str

src/gprofiler/backend/routers/metrics_routes.py

@@ -19,6 +19,8 @@
 from logging import getLogger
 from typing import List, Optional
 
+from botocore.exceptions import ClientError
+
 from backend.models.filters_models import FilterTypes
 from backend.models.flamegraph_models import FGParamsBaseModel
 from backend.models.metrics_models import (
@@ -30,11 +32,14 @@
     MetricNodesCoresSummary,
     MetricSummary,
     SampleCount,
+    HTMLMetadata,
 )
 from backend.utils.filters_utils import get_rql_first_eq_key, get_rql_only_for_one_key
 from backend.utils.request_utils import flamegraph_base_request_params, get_metrics_response, get_query_response
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Depends, Query, HTTPException
 from fastapi.responses import Response
+
+from gprofiler_dev import S3ProfileDal
 from gprofiler_dev.postgres.db_manager import DBManager
 
 logger = getLogger(__name__)
@@ -212,3 +217,21 @@ def calculate_trend_in_cpu(
     )
 
     return response
+
+
+@router.get("/html_metadata", response_model=HTMLMetadata)
+def get_html_metadata(
+    fg_params: FGParamsBaseModel = Depends(flamegraph_base_request_params),
+):
+    host_name_value = get_rql_first_eq_key(fg_params.filter, FilterTypes.HOSTNAME_KEY)
+    if not host_name_value:
+        raise HTTPException(400, detail="Must filter by hostname to get the html metadata")
+    s3_path = get_metrics_response(fg_params, lookup_for="lasthtml")
+    if not s3_path:
+        raise HTTPException(404, detail="The html metadata path not found in CH")
+    s3_dal = S3ProfileDal(logger)
+    try:
+        html_content = s3_dal.get_object(s3_path, is_gzip=True)
+    except ClientError:
+        raise HTTPException(status_code=404, detail="The html metadata file not found in S3")
+    return HTMLMetadata(content=html_content)

src/gprofiler/backend/utils/request_utils.py

@@ -111,7 +111,7 @@ def get_flamegraph_request(db_api_params, stream):
     return response
 
 
-def _common_fg_rest_response(response: Response, db_api_params: Dict) -> Union[List, Dict]:
+def _common_fg_rest_response(response: Response, db_api_params: Dict) -> Union[List, Dict, str]:
     if response.status_code == 401:
         logger.error(f"{response.request.url} request is unauthorized")
     if response.status_code >= 300:
@@ -138,7 +138,7 @@ def _common_fg_rest_response(response: Response, db_api_params: Dict) -> Union[L
 
 def get_query_response(
     fg_params: FGParamsBaseModel, lookup_for: str = "time", resolution=None, interval=None
-) -> Union[List, Dict]:
+) -> Union[List, Dict, str]:
     fg_filter = fg_params.filter.json().encode() if fg_params.filter else None
     db_api_params = get_api_params(
         fg_params.service_name,
@@ -169,8 +169,7 @@ def get_metrics_response(
     interval=None,
     compared_start_datetime=None,
     compared_end_datetime=None,
-    m2m: Optional[bool] = False,
-) -> Union[List, Dict]:
+) -> Union[List, Dict, str]:
     fg_filter = fg_params.filter.json().encode() if fg_params.filter else None
     db_api_params = get_api_params(
         fg_params.service_name,

src/gprofiler/frontend/src/api/hooks/useGetFgMetrics.jsx

@@ -20,6 +20,9 @@ import _ from 'lodash';
 import { stringify } from 'query-string';
 import { useContext, useState } from 'react';
 
+import { isFilterTypeExist } from '@/components/filters/utils';
+import { FILTER_TYPES } from '@/utils/filtersUtils';
+
 import { FilterTagsContext } from '../../states/filters/FiltersTagsContext';
 import { SelectorsContext } from '../../states/selectors';
 import { PROFILES_VIEWS } from '../../utils/consts';
@@ -35,6 +38,7 @@ const useGetFgMetrics = ({ customTimeSelection, customService, disableCoreNodesR
     const { selectedService, timeSelection, viewMode, ignoreZeros } = useContext(SelectorsContext);
     const { activeFilterTag } = useContext(FilterTagsContext);
     const [metricsData, setMetricsData] = useState(undefined);
+    const [lastHtmlData, setLastHtmlData] = useState(undefined);
     const [coresNodesCountData, setCoresNodesCountData] = useState(undefined);
     const [instanceTypeData, setInstanceTypeData] = useState(undefined);
     const timeParams = getStartEndDateTimeFromSelection(customTimeSelection || timeSelection);
@@ -66,6 +70,30 @@ const useGetFgMetrics = ({ customTimeSelection, customService, disableCoreNodesR
         }
     );
 
+    const isHostNameFilterActive = isFilterTypeExist(FILTER_TYPES.HostName.value, activeFilterTag);
+    const { loading: lastHtmlLoading } = useFetchWithRequest(
+        {
+            url: DATA_URLS.GET_LAST_HTML + '?' + stringify(metricsParams),
+        },
+        {
+            refreshDeps: [
+                customService,
+                selectedService,
+                customTimeSelection ? customTimeSelection : timeSelection,
+                activeFilterTag,
+            ],
+            ready:
+                areParamsDefined(customService || selectedService, customTimeSelection || timeSelection) &&
+                isHostNameFilterActive,
+            onSuccess: (result) => {
+                setLastHtmlData(result?.content);
+            },
+            onError: () => {
+                setLastHtmlData(undefined);
+            },
+        }
+    );
+
     const isServiceView = viewMode === PROFILES_VIEWS.service;
     const serviceAndTimeParams = {
         serviceName: customService || selectedService,
@@ -117,6 +145,8 @@ const useGetFgMetrics = ({ customTimeSelection, customService, disableCoreNodesR
     return {
         metricsData,
         metricsLoading,
+        lastHtmlData,
+        lastHtmlLoading,
         coresNodesCountData: coresNodesCountData,
         coresNodesCountLoading,
         instanceTypeData,

src/gprofiler/frontend/src/api/urls.js

@@ -32,6 +32,7 @@ export const DATA_URLS = {
     GET_METRICS: `${API_PREFIX}/metrics/summary`,
     GET_METRICS_CPU_AND_MEMORY_TREND: `${API_PREFIX}/metrics/cpu_trend`,
     GET_NODES_AND_CORES: `${API_PREFIX}/metrics/nodes_cores/summary`,
+    GET_LAST_HTML: `${API_PREFIX}/metrics/html_metadata`,
     GET_INSTANCE_TYPE: `${API_PREFIX}/metrics/instance_type_count`,
     GET_GRAPH_METRICS: `${API_PREFIX}/metrics/graph`,
     GET_FUCNTION_CPU_GRAPH: `${API_PREFIX}/metrics/function_cpu`,

src/gprofiler/frontend/src/components/common/icon/iconsData.js

@@ -18,7 +18,8 @@
 
 import { FILTER_OPERATIONS } from '../../utils/filtersUtils';
 
-const isFilterTypeExist = (type, filter) => {
+export const isFilterTypeExist = (type, filter) => {
+    if (!filter?.filter) return false;
     const [, rules] = Object.entries(filter.filter)[0];
     return rules?.some((rule) => {
         const [ruleType] = Object.entries(rule)[0];

src/gprofiler/frontend/src/components/filters/utils.js

@@ -19,6 +19,9 @@
 import _ from 'lodash';
 import { useCallback, useContext, useEffect, useMemo, useState } from 'react';
 
+import useGetFgMetrics from '@/api/hooks/useGetFgMetrics';
+import HtmlView from '@/components/profiles/views/htmlView/HtmlView';
+
 import {
     countZoomedSearchMatches,
     FgContext,
@@ -52,6 +55,7 @@ import TableView from './views/table/TableView';
 const ProfilesViews = () => {
     const { fgOriginData, lastWeekOriginData, zoomedFgData, setZoomedFgData, framesSelected, setIsFGEmpty, isFGEmpty } =
         useContext(FgContext);
+    const { lastHtmlData } = useGetFgMetrics({});
 
     const { viewMode, selectedService, timeSelection } = useContext(SelectorsContext);
     const { setHoverData } = useContext(FgHoverContext);
@@ -243,6 +247,8 @@ const ProfilesViews = () => {
         <ServiceView />
     ) : viewMode === PROFILES_VIEWS.table ? (
         <TableView timeSelection={timeSelection} rows={tableViewData} filteredData={filteredData} />
+    ) : viewMode === PROFILES_VIEWS.html ? (
+        <HtmlView lastHtml={lastHtmlData} />
     ) : !isFGEmpty ? (
         <FlamegraphView
             flameGraphData={flameGraphData}

src/gprofiler/frontend/src/components/profiles/ProfilesViews.jsx

@@ -89,10 +89,12 @@ const ProfilesTopPanel = () => {
                             </Flexbox>
                         </Flexbox>
                     </Box>
-                    <Flexbox sx={{ px: 5, width: '100%' }} spacing={3}>
-                        <MineSweeper />
-                        <ProfilesMetricsPanel />
-                    </Flexbox>
+                    {viewMode !== PROFILES_VIEWS.html && (
+                        <Flexbox sx={{ px: 5, width: '100%' }} spacing={3}>
+                            <MineSweeper />
+                            <ProfilesMetricsPanel />
+                        </Flexbox>
+                    )}
                 </>
             )}
         </Flexbox>