From 03ebdb1e4e0ab19fd728f84cb365aa4f5da570c8 Mon Sep 17 00:00:00 2001 From: Nikola Milikic Date: Tue, 6 May 2025 10:30:41 +0200 Subject: [PATCH] Remove functions calling the legacy Data API. --- examples/get_data_advanced.py | 76 ------------------ examples/get_data_datasource.py | 106 ------------------------- examples/get_data_simple.py | 116 ---------------------------- examples/list_hosts.py | 93 ---------------------- examples/list_metrics.py | 38 --------- examples/print_conn_table.py | 132 -------------------------------- sdcclient/_common.py | 128 ------------------------------- sdcclient/_monitor.py | 13 ---- 8 files changed, 702 deletions(-) delete mode 100755 examples/get_data_advanced.py delete mode 100755 examples/get_data_datasource.py delete mode 100755 examples/get_data_simple.py delete mode 100755 examples/list_hosts.py delete mode 100755 examples/list_metrics.py delete mode 100755 examples/print_conn_table.py diff --git a/examples/get_data_advanced.py b/examples/get_data_advanced.py deleted file mode 100755 index a1959aa6..00000000 --- a/examples/get_data_advanced.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -# -# This script shows an advanced Sysdig Monitor data request that leverages -# filtering and segmentation. -# -# The request returns the last 10 minutes of CPU utilization for the 5 -# busiest containers inside the given host, with 1 minute data granularity -# - -import json -import sys - -from sdcclient import SdcClient - -# -# Parse arguments -# -if len(sys.argv) != 3: - print(('usage: %s ' % sys.argv[0])) - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - -sdc_token = sys.argv[1] -hostname = sys.argv[2] - -# -# Instantiate the SDC client -# -sdclient = SdcClient(sdc_token) - -# -# Prepare the metrics list. -# -metrics = [ - # The first metric we request is the container name. This is a segmentation - # metric, and you can tell by the fact that we don't specify any aggregation - # criteria. This entry tells Sysdig Monitor that we want to see the CPU - # utilization for each container separately. - {"id": "container.name"}, - # The second metric we request is the CPU. We aggregate it as an average. - {"id": "cpu.used.percent", - "aggregations": { - "time": "avg", - "group": "avg" - } - } -] - -# -# Prepare the filter -# -filter = "host.hostName = '%s'" % hostname - -# -# Paging (from and to included; by default you get from=0 to=9) -# Here we'll get the top 5. -# -paging = {"from": 0, "to": 4} - -# -# Fire the query. -# -ok, res = sdclient.get_data(metrics=metrics, # List of metrics to query - start_ts=-600, # Start of query span is 600 seconds ago - end_ts=0, # End the query span now - sampling_s=60, # 1 data point per minute - filter=filter, # The filter specifying the target host - paging=paging, # Paging to limit to just the 5 most busy - datasource_type='container') # The source for our metrics is the container - -# -# Show the result! -# -print((json.dumps(res, sort_keys=True, indent=4))) -if not ok: - sys.exit(1) diff --git a/examples/get_data_datasource.py b/examples/get_data_datasource.py deleted file mode 100755 index e7625079..00000000 --- a/examples/get_data_datasource.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# -# This script shows the use of the datasource_type argument in the get_data request, -# by providing a few clarifying examples -# - -import sys - -from sdcclient import SdcClient - -# -# Parse arguments -# -if len(sys.argv) != 2: - print(('usage: %s ' % sys.argv[0])) - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - -sdc_token = sys.argv[1] - -# -# Instantiate the SDC client -# -sdclient = SdcClient(sdc_token) - -cpu_metric = [ - {"id": "cpu.used.percent", - "aggregations": { - "time": "avg", - "group": "avg" - } - }] - -# -# First example: CPU by host name -# datasource_type is not necessary since it's infered from the grouping key host.hostName -# -req = [{"id": "host.hostName"}] -req.extend(cpu_metric) -ok, res = sdclient.get_data(req, # metrics list - -600, # start_ts = 600 seconds ago - 0) # end_ts = now - -if ok: - data = res -else: - print(res) - sys.exit(1) - -print("\n\nCPU by host:") -print(data) - -# -# Second example: CPU by container name -# datasource_type is not necessary since it's infered from the grouping key container.name -# -req = [{"id": "container.name"}] -req.extend(cpu_metric) -ok, res = sdclient.get_data(req, # metrics list - -600, # start_ts = 600 seconds ago - 0) # end_ts = now - -if ok: - data = res -else: - print(res) - sys.exit(1) - -print("\n\nCPU by container:") -print(data) - -# -# Third example: CPU average across all hosts -# datasource_type is set to host since no grouping keys or filters are specified (default would be host anyway) -# -ok, res = sdclient.get_data(cpu_metric, # metrics list - -600, # start_ts = 600 seconds ago - 0, # end_ts = now - datasource_type='host') # ask data from hosts - -if ok: - data = res -else: - print(res) - sys.exit(1) - -print("\n\nAverage CPU across all the hosts in the infrastructure:") -print(data) - -# -# Third example: CPU average across all containers -# datasource_type is set to container since no grouping keys or filters are specified (ovverrides the host default) -# -ok, res = sdclient.get_data(cpu_metric, # metrics list - -600, # start_ts = 600 seconds ago - 0, # end_ts = now - datasource_type='container') # ask data from containers - -if ok: - data = res -else: - print(res) - sys.exit(1) - -print("\n\nAverage CPU across all the containers in the infrastructure:") -print(data) diff --git a/examples/get_data_simple.py b/examples/get_data_simple.py deleted file mode 100755 index e6138842..00000000 --- a/examples/get_data_simple.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -# This script shows the basics of getting data out of Sysdig Monitor by creating a -# very simple request that has no filter and no segmentation. -# -# The request queries for the average CPU across all of the instrumented hosts for -# the last 10 minutes, with 1 minute data granularity -# - -import sys - -from sdcclient import SdcClient - -# -# Parse arguments -# -if len(sys.argv) != 2: - print(('usage: %s ' % sys.argv[0])) - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - -sdc_token = sys.argv[1] - -sdclient = SdcClient(sdc_token) - -# -# List of metrics to export. Imagine a SQL data table, with key columns and value columns -# You just need to specify the ID for keys, and ID with aggregation for values. -# -metrics = [ - # {"id": "container.id"}, - # {"id": "agent.tag.env", "aggregations": {"time": "concat", "group": "concat"}}, - {"id": "cpu.used.percent", "aggregations": {"time": "timeAvg", "group": "avg"}} -] - -# -# Data filter or None if you want to see "everything" -# -filter = None - -# -# Time window: -# - for "from A to B": start is equal to A, end is equal to B (expressed in seconds) -# - for "last X seconds": start is equal to -X, end is equal to 0 -# -start = -600 -end = 0 - -# -# Sampling time: -# - for time series: sampling is equal to the "width" of each data point (expressed in seconds) -# - for aggregated data (similar to bar charts, pie charts, tables, etc.): sampling is equal to 0 -# -sampling = 60 - -# -# Load data -# -ok, res = sdclient.get_data(metrics, start, end, sampling, filter=filter) - -# -# Show the result -# -if ok: - # - # Read the response. The JSON looks like this: - # - # { - # start: timestamp, - # end: timestamp, - # data: [ - # { - # t: timestamp, - # d: [ value1, value2, value3, ... ] - # }, - # ... - # ] - # } - # - colLen = 25 - - # - # Print summary (what, when) - # - start = res['start'] - end = res['end'] - data = res['data'] - - print(('Data for %s from %d to %d' % (filter if filter else 'everything', start, end))) - print('') - - # - # Print table headers - # - dataToPrint = ' '.join( - [str(x['id']).ljust(colLen) if len(str(x['id'])) < colLen else str(x['id'])[:(colLen - 3)].ljust( - colLen - 3) + '...' for x in metrics]) - print(('%s %s' % ('timestamp'.ljust(colLen), dataToPrint) if sampling > 0 else dataToPrint)) - print('') - - # - # Print table body - # - for d in data: - timestamp = d['t'] if sampling > 0 else start - values = d['d'] - - dataToPrint = ' '.join( - [str(x).ljust(colLen) if len(str(x)) < colLen else str(x)[:(colLen - 3)].ljust(colLen - 3) + '...' for x in - values]) - - print(('%s %s' % (('' % (timestamp)).ljust(colLen), dataToPrint) if sampling > 0 else dataToPrint)) - -else: - print(res) - sys.exit(1) diff --git a/examples/list_hosts.py b/examples/list_hosts.py deleted file mode 100755 index fb767d86..00000000 --- a/examples/list_hosts.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -# This script shows how to leverage Sysdig data query API to obtain the list of the instrumented -# hosts that have been seen in your infrastructure. -# The output will show the container count (`container.count` metric) in addition to the -# hostnames (`host.hostName` tag) in a format like this: -# -# host-1 12 -# host-2 4 -# -# where the first column is the hostname and the second column is the number of containers running -# in each host. -# -import getopt -import json -import sys - -from sdcclient import SdcClient - - -# -# Parse arguments -# -def usage(): - print(('usage: %s [-j|--json] [-d|--duration ] [-c|--count ] ' % sys.argv[0])) - print('-d|--duration: List hosts seen in the last seconds (default: 3600, ie. last hour)') - print('-c|--count: Number of hosts to print (default: 100)') - print('-j|--json: Print output as json') - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - - -try: - opts, args = getopt.getopt(sys.argv[1:], "jd:c:", ["json", "duration=", "count="]) -except getopt.GetoptError: - usage() - -duration = 3600 -count = 100 -print_json = False -for opt, arg in opts: - if opt in ("-d", "--duration"): - duration = int(arg) - elif opt in ("-c", "--count"): - count = int(arg) - elif opt in ("-j", "--json"): - print_json = True -sdc_token = args[0] - -# Instantiate the SDC client -sdclient = SdcClient(sdc_token) - -# -# Prepare the query's metrics list. -# In this case, we have one tag (used for segmentation) and one metric: -# - host.hostName. This is a tag, to identify each item of the output -# - container.count: This is the metric -# -metrics = [ - {"id": "host.hostName"}, - {"id": "container.count", "aggregations": {"time": "avg", "group": "avg"}} -] - -ok, res = sdclient.get_data( - metrics, # list of metrics - -duration, # start time: either a unix timestamp, or a difference from "now" - 0, # end time: either a unix timestamp, or a difference from "now" (0 means you need "last X seconds") - duration, # sampling time, ie. data granularity; - # if equal to the time window span then the result will contain a single sample - paging={ - "from": 0, - "to": count - 1 - }) - -if not ok: - # data fetch failed - print(res) - sys.exit(1) - -# data fetched successfully -if print_json: - print((json.dumps(res))) -else: - data = res['data'] - output = [] - for i in range(0, len(data)): - sample = data[i] - metrics = sample['d'] - hostName = metrics[0] - count = metrics[1] - output.append('%s\t%d' % (hostName, count)) - - print('\n'.join(output)) diff --git a/examples/list_metrics.py b/examples/list_metrics.py deleted file mode 100755 index a9da394b..00000000 --- a/examples/list_metrics.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# -# Print the list of metrics. -# - -import sys - -from sdcclient import SdcClient - -# -# Parse arguments -# -if len(sys.argv) != 2: - print(('usage: %s ' % sys.argv[0])) - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - -sdc_token = sys.argv[1] - -# -# Instantiate the SDC client -# -sdclient = SdcClient(sdc_token) - -# -# Fire the request. -# -ok, res = sdclient.get_metrics() - -# -# Show the list of metrics -# -if not ok: - print(res) - sys.exit(1) - -for metric_id, metric in res.items(): - print(("Metric name: " + metric_id + ", type: " + metric['type'])) diff --git a/examples/print_conn_table.py b/examples/print_conn_table.py deleted file mode 100755 index dadc71d5..00000000 --- a/examples/print_conn_table.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -# The request prints N entries from the conn table for the filter specified -# mimicking the top connections table in the Sysdig Monitor UI -# - -import sys - -from sdcclient import SdcClient - -# -# Parse arguments -# -if len(sys.argv) not in [2, 3]: - print(('usage: %s ' % sys.argv[0])) - print('You can find your token at https://app.sysdigcloud.com/#/settings/user') - sys.exit(1) - -sdc_token = sys.argv[1] - -if len(sys.argv) == 3: - hostname = sys.argv[2] -else: - hostname = None - -# -# Instantiate the SDC client -# -sdclient = SdcClient(sdc_token) - -# -# Prepare the metrics list. -# -metrics = [ - {"id": "net.local.endpoint"}, - {"id": "net.local.service"}, - {"id": "net.remote.endpoint"}, - {"id": "net.remote.service"}, - {"id": "net.connection.count.total", - "aggregations": { - "time": "timeAvg", - "group": "sum" - } - }, - {"id": "net.bytes.in", - "aggregations": { - "time": "timeAvg", - "group": "avg" - }, - }, - {"id": "net.bytes.out", - "aggregations": { - "time": "timeAvg", - "group": "avg" - } - }, - {"id": "net.bytes.total", - "aggregations": { - "time": "timeAvg", - "group": "avg" - } - }, - {"id": "net.request.count.in", - "aggregations": { - "time": "timeAvg", - "group": "avg" - } - }, - {"id": "net.request.count.out", - "aggregations": { - "time": "timeAvg", - "group": "avg" - } - }, - {"id": "net.request.count", - "aggregations": { - "time": "timeAvg", - "group": "avg" - } - } -] - -# -# Prepare the filter -# - -if hostname is not None: - flt = "host.hostName = '%s'" % hostname -else: - flt = "" - -# -# Time window: -# - for "last X seconds": start is equal to -X, end is equal to 0 -# -start = -7200 -end = 0 - -# -# Fire the query. -# -page_size = 500 -fetch_limit = 10000 - -cur = 0 - -row_format = "{:20.20}\t{:20.20}\t{:20.20}\t{:20.20}\t{:10}\t{:10}\t{:10}\t{:10}\t{:10}\t{:10}\t{:10}" - -print((row_format.format("Source", "Source Process", "Destination", "Destination Process", "Count", - "Bytes In", "Bytes Out", "Bytes", "Req In", "Req Out", "Req"))) - -while cur < fetch_limit: - paging = {'from': cur, 'to': cur + page_size} - ok, res = sdclient.get_data(metrics, - start, - end, - 0, - flt, - 'host', - paging) - - if not ok: - sys.exit(res) - - data = res['data'] - - if len(data) == 0: - break - - cur += len(data) - for line in data: - print((row_format.format(*line['d']))) diff --git a/sdcclient/_common.py b/sdcclient/_common.py index 2f0307fa..77510a1c 100644 --- a/sdcclient/_common.py +++ b/sdcclient/_common.py @@ -314,134 +314,6 @@ def get_data_retention_info(self): res = self.http.get(self.url + '/api/history/timelines/', headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) - def get_topology_map(self, grouping_hierarchy, time_window_s, sampling_time_s): - # - # Craft the time interval section - # - tlines = self.get_data_retention_info() - - for tline in tlines[1]['agents']: - if tline['sampling'] == sampling_time_s * 1000000: - timeinfo = tline - - if timeinfo is None: - return [False, "sampling time " + str(sampling_time_s) + " not supported"] - - timeinfo['from'] = timeinfo['to'] - timeinfo['sampling'] - - # - # Create the grouping hierarchy - # - gby = [{'metric': g} for g in grouping_hierarchy] - - # - # Prepare the json - # - req_json = { - 'format': { - 'type': 'map', - 'exportProcess': True - }, - 'time': timeinfo, - # 'filter': { - # 'filters': [ - # { - # 'metric': 'agent.tag.Tag', - # 'op': '=', - # 'value': 'production-maintenance', - # 'filters': None - # } - # ], - # 'logic': 'and' - # }, - 'limit': { - 'hostGroups': 20, - 'hosts': 20, - 'containers': 20, - 'processes': 10 - }, - 'group': { - 'configuration': { - 'groups': [ - { - 'filters': [], - 'groupBy': gby - } - ] - } - }, - 'nodeMetrics': [ - { - 'id': 'cpu.used.percent', - 'aggregation': 'timeAvg', - 'groupAggregation': 'avg' - } - ], - 'linkMetrics': [ - { - 'id': 'net.bytes.total', - 'aggregation': 'timeAvg', - 'groupAggregation': 'sum' - } - ] - } - - # - # Fire the request - # - res = self.http.post(self.url + '/api/data?format=map', headers=self.hdrs, - data=json.dumps(req_json), verify=self.ssl_verify) - return self._request_result(res) - - def get_data(self, metrics, start_ts, end_ts=0, sampling_s=0, - filter='', datasource_type='host', paging=None): - '''**Description** - Export metric data (both time-series and table-based). - - **Arguments** - - **metrics**: a list of dictionaries, specifying the metrics and grouping keys that the query will return. A metric is any of the entries that can be found in the *Metrics* section of the Explore page in Sysdig Monitor. Metric entries require an *aggregations* section specifying how to aggregate the metric across time and containers/hosts. A grouping key is any of the entries that can be found in the *Show* or *Segment By* sections of the Explore page in Sysdig Monitor. These entries are used to apply single or hierarchical segmentation to the returned data and don't require the aggregations section. Refer to the Example link below for ready-to-use code snippets. - - **start_ts**: the UTC time (in seconds) of the beginning of the data window. A negative value can be optionally used to indicate a relative time in the past from now. For example, -3600 means "one hour ago". - - **end_ts**: the UTC time (in seconds) of the end of the data window, or 0 to indicate "now". A negative value can also be optionally used to indicate a relative time in the past from now. For example, -3600 means "one hour ago". - - **sampling_s**: the duration of the samples that will be returned. 0 means that the whole data will be returned as a single sample. - - **filter**: a boolean expression combining Sysdig Monitor segmentation criteria that defines what the query will be applied to. For example: *kubernetes.namespace.name='production' and container.image='nginx'*. - - **datasource_type**: specify the metric source for the request, can be ``container`` or ``host``. Most metrics, for example ``cpu.used.percent`` or ``memory.bytes.used``, are reported by both hosts and containers. By default, host metrics are used, but if the request contains a container-specific grouping key in the metric list/filter (e.g. ``container.name``), then the container source is used. In cases where grouping keys are missing or apply to both hosts and containers (e.g. ``tag.Name``), *datasource_type* can be explicitly set to avoid any ambiguity and allow the user to select precisely what kind of data should be used for the request. `examples/get_data_datasource.py `_ contains a few examples that should clarify the use of this argument. - - **paging**: if segmentation of the query generates values for several different entities (e.g. containers/hosts), this parameter specifies which to include in the returned result. It's specified as a dictionary of inclusive values for ``from`` and ``to`` with the default being ``{ "from": 0, "to": 9 }``, which will return values for the "top 10" entities. The meaning of "top" is query-dependent, based on points having been sorted via the specified group aggregation, with the results sorted in ascending order if the group aggregation is ``min`` or ``none``, and descending order otherwise. - - **Success Return Value** - A dictionary with the requested data. Data is organized in a list of time samples, each of which includes a UTC timestamp and a list of values, whose content and order reflect what was specified in the *metrics* argument. - - **Examples** - - `examples/get_data_simple.py `_ - - `examples/get_data_advanced.py `_ - - `examples/list_hosts.py `_ - - `examples/get_data_datasource.py `_ - ''' - reqbody = { - 'metrics': metrics, - 'dataSourceType': datasource_type, - } - - if start_ts < 0: - reqbody['last'] = -start_ts - elif start_ts == 0: - return [False, "start_ts cannot be 0"] - else: - reqbody['start'] = start_ts - reqbody['end'] = end_ts - - if filter != '': - reqbody['filter'] = filter - - if paging is not None: - reqbody['paging'] = paging - - if sampling_s != 0: - reqbody['sampling'] = sampling_s - - res = self.http.post(self.url + '/api/data/', headers=self.hdrs, data=json.dumps(reqbody), - verify=self.ssl_verify) - return self._request_result(res) - def get_data_promql(self, query, start, end, step, timeout=None, limit=None): '''**Description** Evaluate an expression query over a specified time range. diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index d8d6d6f8..e3cf6429 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -294,19 +294,6 @@ def set_explore_grouping_hierarchy(self, new_hierarchy) -> Union[Tuple[bool, str else: return True, None - def get_metrics(self) -> Union[Tuple[bool, str], Tuple[bool, Any]]: - '''**Description** - Return the metric list that can be used for data requests/alerts/dashboards. - - **Success Return Value** - A dictionary containing the list of available metrics. - - **Example** - `examples/list_metrics.py `_ - ''' - res = self.http.get(self.url + '/api/data/metrics', headers=self.hdrs, verify=self.ssl_verify) - return self._request_result(res) - @staticmethod def convert_scope_string_to_expression(scope) -> Union[Tuple[bool, str], Tuple[bool, Any]]: '''**Description**