From e65e556ab4024414eaa19d380e15897c7bf79fd8 Mon Sep 17 00:00:00 2001 From: Kim Pohas Date: Thu, 9 May 2024 23:36:36 -0700 Subject: [PATCH 01/19] CloudQuery GCP C2C --- .../cloudquery-gcp-source.md | 104 ++++++++++++++++++ static/files/c2c/cloudquery-gcp/example.json | 27 +++++ 2 files changed, 131 insertions(+) create mode 100644 docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md create mode 100644 static/files/c2c/cloudquery-gcp/example.json diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md new file mode 100644 index 0000000000..8a40ea9a55 --- /dev/null +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -0,0 +1,104 @@ +--- +id: cloudquery-gcp-source +title: CloudQuery GCP Source +sidebar_label: CloudQuery GCP +tags: + - cloud-to-cloud + - cloudquery + - gcp + - google cloud platform +description: Learn how to collect inventory from the GCP APIs using CloudQuery Plugin SDK and send it to Sumo Logic. +--- + + + + + +

Beta

+ +import CodeBlock from '@theme/CodeBlock'; +import AccountExampleJSON from '/files/c2c/cloudquery/example.json'; +import OrgTFExample from '!!raw-loader!/files/c2c/cloudquery/example.tf'; +import useBaseUrl from '@docusaurus/useBaseUrl'; + +cloudquery-iconThumbnail icon + +The CloudQuery integration is used to pull inventory from the AWS APIs and transform them into the CloudQuery schema and send it to Sumo Logic. + +import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; + + + + + +## JSON schema + +Sources can be configured using UTF-8 encoded JSON files with the Collector Management API. See [Use JSON to configure Sources](/docs/send-data/use-json-configure-sources) for details.  + +| Parameter | Type | Required | Description | +|:---|:---|:---|:---| +| config | JSON Object | Yes | Contains the configuration parameters for the source | +| schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}` | +| sourceType | String | Yes | Use “Universal” for CloudQuery GCP source Inventory | + +### Configuration Object + +| Parameter | Type | Required | Default | Description | Access | +|:---|:---|:---|:---|:---|:---| +| name | string | Yes | | Type the desired name of the Source. The name must be unique per Collector. This value is assigned to the metadata field _source. | modifiable | +| description | string | No | Null | Type a description of the Source. | modifiable | +| credentialsJson | JSON object | Yes | | Service account key JSON file. | modifiable | +| projectIds | array | No | | List of Project IDs from which the data needs to be pulled. | modifiable | +| orgIds | array | No | | List of Organization IDs from which the data needs to be pulled. | modifiable | +| limitToServices | array | Yes | | List of services from which the data should be pulled. | modifiable | +| pollingInterval | number | Yes | 12 Hours | How frequently should we pull inventory in Hours | modifiable | + +### JSON example + +{Example} + +[Download example](/files/c2c/cloudquery-gcp/example.json) + + +## FAQ + +:::info +Click [here](/docs/c2c/info) for more information about Cloud-to-Cloud sources. +::: diff --git a/static/files/c2c/cloudquery-gcp/example.json b/static/files/c2c/cloudquery-gcp/example.json new file mode 100644 index 0000000000..6f5ce520e1 --- /dev/null +++ b/static/files/c2c/cloudquery-gcp/example.json @@ -0,0 +1,27 @@ +{ + "api.version": "v1", + "source": { + "config": { + { + "name": "CloudQuery GCP", + "credentialsJson": "", + "projectIds": [ + "test-gcp-435276" + ], + "organizationIds": [ + "24584326357", + "736586988" + ], + "limitToServices": [ + "compute", + "storage" + ], + "pollingInterval": 12 + } + }, + "schemaRef": { + "type": "CloudQuery GCP Inventory" + }, + "sourceType": "Universal" + } +} From 7b086a9375cdd497c20c52eaf996d0ca2a652ba0 Mon Sep 17 00:00:00 2001 From: Kim Pohas Date: Fri, 10 May 2024 08:37:53 -0700 Subject: [PATCH 02/19] fix examples --- .../cloudquery-gcp-source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 8a40ea9a55..92bcdf629a 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -17,8 +17,8 @@ description: Learn how to collect inventory from the GCP APIs using CloudQuery P

Beta

import CodeBlock from '@theme/CodeBlock'; -import AccountExampleJSON from '/files/c2c/cloudquery/example.json'; -import OrgTFExample from '!!raw-loader!/files/c2c/cloudquery/example.tf'; +import Example from '/files/c2c/cloudquery-gcp/example.json'; +import Example from '!!raw-loader!/files/c2c/cloudquery-gcp/example.json'; import useBaseUrl from '@docusaurus/useBaseUrl'; cloudquery-iconThumbnail icon From b247c34741b5ff5fe95de2d1fe000fd524c0bd6c Mon Sep 17 00:00:00 2001 From: Kim Pohas Date: Fri, 10 May 2024 14:26:53 -0700 Subject: [PATCH 03/19] more edits --- cid-redirects.json | 1 + .../cloudquery-gcp-source.md | 48 +++++++------------ static/files/c2c/cloudquery-gcp/example.json | 48 +++++++++---------- 3 files changed, 42 insertions(+), 55 deletions(-) diff --git a/cid-redirects.json b/cid-redirects.json index 65f33f62f9..8edb3ca98e 100644 --- a/cid-redirects.json +++ b/cid-redirects.json @@ -2557,6 +2557,7 @@ "/cid/19901": "/docs/metrics/metrics-operators/topk", "/cid/19902": "/docs/metrics/metrics-operators/where", "/cid/15631": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/citrix-cloud-source", + "/cid/15633": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source", "/cid/14323": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/docusign-source", "/cid/14326": "/docs/integrations/global-intelligence/kubernetes-devops", "/cid/30001": "/docs/integrations/microsoft-azure/azure-batch", diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 92bcdf629a..2f2e6bbbc3 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -17,56 +17,44 @@ description: Learn how to collect inventory from the GCP APIs using CloudQuery P

Beta

import CodeBlock from '@theme/CodeBlock'; -import Example from '/files/c2c/cloudquery-gcp/example.json'; -import Example from '!!raw-loader!/files/c2c/cloudquery-gcp/example.json'; import useBaseUrl from '@docusaurus/useBaseUrl'; +import Example from '/files/c2c/cloudquery-gcp/example.json'; +import MyComponentSource from '!!raw-loader!/files/c2c/cloudquery-gcp/example.json'; cloudquery-iconThumbnail icon -The CloudQuery integration is used to pull inventory from the AWS APIs and transform them into the CloudQuery schema and send it to Sumo Logic. +The goal of this C2C integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and send it to Sumo Logic. import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; - +Cloudquery GCP Source plugin version: v12.4.0 +Cloudquery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a ## JSON schema @@ -92,7 +80,7 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma ### JSON example -{Example} +{MyComponentSource} [Download example](/files/c2c/cloudquery-gcp/example.json) diff --git a/static/files/c2c/cloudquery-gcp/example.json b/static/files/c2c/cloudquery-gcp/example.json index 6f5ce520e1..d82814485e 100644 --- a/static/files/c2c/cloudquery-gcp/example.json +++ b/static/files/c2c/cloudquery-gcp/example.json @@ -1,27 +1,25 @@ { - "api.version": "v1", - "source": { - "config": { - { - "name": "CloudQuery GCP", - "credentialsJson": "", - "projectIds": [ - "test-gcp-435276" - ], - "organizationIds": [ - "24584326357", - "736586988" - ], - "limitToServices": [ - "compute", - "storage" - ], - "pollingInterval": 12 - } - }, - "schemaRef": { - "type": "CloudQuery GCP Inventory" - }, - "sourceType": "Universal" - } + "api.version": "v1", + "source": { + "config": { + "name": "CloudQuery GCP", + "credentialsJson": "aa", + "projectIds": [ + "test-gcp-435276" + ], + "organizationIds": [ + "24584326357", + "736586988" + ], + "limitToServices": [ + "compute", + "storage" + ], + "pollingInterval": 12 + }, + "schemaRef": { + "type": "CloudQuery GCP Inventory" + } + }, + "sourceType": "Universal" } From 300341ec9aa61143df0a916fbd78ac4820dcdd14 Mon Sep 17 00:00:00 2001 From: Kim Pohas Date: Fri, 10 May 2024 19:35:33 -0700 Subject: [PATCH 04/19] more edits --- .../cloudquery-gcp-source.md | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 2f2e6bbbc3..d08d36186a 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -39,22 +39,24 @@ import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; ### Authentication -The GCP plugin authenticates using your Application Default Credentials. +- The GCP plugin authenticates using your Application Default Credentials. +- We will use the credentials JSON file to authenticate with GCP. +- The user will provide the credential JSON file as a string, we will create the `.json` file in a secure environment and will set the environment variable GOOGLE_APPLICATION_CREDENTIALS as the key with the file path as the value. -We will use the credentials JSON file to authenticate with GCP. -The user will provide the credential JSON file as a string, we will create the .json file in a secure environment and will set the environment variable GOOGLE_APPLICATION_CREDENTIALS as the key with the file path as the value. - -``` +```sh GOOGLE_APPLICATION_CREDENTIALS: ``` -The user will provide the list of services and the integration will fetch all the tables supported by CloudQuery plugin(v12.4.0) for each specified service under the specific project and organization(if specified). +- The user will provide the list of services and the integration will fetch all the tables supported by CloudQuery plugin(v12.4.0) for each specified service under the specific project and organization(if specified). +- The CloudQuery GCP plugin contains the [`PluginAutoGeneratedTables()`](https://github.com/cloudquery/cloudquery/blob/plugins-source-gcp-v10.0.0/plugins/source/gcp/resources/plugin/tables.go#L55) function in which we need to specify the required services with the table name to fetch the table schema for an individual service. +- After getting the table schema, we will use the [Sync](https://github.com/cloudquery/plugin-sdk/blob/main/scheduler/scheduler.go#L165) function to populate the table data for that specific project. +- Cloudquery GCP Source plugin version: v12.4.0 +- Cloudquery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a + +## Rate Limiting -The CloudQuery GCP plugin contains the PluginAutoGeneratedTables() function in which we need to specify the required services with the table name to fetch the table schema for an individual service. -After getting the table schema, we will use the Sync function to populate the table data for that specific project. +Rate Limit is not applicable as we will use CloudQuery GCP SDK. -Cloudquery GCP Source plugin version: v12.4.0 -Cloudquery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a ## JSON schema @@ -70,7 +72,7 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma | Parameter | Type | Required | Default | Description | Access | |:---|:---|:---|:---|:---|:---| -| name | string | Yes | | Type the desired name of the Source. The name must be unique per Collector. This value is assigned to the metadata field _source. | modifiable | +| name | string | Yes | | Type the desired name of the Source. The name must be unique per Collector. This value is assigned to the metadata field `_source`. | modifiable | | description | string | No | Null | Type a description of the Source. | modifiable | | credentialsJson | JSON object | Yes | | Service account key JSON file. | modifiable | | projectIds | array | No | | List of Project IDs from which the data needs to be pulled. | modifiable | @@ -85,6 +87,14 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma [Download example](/files/c2c/cloudquery-gcp/example.json) +## Error Handling + +- For the SDK, if the response code is 401 and 403. + - Config error and quit the integration +- For any other error returned by CloudQuery, we will set ErrorGeneric as an error in C2C. + - Integration will not stop it’s execution + + ## FAQ :::info From e6dd8ac42a5c09f480b14bd60a4b0d57efd9ca8f Mon Sep 17 00:00:00 2001 From: Kim Pohas Date: Sat, 11 May 2024 12:35:43 -0700 Subject: [PATCH 05/19] minor formatting --- .../cloudquery-gcp-source.md | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index d08d36186a..18df08ab90 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -23,7 +23,7 @@ import MyComponentSource from '!!raw-loader!/files/c2c/cloudquery-gcp/example.js cloudquery-iconThumbnail icon -The goal of this C2C integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and send it to Sumo Logic. +The goal of this C2C integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and then send it to Sumo Logic. import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; @@ -41,22 +41,19 @@ import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; - The GCP plugin authenticates using your Application Default Credentials. - We will use the credentials JSON file to authenticate with GCP. -- The user will provide the credential JSON file as a string, we will create the `.json` file in a secure environment and will set the environment variable GOOGLE_APPLICATION_CREDENTIALS as the key with the file path as the value. - -```sh -GOOGLE_APPLICATION_CREDENTIALS: -``` - +- The user will provide the credential JSON file as a string, we will create the `.json` file in a secure environment and will set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` as the key with the file path as the value. + ```sh + GOOGLE_APPLICATION_CREDENTIALS: + ``` - The user will provide the list of services and the integration will fetch all the tables supported by CloudQuery plugin(v12.4.0) for each specified service under the specific project and organization(if specified). - The CloudQuery GCP plugin contains the [`PluginAutoGeneratedTables()`](https://github.com/cloudquery/cloudquery/blob/plugins-source-gcp-v10.0.0/plugins/source/gcp/resources/plugin/tables.go#L55) function in which we need to specify the required services with the table name to fetch the table schema for an individual service. - After getting the table schema, we will use the [Sync](https://github.com/cloudquery/plugin-sdk/blob/main/scheduler/scheduler.go#L165) function to populate the table data for that specific project. -- Cloudquery GCP Source plugin version: v12.4.0 -- Cloudquery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a +- CloudQuery GCP Source plugin version: v12.4.0 +- CloudQuery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a -## Rate Limiting - -Rate Limit is not applicable as we will use CloudQuery GCP SDK. +## Rate limiting +Rate Limit is not applicable as we're using CloudQuery GCP SDK. ## JSON schema @@ -68,7 +65,7 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma | schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}` | | sourceType | String | Yes | Use “Universal” for CloudQuery GCP source Inventory | -### Configuration Object +### Configuration object | Parameter | Type | Required | Default | Description | Access | |:---|:---|:---|:---|:---|:---| @@ -86,14 +83,11 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma [Download example](/files/c2c/cloudquery-gcp/example.json) +## Error handling -## Error Handling - -- For the SDK, if the response code is 401 and 403. - - Config error and quit the integration -- For any other error returned by CloudQuery, we will set ErrorGeneric as an error in C2C. - - Integration will not stop it’s execution +For the SDK, if the response code is `401` or `403`, this is config error. You'll need to quit the integration. +For any other error returned by CloudQuery, we will set ErrorGeneric as an error in C2C. Integration will not stop its execution. ## FAQ From e9e346623c7c0d55fa83da41b9e333e7191215f9 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Wed, 15 May 2024 11:52:18 +0530 Subject: [PATCH 06/19] Added more information as per the template --- .../cloudquery-gcp-source.md | 64 +++++++++++-------- static/files/c2c/cloudquery-gcp/example.tf | 26 ++++++++ 2 files changed, 65 insertions(+), 25 deletions(-) create mode 100644 static/files/c2c/cloudquery-gcp/example.tf diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 18df08ab90..d9e5667819 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -1,6 +1,6 @@ --- id: cloudquery-gcp-source -title: CloudQuery GCP Source +title: CloudQuery GCP (Beta) sidebar_label: CloudQuery GCP tags: - cloud-to-cloud @@ -18,12 +18,13 @@ description: Learn how to collect inventory from the GCP APIs using CloudQuery P import CodeBlock from '@theme/CodeBlock'; import useBaseUrl from '@docusaurus/useBaseUrl'; -import Example from '/files/c2c/cloudquery-gcp/example.json'; +import ExampleJSON from '/files/c2c/cloudquery-gcp/example.json'; import MyComponentSource from '!!raw-loader!/files/c2c/cloudquery-gcp/example.json'; +import TerraformExample from '!!raw-loader!/files/c2c/cloudquery-gcp/example.tf'; -cloudquery-iconThumbnail icon +cloudquery-icon -The goal of this C2C integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and then send it to Sumo Logic. +The CloudQuery GCP integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and then send it to Sumo Logic. import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; @@ -37,23 +38,30 @@ import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; ## Setup -### Authentication +### Prerequisites -- The GCP plugin authenticates using your Application Default Credentials. -- We will use the credentials JSON file to authenticate with GCP. -- The user will provide the credential JSON file as a string, we will create the `.json` file in a secure environment and will set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` as the key with the file path as the value. - ```sh - GOOGLE_APPLICATION_CREDENTIALS: - ``` -- The user will provide the list of services and the integration will fetch all the tables supported by CloudQuery plugin(v12.4.0) for each specified service under the specific project and organization(if specified). -- The CloudQuery GCP plugin contains the [`PluginAutoGeneratedTables()`](https://github.com/cloudquery/cloudquery/blob/plugins-source-gcp-v10.0.0/plugins/source/gcp/resources/plugin/tables.go#L55) function in which we need to specify the required services with the table name to fetch the table schema for an individual service. -- After getting the table schema, we will use the [Sync](https://github.com/cloudquery/plugin-sdk/blob/main/scheduler/scheduler.go#L165) function to populate the table data for that specific project. -- CloudQuery GCP Source plugin version: v12.4.0 -- CloudQuery GCP git source code version: v0.10.2-pre.0.0.20231122092836-a61fa0d83d1a +[Enable](https://cloud.google.com/endpoints/docs/openapi/enable-api) all the service APIs in the authenticated GCP account related with the services from which you want to fetch data via CloudQuery GCP plugin. -## Rate limiting +### Vendor configuration -Rate Limit is not applicable as we're using CloudQuery GCP SDK. +Download the user credentials JSON file by using the gcloud CLI, refer to the [Google Cloud Documentation](https://cloud.google.com/docs/authentication/application-default-credentials#personal). + +### Source configuration + +When you create an CloudQuery GCP source, you add it to a Hosted Collector. Before creating the source, identify the Hosted Collector you want to use or create a new Hosted Collector. For instructions, see [Configure a Hosted Collector](/docs/send-data/hosted-collectors/configure-hosted-collector). + +To configure a CloudQuery GCP Source: +1. In Sumo Logic, select **Manage Data** > **Collection** > **Collection**.  +1. On the Collection page, click **Add Source** next to a Hosted Collector. +1. Search for and select **CloudQuery GCP**. +1. Enter a **Name** for the source. The description is optional. +1. **GCP Credentials**. Upload the credentials JSON file downloaded by running the gcloud CLI command. +1. **(Optional) Project ID(s)**. Enter the unique identifier number for your project. You can find this from the Google Cloud Console. If not specified, all the available active projects will be considered by default. +1. **(Optional) Organization ID(s)**. Enter the unique identifier number for your organization. You can find this from the Google Cloud Console. If not specified, all the available active projects will be considered by default. +1. **Services**. Enter the type of service from which the GCP data should be fetched via CloudQuery GCP plugin. +1. By default, **Polling Interval** is set to 12 hours. +1. **(Optional) Processing Rules for Logs**. Configure any desired filters, such as allowlist, denylist, hash, or mask, as described in [Create a Processing Rule](/docs/send-data/collection/processing-rules/create-processing-rule). +1. When you are finished configuring the Source, click **Save**. ## JSON schema @@ -69,13 +77,13 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma | Parameter | Type | Required | Default | Description | Access | |:---|:---|:---|:---|:---|:---| -| name | string | Yes | | Type the desired name of the Source. The name must be unique per Collector. This value is assigned to the metadata field `_source`. | modifiable | -| description | string | No | Null | Type a description of the Source. | modifiable | -| credentialsJson | JSON object | Yes | | Service account key JSON file. | modifiable | -| projectIds | array | No | | List of Project IDs from which the data needs to be pulled. | modifiable | -| orgIds | array | No | | List of Organization IDs from which the data needs to be pulled. | modifiable | -| limitToServices | array | Yes | | List of services from which the data should be pulled. | modifiable | -| pollingInterval | number | Yes | 12 Hours | How frequently should we pull inventory in Hours | modifiable | +| name | string | Yes | `null` | Type the desired name of the Source. The name must be unique per Collector. This value is assigned to the metadata field `_source`. | modifiable | +| description | string | No | `null` | Type a description of the Source. | modifiable | +| credentialsJson | JSON object | Yes | `null` | Service account key JSON file. | modifiable | +| projectIds | array | No | `null` | List of Project IDs from which the data needs to be pulled. | modifiable | +| orgIds | array | No | `null` | List of Organization IDs from which the data needs to be pulled. | modifiable | +| limitToServices | array | Yes | `null` | List of services from which the data should be pulled. | modifiable | +| pollingInterval | number | Yes | 12 Hours | This sets how often the Source checks for data. | modifiable | ### JSON example @@ -83,6 +91,12 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma [Download example](/files/c2c/cloudquery-gcp/example.json) +### Terraform example + +{TerraformExample} + +[Download example](/files/c2c/cloudquery-gcp/example.tf) + ## Error handling For the SDK, if the response code is `401` or `403`, this is config error. You'll need to quit the integration. diff --git a/static/files/c2c/cloudquery-gcp/example.tf b/static/files/c2c/cloudquery-gcp/example.tf new file mode 100644 index 0000000000..b5c0935c8e --- /dev/null +++ b/static/files/c2c/cloudquery-gcp/example.tf @@ -0,0 +1,26 @@ +resource "sumologic_cloud_to_cloud_source" "asana_source" { + collector_id = sumologic_collector.collector.id + schema_ref = { + type = "CloudQuery GCP" + } + config = jsonencode({ + "name": "CloudQuery GCP", + "credentialsJson": "aa", + "projectIds": { + "test-gcp-435276" + }, + "organizationIds": { + "24584326357", + "736586988" + }, + "limitToServices": { + "compute", + "storage" + }, + "pollingInterval": 12 + }) +} +resource "sumologic_collector" "collector" { + name = "my-collector" + description = "Just testing this" +} \ No newline at end of file From 374c276d6d388cd8f9bd083bd2a156ddc1bd564f Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Wed, 15 May 2024 18:35:12 +0530 Subject: [PATCH 07/19] Update cid-redirects.json --- cid-redirects.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid-redirects.json b/cid-redirects.json index 003dc7ddb0..d19cf22cf5 100644 --- a/cid-redirects.json +++ b/cid-redirects.json @@ -2558,7 +2558,7 @@ "/cid/19901": "/docs/metrics/metrics-operators/topk", "/cid/19902": "/docs/metrics/metrics-operators/where", "/cid/15631": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/citrix-cloud-source", - "/cid/15633": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source", + "/cid/15634": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source", "/cid/15632": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-azure-plugin-source", "/cid/15633": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-source", "/cid/14323": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/docusign-source", From e5111f001c6abcf1ff73faef89b643c6ad70155c Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Wed, 15 May 2024 18:41:03 +0530 Subject: [PATCH 08/19] Update cloudquery-gcp-source.md --- .../cloudquery-gcp-source.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index d9e5667819..6aa2d9ea63 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -97,7 +97,21 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma [Download example](/files/c2c/cloudquery-gcp/example.tf) -## Error handling +## Troubleshooting + +### Warning message + +#### Message + +``` +transport: received unexpected content-type \"text/html; charset=UTF-8 +``` + +#### Solution + +If you come across this warning message while fetching service data for the configured source then please contact CloudQuery or GCP team. + +### Error handling For the SDK, if the response code is `401` or `403`, this is config error. You'll need to quit the integration. From 6b991a3c5014adee7b3c45711e6c65eb228fa3d5 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:01 +0530 Subject: [PATCH 09/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 6aa2d9ea63..8361f0b5d0 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -7,7 +7,7 @@ tags: - cloudquery - gcp - google cloud platform -description: Learn how to collect inventory from the GCP APIs using CloudQuery Plugin SDK and send it to Sumo Logic. +description: Learn how to collect inventory from GCP APIs using the CloudQuery Plugin SDK and send it to Sumo Logic. --- From 942db4e5b8b1023062ba81c82a492d1b71747abe Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:07 +0530 Subject: [PATCH 10/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 8361f0b5d0..2d3a2c88f4 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -24,7 +24,7 @@ import TerraformExample from '!!raw-loader!/files/c2c/cloudquery-gcp/example.tf' cloudquery-icon -The CloudQuery GCP integration is to pull inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transform them into the CloudQuery schema, and then send it to Sumo Logic. +The CloudQuery GCP integration pulls inventory from various Google Cloud Platform (GCP) APIs via the CloudQuery GCP plugin, transforms it into the CloudQuery schema, and then sends it to Sumo Logic. import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; From f4eb348481ea98a278eb40d21d12936493d53dd3 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:26 +0530 Subject: [PATCH 11/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 2d3a2c88f4..6d324aec5c 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -44,7 +44,7 @@ import FedDeploymentNote from '../../../reuse/fed-deployment-note.md'; ### Vendor configuration -Download the user credentials JSON file by using the gcloud CLI, refer to the [Google Cloud Documentation](https://cloud.google.com/docs/authentication/application-default-credentials#personal). +To download the user credentials JSON file by using the gcloud CLI, refer to the [Google Cloud documentation](https://cloud.google.com/docs/authentication/application-default-credentials#personal). ### Source configuration From b2f23a9c299ed793c300eec48216332112b0d81c Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:32 +0530 Subject: [PATCH 12/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 6d324aec5c..406cbaf84a 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -48,7 +48,7 @@ To download the user credentials JSON file by using the gcloud CLI, refer to the ### Source configuration -When you create an CloudQuery GCP source, you add it to a Hosted Collector. Before creating the source, identify the Hosted Collector you want to use or create a new Hosted Collector. For instructions, see [Configure a Hosted Collector](/docs/send-data/hosted-collectors/configure-hosted-collector). +When you create a CloudQuery GCP source, you add it to a Hosted Collector. Before creating the source, identify the Hosted Collector you want to use or create a new Hosted Collector. For instructions, see [Configure a Hosted Collector and Source](/docs/send-data/hosted-collectors/configure-hosted-collector). To configure a CloudQuery GCP Source: 1. In Sumo Logic, select **Manage Data** > **Collection** > **Collection**.  From 3b9e5833e0246fbde1bf7e3702034b547b2a8f43 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:38 +0530 Subject: [PATCH 13/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 406cbaf84a..ad5b30b4f4 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -65,7 +65,7 @@ To configure a CloudQuery GCP Source: ## JSON schema -Sources can be configured using UTF-8 encoded JSON files with the Collector Management API. See [Use JSON to configure Sources](/docs/send-data/use-json-configure-sources) for details.  +Sources can be configured using UTF-8 encoded JSON files with the Collector Management API. See [Use JSON to Configure Sources](/docs/send-data/use-json-configure-sources) for details.  | Parameter | Type | Required | Description | |:---|:---|:---|:---| From deed9373cb007d216e5d1764240b82ec8345fa5b Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:45 +0530 Subject: [PATCH 14/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index ad5b30b4f4..ffc372408d 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -113,7 +113,7 @@ If you come across this warning message while fetching service data for the conf ### Error handling -For the SDK, if the response code is `401` or `403`, this is config error. You'll need to quit the integration. +For the SDK, if the response code is `401` or `403`, it indicates a config error. You'll need to quit the integration. For any other error returned by CloudQuery, we will set ErrorGeneric as an error in C2C. Integration will not stop its execution. From cf3d92ba77e4651ad444991a846e134eb780baf5 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:51 +0530 Subject: [PATCH 15/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index ffc372408d..23d7c2e1e1 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -115,7 +115,7 @@ If you come across this warning message while fetching service data for the conf For the SDK, if the response code is `401` or `403`, it indicates a config error. You'll need to quit the integration. -For any other error returned by CloudQuery, we will set ErrorGeneric as an error in C2C. Integration will not stop its execution. +For any other error returned by CloudQuery, we will set ErrorGeneric as an error in the source. The integration will not stop its execution. ## FAQ From 68ab43341c1f757ddf0f16a69b32771c5701cd9e Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:27:58 +0530 Subject: [PATCH 16/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 23d7c2e1e1..576ddfbed9 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -109,7 +109,7 @@ transport: received unexpected content-type \"text/html; charset=UTF-8 #### Solution -If you come across this warning message while fetching service data for the configured source then please contact CloudQuery or GCP team. +If you come across this warning message while fetching service data for the configured source, then contact the CloudQuery or GCP team. ### Error handling From 88f28dfbcd461221c687565eb9554c1c9d2a6f52 Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 09:28:08 +0530 Subject: [PATCH 17/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: John Pipkin (Sumo Logic) --- .../cloudquery-gcp-source.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 576ddfbed9..aacc291a81 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -69,9 +69,9 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma | Parameter | Type | Required | Description | |:---|:---|:---|:---| -| config | JSON Object | Yes | Contains the configuration parameters for the source | -| schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}` | -| sourceType | String | Yes | Use “Universal” for CloudQuery GCP source Inventory | +| config | JSON Object | Yes | Contains the configuration parameters for the source. | +| schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}.` | +| sourceType | String | Yes | Use “Universal” for CloudQuery GCP source Inventory. | ### Configuration object From 8d2849da8c878896ff67d9be4b224dc6d36e522f Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 14:25:57 +0530 Subject: [PATCH 18/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: Kim (Sumo Logic) <56411016+kimsauce@users.noreply.github.com> --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index aacc291a81..563e626aae 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -70,7 +70,7 @@ Sources can be configured using UTF-8 encoded JSON files with the Collector Ma | Parameter | Type | Required | Description | |:---|:---|:---|:---| | config | JSON Object | Yes | Contains the configuration parameters for the source. | -| schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}.` | +| schemaRef | JSON Object | Yes | Use `{“type”: “CloudQuery GCP Inventory”}`. | | sourceType | String | Yes | Use “Universal” for CloudQuery GCP source Inventory. | ### Configuration object From e9d305597d5dabc24d78c1e02fe334062282916d Mon Sep 17 00:00:00 2001 From: Jagadisha V <129049263+JV0812@users.noreply.github.com> Date: Thu, 16 May 2024 14:26:10 +0530 Subject: [PATCH 19/19] Update docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md Co-authored-by: Kim (Sumo Logic) <56411016+kimsauce@users.noreply.github.com> --- .../cloudquery-gcp-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md index 563e626aae..b42fdf8f0e 100644 --- a/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md +++ b/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/cloudquery-gcp-source.md @@ -113,7 +113,7 @@ If you come across this warning message while fetching service data for the conf ### Error handling -For the SDK, if the response code is `401` or `403`, it indicates a config error. You'll need to quit the integration. +For the SDK, if the response code is `401` or `403`, it indicates a config error. You'll need to quit and restart the integration. For any other error returned by CloudQuery, we will set ErrorGeneric as an error in the source. The integration will not stop its execution.