diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index a74af3f66b2..d145fad51ec 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -257,7 +257,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes --data @index_config.json -H "C ```json title="index_config.json { - "version": "0.8", + "version": "0.9", "index_id": "hdfs-logs", "doc_mapping": { "field_mappings": [ @@ -355,7 +355,7 @@ curl -XPUT http://localhost:7280/api/v1/indexes/hdfs-logs --data @updated_index_ ```json title="updated_index_update.json { - "version": "0.8", + "version": "0.9", "index_id": "hdfs-logs", "doc_mapping": { "field_mappings": [ @@ -513,7 +513,7 @@ GET /api/v1/indexes/stackoverflow/splits?offset=0&limit=10 "split_state": "Published", "update_timestamp": 1695642901, "publish_timestamp": 1695642901, - "version": "0.7", + "version": "0.9", "split_id": "01HB632HD8W6WHNM7CZFH3KG1X", "index_uid": "stackoverflow:01HB6321TDT3SP58D4EZP14KSX", "partition_id": 0, @@ -617,7 +617,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source ```json title="source_config.json { - "version": "0.8", + "version": "0.9", "source_id": "kafka-source", "source_type": "kafka", "params": { @@ -664,7 +664,7 @@ curl -XPOST http://localhost:7280/api/v1/indexes/my-index/sources --data @source ```json title="source_config.json { - "version": "0.8", + "version": "0.9", "source_id": "kafka-source", "source_type": "kafka", "params": { diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 6fca85aa021..20e604e37b7 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -6373,7 +6373,7 @@ dependencies = [ [[package]] name = "quickwit-actors" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6393,7 +6393,7 @@ dependencies = [ [[package]] name = "quickwit-aws" -version = "0.8.0" +version = "0.9.0" dependencies = [ "aws-config", "aws-runtime", @@ -6412,7 +6412,7 @@ dependencies = [ [[package]] name = "quickwit-cli" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "bytesize", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "quickwit-cluster" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6491,7 +6491,7 @@ dependencies = [ [[package]] name = "quickwit-codegen" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "futures", @@ -6507,7 +6507,7 @@ dependencies = [ [[package]] name = "quickwit-codegen-example" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6531,7 +6531,7 @@ dependencies = [ [[package]] name = "quickwit-common" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-speed-limit", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "quickwit-config" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "bytes", @@ -6603,7 +6603,7 @@ dependencies = [ [[package]] name = "quickwit-control-plane" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6636,7 +6636,7 @@ dependencies = [ [[package]] name = "quickwit-datetime" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -6649,7 +6649,7 @@ dependencies = [ [[package]] name = "quickwit-directories" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6666,7 +6666,7 @@ dependencies = [ [[package]] name = "quickwit-doc-mapper" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -6698,7 +6698,7 @@ dependencies = [ [[package]] name = "quickwit-index-management" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "futures", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "quickwit-indexing" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "arc-swap", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "quickwit-ingest" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6818,7 +6818,7 @@ dependencies = [ [[package]] name = "quickwit-integration-tests" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "aws-sdk-sqs", @@ -6847,7 +6847,7 @@ dependencies = [ [[package]] name = "quickwit-jaeger" -version = "0.8.0" +version = "0.9.0" dependencies = [ "async-trait", "itertools 0.13.0", @@ -6879,7 +6879,7 @@ dependencies = [ [[package]] name = "quickwit-janitor" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -6911,7 +6911,7 @@ dependencies = [ [[package]] name = "quickwit-lambda" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "aws_lambda_events 0.15.1", @@ -6957,7 +6957,7 @@ dependencies = [ [[package]] name = "quickwit-macros" -version = "0.8.0" +version = "0.9.0" dependencies = [ "proc-macro2", "quote", @@ -6966,7 +6966,7 @@ dependencies = [ [[package]] name = "quickwit-metastore" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -7010,7 +7010,7 @@ dependencies = [ [[package]] name = "quickwit-opentelemetry" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -7033,7 +7033,7 @@ dependencies = [ [[package]] name = "quickwit-proto" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -7069,7 +7069,7 @@ dependencies = [ [[package]] name = "quickwit-query" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -7095,7 +7095,7 @@ dependencies = [ [[package]] name = "quickwit-rest-client" -version = "0.8.0" +version = "0.9.0" dependencies = [ "bytes", "quickwit-cluster", @@ -7118,7 +7118,7 @@ dependencies = [ [[package]] name = "quickwit-search" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "assert-json-diff 2.0.2", @@ -7163,7 +7163,7 @@ dependencies = [ [[package]] name = "quickwit-serve" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "assert-json-diff 2.0.2", @@ -7229,7 +7229,7 @@ dependencies = [ [[package]] name = "quickwit-storage" -version = "0.8.0" +version = "0.9.0" dependencies = [ "anyhow", "async-trait", @@ -7277,7 +7277,7 @@ dependencies = [ [[package]] name = "quickwit-telemetry" -version = "0.8.0" +version = "0.9.0" dependencies = [ "async-trait", "encoding_rs", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index b3b0da4f933..03ad480f866 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -70,7 +70,7 @@ default-members = [ ] [workspace.package] -version = "0.8.0" +version = "0.9.0" edition = "2021" homepage = "https://quickwit.io/" documentation = "https://quickwit.io/docs/" diff --git a/quickwit/quickwit-cli/src/cli_doc_ext.toml b/quickwit/quickwit-cli/src/cli_doc_ext.toml index 2f1c5b273f6..18f56ae933c 100644 --- a/quickwit/quickwit-cli/src/cli_doc_ext.toml +++ b/quickwit/quickwit-cli/src/cli_doc_ext.toml @@ -232,7 +232,7 @@ quickwit run --service metastore --config=./config/quickwit.yaml # Open a new terminal and run: cat << EOF > wikipedia-kafka-source.json { - "version": "0.7", + "version": "0.9", "source_id": "kafka-source", "source_type": "kafka", "params": { diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml index 25acef0a2b1..b6928dee494 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml +++ b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs-create-config.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 doc_mapping: field_mappings: diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json index 6c256c165bf..e3d3a74d5d7 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json +++ b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.json @@ -1,6 +1,6 @@ # Comments are supported. { - "version": "0.7", + "version": "0.9", "index_id": "hdfs-logs", "index_uri": "s3://quickwit-indexes/hdfs-logs", "doc_mapping": { diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml index 3a9d37f1c28..fc15554fc5d 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml +++ b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.toml @@ -1,4 +1,4 @@ -version = "0.7" +version = "0.9" index_id = "hdfs-logs" index_uri = "s3://quickwit-indexes/hdfs-logs" diff --git a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml index ea01ed92c8d..ce85b84882f 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml +++ b/quickwit/quickwit-config/resources/tests/index_config/hdfs-logs.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 index_id: hdfs-logs index_uri: s3://quickwit-indexes/hdfs-logs diff --git a/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml index 4f628e8baad..40fb76af097 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml +++ b/quickwit/quickwit-config/resources/tests/index_config/minimal-hdfs-logs.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 index_id: hdfs-logs index_uri: s3://quickwit-indexes/hdfs-logs diff --git a/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml b/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml index 21ad7415516..7cb7d94dfb3 100644 --- a/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml +++ b/quickwit/quickwit-config/resources/tests/index_config/partial-hdfs-logs.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 index_id: hdfs-logs index_uri: s3://quickwit-indexes/hdfs-logs diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.json b/quickwit/quickwit-config/resources/tests/node_config/quickwit.json index 0b881f85a13..507aa30e450 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.json +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.json @@ -1,6 +1,6 @@ # Comments are supported. { - "version": "0.7", + "version": "0.9", "cluster_id": "quickwit-cluster", "node_id": "my-unique-node-id", "enabled_services": [ diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml index 97b44376a75..a0d42161712 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.toml @@ -1,4 +1,4 @@ -version = "0.7" +version = "0.9" cluster_id = "quickwit-cluster" node_id = "my-unique-node-id" diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml index 8dd847ff4ac..a2dbf6dd1cb 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.wrongkey.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 searcher: fast_field_cache_capacity: 10G # Typo here. It is supposed to be searches. diff --git a/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml b/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml index c23ec0c0e3f..65967072cd9 100644 --- a/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml +++ b/quickwit/quickwit-config/resources/tests/node_config/quickwit.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 cluster_id: quickwit-cluster node_id: my-unique-node-id diff --git a/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json b/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json index 983172284a0..218490c0bfe 100644 --- a/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json +++ b/quickwit/quickwit-config/resources/tests/source_config/ingest-api-source.json @@ -1,5 +1,5 @@ { - "version": "0.7", + "version": "0.9", "source_id": "_ingest-api-source", "enabled": true, "source_type": "ingest-api", diff --git a/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json b/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json index cd3380fe8db..e4f98e06d3f 100644 --- a/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json +++ b/quickwit/quickwit-config/resources/tests/source_config/kafka-source.json @@ -1,7 +1,7 @@ { - "version": "0.7", + "version": "0.9", "source_id": "hdfs-logs-kafka-source", - "desired_num_pipelines": 2, + "num_pipelines": 2, "source_type": "kafka", "params": { "topic": "cloudera-cluster-logs", diff --git a/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml b/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml index 132785dafe4..fb440d6f318 100644 --- a/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml +++ b/quickwit/quickwit-config/resources/tests/source_config/kinesis-source.yaml @@ -1,4 +1,4 @@ -version: 0.8 +version: 0.9 source_id: hdfs-logs-kinesis-source source_type: kinesis params: diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index c731e285a35..794ea807384 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -74,7 +74,7 @@ pub use crate::node_config::{ IndexerConfig, IngestApiConfig, JaegerConfig, NodeConfig, SearcherConfig, SplitCacheLimits, StorageTimeoutPolicy, DEFAULT_QW_CONFIG_PATH, }; -use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig}; +use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_9, VersionedSourceConfig}; pub use crate::storage_config::{ AzureStorageConfig, FileStorageConfig, GoogleCloudStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig, StorageConfigs, @@ -104,7 +104,7 @@ pub fn disable_ingest_v1() -> bool { DocMapping, VersionedSourceConfig, SourceConfigV0_7, - SourceConfigV0_8, + SourceConfigV0_9, VersionedIndexConfig, IndexConfigV0_8, VersionedIndexTemplate, diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index faf159fe620..d3c8fb244d4 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -145,16 +145,16 @@ pub async fn load_node_config_with_env( #[derive(Debug, Deserialize)] #[serde(tag = "version")] enum VersionedNodeConfig { - #[serde(rename = "0.8")] + #[serde(rename = "0.9")] // Retro compatibility. - #[serde(alias = "0.7")] - V0_8(NodeConfigBuilder), + #[serde(alias = "0.7", alias = "0.8")] + V0_9(NodeConfigBuilder), } impl From for NodeConfigBuilder { fn from(versioned_node_config: VersionedNodeConfig) -> Self { match versioned_node_config { - VersionedNodeConfig::V0_8(node_config_builder) => node_config_builder, + VersionedNodeConfig::V0_9(node_config_builder) => node_config_builder, } } } diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs index a3a783ab077..9e6bd2baa64 100644 --- a/quickwit/quickwit-config/src/source_config/mod.rs +++ b/quickwit/quickwit-config/src/source_config/mod.rs @@ -861,41 +861,22 @@ mod tests { { let content = r#" { - "version": "0.7", + "version": "0.9", "source_id": "hdfs-logs-void-source", - "desired_num_pipelines": 0, - "max_num_pipelines_per_indexer": 1, + "num_pipelines": 0, "source_type": "void", "params": {} } "#; let error = load_source_config_from_user_config(ConfigFormat::Json, content.as_bytes()) .unwrap_err(); - assert!(error - .to_string() - .contains("`desired_num_pipelines` must be")); + println!("error: {error}"); + assert!(error.to_string().contains("`num_pipelines` must be")); } - // { - // let content = r#" - // { - // "version": "0.7", - // "source_id": "hdfs-logs-void-source", - // "desired_num_pipelines": 1, - // "max_num_pipelines_per_indexer": 0, - // "source_type": "void", - // "params": {} - // } - // "#; - // let error = load_source_config_from_user_config(ConfigFormat::Json, - // content.as_bytes()) .unwrap_err(); - // assert!(error - // .to_string() - // .contains("`max_num_pipelines_per_indexer` must be")); - // } { let content = r#" { - "version": "0.8", + "version": "0.9", "source_id": "hdfs-logs-void-source", "num_pipelines": 2, "source_type": "void", @@ -909,10 +890,9 @@ mod tests { { let content = r#" { - "version": "0.7", + "version": "0.9", "source_id": "hdfs-logs-void-source", - "desired_num_pipelines": 2, - "max_num_pipelines_per_indexer": 1, + "num_pipelines": 2, "source_type": "void", "params": {} } @@ -970,7 +950,7 @@ mod tests { { let content = r#" { - "version": "0.8", + "version": "0.9", "source_id": "hdfs-logs-kafka-source", "num_pipelines": 3, "source_type": "kafka", @@ -1461,10 +1441,9 @@ mod tests { #[tokio::test] async fn test_source_config_plain_text_input_format() { let file_content = r#"{ - "version": "0.7", + "version": "0.9", "source_id": "logs-file-source", - "desired_num_pipelines": 1, - "max_num_pipelines_per_indexer": 1, + "num_pipelines": 1, "source_type": "file", "params": { "filepath": "s3://mybucket/test_non_json_corpus.txt" diff --git a/quickwit/quickwit-config/src/source_config/serialize.rs b/quickwit/quickwit-config/src/source_config/serialize.rs index 1f2c6e326a7..5d31f50694e 100644 --- a/quickwit/quickwit-config/src/source_config/serialize.rs +++ b/quickwit/quickwit-config/src/source_config/serialize.rs @@ -24,7 +24,7 @@ use crate::{ SourceParams, }; -type SourceConfigForSerialization = SourceConfigV0_8; +type SourceConfigForSerialization = SourceConfigV0_9; #[derive(Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] @@ -32,7 +32,7 @@ type SourceConfigForSerialization = SourceConfigV0_8; pub enum VersionedSourceConfig { #[serde(rename = "0.9")] #[serde(alias = "0.8")] - V0_8(SourceConfigV0_8), + V0_9(SourceConfigV0_9), // Retro compatibility. #[serde(rename = "0.7")] V0_7(SourceConfigV0_7), @@ -42,7 +42,7 @@ impl From for SourceConfigForSerialization { fn from(versioned_source_config: VersionedSourceConfig) -> Self { match versioned_source_config { VersionedSourceConfig::V0_7(v0_7) => v0_7.into(), - VersionedSourceConfig::V0_8(v0_8) => v0_8, + VersionedSourceConfig::V0_9(v0_9) => v0_9, } } } @@ -102,7 +102,7 @@ impl SourceConfigForSerialization { validate_identifier("source", &self.source_id)?; } let num_pipelines = NonZeroUsize::new(self.num_pipelines) - .ok_or_else(|| anyhow::anyhow!("`desired_num_pipelines` must be strictly positive"))?; + .ok_or_else(|| anyhow::anyhow!("`num_pipelines` must be strictly positive"))?; match &self.source_params { SourceParams::Stdin => { bail!( @@ -158,9 +158,9 @@ impl SourceConfigForSerialization { } } -impl From for SourceConfigV0_8 { +impl From for SourceConfigV0_9 { fn from(source_config: SourceConfig) -> Self { - SourceConfigV0_8 { + SourceConfigV0_9 { source_id: source_config.source_id, num_pipelines: source_config.num_pipelines.get(), enabled: source_config.enabled, @@ -173,7 +173,7 @@ impl From for SourceConfigV0_8 { impl From for VersionedSourceConfig { fn from(source_config: SourceConfig) -> Self { - VersionedSourceConfig::V0_8(source_config.into()) + VersionedSourceConfig::V0_9(source_config.into()) } } @@ -181,7 +181,7 @@ impl TryFrom for SourceConfig { type Error = anyhow::Error; fn try_from(versioned_source_config: VersionedSourceConfig) -> anyhow::Result { - let v1: SourceConfigV0_8 = versioned_source_config.into(); + let v1: SourceConfigV0_9 = versioned_source_config.into(); v1.validate_and_build() } } @@ -230,7 +230,7 @@ pub struct SourceConfigV0_7 { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] -pub struct SourceConfigV0_8 { +pub struct SourceConfigV0_9 { #[schema(value_type = String)] pub source_id: SourceId, @@ -252,7 +252,7 @@ pub struct SourceConfigV0_8 { pub input_format: SourceInputFormat, } -impl From for SourceConfigV0_8 { +impl From for SourceConfigV0_9 { fn from(source_config_v0_7: SourceConfigV0_7) -> Self { let SourceConfigV0_7 { source_id, @@ -263,7 +263,7 @@ impl From for SourceConfigV0_8 { transform, input_format, } = source_config_v0_7; - SourceConfigV0_8 { + SourceConfigV0_9 { source_id, num_pipelines: desired_num_pipelines, enabled, diff --git a/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs b/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs index 29bbe244ed9..57298076f97 100644 --- a/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/update_tests/restart_indexer_tests.rs @@ -64,7 +64,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() { .indexes() .create( json!({ - "version": "0.8", + "version": "0.9", "index_id": index_id, "doc_mapping": { "doc_mapping_uid": initial_mapping_uid, @@ -141,7 +141,7 @@ async fn test_update_doc_mapping_restart_indexing_pipeline() { .update( index_id, json!({ - "version": "0.8", + "version": "0.9", "index_id": index_id, "doc_mapping": { "doc_mapping_uid": final_mapping_uid, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json index ffa5699989d..f2f8d077078 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json @@ -1,197 +1,197 @@ { - "version": "0.9", + "delete_tasks": [ + { + "create_timestamp": 0, + "delete_query": { + "index_uid": "my-index:00000000000000000000000001", + "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}" + }, + "opstamp": 10 + } + ], "index": { - "version": "0.9", - "index_uid": "my-index:00000000000000000000000001", + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, "index_config": { - "version": "0.9", - "index_id": "my-index", - "index_uri": "s3://quickwit-indexes/my-index", "doc_mapping": { "doc_mapping_uid": "00000000000000000000000001", - "mode": "dynamic", "dynamic_mapping": { - "indexed": true, - "tokenizer": "raw", - "record": "basic", - "stored": true, "expand_dots": true, "fast": { "normalizer": "raw" - } + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" }, "field_mappings": [ { + "coerce": true, + "fast": true, + "indexed": true, "name": "tenant_id", - "type": "u64", + "output_format": "number", "stored": true, - "indexed": true, - "fast": true, - "coerce": true, - "output_format": "number" + "type": "u64" }, { - "name": "timestamp", - "type": "datetime", + "fast": true, + "fast_precision": "seconds", + "indexed": true, "input_formats": [ "rfc3339", "unix_timestamp" ], + "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", - "indexed": true, "stored": true, - "fast": true + "type": "datetime" }, { - "name": "log_level", - "type": "text", + "fast": false, + "fieldnorms": false, "indexed": true, - "tokenizer": "raw", + "name": "log_level", "record": "basic", - "fieldnorms": false, "stored": true, - "fast": false + "tokenizer": "raw", + "type": "text" }, { - "name": "message", - "type": "text", + "fast": false, + "fieldnorms": false, "indexed": true, - "tokenizer": "default", + "name": "message", "record": "position", - "fieldnorms": false, "stored": true, - "fast": false + "tokenizer": "default", + "type": "text" } ], - "timestamp_field": "timestamp", + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_document_size": false, + "store_source": true, "tag_fields": [ "log_level", "tenant_id" ], - "partition_key": "tenant_id", - "max_num_partitions": 100, - "index_field_presence": true, - "store_document_size": false, - "store_source": true, + "timestamp_field": "timestamp", "tokenizers": [ { + "filters": [], "name": "custom_tokenizer", - "type": "regex", "pattern": "[^\\p{L}\\p{N}]+", - "filters": [] + "type": "regex" } ] }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", "indexing_settings": { "commit_timeout_secs": 301, - "docstore_compression_level": 8, "docstore_blocksize": 1000000, - "split_num_docs_target": 10000001, + "docstore_compression_level": 8, "merge_policy": { - "type": "stable_log", - "min_level_num_docs": 100000, - "merge_factor": 9, + "maturation_period": "2days", "max_merge_factor": 11, - "maturation_period": "2days" + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" }, "resources": { "heap_size": "50.0 MB" - } + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" }, "search_settings": { "default_search_fields": [ "message" ] }, - "retention": { - "period": "90 days", - "schedule": "daily" - } - }, - "checkpoint": { - "kafka-source": { - "00000000000000000000": "00000000000000000042" - } + "version": "0.9" }, - "create_timestamp": 1789, + "index_uid": "my-index:00000000000000000000000001", "sources": [ { - "version": "0.9", - "source_id": "kafka-source", - "num_pipelines": 2, "enabled": true, - "source_type": "kafka", + "input_format": "json", + "num_pipelines": 2, "params": { - "topic": "kafka-topic", - "client_params": {} + "client_params": {}, + "topic": "kafka-topic" }, + "source_id": "kafka-source", + "source_type": "kafka", "transform": { "script": ".message = downcase(string!(.message))", "timezone": "UTC" }, - "input_format": "json" + "version": "0.9" + } + ], + "version": "0.9" + }, + "shards": { + "_ingest-source": [ + { + "doc_mapping_uid": "00000000000000000000000001", + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000001", + "leader_id": "leader-ingester", + "publish_position_inclusive": "", + "shard_id": "00000000000000000001", + "shard_state": 1, + "source_id": "_ingest-source", + "update_timestamp": 1724240908 } ] }, "splits": [ { - "split_state": "Published", - "update_timestamp": 1789, - "publish_timestamp": 1789, - "version": "0.9", - "split_id": "split", - "index_uid": "my-index:00000000000000000000000001", - "partition_id": 7, - "source_id": "source", - "node_id": "node", - "num_docs": 12303, - "uncompressed_docs_size_in_bytes": 234234, - "time_range": { - "start": 121000, - "end": 130198 - }, "create_timestamp": 3, + "delete_opstamp": 10, + "doc_mapping_uid": "00000000000000000000000000", + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000001", "maturity": { - "type": "immature", - "maturation_period_millis": 4000 + "maturation_period_millis": 4000, + "type": "immature" }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "publish_timestamp": 1789, + "source_id": "source", + "split_id": "split", + "split_state": "Published", "tags": [ "234", "aaa" ], - "footer_offsets": { - "start": 1000, - "end": 2000 + "time_range": { + "end": 130198, + "start": 121000 }, - "delete_opstamp": 10, - "num_merge_ops": 3, - "doc_mapping_uid": "00000000000000000000000000" + "uncompressed_docs_size_in_bytes": 234234, + "update_timestamp": 1789, + "version": "0.9" } ], - "shards": { - "_ingest-source": [ - { - "index_uid": "my-index:00000000000000000000000001", - "source_id": "_ingest-source", - "shard_id": "00000000000000000001", - "leader_id": "leader-ingester", - "follower_id": "follower-ingester", - "shard_state": 1, - "publish_position_inclusive": "", - "doc_mapping_uid": "00000000000000000000000001", - "update_timestamp": 1724240908 - } - ] - }, - "delete_tasks": [ - { - "create_timestamp": 0, - "opstamp": 10, - "delete_query": { - "index_uid": "my-index:00000000000000000000000001", - "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}" - } - } - ] + "version": "0.9" } diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json index ffa5699989d..f2f8d077078 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json @@ -1,197 +1,197 @@ { - "version": "0.9", + "delete_tasks": [ + { + "create_timestamp": 0, + "delete_query": { + "index_uid": "my-index:00000000000000000000000001", + "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}" + }, + "opstamp": 10 + } + ], "index": { - "version": "0.9", - "index_uid": "my-index:00000000000000000000000001", + "checkpoint": { + "kafka-source": { + "00000000000000000000": "00000000000000000042" + } + }, + "create_timestamp": 1789, "index_config": { - "version": "0.9", - "index_id": "my-index", - "index_uri": "s3://quickwit-indexes/my-index", "doc_mapping": { "doc_mapping_uid": "00000000000000000000000001", - "mode": "dynamic", "dynamic_mapping": { - "indexed": true, - "tokenizer": "raw", - "record": "basic", - "stored": true, "expand_dots": true, "fast": { "normalizer": "raw" - } + }, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" }, "field_mappings": [ { + "coerce": true, + "fast": true, + "indexed": true, "name": "tenant_id", - "type": "u64", + "output_format": "number", "stored": true, - "indexed": true, - "fast": true, - "coerce": true, - "output_format": "number" + "type": "u64" }, { - "name": "timestamp", - "type": "datetime", + "fast": true, + "fast_precision": "seconds", + "indexed": true, "input_formats": [ "rfc3339", "unix_timestamp" ], + "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", - "indexed": true, "stored": true, - "fast": true + "type": "datetime" }, { - "name": "log_level", - "type": "text", + "fast": false, + "fieldnorms": false, "indexed": true, - "tokenizer": "raw", + "name": "log_level", "record": "basic", - "fieldnorms": false, "stored": true, - "fast": false + "tokenizer": "raw", + "type": "text" }, { - "name": "message", - "type": "text", + "fast": false, + "fieldnorms": false, "indexed": true, - "tokenizer": "default", + "name": "message", "record": "position", - "fieldnorms": false, "stored": true, - "fast": false + "tokenizer": "default", + "type": "text" } ], - "timestamp_field": "timestamp", + "index_field_presence": true, + "max_num_partitions": 100, + "mode": "dynamic", + "partition_key": "tenant_id", + "store_document_size": false, + "store_source": true, "tag_fields": [ "log_level", "tenant_id" ], - "partition_key": "tenant_id", - "max_num_partitions": 100, - "index_field_presence": true, - "store_document_size": false, - "store_source": true, + "timestamp_field": "timestamp", "tokenizers": [ { + "filters": [], "name": "custom_tokenizer", - "type": "regex", "pattern": "[^\\p{L}\\p{N}]+", - "filters": [] + "type": "regex" } ] }, + "index_id": "my-index", + "index_uri": "s3://quickwit-indexes/my-index", "indexing_settings": { "commit_timeout_secs": 301, - "docstore_compression_level": 8, "docstore_blocksize": 1000000, - "split_num_docs_target": 10000001, + "docstore_compression_level": 8, "merge_policy": { - "type": "stable_log", - "min_level_num_docs": 100000, - "merge_factor": 9, + "maturation_period": "2days", "max_merge_factor": 11, - "maturation_period": "2days" + "merge_factor": 9, + "min_level_num_docs": 100000, + "type": "stable_log" }, "resources": { "heap_size": "50.0 MB" - } + }, + "split_num_docs_target": 10000001 + }, + "retention": { + "period": "90 days", + "schedule": "daily" }, "search_settings": { "default_search_fields": [ "message" ] }, - "retention": { - "period": "90 days", - "schedule": "daily" - } - }, - "checkpoint": { - "kafka-source": { - "00000000000000000000": "00000000000000000042" - } + "version": "0.9" }, - "create_timestamp": 1789, + "index_uid": "my-index:00000000000000000000000001", "sources": [ { - "version": "0.9", - "source_id": "kafka-source", - "num_pipelines": 2, "enabled": true, - "source_type": "kafka", + "input_format": "json", + "num_pipelines": 2, "params": { - "topic": "kafka-topic", - "client_params": {} + "client_params": {}, + "topic": "kafka-topic" }, + "source_id": "kafka-source", + "source_type": "kafka", "transform": { "script": ".message = downcase(string!(.message))", "timezone": "UTC" }, - "input_format": "json" + "version": "0.9" + } + ], + "version": "0.9" + }, + "shards": { + "_ingest-source": [ + { + "doc_mapping_uid": "00000000000000000000000001", + "follower_id": "follower-ingester", + "index_uid": "my-index:00000000000000000000000001", + "leader_id": "leader-ingester", + "publish_position_inclusive": "", + "shard_id": "00000000000000000001", + "shard_state": 1, + "source_id": "_ingest-source", + "update_timestamp": 1724240908 } ] }, "splits": [ { - "split_state": "Published", - "update_timestamp": 1789, - "publish_timestamp": 1789, - "version": "0.9", - "split_id": "split", - "index_uid": "my-index:00000000000000000000000001", - "partition_id": 7, - "source_id": "source", - "node_id": "node", - "num_docs": 12303, - "uncompressed_docs_size_in_bytes": 234234, - "time_range": { - "start": 121000, - "end": 130198 - }, "create_timestamp": 3, + "delete_opstamp": 10, + "doc_mapping_uid": "00000000000000000000000000", + "footer_offsets": { + "end": 2000, + "start": 1000 + }, + "index_uid": "my-index:00000000000000000000000001", "maturity": { - "type": "immature", - "maturation_period_millis": 4000 + "maturation_period_millis": 4000, + "type": "immature" }, + "node_id": "node", + "num_docs": 12303, + "num_merge_ops": 3, + "partition_id": 7, + "publish_timestamp": 1789, + "source_id": "source", + "split_id": "split", + "split_state": "Published", "tags": [ "234", "aaa" ], - "footer_offsets": { - "start": 1000, - "end": 2000 + "time_range": { + "end": 130198, + "start": 121000 }, - "delete_opstamp": 10, - "num_merge_ops": 3, - "doc_mapping_uid": "00000000000000000000000000" + "uncompressed_docs_size_in_bytes": 234234, + "update_timestamp": 1789, + "version": "0.9" } ], - "shards": { - "_ingest-source": [ - { - "index_uid": "my-index:00000000000000000000000001", - "source_id": "_ingest-source", - "shard_id": "00000000000000000001", - "leader_id": "leader-ingester", - "follower_id": "follower-ingester", - "shard_state": 1, - "publish_position_inclusive": "", - "doc_mapping_uid": "00000000000000000000000001", - "update_timestamp": 1724240908 - } - ] - }, - "delete_tasks": [ - { - "create_timestamp": 0, - "opstamp": 10, - "delete_query": { - "index_uid": "my-index:00000000000000000000000001", - "query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}" - } - } - ] + "version": "0.9" } diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json index 3316f18fde5..15d9c3979a4 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.expected.json @@ -16,7 +16,7 @@ "indexed": true, "record": "basic", "stored": true, - "tokenizer": "raw" + "tokenizer": "default" }, "field_mappings": [ { diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json index 3316f18fde5..15d9c3979a4 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.9.json @@ -16,7 +16,7 @@ "indexed": true, "record": "basic", "stored": true, - "tokenizer": "raw" + "tokenizer": "default" }, "field_mappings": [ { diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json index a21adb65ebe..7998fdb1841 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.9.expected.json @@ -17,7 +17,7 @@ "indexed": true, "record": "basic", "stored": true, - "tokenizer": "raw" + "tokenizer": "default" }, "field_mappings": [ { diff --git a/quickwit/quickwit-metastore/test-data/manifest/v0.9.json b/quickwit/quickwit-metastore/test-data/manifest/v0.9.json index a21adb65ebe..7998fdb1841 100644 --- a/quickwit/quickwit-metastore/test-data/manifest/v0.9.json +++ b/quickwit/quickwit-metastore/test-data/manifest/v0.9.json @@ -17,7 +17,7 @@ "indexed": true, "record": "basic", "stored": true, - "tokenizer": "raw" + "tokenizer": "default" }, "field_mappings": [ { diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs index fbdc8a733dd..c497bec4232 100644 --- a/quickwit/quickwit-serve/src/index_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/index_api/rest_handler.rs @@ -703,7 +703,7 @@ mod tests { .path("/indexes?overwrite=true") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -713,7 +713,7 @@ mod tests { .path("/indexes?overwrite=true") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -723,7 +723,7 @@ mod tests { .path("/indexes") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 400); @@ -742,7 +742,7 @@ mod tests { .path("/indexes") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -756,7 +756,7 @@ mod tests { assert_json_include!(actual: resp_json, expected: expected_response_json); // Create source. - let source_config_body = r#"{"version": "0.7", "source_id": "vec-source", "source_type": "vec", "params": {"docs": [], "batch_num_docs": 10}}"#; + let source_config_body = r#"{"version": "0.9", "source_id": "vec-source", "source_type": "vec", "params": {"docs": [], "batch_num_docs": 10}}"#; let resp = warp::test::request() .path("/indexes/hdfs-logs/sources") .method("POST") @@ -906,7 +906,7 @@ mod tests { .header("content-type", "application/toml") .body( r#" - version = "0.7" + version = "0.9" index_id = "hdfs-logs" [doc_mapping] field_mappings = [ @@ -962,7 +962,7 @@ mod tests { .method("POST") .json(&true) .body( - r#"{"version": "0.7", "index_id": "hdfs-log", "doc_mapping": + r#"{"version": "0.9", "index_id": "hdfs-log", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "unknown", "fast": true, "indexed": true}]}}"#, ) @@ -987,7 +987,7 @@ mod tests { .path("/indexes") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]},"search_settings":{"default_search_fields":["body"]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]},"search_settings":{"default_search_fields":["body"]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -1006,7 +1006,7 @@ mod tests { .path("/indexes/hdfs-logs") .method("PUT") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]},"search_settings":{"default_search_fields":["severity_text", "body"]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]},"search_settings":{"default_search_fields":["severity_text", "body"]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -1063,7 +1063,7 @@ mod tests { .method("POST") .json(&true) .body( - r#"{"version": "0.8", "source_id": "pulsar-source", + r#"{"version": "0.9", "source_id": "pulsar-source", "num_pipelines": 2, "source_type": "pulsar", "params": {"topics": ["my-topic"], "address": "pulsar://localhost:6650" }}"#, ) @@ -1081,7 +1081,7 @@ mod tests { .path("/indexes/hdfs-logs/sources") .method("POST") .body( - r#"{"version": "0.8", "source_id": "my-stdin-source", "source_type": "stdin"}"#, + r#"{"version": "0.9", "source_id": "my-stdin-source", "source_type": "stdin"}"#, ) .reply(&index_management_handler) .await; @@ -1096,7 +1096,7 @@ mod tests { .path("/indexes/hdfs-logs/sources") .method("POST") .body( - r#"{"version": "0.8", "source_id": "my-local-file-source", "source_type": "file", "params": {"filepath": "localfile"}}"#, + r#"{"version": "0.9", "source_id": "my-local-file-source", "source_type": "file", "params": {"filepath": "localfile"}}"#, ) .reply(&index_management_handler) .await; @@ -1122,7 +1122,7 @@ mod tests { .path("/indexes") .method("POST") .json(&true) - .body(r#"{"version": "0.7", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) + .body(r#"{"version": "0.9", "index_id": "hdfs-logs", "doc_mapping": {"field_mappings":[{"name": "timestamp", "type": "i64", "fast": true, "indexed": true}]}}"#) .reply(&index_management_handler) .await; assert_eq!(resp.status(), 200); @@ -1137,7 +1137,7 @@ mod tests { // Create source. let source_config_body = serde_json::json!({ - "version": "0.7", + "version": "0.9", "source_id": "sqs-source", "source_type": "file", "params": {"notifications": [{"type": "sqs", "queue_url": queue_url, "message_type": "s3_notification"}]}, @@ -1154,7 +1154,7 @@ mod tests { { // Update the source. let update_source_config_body = serde_json::json!({ - "version": "0.7", + "version": "0.9", "source_id": "sqs-source", "source_type": "file", "params": {"notifications": [{"type": "sqs", "queue_url": queue_url, "message_type": "s3_notification"}]}, @@ -1184,7 +1184,7 @@ mod tests { { // Update the source with a different source_id (forbidden) let update_source_config_body = serde_json::json!({ - "version": "0.7", + "version": "0.9", "source_id": "new-source-id", "source_type": "file", "params": {"notifications": [{"type": "sqs", "queue_url": queue_url, "message_type": "s3_notification"}]}, diff --git a/quickwit/quickwit-serve/src/template_api/rest_handler.rs b/quickwit/quickwit-serve/src/template_api/rest_handler.rs index ddc3fec9998..3ae23222f04 100644 --- a/quickwit/quickwit-serve/src/template_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/template_api/rest_handler.rs @@ -298,7 +298,7 @@ mod tests { .path("/templates") .method("POST") .json(&json!({ - "version": "0.7", + "version": "0.9", "template_id": "test-template-foo", "index_id_patterns": ["test-index-foo*"], "doc_mapping": {}, @@ -379,7 +379,7 @@ mod tests { .path("/templates/test-template-foo") .method("PUT") .json(&json!({ - "version": "0.7", + "version": "0.9", "template_id": "test-template-bar", // This `template_id` should be ignored and overridden by the path parameter. "index_id_patterns": ["test-index-foo*"], "doc_mapping": {}, diff --git a/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml index 60580ca576a..aa187508b31 100644 --- a/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/aggregations/_setup.quickwit.yaml @@ -11,7 +11,7 @@ status_code: null method: POST endpoint: indexes/ json: - version: "0.8" + version: "0.9" index_id: aggregations doc_mapping: mode: dynamic @@ -34,7 +34,7 @@ json: method: POST endpoint: indexes/ json: - version: "0.8" + version: "0.9" index_id: empty_aggregations doc_mapping: mode: dynamic diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml index 82febfc2517..6a26060c34d 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml @@ -18,7 +18,7 @@ method: POST api_root: http://localhost:7280/api/v1/ endpoint: indexes/ json: - version: "0.8" + version: "0.9" index_id: test-index doc_mapping: field_mappings: @@ -33,7 +33,7 @@ method: POST api_root: http://localhost:7280/api/v1/ endpoint: templates json: - version: "0.8" + version: "0.9" template_id: test-index-template index_id_patterns: - test-index-pattern-*