feat: introduce new setting enable_last_snapshot_location_hint (#16226)

* feat: introduce new setting `enable_last_snapshot_location_hint`

- Which indicates if `last_snapshot_location_hint` should be kept or not.
- It is enabled by default

* refactor: let `write_last_snapshot_hint` handles the setting

* chore: license info

* logic test for setting enable_last_snapshot_location_hint

* chore: fix typo in test script

---------

Co-authored-by: Sky Fan <3374614481@qq.com>

Author: dantengsky

src/query/service/src/interpreters/interpreter_table_add_column.rs

@@ -112,7 +112,7 @@ impl Interpreter for AddTableColumnInterpreter {
         };
         new_table_meta.add_column(&field, &self.plan.comment, index)?;
 
-        let _ = generate_new_snapshot(tbl.as_ref(), &mut new_table_meta).await?;
+        let _ = generate_new_snapshot(self.ctx.as_ref(), tbl.as_ref(), &mut new_table_meta).await?;
         let table_id = table_info.ident.table_id;
         let table_version = table_info.ident.seq;
 
@@ -162,6 +162,7 @@ impl Interpreter for AddTableColumnInterpreter {
 }
 
 pub(crate) async fn generate_new_snapshot(
+    ctx: &QueryContext,
     table: &dyn Table,
     new_table_meta: &mut TableMeta,
 ) -> Result<()> {
@@ -188,16 +189,16 @@ pub(crate) async fn generate_new_snapshot(
 
             // write down hint
             FuseTable::write_last_snapshot_hint(
+                ctx,
                 fuse_table.get_operator_ref(),
                 fuse_table.meta_location_generator(),
-                new_snapshot_location.clone(),
+                &new_snapshot_location,
             )
             .await;
 
-            new_table_meta.options.insert(
-                OPT_KEY_SNAPSHOT_LOCATION.to_owned(),
-                new_snapshot_location.clone(),
-            );
+            new_table_meta
+                .options
+                .insert(OPT_KEY_SNAPSHOT_LOCATION.to_owned(), new_snapshot_location);
         } else {
             info!("Snapshot not found, no need to generate new snapshot");
         }

src/query/service/src/interpreters/interpreter_table_drop_column.rs

@@ -130,7 +130,7 @@ impl Interpreter for DropTableColumnInterpreter {
         let table_id = table_info.ident.table_id;
         let table_version = table_info.ident.seq;
 
-        generate_new_snapshot(table.as_ref(), &mut new_table_meta).await?;
+        generate_new_snapshot(self.ctx.as_ref(), table.as_ref(), &mut new_table_meta).await?;
 
         let req = UpdateTableMetaReq {
             table_id,

src/query/settings/src/settings_default.rs

@@ -813,6 +813,12 @@ impl DefaultSettings {
                     mode: SettingMode::Both,
                     range: Some(SettingRange::Numeric(0..=1)),
                 }),
+                ("enable_last_snapshot_location_hint", DefaultSettingValue {
+                    value: UserSettingValue::UInt64(1),
+                    desc: "Enables writing last_snapshot_location_hint object",
+                    mode: SettingMode::Both,
+                    range: Some(SettingRange::Numeric(0..=1)),
+                }),
                 ("format_null_as_str", DefaultSettingValue {
                     value: UserSettingValue::UInt64(1),
                     desc: "Format NULL as str in query api response",

src/query/settings/src/settings_getter_setter.rs

@@ -677,4 +677,8 @@ impl Settings {
     pub fn get_format_null_as_str(&self) -> Result<bool> {
         Ok(self.try_get_u64("format_null_as_str")? == 1)
     }
+
+    pub fn get_enable_last_snapshot_location_hint(&self) -> Result<bool> {
+        Ok(self.try_get_u64("enable_last_snapshot_location_hint")? == 1)
+    }
 }

src/query/storages/fuse/src/operations/commit.rs

@@ -149,6 +149,7 @@ impl FuseTable {
         let catalog = ctx.get_catalog(table_info.catalog()).await?;
         // 2. update table meta
         let res = Self::update_table_meta(
+            ctx,
             catalog,
             table_info,
             location_generator,
@@ -207,6 +208,7 @@ impl FuseTable {
     #[allow(clippy::too_many_arguments)]
     #[async_backtrace::framed]
     pub async fn update_table_meta(
+        ctx: &dyn TableContext,
         catalog: Arc<dyn Catalog>,
         table_info: &TableInfo,
         location_generator: &TableMetaLocationGenerator,
@@ -242,18 +244,27 @@ impl FuseTable {
 
         // update_table_meta succeed, populate the snapshot cache item and try keeping a hit file of last snapshot
         TableSnapshot::cache().put(snapshot_location.clone(), Arc::new(snapshot));
-        Self::write_last_snapshot_hint(operator, location_generator, snapshot_location).await;
+        Self::write_last_snapshot_hint(ctx, operator, location_generator, &snapshot_location).await;
 
         Ok(())
     }
 
     // Left a hint file which indicates the location of the latest snapshot
     #[async_backtrace::framed]
     pub async fn write_last_snapshot_hint(
+        ctx: &dyn TableContext,
         operator: &Operator,
         location_generator: &TableMetaLocationGenerator,
-        last_snapshot_path: String,
+        last_snapshot_path: &str,
     ) {
+        if let Ok(false) = ctx.get_settings().get_enable_last_snapshot_location_hint() {
+            info!(
+                "Write last_snapshot_location_hint disabled. Snapshot {}",
+                last_snapshot_path
+            );
+            return;
+        }
+
         // Just try our best to write down the hint file of last snapshot
         // - will retry in the case of temporary failure
         // but

src/query/storages/fuse/src/operations/common/processors/sink_commit.rs

@@ -365,6 +365,7 @@ where F: SnapshotGenerator + Send + 'static
 
                 let catalog = self.ctx.get_catalog(table_info.catalog()).await?;
                 match FuseTable::update_table_meta(
+                    self.ctx.as_ref(),
                     catalog.clone(),
                     &table_info,
                     &self.location_gen,

src/query/storages/fuse/src/operations/revert.rs

@@ -63,14 +63,15 @@ impl FuseTable {
         // 4. let's roll
         let reply = catalog.update_single_table_meta(req, table_info).await;
         if reply.is_ok() {
-            // try keep the snapshot hit
+            // try keeping the snapshot hit
             let snapshot_location = table_reverting_to.snapshot_loc().await?.ok_or_else(|| {
                     ErrorCode::Internal("internal error, fuse table which navigated to given point has no snapshot location")
                 })?;
             Self::write_last_snapshot_hint(
+                ctx.as_ref(),
                 &table_reverting_to.operator,
                 &table_reverting_to.meta_location_generator,
-                snapshot_location,
+                &snapshot_location,
             )
             .await;
         };

tests/sqllogictests/suites/base/20+_others/20_0019_snapshot_hint.test

@@ -0,0 +1,32 @@
+
+
+statement ok
+create or replace database snapshot_hint;
+
+statement ok
+use snapshot_hint;
+
+# Disable keeping snapshot location hint
+statement ok
+set enable_last_snapshot_location_hint = 0;
+
+statement ok
+create or replace table t (c int) 'fs:///tmp/snapshot_hint_disabled/' as select * from numbers(1);
+
+# create a stage that points to the same location of table `t`
+statement ok
+create or replace stage hint_stage url='fs:///tmp/snapshot_hint_disabled/';
+
+
+# Table created with CTAS, should have no snapshot hint file
+query T
+select name from list_stage(location => '@hint_stage') where name like '%last_snapshot_location_hint%';
+----
+
+statement ok
+insert into t values(2);
+
+# After insertion, there should be no snapshot hint file
+query T
+select name from list_stage(location => '@hint_stage') where name like '%last_snapshot_location_hint%';
+----

tests/sqllogictests/suites/ee/03_ee_vacuum/03_0000_vacuum_ctas.test

@@ -1,3 +1,17 @@
+## Copyright 2023 Databend Cloud
+##
+## Licensed under the Elastic License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     https://www.elastic.co/licensing/elastic-license
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
 # test orphan data created by failed CTAS could be vacuumed
 statement ok
 drop database if exists ctas_test;

tests/sqllogictests/suites/ee/03_ee_vacuum/03_0000_vacuum_temporary_files.test

@@ -1,3 +1,17 @@
+## Copyright 2023 Databend Cloud
+##
+## Licensed under the Elastic License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     https://www.elastic.co/licensing/elastic-license
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
 onlyif mysql
 statement ok
 set max_threads = 8;