Catch exception in LoadDynamicMetadata (#17500)

Author: yumkam

ydb/core/external_sources/object_storage.cpp

@@ -344,7 +344,7 @@ struct TObjectStorageExternalSource : public IExternalSource {
         std::shared_ptr<TMetadata> Metadata;
     };
 
-    virtual NThreading::TFuture<std::shared_ptr<TMetadata>> LoadDynamicMetadata(std::shared_ptr<TMetadata> meta) override {
+    virtual NThreading::TFuture<std::shared_ptr<TMetadata>> LoadDynamicMetadata(std::shared_ptr<TMetadata> meta) override try {
         auto format = meta->Attributes.FindPtr("format");
         if (!format || !meta->Attributes.contains("withinfer")) {
             return NThreading::MakeFuture(std::move(meta));
@@ -363,11 +363,7 @@ struct TObjectStorageExternalSource : public IExternalSource {
             structuredTokenBuilder.SetBasicAuth(params.SerializeAsString(), awsAuth.SecretAccessKey);
         } else if (std::holds_alternative<NAuth::TServiceAccount>(meta->Auth)) {
             if (!CredentialsFactory) {
-                try {
-                    throw yexception{} << "trying to authenticate with service account credentials, internal error";
-                } catch (const yexception& error) {
-                    return NThreading::MakeErrorFuture<std::shared_ptr<TMetadata>>(std::current_exception());
-                }
+                throw yexception{} << "trying to authenticate with service account credentials, internal error";
             }
             auto& saAuth = std::get<NAuth::TServiceAccount>(meta->Auth);
             structuredTokenBuilder.SetServiceAccountIdAuth(saAuth.ServiceAccountId, saAuth.ServiceAccountIdSignature);
@@ -548,6 +544,8 @@ struct TObjectStorageExternalSource : public IExternalSource {
             }
             throw TExternalSourceException{} << value.Issues().ToOneLineString();
         });
+    } catch (const std::exception&) {
+        return NThreading::MakeErrorFuture<std::shared_ptr<TMetadata>>(std::current_exception());
     }
 
     virtual bool CanLoadDynamicMetadata() const override {

ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp

@@ -333,6 +333,22 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) {
             UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
             UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
         }
+
+        {
+            auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+                SELECT * FROM `{external_source}`.`{path}/` WITH (
+                    format="csv_with_names",
+                    `with_infer`="true",
+                    `data.datetime.format`="%Y-%m-%dT%H-%M"
+                )
+            )", "external_source"_a = externalDataSourceName, "path"_a = path)).ExtractValueSync();
+            UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+            UNIT_ASSERT(!scriptExecutionOperation.Metadata().ExecutionId.empty());
+
+            NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+            UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Failed, readyOp.Status().GetIssues().ToString());
+            UNIT_ASSERT_STRING_CONTAINS_C(readyOp.Status().GetIssues().ToString(), "parameter is not supported with type inference: data.datetime.format", readyOp.Status().GetIssues().ToString());
+        }
     }
 }
 

ydb/tests/fq/s3/test_formats.py

@@ -548,3 +548,44 @@ def test_raw_empty_schema_query(self, kikimr, s3, client, unique_prefix):
         describe_result = client.describe_query(query_id).result
         describe_string = "{}".format(describe_result)
         assert r"Only one column in schema supported in raw format" in describe_string
+
+    @yq_v2
+    @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True)
+    def test_with_infer_and_unsupported_option(self, kikimr, s3, client, unique_prefix):
+        resource = boto3.resource(
+            "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key"
+        )
+
+        bucket = resource.Bucket("fbucket")
+        bucket.create(ACL='public-read')
+
+        s3_client = boto3.client(
+            "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key"
+        )
+
+        fruits = '''Fruit,Price,Weight
+Banana,3,100
+Apple,2,22
+Pear,15,33'''
+        s3_client.put_object(Body=fruits, Bucket='fbucket', Key='fruits.csv', ContentType='text/plain')
+        kikimr.control_plane.wait_bootstrap(1)
+
+        storage_connection_name = unique_prefix + "fruitbucket"
+        client.create_storage_connection(storage_connection_name, "fbucket")
+
+        # XXX replace with other unsupported parameter when/if this one become supported
+        sql = f'''
+            SELECT *
+            FROM `{storage_connection_name}`.`fruits.csv`
+            WITH (format="csv_with_names", with_infer="true", `data.datetime.format`="%Y-%m-%dT%H-%M");
+            '''
+
+        query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id
+        client.wait_query_status(query_id, fq.QueryMeta.FAILED)
+        describe_result = client.describe_query(query_id).result
+        logging.debug("Describe result: {}".format(describe_result))
+        describe_string = "{}".format(describe_result)
+        assert (
+            "couldn\\'t load table metadata: parameter is not supported with type inference: data.datetime.format"
+            in describe_string
+        )