From 69c85aa82935370f8f12145822c7ea558b6d4f12 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 19 Feb 2025 10:27:08 +0100 Subject: [PATCH 1/3] adding failed test --- .../unit/source_code/linters/test_pyspark.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/source_code/linters/test_pyspark.py b/tests/unit/source_code/linters/test_pyspark.py index 5863b49a87..f5e839d6b3 100644 --- a/tests/unit/source_code/linters/test_pyspark.py +++ b/tests/unit/source_code/linters/test_pyspark.py @@ -30,6 +30,23 @@ def test_spark_dynamic_sql(empty_index) -> None: sqf = SparkTableNamePyLinter(ftf, empty_index, session_state) assert not list(sqf.lint(source)) +def test_dbfs_stuff(empty_index) -> None: + source = """ + # Databricks notebook source + # MAGIC %run ./_param_connexion_formation + + # COMMAND ---------- + + from pyspark.sql import functions as F + + # COMMAND ---------- + + path_dossierLIZZY = searchPathRecent('/mnt/mnt_datalake_smartCompetencies/', repLIZZY) + """ + session_state = CurrentSessionState() + ftf = FromTableSqlLinter(empty_index, session_state) + sqf = SparkTableNamePyLinter(ftf, empty_index, session_state) + assert not list(sqf.lint(source)) def test_linter_context_python_linter_lints_table_pending_migration_with_empty_index(empty_index) -> None: source_code = """ @@ -526,6 +543,20 @@ def test_from_table_sql_py_linter_fixes_migrated_table(migration_index) -> None: ) ], ), + # Test for mount calls inside a function + ( + """variable = searchPath('/mnt/something/', some_variable)""", + [ + Deprecation( + code='direct-filesystem-access', + message="The use of direct filesystem references is deprecated: /mnt/something/", + start_line=0, + start_col=0, + end_line=0, + end_col=41, + ) + ], + ), ], ) def test_spark_cloud_direct_access(empty_index, code, expected) -> None: @@ -656,3 +687,4 @@ def test_spark_collect_tables_ignores_dfsas(source_code, expected, migration_ind for used_table in used_tables: actual = (used_table.catalog_name, used_table.schema_name, used_table.table_name) assert actual in expected + From 4d5cb6de91b1b5940497029c8a3a5da5bf2dd6a8 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 19 Feb 2025 10:29:23 +0100 Subject: [PATCH 2/3] Adding failed test --- tests/unit/source_code/linters/test_pyspark.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unit/source_code/linters/test_pyspark.py b/tests/unit/source_code/linters/test_pyspark.py index f5e839d6b3..ec785b7602 100644 --- a/tests/unit/source_code/linters/test_pyspark.py +++ b/tests/unit/source_code/linters/test_pyspark.py @@ -30,24 +30,6 @@ def test_spark_dynamic_sql(empty_index) -> None: sqf = SparkTableNamePyLinter(ftf, empty_index, session_state) assert not list(sqf.lint(source)) -def test_dbfs_stuff(empty_index) -> None: - source = """ - # Databricks notebook source - # MAGIC %run ./_param_connexion_formation - - # COMMAND ---------- - - from pyspark.sql import functions as F - - # COMMAND ---------- - - path_dossierLIZZY = searchPathRecent('/mnt/mnt_datalake_smartCompetencies/', repLIZZY) - """ - session_state = CurrentSessionState() - ftf = FromTableSqlLinter(empty_index, session_state) - sqf = SparkTableNamePyLinter(ftf, empty_index, session_state) - assert not list(sqf.lint(source)) - def test_linter_context_python_linter_lints_table_pending_migration_with_empty_index(empty_index) -> None: source_code = """ for i in range(10): From 3e10dc146c63e8ed76c499abb4ed01c51c9293b8 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 19 Feb 2025 15:11:40 +0100 Subject: [PATCH 3/3] adding method definition --- tests/unit/source_code/linters/test_pyspark.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/source_code/linters/test_pyspark.py b/tests/unit/source_code/linters/test_pyspark.py index ec785b7602..970af7969b 100644 --- a/tests/unit/source_code/linters/test_pyspark.py +++ b/tests/unit/source_code/linters/test_pyspark.py @@ -527,7 +527,10 @@ def test_from_table_sql_py_linter_fixes_migrated_table(migration_index) -> None: ), # Test for mount calls inside a function ( - """variable = searchPath('/mnt/something/', some_variable)""", + """ + def searchPath(rep, dateAlternative = None): + return dbutils.fs.ls(rep) + variable = searchPath('/mnt/something/', some_variable)""", [ Deprecation( code='direct-filesystem-access',