Support fixing LocalFile's with FileLinter (#3660)

## Changes
Add fixing files to `LocalFileLinter`:
- Add `apply` method to `LocalFileLinter` to fix files using the linters
- Support writing back to a `LocalFile` container

### Linked issues

Progresses #3514

### Functionality

- [x] modified existing command: `databricks labs ucx
migrate-local-code`

### Tests

- [ ] manually tested
- [x] modified and added unit tests
- [x] modified and added integration tests

Author: JCZuurmond

src/databricks/labs/ucx/source_code/base.py

@@ -4,6 +4,8 @@
 import dataclasses
 import io
 import logging
+import os
+import shutil
 import sys
 from abc import abstractmethod, ABC
 from collections.abc import Iterable
@@ -14,11 +16,10 @@
 
 from astroid import NodeNG  # type: ignore
 
+from databricks.labs.blueprint.paths import WorkspacePath
 from databricks.sdk.service import compute
 from databricks.sdk.service.workspace import Language
 
-from databricks.labs.blueprint.paths import WorkspacePath
-
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -412,6 +413,40 @@ def safe_read_text(path: Path, size: int = -1) -> str | None:
         return None
 
 
+def write_text(path: Path, contents: str, *, encoding: str | None = None) -> int:
+    """Write content to a file as text, encode according to the BOM marker if that is present.
+
+    This differs to the normal `.read_text()` method on path which does not support BOM markers.
+
+    Arguments:
+        path (Path): The file path to write text to.
+        contents (str) : The content to write to the file.
+        encoding (str) : Force encoding with a specific locale. If not present the file BOM and
+            system locale are used.
+
+    Returns:
+        int : The number of characters written to the file.
+    """
+    if not encoding and path.exists():
+        with path.open("rb") as binary_io:
+            encoding = _detect_encoding_bom(binary_io, preserve_position=False)
+    # If encoding=None, the system locale is used for encoding (as per open()).
+    return path.write_text(contents, encoding=encoding)
+
+
+def safe_write_text(path: Path, contents: str, *, encoding: str | None = None) -> int | None:
+    """Safe write content to a file by handling writing exceptions, see :func:write_text.
+
+    Returns:
+        int | None : The number of characters written to the file. If None, no content was written.
+    """
+    try:
+        return write_text(path, contents, encoding=encoding)
+    except OSError as e:
+        logger.warning(f"Cannot write to file: {path}", exc_info=e)
+        return None
+
+
 # duplicated from CellLanguage to prevent cyclic import
 LANGUAGE_COMMENT_PREFIXES = {Language.PYTHON: '#', Language.SCALA: '//', Language.SQL: '--'}
 NOTEBOOK_HEADER = "Databricks notebook source"
@@ -430,3 +465,64 @@ def is_a_notebook(path: Path, content: str | None = None) -> bool:
         return content.startswith(magic_header)
     file_header = safe_read_text(path, size=len(magic_header))
     return file_header == magic_header
+
+
+def _add_backup_suffix(path: Path) -> Path:
+    """Add a backup suffix to a path.
+
+    The backed up path is the same as the original path with an additional
+    `.bak` appended to the suffix.
+
+    Reuse this method so that the backup path is consistent in this module.
+    """
+    # Not checking for the backup suffix to allow making backups of backups.
+    return path.with_suffix(path.suffix + ".bak")
+
+
+def back_up_path(path: Path) -> Path | None:
+    """Back up a path.
+
+    The backed up path is the same as the original path with an additional
+    `.bak` appended to the suffix.
+
+    Returns :
+        path | None : The backed up path. If None, the backup failed.
+    """
+    path_backed_up = _add_backup_suffix(path)
+    try:
+        shutil.copyfile(path, path_backed_up)
+    except OSError as e:
+        logger.warning(f"Cannot back up file: {path}", exc_info=e)
+        return None
+    return path_backed_up
+
+
+def revert_back_up_path(path: Path) -> bool | None:
+    """Revert a backed up path, see :func:back_up_path.
+
+    The backed up path is the same as the original path with an additional
+    `.bak` appended to the suffix.
+
+    Args :
+        path : The original path, NOT the backed up path.
+
+    Returns :
+        bool : Flag if the revert was successful. If None, the backed up path
+        does not exist, thus it cannot be reverted and the operation is not
+        successful nor failed.
+    """
+    path_backed_up = _add_backup_suffix(path)
+    if not path_backed_up.exists():
+        logger.warning(f"Backup is missing: {path_backed_up}")
+        return None
+    try:
+        shutil.copyfile(path_backed_up, path)
+    except OSError as e:
+        logger.warning(f"Cannot revert backup: {path}", exc_info=e)
+        return False
+    try:
+        os.unlink(path_backed_up)
+    except OSError as e:
+        # The backup revert is successful, but the backup file cannot be removed
+        logger.warning(f"Cannot remove backup file: {path_backed_up}", exc_info=e)
+    return True

src/databricks/labs/ucx/source_code/files.py

@@ -6,7 +6,13 @@
 
 from databricks.sdk.service.workspace import Language
 
-from databricks.labs.ucx.source_code.base import infer_file_language_if_supported, safe_read_text
+from databricks.labs.ucx.source_code.base import (
+    back_up_path,
+    infer_file_language_if_supported,
+    revert_back_up_path,
+    safe_read_text,
+    safe_write_text,
+)
 from databricks.labs.ucx.source_code.graph import (
     SourceContainer,
     DependencyGraph,
@@ -33,13 +39,53 @@ class LocalFile(SourceContainer):
     def __init__(self, path: Path, source: str, language: Language):
         self._path = path
         self._original_code = source
+        self._migrated_code = source
         self.language = language
 
     @property
-    def content(self) -> str:
-        """The local file content"""
+    def original_code(self) -> str:
+        """The source code when creating the container."""
         return self._original_code
 
+    @property
+    def migrated_code(self) -> str:
+        """The source code after fixing with a linter."""
+        return self._migrated_code
+
+    @migrated_code.setter
+    def migrated_code(self, source: str) -> None:
+        """Set the source code after fixing with a linter."""
+        self._migrated_code = source
+
+    def _safe_write_text(self, contents: str) -> int | None:
+        """Write content to the local file."""
+        return safe_write_text(self._path, contents)
+
+    def _back_up_path(self) -> Path | None:
+        """Back up the original file."""
+        return back_up_path(self._path)
+
+    def back_up_original_and_flush_migrated_code(self) -> int | None:
+        """Back up the original file and flush the migrated code to the file.
+
+        This is a single method to avoid overwriting the original file without a backup.
+
+        Returns :
+            int : The number of characters written. If None, nothing is written to the file.
+        """
+        if self._original_code == self._migrated_code:
+            # Avoiding unnecessary back up and flush
+            return len(self._migrated_code)
+        backed_up_path = self._back_up_path()
+        if not backed_up_path:
+            # Failed to back up the original file, avoid overwriting existing file
+            return None
+        number_of_characters_written = self._safe_write_text(self._migrated_code)
+        if number_of_characters_written is None:
+            # Failed to overwrite original file, clean up by reverting backup
+            revert_back_up_path(self._path)
+        return number_of_characters_written
+
     def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProblem]:
         """The dependency graph for the local file."""
         if self.language == Language.PYTHON:

src/databricks/labs/ucx/source_code/linters/files.py

@@ -213,48 +213,6 @@ def _resolve_and_parse_notebook_path(self, path: Path | None) -> Notebook | None
 
 
 class FileLinter:
-    _NOT_YET_SUPPORTED_SUFFIXES = {
-        '.scala',
-        '.sh',
-        '.r',
-    }
-    _IGNORED_SUFFIXES = {
-        '.json',
-        '.md',
-        '.txt',
-        '.xml',
-        '.yml',
-        '.toml',
-        '.cfg',
-        '.bmp',
-        '.gif',
-        '.png',
-        '.tif',
-        '.tiff',
-        '.svg',
-        '.jpg',
-        '.jpeg',
-        '.pyc',
-        '.whl',
-        '.egg',
-        '.class',
-        '.iml',
-        '.gz',
-    }
-    _IGNORED_NAMES = {
-        '.ds_store',
-        '.gitignore',
-        '.coverage',
-        'license',
-        'codeowners',
-        'makefile',
-        'pkg-info',
-        'metadata',
-        'wheel',
-        'record',
-        'notice',
-        'zip-safe',
-    }
 
     def __init__(
         self,
@@ -270,15 +228,10 @@ def __init__(
 
     def lint(self) -> Iterable[Advice]:
         """Lint the file."""
-        if self._dependency.path.suffix.lower() in self._IGNORED_SUFFIXES:
-            return
-        if self._dependency.path.name.lower() in self._IGNORED_NAMES:
-            return
-        if self._dependency.path.suffix.lower() in self._NOT_YET_SUPPORTED_SUFFIXES:
-            message = f"Unsupported language for suffix: {self._dependency.path.suffix}"
-            yield Failure("unsupported-language", message, -1, -1, -1, -1)
-            return
         source_container = self._dependency.load(self._path_lookup)
+        if not source_container:
+            # The linter only reports **linting** errors, not loading errors
+            return
         if isinstance(source_container, Notebook):
             yield from self._lint_notebook(source_container)
         elif isinstance(source_container, LocalFile):
@@ -290,7 +243,7 @@ def _lint_file(self, local_file: LocalFile) -> Iterable[Advice]:
         """Lint a local file."""
         try:
             linter = self._context.linter(local_file.language)
-            yield from linter.lint(local_file.content)
+            yield from linter.lint(local_file.original_code)
         except ValueError:
             # TODO: Remove when implementing: https://github.com/databrickslabs/ucx/issues/3544
             yield Failure("unsupported-language", f"Unsupported language: {local_file.language}", -1, -1, -1, -1)
@@ -300,6 +253,18 @@ def _lint_notebook(self, notebook: Notebook) -> Iterable[Advice]:
         notebook_linter = NotebookLinter(notebook, self._path_lookup, self._context, self._inherited_tree)
         yield from notebook_linter.lint()
 
+    def apply(self) -> None:
+        """Apply changes to the file."""
+        source_container = self._dependency.load(self._path_lookup)
+        if isinstance(source_container, LocalFile):
+            self._apply_file(source_container)
+
+    def _apply_file(self, local_file: LocalFile) -> None:
+        """Apply changes to a local file."""
+        fixed_code = self._context.apply_fixes(local_file.language, local_file.original_code)
+        local_file.migrated_code = fixed_code
+        local_file.back_up_original_and_flush_migrated_code()
+
 
 class NotebookMigrator:
     def __init__(self, languages: LinterContext):