feat(parser): support distant headers and flexible hash formats

This commit introduces a major enhancement to the markdown parser, enabling it to associate content-related headers (`File:`, `Append File:`, etc.) with code blocks that are not immediately adjacent, and adds more flexibility to the hash-based header format.

The parser is refactored from a two-pass to a three-pass architecture to support this new "distant header" functionality:

-   **Pass 1:** Associates code blocks with immediately adjacent, internal, or wrapped headers. It now only marks a code block's range as "processed" if it was successfully associated with a header, leaving unassociated blocks available for the next pass.

-   **Pass 2 (New):** Scans for any remaining unassociated content headers. For each one found, it searches forward for the next available (unprocessed) code block and links them. This allows for descriptive paragraphs or other content between a header and its corresponding code block.

-   **Pass 3 (Formerly Pass 2):** Finds standalone headers that do not require a code block (`Deleted File:`, `Moved File:`). This pass now correctly ignores headers that appear inside *any* code block discovered in Pass 1, not just those that were associated with an action.

Additionally, the header-matching regular expression has been improved to:
-   Support any number of hash symbols (e.g., `###`, `####`).
-   Allow for arbitrary descriptive text on the same line before the action keyword (e.g., `### 1. New File: path/to/file.txt`).

Fixes included in this refactor:
-   Headers with invalid paths are now correctly marked as processed in Pass 1 to prevent them from being incorrectly re-evaluated by Pass 2.
-   A new suite of integration tests (`tests/parser/create_distant.rs`) has been added to validate the new distant header functionality and its edge cases.

Author: Romelium

src/parser/mod.rs

@@ -10,7 +10,8 @@ mod header_utils;
 mod helpers;
 mod internal_comment;
 mod pass1;
-mod pass2;
+mod pass2; // Find unassociated content headers and link forward
+mod pass3; // Find standalone Delete/Move headers
 mod path_utils;
 mod regex; // Contains regex definitions
 
@@ -38,6 +39,7 @@ pub fn parse_markdown(markdown_content: &str) -> Result<Vec<Action>, ParseError>
     let mut actions_with_pos: Vec<(usize, Action)> = Vec::new();
     let mut processed_header_starts: HashSet<usize> = HashSet::new();
     // Store (start, end) byte indices relative to content_to_parse
+    let mut all_code_block_ranges: HashSet<(usize, usize)> = HashSet::new();
     let mut processed_code_block_ranges: HashSet<(usize, usize)> = HashSet::new();
 
     let (content_to_parse, parse_offset) = helpers::preprocess_markdown(markdown_content);
@@ -48,25 +50,39 @@ pub fn parse_markdown(markdown_content: &str) -> Result<Vec<Action>, ParseError>
     }
 
     // --- Pass 1: Find Code Blocks and associate actions ---
-    println!("Step 1: Locating code blocks and associating actions...");
+    println!(
+        "Step 1: Locating code blocks and associating with adjacent/internal/wrapped headers..."
+    );
     pass1::run_pass1(
         // Now calls the function in the pass1 module
         content_to_parse,
         parse_offset,
         &mut actions_with_pos,
         &mut processed_header_starts,
+        &mut all_code_block_ranges,
         &mut processed_code_block_ranges,
     )?;
 
-    // --- Pass 2: Find standalone Delete headers and orphaned Create ---
-    println!("\nStep 2: Locating standalone Delete headers and orphaned Create...");
+    // --- Pass 2: Find unassociated content headers and link to next block ---
+    println!(
+        "\nStep 2: Locating unassociated content headers and linking to subsequent code blocks..."
+    );
     pass2::run_pass2(
-        // Now calls the function in the pass2 module
         content_to_parse,
         parse_offset,
         &mut actions_with_pos,
-        &processed_header_starts,     // Pass as immutable ref
-        &processed_code_block_ranges, // Pass as immutable ref
+        &mut processed_header_starts,
+        &mut processed_code_block_ranges,
+    )?;
+
+    // --- Pass 3: Find standalone Delete/Move headers ---
+    println!("\nStep 3: Locating standalone Delete/Move headers...");
+    pass3::run_pass3(
+        content_to_parse,
+        parse_offset,
+        &mut actions_with_pos,
+        &processed_header_starts, // Pass as immutable ref
+        &all_code_block_ranges,   // Pass as immutable ref
     )?;
 
     // --- Sort actions by original position ---

src/parser/pass1/action_adder.rs

@@ -1,47 +1,26 @@
-//! Handles adding a successfully parsed action to the list or logging skips in Pass 1.
+//! Handles adding a successfully parsed action to the list.
 
 use crate::core_types::Action;
 use std::collections::HashSet;
 
-/// Adds the action if found, sets its original position, and marks the header as processed.
-/// Logs a skip message if no action was found and the block wasn't skipped for other reasons.
-#[allow(clippy::too_many_arguments)]
-pub(crate) fn add_action_or_log_skip(
-    current_action: Option<Action>,
-    header_line_start_pos_rel: Option<usize>,
+/// Adds the action, sets its final original position, and marks the header as processed.
+pub(crate) fn add_action(
+    mut action: Action,
+    header_start_rel: usize,
     action_source: &str,
     parse_offset: usize,
-    block_content_start: usize,
-    original_block_start: usize,
     actions_with_pos: &mut Vec<(usize, Action)>,
     processed_header_starts: &mut HashSet<usize>,
 ) {
-    if let (Some(mut action), Some(header_start_rel)) = (current_action, header_line_start_pos_rel)
-    {
-        let original_pos = header_start_rel + parse_offset;
-        // Ensure original_pos wasn't already set by wrapped header logic
-        if action.original_pos == 0 {
-            action.original_pos = original_pos;
-        }
-        println!(
-            "    -> Adding action from source '{}' with original_pos {}",
-            action_source, action.original_pos
-        );
-        actions_with_pos.push((action.original_pos, action)); // Use final original_pos for sorting
-        processed_header_starts.insert(original_pos); // Mark header associated with action
-    } else if action_source == "unknown" {
-        // Only log skip if no action was attempted from any source.
-        // Check if the block was skipped because of an *ignored* internal header
-        // (which would have been marked in processed_header_starts by handle_internal_header).
-        let first_line_start_original = block_content_start + parse_offset;
-        if !processed_header_starts.contains(&first_line_start_original) {
-            println!(
-                "    Code block at original pos {} has no associated action header (checked external, wrapped, internal). Skipping.",
-                original_block_start
-            );
-        }
+    let original_pos = header_start_rel + parse_offset;
+    // Ensure original_pos wasn't already set by wrapped header logic
+    if action.original_pos == 0 {
+        action.original_pos = original_pos;
     }
-    // If action_source is known but current_action is None, it means a potential header
-    // was found but ignored (e.g., invalid path, wrapped header without next block),
-    // and the warning/info message was already printed by the respective handler.
+    println!(
+        "    -> Adding action from source '{}' with original_pos {}",
+        action_source, action.original_pos
+    );
+    actions_with_pos.push((action.original_pos, action)); // Use final original_pos for sorting
+    processed_header_starts.insert(original_pos); // Mark header associated with action
 }

src/parser/pass1/action_determiner.rs

@@ -28,6 +28,7 @@ pub(crate) fn determine_block_action(
         block_content_start,
         block_content_end,
         parse_offset,
+        processed_header_starts,
     )? {
         return Ok(Some((action, header_pos, "external")));
     }

src/parser/pass1/block_processor.rs

@@ -21,6 +21,7 @@ pub(crate) fn process_single_block(
     original_block_start: usize,
     actions_with_pos: &mut Vec<(usize, Action)>,
     processed_header_starts: &mut HashSet<usize>,
+    all_code_block_ranges: &mut HashSet<(usize, usize)>,
     processed_code_block_ranges: &mut HashSet<(usize, usize)>,
 ) -> Result<(), ParseError> {
     // Determine the action type and source associated with this block
@@ -36,41 +37,32 @@ pub(crate) fn process_single_block(
         processed_header_starts,
         processed_code_block_ranges,
     )?; // Use ? here
+    let action_found = determination_result.is_some();
 
-    // Process the result
-    match determination_result {
-        // Destructuring the tuple works the same with the type alias
-        Some((action, header_pos_rel, action_source)) => {
-            // Add action if found and log skips
-            action_adder::add_action_or_log_skip(
-                Some(action), // Pass action
-                Some(header_pos_rel),
-                action_source,
-                parse_offset,
-                block_content_start,
-                original_block_start,
-                actions_with_pos,
-                processed_header_starts,
-            );
-        }
-        None => {
-            // No action associated with this block from any known source
-            action_adder::add_action_or_log_skip(
-                None,
-                None,
-                "unknown", // Mark source as unknown
-                parse_offset,
-                block_content_start,
-                original_block_start,
-                actions_with_pos,
-                processed_header_starts,
-            );
-        }
+    if let Some((action, header_pos_rel, action_source)) = determination_result {
+        action_adder::add_action(
+            action,
+            header_pos_rel,
+            action_source,
+            parse_offset,
+            actions_with_pos,
+            processed_header_starts,
+        );
+    } else {
+        // The block was unassociated in this pass. Log it.
+        println!(
+            "    Code block at original pos {} has no associated action header (checked external, wrapped, internal). Leaving for Pass 2.",
+            original_block_start
+        );
     }
 
-    // Always record the block range if we successfully found opening and closing fences,
-    // unless it was already added by the wrapped header logic pairing it with *this* block.
-    if !processed_code_block_ranges.contains(&(fence_start_pos, block_outer_end)) {
+    // Always record that this range is a code block.
+    all_code_block_ranges.insert((fence_start_pos, block_outer_end));
+
+    // ONLY record the block range if an action was found for it in this pass.
+    // Wrapped actions mark their content blocks inside the determiner, so this
+    // correctly handles external/internal actions.
+    if action_found && !processed_code_block_ranges.contains(&(fence_start_pos, block_outer_end)) {
         processed_code_block_ranges.insert((fence_start_pos, block_outer_end));
     }
 

src/parser/pass1/external_header.rs

@@ -8,6 +8,7 @@ use crate::parser::helpers::ensure_trailing_newline;
 use crate::parser::pass1::external_delete_special;
 use crate::parser::path_utils::validate_path_format;
 use crate::parser::regex::HEADER_REGEX; // Import the new module
+use std::collections::HashSet;
 
 /// Checks for and handles an external header preceding a code block.
 /// This applies to *any* code block, including ```markdown.
@@ -18,6 +19,7 @@ pub(crate) fn handle_external_header(
     block_content_start: usize,
     block_content_end: usize,
     parse_offset: usize,
+    processed_header_starts: &mut HashSet<usize>,
 ) -> Result<Option<(Action, usize)>, ParseError> {
     // Returns (Action, header_start_pos_rel)
     let fence_line_start_rel = content[..fence_start_pos].rfind('\n').map_or(0, |n| n + 1);
@@ -77,6 +79,7 @@ pub(crate) fn handle_external_header(
                     "Warning: Invalid path format in external header '{}'. Skipping.",
                     stripped_prev_line
                 );
+                processed_header_starts.insert(prev_line_start_rel + parse_offset);
                 return Ok(None);
             }
 

src/parser/pass1/mod.rs

@@ -11,14 +11,14 @@ mod action_determiner;
 mod block_processor;
 mod external_delete_special;
 mod external_header;
-mod fence_finder;
+pub(crate) mod fence_finder;
 #[cfg(test)] // Conditionally compile the test module
 mod fence_finder_tests; // ADDED test module
 mod internal_comment_handler;
 mod internal_header;
 mod internal_standard_handler;
 mod types;
-mod utils;
+pub(crate) mod utils;
 mod wrapped_create_handler;
 mod wrapped_header;
 
@@ -29,6 +29,7 @@ pub(crate) fn run_pass1(
     parse_offset: usize,
     actions_with_pos: &mut Vec<(usize, Action)>,
     processed_header_starts: &mut HashSet<usize>,
+    all_code_block_ranges: &mut HashSet<(usize, usize)>,
     processed_code_block_ranges: &mut HashSet<(usize, usize)>,
 ) -> Result<(), ParseError> {
     let mut current_search_pos = 0;
@@ -90,6 +91,7 @@ pub(crate) fn run_pass1(
             original_block_start,
             actions_with_pos,
             processed_header_starts,
+            all_code_block_ranges,
             processed_code_block_ranges,
         )?;