Skip to content

fix: Resolve chunking issue during deployment when enabling advanced image… #1633

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 21, 2025
Merged

fix: Resolve chunking issue during deployment when enabling advanced image… #1633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5078f3b
Resolve chunking issue during deployment when enabling advanced image…
Priyanka-Microsoft Jan 24, 2025
471d9b0
Merge branch 'dev' into bug/fix-advance-img-proccessing-chunking-error
Roopan-Microsoft Apr 21, 2025
0d4e962
updated test case
Apr 21, 2025
c47b64d
test case updated2
Apr 21, 2025
799ae9a
test update 2
Apr 21, 2025
0569b0e
updated the test file
Apr 21, 2025
ee5f46f
solved linting issue
Apr 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions code/backend/batch/utilities/helpers/config/config_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,10 @@ def clear_config():
@staticmethod
def _append_advanced_image_processors():
image_file_types = ["jpeg", "jpg", "png", "tiff", "bmp"]
ConfigHelper._remove_processors_for_file_types(image_file_types)
# ConfigHelper._remove_processors_for_file_types(image_file_types)
ConfigHelper._default_config["document_processors"].extend(
[
{"document_type": file_type, "use_advanced_image_processing": True}
{"document_type": file_type, "chunking" : ConfigHelper._default_config["document_processors"][0]["chunking"], "loading" : ConfigHelper._default_config["document_processors"][0]["loading"], "use_advanced_image_processing": True}
for file_type in image_file_types
]
)
Expand Down
70 changes: 29 additions & 41 deletions code/tests/utilities/helpers/test_config_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,63 +178,51 @@ def test_default_config_is_cached():
assert default_config_one is default_config_two


@patch("backend.batch.utilities.helpers.config.config_helper.EnvHelper")
def test_default_config_when_use_advanced_image_processing(env_helper_mock):
# given
ConfigHelper._default_config = None
env_helper_mock.return_value.USE_ADVANCED_IMAGE_PROCESSING = True

# when
config = ConfigHelper.get_default_config()

# then
expected_chunking = {"strategy": "layout", "size": 500, "overlap": 100}
assert config["document_processors"] == [
{
"document_type": "pdf",
"chunking": expected_chunking,
"loading": {"strategy": "layout"},
},
{
"document_type": "txt",
"chunking": expected_chunking,
"loading": {"strategy": "web"},
},
{
"document_type": "url",
"chunking": expected_chunking,
"loading": {"strategy": "web"},
},
{
"document_type": "md",
"chunking": expected_chunking,
"loading": {"strategy": "web"},
},
{
"document_type": "html",
"chunking": expected_chunking,
"loading": {"strategy": "web"},
},
{
"document_type": "htm",
"chunking": expected_chunking,
"loading": {"strategy": "web"},
},
{
"document_type": "docx",
"chunking": expected_chunking,
"loading": {"strategy": "docx"},
},
expected_loading = {"strategy": "layout"}
expected_image_processor = {
"chunking": expected_chunking,
"loading": expected_loading,
"use_advanced_image_processing": True,
}

actual_processors = config["document_processors"]

expected_processors = [
{"document_type": "pdf", "chunking": expected_chunking, "loading": expected_loading},
{"document_type": "txt", "chunking": expected_chunking, "loading": {"strategy": "web"}},
{"document_type": "url", "chunking": expected_chunking, "loading": {"strategy": "web"}},
{"document_type": "md", "chunking": expected_chunking, "loading": {"strategy": "web"}},
{"document_type": "html", "chunking": expected_chunking, "loading": {"strategy": "web"}},
{"document_type": "htm", "chunking": expected_chunking, "loading": {"strategy": "web"}},
{"document_type": "docx", "chunking": expected_chunking, "loading": {"strategy": "docx"}},
{
"document_type": "json",
"chunking": {"strategy": "json", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
},
{"document_type": "jpeg", "use_advanced_image_processing": True},
{"document_type": "jpg", "use_advanced_image_processing": True},
{"document_type": "png", "use_advanced_image_processing": True},
{"document_type": "tiff", "use_advanced_image_processing": True},
{"document_type": "bmp", "use_advanced_image_processing": True},
{"document_type": "jpg", "chunking": expected_chunking, "loading": expected_loading},
{"document_type": "jpeg", "chunking": expected_chunking, "loading": expected_loading},
{"document_type": "png", "chunking": expected_chunking, "loading": expected_loading},
{"document_type": "jpeg", **expected_image_processor},
{"document_type": "jpg", **expected_image_processor},
{"document_type": "png", **expected_image_processor},
{"document_type": "tiff", **expected_image_processor},
{"document_type": "bmp", **expected_image_processor},
]

assert actual_processors == expected_processors


def test_get_config_from_azure(
AzureBlobStorageClientMock: MagicMock,
Expand Down