update master (#465)

Co-authored-by: Alexander Golodkov <55749660+alexander1999-hub@users.noreply.github.com>
Co-authored-by: Alexander Golodkov <golodkov@ispras.ru>
Co-authored-by: Andrew Perminov <perminov@ispras.ru>
Co-authored-by: Oksana Belyaeva <belyaeva@ispras.ru>
Co-authored-by: RichardScottOZ <Richard.Scott@ozminerals.com>
Co-authored-by: Andrey Mikhailov <mikhailov@icc.ru>

Author: 7 people

.flake8

@@ -7,6 +7,19 @@ inline-quotes = "
 application-import-names = dedoc, tests, scripts, train_dataset
 import-order-style = pycharm
 
+extend-immutable-calls = File, Depends
+
+banned-modules =
+    dedoc = Use full path
+    dedoc.data_structures = Use full path
+    dedoc.attachments_extractors = Use full path
+    dedoc.attachments_handler = Use full path
+    dedoc.converters = Use full path
+    dedoc.metadata_extractors = Use full path
+    dedoc.readers = Use full path
+    dedoc.structure_constructors = Use full path
+    dedoc.structure_extractors = Use full path
+
 exclude =
     .git,
     __pycache__,
@@ -28,9 +41,11 @@ exclude =
 # ANN202 - Missing return type annotation for protected function
 # ANN204 - Missing return type annotation for special method
 # N802 - function name should be lowercase
+# I251 - Banned import (Use full path)
 ignore =
     ANN101
 per-file-ignores =
     scripts/*:T201
     scripts/benchmark_pdf_performance*:JS101
     tests/custom_test_runner.py:ANN001,ANN201,ANN202,ANN204,N802
+    docs/source/_static/code_examples/*:I251

.github/check_version.py

@@ -23,7 +23,7 @@ def is_correct_version(version: str, tag: str, old_version: str, regexp: Pattern
     args = parser.parse_args()
 
     print(f"Old version: {args.old_version}, new version: {args.new_version}, "
-          f"branch: {args.branch}, tag: {args.tag}, pre_release: {args.pre_release}")  # noqa
+          f"branch: {args.branch}, tag: {args.tag}, pre_release: {args.pre_release}")
 
     master_version_pattern = re.compile(r"^\d+\.\d+(\.\d+)?$")
     develop_version_pattern = re.compile(r"^\d+\.\d+\.\d+rc\d+$")
@@ -43,4 +43,4 @@ def is_correct_version(version: str, tag: str, old_version: str, regexp: Pattern
         is_correct_version(args.new_version, args.tag, args.old_version, master_version_pattern)
         assert args.pre_release != "true", "Pre-releases are not allowed on master"
 
-    print("Version is correct")  # noqa
+    print("Version is correct")

.pre-commit-config.yaml

@@ -15,6 +15,7 @@ repos:
             flake8-import-order==0.18.2,
             flake8-multiline-containers==0.0.19,
             flake8-print==5.0.0,
+            flake8-tidy-imports==4.10.0,
             flake8-quotes==3.3.2,
             flake8-use-fstring==1.4,
             pycodestyle==2.9.0,

Dockerfile

@@ -1,5 +1,7 @@
 ARG REPOSITORY="docker.io"
 FROM dedocproject/dedoc_p3.9_base:version_2023_08_28
+ARG LANGUAGES=""
+RUN for lang in $LANGUAGES; do apt install -y tesseract-ocr-$lang; done
 
 ENV PYTHONPATH "${PYTHONPATH}:/dedoc_root"
 ENV RESOURCES_PATH "/dedoc_root/resources"

README.md

@@ -1,17 +1,18 @@
 # Dedoc
 
+[![GitHub release](https://img.shields.io/github/release/ispras/dedoc.svg)](https://github.com/ispras/dedoc/releases/)
 [![PyPI version](https://badge.fury.io/py/dedoc.svg)](https://badge.fury.io/py/dedoc)
+[![PyPI downloads](https://pepy.tech/badge/dedoc)](https://pepy.tech/project/dedoc)
+[![Docker Hub](https://img.shields.io/docker/pulls/dedocproject/dedoc.svg)](https://hub.docker.com/r/dedocproject/dedoc/ "Docker Pulls")
 [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
-[![Documentation Status](https://readthedocs.org/projects/dedoc/badge/?version=latest)](https://dedoc.readthedocs.io/en/latest/?badge=latest)
-[![GitHub release](https://img.shields.io/github/release/ispras/dedoc.svg)](https://github.com/ispras/dedoc/releases/)
 [![Demo dedoc-readme.hf.space](https://img.shields.io/website-up-down-green-red/https/huggingface.co/spaces/dedoc/README.svg)](https://dedoc-readme.hf.space)
-[![Docker Hub](https://img.shields.io/docker/pulls/dedocproject/dedoc.svg)](https://hub.docker.com/r/dedocproject/dedoc/ "Docker Pulls")
+[![Documentation Status](https://readthedocs.org/projects/dedoc/badge/?version=latest)](https://dedoc.readthedocs.io/en/latest/?badge=latest)
 [![CI tests](https://github.com/ispras/dedoc/workflows/CI/badge.svg)](https://github.com/ispras/dedoc/actions)
 
 ![Dedoc](https://github.com/ispras/dedoc/raw/master/dedoc_logo.png)
 
 Dedoc is an open universal system for converting documents to a unified output format. 
-It extracts a document’s logical structure and content, its tables, text formatting and metadata. 
+It extracts a document’s logical structure and content: tables, text formatting and metadata. 
 The document’s content is represented as a tree storing headings and lists of any level. 
 Dedoc can be integrated in a document contents and structure analysis system as a separate module.
 
@@ -22,14 +23,14 @@ Dedoc can be integrated in a document contents and structure analysis system as
 Workflow description is given [`here`](https://dedoc.readthedocs.io/en/latest/?badge=latest#workflow)
 
 ## Features and advantages
-Dedoc is implemented in Python and works with semi-structured data formats (DOC/DOCX, ODT, XLS/XLSX, CSV, TXT, JSON) and none-structured data formats like images (PNG, JPG etc.), archives (ZIP, RAR etc.), PDF and HTML formats. 
+Dedoc is implemented in Python and works with semi-structured data formats (DOC/DOCX, ODT, XLS/XLSX, CSV, TXT, JSON) and unstructured data formats like images (PNG, JPG etc.), archives (ZIP, RAR etc.), PDF and HTML formats. 
 Document structure extraction is fully automatic regardless of input data type. 
 Metadata and text formatting are also extracted automatically. 
 
 In 2022, the system won a grant to support the development of promising AI projects from the [Innovation Assistance Foundation (Фонд содействия инновациям)](https://fasie.ru/).
 
 ## Dedoc provides:
-* Extensibility due to a flexible addition of new document formats and to an easy change of an output data format. 
+* Extensibility due to flexible addition of new document formats and easy change of an output data format. 
 * Support for extracting document structure out of nested documents having different formats. 
 * Extracting various text formatting features (indentation, font type, size, style etc.). 
 * Working with documents of various origin (statements of work, legal documents, technical reports, scientific papers) allowing flexible tuning for new domains. 
@@ -68,7 +69,7 @@ The system processes different document formats. The main formats are listed bel
 
 
 ## Impact
-This project may be useful as a first step of automatic document analysis pipeline (e.g. before the NLP part).
+This project may be useful as a first step of an automatic document analysis pipeline (e.g. before the NLP part).
 Dedoc is in demand for information analytic systems, information leak monitoring systems, as well as for natural language processing systems.
 The library is intended for application use by developers of systems for automatic analysis and structuring of electronic documents, including for further search in electronic documents. 
 
@@ -92,7 +93,7 @@ Relevant documentation of dedoc is available [here](https://dedoc.readthedocs.io
 
 # Installation instructions
 
-This project has REST Api and you can run it in Docker container.
+This project has a REST api and you can run it in Docker container.
 Also, dedoc can be installed as a library via `pip`.
 There are two ways to install and run dedoc as a web application or a library that are described below.
 
@@ -149,7 +150,7 @@ If you need to change some application settings, you may update `config.py` acco
 
 If you don't want to use docker for running the application, it's possible to run dedoc locally.
 However, it isn't suitable for any operating system (`Ubuntu 20+` is recommended) and
-there may be not enough machine's resources for its work.
+there may be not enough machine resources for its work.
 You should have `python` (`python3.8`, `python3.9` are recommended) and `pip` installed.
 Installation instructions via pip are available [here](https://dedoc.readthedocs.io/en/latest/getting_started/installation.html#install-dedoc-using-pip).
 

VERSION

@@ -1 +1 @@
-2.2.3
+2.2.4

dedoc/__init__.py

@@ -1,2 +1,2 @@
-from .dedoc_manager import DedocManager  # noqa
-from .version import __version__  # noqa
+from .dedoc_manager import DedocManager
+from .version import __version__

dedoc/api/api_args.py

@@ -28,7 +28,7 @@ class QueryParameters:
     # pdf handling
     pdf_with_text_layer: str = Form("auto_tabby", enum=["true", "false", "auto", "auto_tabby", "tabby"],
                                     description="Extract text from a text layer of PDF or using OCR methods for image-like documents")
-    language: str = Form("rus+eng", enum=["rus+eng", "rus", "eng", "fra", "spa"], description="Recognition language")
+    language: str = Form("rus+eng", description="Recognition language ('rus+eng', 'rus', 'eng', 'fra', 'spa')")
     pages: str = Form(":", description='Page numbers range for reading PDF or images, "left:right" means read pages from left to right')
     is_one_column_document: str = Form("auto", enum=["auto", "true", "false"],
                                        description='One or multiple column document, "auto" - predict number of page columns automatically')

dedoc/api/api_utils.py

@@ -120,12 +120,20 @@ def json2html(text: str,
               attachments: Optional[List[ParsedDocument]],
               tabs: int = 0,
               table2id: Dict[str, int] = None,
-              attach2id: Dict[str, int] = None) -> str:
+              attach2id: Dict[str, int] = None,
+              prev_page_id: Optional[List[int]] = None) -> str:
+    if prev_page_id is None:
+        prev_page_id = [0]
+
     tables = [] if tables is None else tables
     attachments = [] if attachments is None else attachments
     table2id = {table.metadata.uid: table_id for table_id, table in enumerate(tables)} if table2id is None else table2id
     attach2id = {attachment.metadata.uid: attachment_id for attachment_id, attachment in enumerate(attachments)} if attach2id is None else attach2id
 
+    if paragraph.metadata.page_id != prev_page_id[0]:
+        text += f"<center><small><b>Page {prev_page_id[0] + 1}</b></small></center><hr>"
+        prev_page_id[0] = paragraph.metadata.page_id
+
     ptext = __annotations2html(paragraph=paragraph, table2id=table2id, attach2id=attach2id, tabs=tabs)
 
     if paragraph.metadata.hierarchy_level.line_type in [HierarchyLevel.header, HierarchyLevel.root]:
@@ -141,7 +149,8 @@ def json2html(text: str,
     text += ptext
 
     for subparagraph in paragraph.subparagraphs:
-        text = json2html(text=text, paragraph=subparagraph, tables=None, attachments=None, tabs=tabs + 4, table2id=table2id, attach2id=attach2id)
+        text = json2html(text=text, paragraph=subparagraph, tables=None, attachments=None, tabs=tabs + 4, table2id=table2id, attach2id=attach2id,
+                         prev_page_id=prev_page_id)
 
     if tables is not None and len(tables) > 0:
         text += "<h3> Tables: </h3>"

dedoc/api/dedoc_api.py

@@ -7,13 +7,12 @@
 import traceback
 from typing import Optional
 
-import uvicorn
 from fastapi import Depends, FastAPI, File, Request, Response, UploadFile
 from fastapi.responses import ORJSONResponse, UJSONResponse
 from fastapi.staticfiles import StaticFiles
 from starlette.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse
 
-import dedoc
+import dedoc.version
 from dedoc.api.api_args import QueryParameters
 from dedoc.api.api_utils import json2collapsed_tree, json2html, json2tree, json2txt
 from dedoc.api.schema.parsed_document import ParsedDocument
@@ -53,7 +52,7 @@ def get_static_file(request: Request) -> Response:
 
 @app.get("/version")
 def get_version() -> Response:
-    return PlainTextResponse(dedoc.__version__)
+    return PlainTextResponse(dedoc.version.__version__)
 
 
 def _get_static_file_path(request: Request) -> str:
@@ -70,10 +69,10 @@ def __add_base64_info_to_attachments(document_tree: ParsedDocument, attachments_
 
 
 @app.post("/upload", response_model=ParsedDocument)
-async def upload(file: UploadFile = File(...), query_params: QueryParameters = Depends()) -> Response:  # noqa
+async def upload(file: UploadFile = File(...), query_params: QueryParameters = Depends()) -> Response:
     parameters = dataclasses.asdict(query_params)
     if not file or file.filename == "":
-        raise MissingFileError("Error: Missing content in request_post file parameter", version=dedoc.__version__)
+        raise MissingFileError("Error: Missing content in request_post file parameter", version=dedoc.version.__version__)
 
     return_format = str(parameters.get("return_format", "json")).lower()
 
@@ -152,4 +151,5 @@ def get_api() -> FastAPI:
 
 
 def run_api(app: FastAPI) -> None:
+    import uvicorn
     uvicorn.run(app=app, host="0.0.0.0", port=int(PORT))

dedoc/api/web/index.html

@@ -129,14 +129,15 @@ <h4>PDF handling</h4>
                 </p>
 
                 <p>
-                    <label>
-                        <select name="language">
+                    <label> language
+                        <input name="language" list="language" size="8" placeholder="rus+eng">
+                        <datalist id="language">
+                            <option value="rus+eng" selected>rus+eng</option>
                             <option value="rus">rus</option>
                             <option value="eng">eng</option>
-                            <option value="rus+eng" selected>rus+eng</option>
                             <option value="fra">fra</option>
                             <option value="spa">spa</option>
-                        </select> language
+                        </datalist>
                     </label>
                 </p>
 

dedoc/attachments_extractors/abstract_attachment_extractor.py

@@ -1,12 +1,7 @@
-import logging
-import os
-import uuid
 from abc import ABC, abstractmethod
 from typing import List, Optional, Set, Tuple
 
 from dedoc.data_structures.attached_file import AttachedFile
-from dedoc.utils.parameter_utils import get_param_attachments_dir
-from dedoc.utils.utils import get_mime_extension, save_data_to_unique_file
 
 
 class AbstractAttachmentsExtractor(ABC):
@@ -19,6 +14,8 @@ def __init__(self, *, config: Optional[dict] = None, recognized_extensions: Opti
         :param recognized_extensions: set of supported files extensions with a dot, for example {.doc, .pdf}
         :param recognized_mimes: set of supported MIME types of files
         """
+        import logging
+
         self.config = {} if config is None else config
         self.logger = self.config.get("logger", logging.getLogger())
         self._recognized_extensions = {} if recognized_extensions is None else recognized_extensions
@@ -39,6 +36,7 @@ def can_extract(self,
         :param parameters: any additional parameters for the given document
         :return: the indicator of possibility to get attachments of this file
         """
+        from dedoc.utils.utils import get_mime_extension
         mime, extension = get_mime_extension(file_path=file_path, mime=mime, extension=extension)
         return extension.lower() in self._recognized_extensions or mime in self._recognized_mimes
 
@@ -66,7 +64,13 @@ def with_attachments(parameters: dict) -> bool:
         return str(parameters.get("with_attachments", "false")).lower() == "true"
 
     def _content2attach_file(self, content: List[Tuple[str, bytes]], tmpdir: str, need_content_analysis: bool, parameters: dict) -> List[AttachedFile]:
+        import os
+        import uuid
+        from dedoc.utils.parameter_utils import get_param_attachments_dir
+        from dedoc.utils.utils import save_data_to_unique_file
+
         attachments = []
+
         attachments_dir = get_param_attachments_dir(parameters, tmpdir)
 
         for original_name, contents in content:

dedoc/attachments_extractors/concrete_attachments_extractors/abstract_office_attachments_extractor.py

@@ -1,14 +1,8 @@
-import os
-import zipfile
 from abc import ABC
 from typing import List, Optional, Set, Tuple
 
-import olefile
-from charset_normalizer import from_bytes
-
 from dedoc.attachments_extractors.abstract_attachment_extractor import AbstractAttachmentsExtractor
 from dedoc.data_structures.attached_file import AttachedFile
-from dedoc.utils.parameter_utils import get_param_need_content_analysis
 
 
 class AbstractOfficeAttachmentsExtractor(AbstractAttachmentsExtractor, ABC):
@@ -25,6 +19,8 @@ def __parse_ole_contents(self, stream: bytes) -> Tuple[str, bytes]:
         :param stream: binary content of olefile
         :return: tuple of (name of original file and binary file content)
         """
+        from charset_normalizer import from_bytes
+
         # original filename in ANSI starts at byte 7 and is null terminated
         stream = stream[6:]
 
@@ -65,6 +61,11 @@ def __parse_ole_contents(self, stream: bytes) -> Tuple[str, bytes]:
         return filename, contents
 
     def _get_attachments(self, tmpdir: str, filename: str, parameters: dict, attachments_dir: str) -> List[AttachedFile]:
+        import olefile
+        import os
+        import zipfile
+        from dedoc.utils.parameter_utils import get_param_need_content_analysis
+
         result = []
 
         with zipfile.ZipFile(os.path.join(tmpdir, filename), "r") as zfile: