new version 2.2.7 (#486)

Co-authored-by: Andrey Mikhailov <mikhailov@icc.ru>
Co-authored-by: Alexander Golodkov <55749660+alexander1999-hub@users.noreply.github.com>
Co-authored-by: Alexander Golodkov <golodkov@ispras.ru>
Co-authored-by: Nikita Shevtsov <61932814+Travvy88@users.noreply.github.com>
Co-authored-by: Nikita Shevtsov <shevtsov@ispras.ru>
Co-authored-by: Oksana Belyaeva <belyaeva@ispras.ru>

Author: 7 people

.github/workflows/test_labeling.yaml

@@ -31,4 +31,4 @@ jobs:
         python-version: '3.9'
     - name: Run tests for labeling
       run: |
-        test="true" docker-compose -f labeling/docker-compose.yml up --build --exit-code-from test
+        test="true" docker compose -f labeling/docker-compose.yml up --build --exit-code-from test

.github/workflows/test_on_push.yaml

@@ -36,4 +36,4 @@ jobs:
         flake8 .
     - name: Run tests
       run: |
-        test="true" docker-compose up --build --exit-code-from test
+        test="true" docker compose up --build --exit-code-from test

README.md

@@ -17,8 +17,10 @@ It extracts a document’s logical structure and content: tables, text formattin
 The document’s content is represented as a tree storing headings and lists of any level. 
 Dedoc can be integrated in a document contents and structure analysis system as a separate module.
 
-## Workflow
+## Star History
+[![Star History Chart](https://api.star-history.com/svg?repos=ispras/dedoc&type=Date)](https://tar-history.com/#ispras/dedoc&Date)
 
+## Workflow
 ![Workflow](https://github.com/ispras/dedoc/raw/master/docs/source/_static/workflow.png)
 
 Workflow description is given [`here`](https://dedoc.readthedocs.io/en/latest/?badge=latest#workflow)
@@ -136,12 +138,12 @@ cd dedoc
 
 ### 3. Build the image and run the application
 ```shell
-docker-compose up --build
+docker compose up --build
 ```
 
 ### 4. Run container with tests
 ```shell
-test="true" docker-compose up --build
+test="true" docker compose up --build
 ```
 
 If you need to change some application settings, you may update `config.py` according to your needs and re-build the image.

VERSION

@@ -1 +1 @@
-2.2.6
+2.2.7

dedoc/api/api_args.py

@@ -28,6 +28,8 @@ class QueryParameters:
     # pdf handling
     pdf_with_text_layer: str = Form("auto_tabby", enum=["true", "false", "auto", "auto_tabby", "tabby"],
                                     description="Extract text from a text layer of PDF or using OCR methods for image-like documents")
+    fast_textual_layer_detection: str = Form("false", enum=["true", "false"],
+                                             description="Use non-ML solution to detect textual layer. Much faster but less accurate.")
     language: str = Form("rus+eng", description="Recognition language ('rus+eng', 'rus', 'eng', 'fra', 'spa')")
     pages: str = Form(":", description='Page numbers range for reading PDF or images, "left:right" means read pages from left to right')
     is_one_column_document: str = Form("auto", enum=["auto", "true", "false"],

dedoc/api/web/index.html

@@ -128,6 +128,10 @@ <h4>PDF handling</h4>
                     </label>
                 </p>
 
+                <p>
+                    <label><input name="fast_textual_layer_detection" type="checkbox" value="true"> fast_textual_layer_detection</label>
+                </p>
+                
                 <p>
                     <label> language
                         <input name="language" list="language" size="8" placeholder="rus+eng">

dedoc/data_structures/concrete_annotations/table_annotation.py

@@ -8,10 +8,10 @@ class TableAnnotation(Annotation):
     """
     name = "table"
 
-    def __init__(self, name: str, start: int, end: int) -> None:
+    def __init__(self, value: str, start: int, end: int) -> None:
         """
-        :param name: unique identifier of the table which is referenced inside this annotation
+        :param value: unique identifier of the table which is referenced inside this annotation
         :param start: start of the annotated text (usually zero)
         :param end: end of the annotated text (usually end of the line)
         """
-        super().__init__(start=start, end=end, name=TableAnnotation.name, value=name, is_mergeable=False)
+        super().__init__(start=start, end=end, name=TableAnnotation.name, value=value, is_mergeable=False)

dedoc/data_structures/line_with_meta.py

@@ -136,6 +136,9 @@ def uid(self) -> str:
     def set_line(self, line: str) -> None:
         self._line = line
 
+    def set_metadata(self, metadata: LineMetadata) -> None:
+        self._metadata = metadata
+
     def __repr__(self) -> str:
         return (f"LineWithMeta({self.line[:65]}, "
                 f"tagHL={self.metadata.tag_hierarchy_level.level_1, self.metadata.tag_hierarchy_level.level_2, self.metadata.tag_hierarchy_level.line_type})")

dedoc/download_models.py

@@ -6,10 +6,10 @@
 """
 model_hash_dict = dict(
     txtlayer_classifier="9ca1de749d8d37147b00a3a228e03ee1776c695f",
-    scan_orientation_efficient_net_b0="9ea283f3d346ae4fdd82463a9f60b5369a3ffb58",
+    scan_orientation_efficient_net_b0="c60812552a1be624476c1e5b58599867b36f8d4e",
     font_classifier="db4481ad60ab050cbb42079b64f97f9e431feb07",
-    paragraph_classifier="c26a10193499d3cbc77ffec9842bece24fa8950b",
-    line_type_classifiers="0568c6e1f49612c0c351f10b80a26dc05f796683",
+    paragraph_classifier="97c4b78bc20d87ec7d53389e09f1ca35c6ade067",
+    line_type_classifiers="6ad0eacbfdea065b658cb6f039d13f75245d51ae",
     fintoc_classifiers="6a907b7d2437c3f61ac9c506f67175207982fae8"
 )
 

dedoc/readers/article_reader/article_reader.py

@@ -235,7 +235,7 @@ def __create_line_with_refs(self, content: List[Tuple[str, Tag]], bib2uid: dict,
                 if subpart.get("type") == "bibr" and target in bib2uid:
                     annotations.append(ReferenceAnnotation(value=bib2uid[target], start=start, end=start + len(sub_text)))
                 if subpart.get("type") == "table" and target in table2uid:
-                    annotations.append(TableAnnotation(name=table2uid[target], start=start, end=start + len(sub_text)))
+                    annotations.append(TableAnnotation(value=table2uid[target], start=start, end=start + len(sub_text)))
                 if subpart.get("type") == "figure" and target in attachment2uid:
                     annotations.append(AttachAnnotation(attach_uid=attachment2uid[target], start=start, end=start + len(sub_text)))
             else: