33import math
44import os
55import subprocess
6+ from collections import namedtuple
67from typing import List , Optional , Tuple
78
89import numpy as np
910
1011from dedoc .common .exceptions .java_not_found_error import JavaNotFoundError
1112from dedoc .common .exceptions .tabby_pdf_error import TabbyPdfError
1213from dedoc .data_structures .bbox import BBox
14+ from dedoc .data_structures .concrete_annotations .bbox_annotation import BBoxAnnotation
1315from dedoc .data_structures .concrete_annotations .bold_annotation import BoldAnnotation
1416from dedoc .data_structures .concrete_annotations .indentation_annotation import IndentationAnnotation
1517from dedoc .data_structures .concrete_annotations .italic_annotation import ItalicAnnotation
3335from dedoc .utils .parameter_utils import get_param_page_slice
3436from dedoc .utils .utils import calculate_file_hash
3537
38+ CellPropertyInfo = namedtuple ("NamedTuple" , "colspan, rowspan, invisible" )
39+
3640
3741class PdfTabbyReader (PdfBaseReader ):
3842 """
@@ -76,7 +80,7 @@ def read(self, path: str, document_type: Optional[str] = None, parameters: Optio
7680 Look to the documentation of :meth:`~dedoc.readers.BaseReader.read` to get information about the method's parameters.
7781 """
7882 parameters = {} if parameters is None else parameters
79- lines , scan_tables = self .__extract (path = path )
83+ lines , scan_tables , tables_cell_properties = self .__extract (path = path )
8084 warnings = []
8185 document_metadata = None
8286
@@ -93,10 +97,12 @@ def read(self, path: str, document_type: Optional[str] = None, parameters: Optio
9397
9498 lines = self .linker .link_objects (lines = lines , tables = scan_tables , images = [])
9599 tables = []
96- for scan_table in scan_tables :
100+ assert len (scan_tables ) == len (tables_cell_properties )
101+ for scan_table , table_cells_property in zip (scan_tables , tables_cell_properties ):
102+ cell_properties = [[cellp for cellp in row ] for row in table_cells_property ]
97103 metadata = TableMetadata (page_id = scan_table .page_number , uid = scan_table .name )
98104 cells = [[cell for cell in row ] for row in scan_table .matrix_cells ]
99- table = Table (metadata = metadata , cells = cells )
105+ table = Table (metadata = metadata , cells = cells , cells_properties = cell_properties )
100106 tables .append (table )
101107
102108 attachments = []
@@ -111,23 +117,26 @@ def read(self, path: str, document_type: Optional[str] = None, parameters: Optio
111117
112118 return self ._postprocess (result )
113119
114- def __extract (self , path : str , start_page : int = None , end_page : int = None ) -> Tuple [List [LineWithMeta ], List [ScanTable ]]:
120+ def __extract (self , path : str , start_page : int = None , end_page : int = None ) -> Tuple [List [LineWithMeta ], List [ScanTable ], List [ List [ CellPropertyInfo ]] ]:
115121 file_hash = calculate_file_hash (path = path )
116122 document = self .__process_pdf (path = path , start_page = start_page , end_page = end_page )
117123 all_lines = []
118124 all_tables = []
125+ all_cell_properties = []
119126 for page in document .get ("pages" , []):
120127 lines = self .__get_lines_with_location (page , file_hash )
121128 if lines :
122129 all_lines .extend (lines )
123- tables = self .__get_tables (page , file_hash )
130+ tables , cell_properties = self .__get_tables (page , file_hash )
124131 if tables :
125132 all_tables .extend (tables )
133+ all_cell_properties .extend (cell_properties )
126134
127- return all_lines , all_tables
135+ return all_lines , all_tables , all_cell_properties
128136
129137 def __get_tables (self , page : dict , file_hash : str ) -> List [ScanTable ]:
130138 tables = []
139+ cell_properties = []
131140 page_number = page ["number" ]
132141 i = 0
133142 for table in page ["tables" ]:
@@ -138,26 +147,44 @@ def __get_tables(self, page: dict, file_hash: str) -> List[ScanTable]:
138147 y_bottom_right = y_top_left + table ["height" ]
139148 order = table ["order" ]
140149 rows = table ["rows" ]
150+ cell_properties_json = table ["cell_properties" ]
151+ cell_property_list = []
152+
153+ for cell_properties_row in cell_properties_json :
154+ cell_property_row_list = []
155+
156+ for cell_property in cell_properties_row :
157+ cell_property_info = CellPropertyInfo (cell_property ["col_span" ],
158+ cell_property ["row_span" ],
159+ bool (cell_property ["invisible" ]))
160+
161+ cell_property_row_list .append (cell_property_info )
162+
163+ cell_property_list .append (cell_property_row_list )
164+
141165 cells = [row for row in rows ]
142166 bbox = BBox .from_two_points ((x_top_left , y_top_left ), (x_bottom_right , y_bottom_right ))
143167
144168 tables .append (ScanTable (matrix_cells = cells , page_number = page_number , bbox = bbox , name = file_hash + str (page_number ) + str (i ), order = order ))
169+ cell_properties .append (cell_property_list )
145170
146- return tables
171+ return tables , cell_properties
147172
148173 def __get_lines_with_location (self , page : dict , file_hash : str ) -> List [LineWithLocation ]:
149174 lines = []
150175 page_number = page ["number" ]
176+ page_width = int (page ["width" ])
177+ page_height = int (page ["height" ])
151178 prev_line = None
152179
153180 for block in page ["blocks" ]:
154181 annotations = []
155182 order = block ["order" ]
156183 block_text = block ["text" ]
157- bx_top_left = block ["x_top_left" ]
158- by_top_left = block ["y_top_left" ]
159- bx_bottom_right = bx_top_left + block ["width" ]
160- by_bottom_right = by_top_left + block ["height" ]
184+ bx_top_left = int ( block ["x_top_left" ])
185+ by_top_left = int ( block ["y_top_left" ])
186+ bx_bottom_right = bx_top_left + int ( block ["width" ])
187+ by_bottom_right = by_top_left + int ( block ["height" ])
161188 indent = block ["indent" ]
162189 spacing = block ["spacing" ]
163190 len_block = len (block_text )
@@ -173,7 +200,12 @@ def __get_lines_with_location(self, page: dict, file_hash: str) -> List[LineWith
173200 url = annotation ["url" ]
174201 start = annotation ["start" ]
175202 end = annotation ["end" ]
176-
203+ x_top_left = int (annotation ["x_top_left" ])
204+ y_top_left = int (annotation ["y_top_left" ])
205+ x_bottom_right = bx_top_left + int (annotation ["width" ])
206+ y_bottom_right = by_top_left + int (annotation ["height" ])
207+ box = BBox .from_two_points ((x_top_left , y_top_left ), (x_bottom_right , y_bottom_right ))
208+ annotations .append (BBoxAnnotation (start , end , box , page_width = page_width , page_height = page_height ))
177209 annotations .append (SizeAnnotation (start , end , str (font_size )))
178210 annotations .append (StyleAnnotation (start , end , font_name ))
179211
@@ -189,6 +221,7 @@ def __get_lines_with_location(self, page: dict, file_hash: str) -> List[LineWith
189221 meta = block ["metadata" ].lower ()
190222 uid = f"txt_{ file_hash } { order }
191223 bbox = BBox .from_two_points ((bx_top_left , by_top_left ), (bx_bottom_right , by_bottom_right ))
224+ annotations .append (BBoxAnnotation (0 , len_block , bbox , page_width = page_width , page_height = page_height ))
192225
193226 metadata = LineMetadata (page_id = page_number , line_id = order )
194227 line_with_location = LineWithLocation (line = block_text ,
0 commit comments