Extraction of text from tagged PDF.

Author: sambitdash

src/CosDoc.jl

@@ -46,6 +46,25 @@ mutable struct CosDocImpl <: CosDoc
   end
 end
 
+"""
+```
+    show(io::IO, doc::CosDoc)
+```
+Prints the CosDoc. The intent is to print lesser information from the structure.
+"""
+function show(io::IO, doc::CosDoc)
+    print(io, "\nCosDoc ==>\n")
+    print(io, "\tfilepath:\t\t$(doc.filepath)\n")
+    print(io, "\tsize:\t\t\t$(doc.size)\n")
+    print(io, "\thasNativeXRefStm:\t $(doc.hasNativeXRefStm)\n")
+    print(io, "\tTrailer dictionaries: \n")
+    for t in doc.trailer
+        print(io, '\t')
+        print(io, t)
+        print(io, '\n')
+    end
+end
+
 """
 ```
     cosDocClose(doc::CosDoc)
@@ -432,7 +451,7 @@ cosDocGetPageNumbers(doc::CosDoc, catalog::CosObject, label::AbstractString) ->
 ```
 PDF utilizes two pagination schemes. An internal global page number that is maintained
 serially as an integer and `PageLabel` that is shown by the viewers. Given a `label` this
-method returns a `range` of valid page numbers for the given label. 
+method returns a `range` of valid page numbers for the given label.
 """
 function cosDocGetPageNumbers(doc::CosDoc, catalog::CosObject, label::AbstractString)
     ref = get(catalog, cn"PageLabels")

src/CosObject.jl

@@ -43,6 +43,8 @@ abstract type CosObject end
 ```
 """
 abstract type CosString <: CosObject end
+
+@inline get{T<:CosString}(o::T) = copy(o.val)
 """
 ```
     CosNumeric
@@ -382,7 +384,7 @@ function show(io::IO, o::CosDict)
 end
 
 show(io::IO, stm::CosStream) =
-  (show(io, stm.extent); print(io, "stream\n...\nendstream\n"))
+  (show(io, stm.extent); print(io, "\nstream\n...\nendstream"))
 
 show(io::IO, os::CosObjectStream) = show(io, os.stm)
 

src/CosObjectHelpers.jl

@@ -8,18 +8,21 @@ function convert(::Type{CDTextString}, xstr::CosXString)
     const feff = [LATIN_F, LATIN_E, LATIN_F, LATIN_F]
     const FEFF = [LATIN_UPPER_F, LATIN_UPPER_E, LATIN_UPPER_F, LATIN_UPPER_F]
     prefix = xstr.val[1:4]
+    hasPrefix = (prefix == feff || prefix == FEFF)
+    isUTF16   = hasPrefix || prefix[1:2] == UInt8[0x00, 0x00]
     data = xstr.val
     buffer = data |> String |> hex2bytes
-    if prefix == feff || prefix == FEFF
+    if isUTF16
         len2 = div(length(buffer),2)
-        utf_16_arr = Vector{UInt16}(len2-1)
+        utf_16_arr = Vector{UInt16}(hasPrefix ? len2-1 : len2)
         utf_16_data = reinterpret(UInt8, utf_16_arr)
         if (0x04030201 == ENDIAN_BOM)
             for i=1:len2
                 (buffer[2i-1], buffer[2i]) = (buffer[2i], buffer[2i-1])
             end
         end
-        copy!(utf_16_data, 1, buffer, 3, 2len2-2)
+        hasPrefix ? copy!(utf_16_data, 1, buffer, 3, 2len2-2) :
+            copy!(utf_16_data, 1, buffer, 1, 2len2)
         str = transcode(String, utf_16_arr)
     else
         # Assume PDFDocEncoding (ISO-8859-1)

src/PDDocImpl.jl

@@ -2,14 +2,30 @@ mutable struct PDDocImpl <: PDDoc
     cosDoc::CosDoc
     catalog::CosObject
     pages::CosObject
+    structTreeRoot::CosObject
     isTagged::Symbol #Valid values :tagged, :none and :suspect
     function PDDocImpl(fp::String)
         cosDoc = cosDocOpen(fp)
         catalog = cosDocGetRoot(cosDoc)
-        new(cosDoc,catalog,CosNull,:none)
+        new(cosDoc,catalog,CosNull,CosNull,:none)
     end
 end
 
+"""
+```
+    show(io::IO, doc::PDDoc)
+```
+Prints the PDDoc. The intent is to print lesser information from the structure.
+"""
+function Base.show(io::IO, doc::PDDoc)
+    print(io, "\nPDDoc ==>\n")
+    print(io, doc.cosDoc)
+    print(io, '\n')
+    print(io, "Catalog:")
+    print(io, doc.catalog)
+    print(io, "isTagged: $(doc.isTagged)\n")
+end
+
 """
 Recursively reads the page object and populates the indirect objects
 Ensures indirect objects are read and updated in the xref Dictionary.
@@ -66,6 +82,34 @@ function find_page_from_treenode(node::CosObject, pageno::Int)
     throw(ErrorException(E_INVALID_PAGE_NUMBER))
 end
 
+mutable struct StructTreeRoot
+    k::CosObject                                    # Dict, Array or null
+    idTree::Nullable{CosTreeNode{CDTextString}}     # Name tree
+    parentTree::Nullable{CosTreeNode{Int}}          # Number tree
+    parentTreeNext::Int
+    roleMap::CosObject                              # Dict or null
+    classMap::CosObject                             # Dict or null
+end
+
+mutable struct StructElem
+    s::CosName
+    p::CosObject                                    # Indirect Dict
+    id::Vector{UInt8}
+    pg::CosObject                                   # Dict
+    k::Union{StructElem, CosObject}
+    a::CosObject
+    r::Int
+    t::CDTextString
+    lang::CDTextString
+    alt::CDTextString
+    e::CDTextString
+    actualText::CDTextString
+end
+
+
+# The structure tree is not fully loaded but the object linkages are established for future
+# correlations during text extraction.
+
 function update_structure_tree(doc::PDDocImpl)
     catalog = pdDocGetCatalog(doc)
     marking = get(catalog, cn"MarkInfo")
@@ -76,5 +120,8 @@ function update_structure_tree(doc::PDDocImpl)
         doc.isTagged = (suspect === CosTrue) ? (:suspect) :
                        (tagged  === CosTrue) ? (:tagged)  : (:none)
     end
+
+    structTreeRef = get(catalog, cn"StructTreeRoot")
+    doc.structTreeRoot = cosDocGetObject(doc.cosDoc, structTreeRef)
     return nothing
 end

src/PDFIO.jl

@@ -29,7 +29,8 @@ export  PDDoc,
             pdPageGetContents,
             pdPageIsEmpty,
             pdPageGetCosObject,
-            pdPageGetContentObjects
+            pdPageGetContentObjects,
+            pdPageExtractText
 
 using .Cos
 export  CosDoc,

src/PDPage.jl

@@ -2,7 +2,8 @@ export PDPage,
        pdPageGetContents,
        pdPageIsEmpty,
        pdPageGetCosObject,
-       pdPageGetContentObjects
+       pdPageGetContentObjects,
+       pdPageExtractText
 
 abstract type PDPage end
 
@@ -56,6 +57,18 @@ function pdPageGetContentObjects(page::PDPage)
     return get(page.content_objects)
 end
 
+"""
+```
+    pdPageExtractText(io::IO, page::PDPage) -> IO
+```
+Extracts the text from the `page`. This extraction works only for tagged PDF files only.
+"""
+function pdPageExtractText(io::IO, page::PDPage)
+    page.doc.isTagged != :tagged && throw(ErrorException(E_NOT_TAGGED_PDF))
+    showtext(io, pdPageGetContentObjects(page))
+    return io
+end
+
 mutable struct PDPageImpl <: PDPage
   doc::PDDocImpl
   cospage::CosObject