add ocr-text endpoint to get document page(s) ocred text

add ocr-text endpoint to get document page(s) ocred text

Author: ciur

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 .papermerge.toml
 .env
+.envrc
 .env_shell
 .env_services
 config/settings/local/*

docker/dev/.envrc

@@ -12,10 +12,6 @@ handlers:
     formatter: verbose
 
 loggers:
-  django.db.backends:
-    level: DEBUG
-    handlers: [console]
-    propagate: no
   papermerge:
     level: DEBUG
     handlers: [console]

docker/dev/logging.yaml

@@ -12,7 +12,13 @@
 
 
 class DocumentVersion(models.Model):
+    """Document Version
 
+    Document can have one or multiple versions.
+    Document has at least one version associated (the original).
+    Each document version has a number - which starts with 1 (one) i.e.
+    original document version is - document version 1 (one).
+    """
     id = models.UUIDField(primary_key=True, default=uuid.uuid4)
 
     document = models.ForeignKey(
@@ -31,7 +37,7 @@ class DocumentVersion(models.Model):
     )
     # version number
     number = models.IntegerField(
-        default=1,
+        default=1,  # Document versioning starts with 1
         verbose_name=_('Version number')
     )
     #: basename + ext of uploaded file.
@@ -64,8 +70,8 @@ class Meta:
         verbose_name = _('Document version')
         verbose_name_plural = _('Document versions')
 
-    def __str__(self):
-        return f"id={self.pk} number={self.number}"
+    def __repr__(self):
+        return f"DocumentVersion(id={self.pk}, number={self.number})"
 
     def abs_file_path(self):
         return abs_path(
@@ -169,3 +175,29 @@ def update_text_field(self, streams):
             self.save()
 
         return self.has_combined_text
+
+    def get_ocred_text(
+        self,
+        page_numbers: list = (),
+        page_ids: list = ()
+    ) -> str:
+        """
+        Returns OCRed text of given pages.
+
+        You can filter pages for which OCRed is requested either be page numbers
+        or by page_ids.
+        If both page_numbers and page_ids are empty i.e. no filters, then
+        return `self.text`.
+        """
+        pages_text = " ".join([
+            page.text for page in self.pages.all()
+            if page.number in page_numbers or str(page.pk) in page_ids
+        ])
+
+        if page_ids or page_numbers:
+            result = pages_text.strip()
+        else:
+            # when both filters are empty, return the `self.text` field
+            result = self.text.strip()
+
+        return result

papermerge/core/models/document_version.py

@@ -1,7 +1,10 @@
 from .automate import AutomateSerializer
 from .document import DocumentSerializer
 from .document import DocumentDetailsSerializer, DocumentsMergeSerializer
-from .document_version import DocumentVersionSerializer
+from .document_version import (
+    DocumentVersionSerializer,
+    DocumentVersionOcrTextSerializer
+)
 from .folder import FolderSerializer
 from .node import (
     NodeSerializer,
@@ -42,6 +45,7 @@
     'DocumentDetailsSerializer',
     'DocumentsMergeSerializer',
     'DocumentVersionSerializer',
+    'DocumentVersionOcrTextSerializer',
     'FolderSerializer',
     'NodeSerializer',
     'NodeMoveSerializer',

papermerge/core/serializers/__init__.py

@@ -1,4 +1,5 @@
 from rest_framework_json_api import serializers
+from rest_framework import serializers as rest_serializers
 from papermerge.core.models import DocumentVersion
 
 
@@ -18,3 +19,8 @@ class Meta:
             'short_description',
             'document',
         )
+
+
+class DocumentVersionOcrTextSerializer(rest_serializers.Serializer):
+    """Returns OCRed Text of the document"""
+    text = serializers.CharField(required=False, allow_blank=True)

papermerge/core/serializers/document_version.py

@@ -77,6 +77,17 @@ def ocr_document_task(
 
 
 @shared_task
+def post_ocr_document_task(document_id, namespace=None):
+    """
+    Task to run immediately after document OCR is complete
+
+    This task guarantees that `increment_document_version` will run
+    before `update_document_pages`.
+    """
+    increment_document_version(document_id, namespace)
+    update_document_pages(document_id, namespace)
+
+
 def increment_document_version(document_id, namespace=None):
     logger.debug(
         'increment_document_version: '
@@ -113,7 +124,6 @@ def increment_document_version(document_id, namespace=None):
         )
 
 
-@shared_task
 def update_document_pages(document_id, namespace=None):
     """
     Updates document latest versions's ``text`` field

papermerge/core/tasks.py

@@ -7,7 +7,6 @@
 
 router = routers.DefaultRouter()
 
-router.register(r"automates", views.AutomatesViewSet, basename="automate")
 router.register(r"tokens", views.TokensViewSet, basename="token")
 router.register(r"tags", views.TagsViewSet, basename="tag")
 router.register("nodes", views.NodesViewSet, basename="node")
@@ -29,6 +28,11 @@
         views.DocumentUploadView.as_view(),
         name='documents_upload'
     ),
+    path(
+        'documents/<uuid:pk>/ocr-text',
+        views.DocumentOcrTextView.as_view(),
+        name='document-ocr-text'
+    ),
     path(
         'documents/merge/',
         views.DocumentsMergeView.as_view(),

papermerge/core/urls.py

@@ -15,7 +15,11 @@
     InboxCountView,
     NodeTagsView,
 )
-from .documents import DocumentUploadView, DocumentsMergeView
+from .documents import (
+    DocumentUploadView,
+    DocumentsMergeView,
+    DocumentOcrTextView
+)
 from .document_versions import DocumentVersionsDownloadView
 from .documents import DocumentDetailsViewSet
 from .folders import FoldersViewSet
@@ -46,6 +50,7 @@
     'NodesDownloadView',
     'DocumentUploadView',
     'DocumentsMergeView',
+    'DocumentOcrTextView',
     'DocumentVersionsDownloadView',
     'DocumentDetailsViewSet',
     'FoldersViewSet',

papermerge/core/views/__init__.py

@@ -11,18 +11,21 @@
 from rest_framework.parsers import FileUploadParser
 from rest_framework_json_api.views import ModelViewSet
 from rest_framework_json_api.renderers import JSONRenderer
-from drf_spectacular.utils import extend_schema
+from drf_spectacular.utils import (
+    extend_schema,
+    OpenApiParameter
+)
 
 from papermerge.core.serializers import (
     DocumentDetailsSerializer,
-    DocumentsMergeSerializer
+    DocumentsMergeSerializer,
+    DocumentVersionOcrTextSerializer
 )
 from papermerge.core.storage import get_storage_instance
 from papermerge.core.models import Document
 from papermerge.core.tasks import (
     ocr_document_task,
-    update_document_pages,
-    increment_document_version
+    post_ocr_document_task
 )
 from papermerge.core.exceptions import APIBadRequest
 
@@ -71,8 +74,7 @@ def put(self, request, document_id, file_name):
                         'user_id': str(request.user.id)
                     },
                     link=[
-                        increment_document_version.s(namespace),
-                        update_document_pages.s(namespace)
+                        post_ocr_document_task.s(namespace),
                     ]
                 )
             except OperationalError as ex:
@@ -90,6 +92,70 @@ def put(self, request, document_id, file_name):
         return Response({}, status=status.HTTP_201_CREATED)
 
 
+class DocumentOcrTextView(RequireAuthMixin, GenericAPIView):
+    serializer_class = DocumentVersionOcrTextSerializer
+    parser_classes = (rest_framework_JSONParser,)
+    renderer_classes = (rest_framework_JSONRenderer,)
+    queryset = Document.objects.all()
+
+    @extend_schema(
+        operation_id="Document OCR Text",
+        parameters=[
+            OpenApiParameter(
+                name='page_numbers[]',
+                description=(
+                    "Filter pages by provided page numbers"
+                ),
+                required=False,
+                type={'type': 'array', 'items': {'type': 'number'}}
+            ),
+            OpenApiParameter(
+                name='page_ids[]',
+                description=(
+                    "Filter pages by provided page ids"
+                ),
+                required=False,
+                type={'type': 'array', 'items': {'type': 'string'}}
+            ),
+        ]
+    )
+    def get(self, request, pk, *args, **kwargs):
+        """Retrieve OCRed text of the document
+
+        You can filter pages for which OCRed text is to be received either by
+        page numbers or by page ids. When both filters are empty - retrieve
+        OCRed text of the whole document (i.e. of its last document version)
+        """
+
+        # Document instance
+        instance = self.get_object()
+        document_version = instance.versions.last()
+        # For what page number does user want to get OCR text ?
+        # If page_numbers parameter is empty - get OCR text for all pages
+        # of the document version
+        try:
+            page_numbers = self.request.GET.getlist('page_numbers[]', [])
+            page_numbers = [int(number) for number in page_numbers]
+        except ValueError:
+            page_numbers = []
+
+        page_ids = self.request.GET.getlist('page_ids[]', [])
+
+        text = document_version.get_ocred_text(
+            page_numbers=page_numbers,
+            page_ids=page_ids
+        )
+        serializer = self.get_serializer(data={'text': text})
+
+        if serializer.is_valid():
+            return Response(data=serializer.data)
+
+        return Response(
+            serializer.errors,
+            status=status.HTTP_400_BAD_REQUEST
+        )
+
+
 class DocumentsMergeView(RequireAuthMixin, GenericAPIView):
     serializer_class = DocumentsMergeSerializer
     parser_classes = (rest_framework_JSONParser,)