Merge pull request #1414 from redis/DOC-5093-rdi-image-update

DOC-5093 RDI image update

Author: andy-stark-redis

build/image_report.py

@@ -0,0 +1,63 @@
+"""Image report
+"""
+
+from pylibs.hugotools import ShortcodeIterator
+
+import argparse
+import os
+
+
+def scan_file(path: str) -> int:
+    """Scans a file for all `image` shortcodes.
+
+    Args:
+        path (str): Path to file.
+
+    Returns:
+        (int) Number of shortcodes found.
+    """
+
+    img_list = []
+
+    with open(path, encoding="utf_8") as md_file:
+        text = md_file.read()
+
+        for img, pos_info in ShortcodeIterator(
+            text, lambda t: t.tag == "image"
+        ):
+            img_list.append((img, pos_info))
+
+    if len(img_list) > 0:
+        print(f"File '{path}':")
+
+        for img in img_list:
+            print(
+                f"    Line {img[1].line}: '{img[0].named_params['filename']}'"
+            )
+
+    return len(img_list)
+
+
+parser = argparse.ArgumentParser(
+    "Image report",
+    "Scans a folder and report all Hugo image shortcodes found"
+)
+
+parser.add_argument("pathname", help="Path of the folder to scan")
+
+args = parser.parse_args()
+
+print(f"Scanning '{args.pathname}'")
+
+num_found = 0
+
+for root, dirs, files in os.walk(args.pathname):
+    for file in files:
+        if file.endswith(".md"):
+            fullpath = os.path.join(root, file)
+            num_found += scan_file(fullpath)
+
+if num_found == 0:
+    print(f"No image shortcodes found in '{args.pathname}'")
+else:
+    print(f"Found {num_found} image shortcodes.")

build/pylibs/hugotools.py

@@ -0,0 +1,155 @@
+from enum import Enum
+from typing import Iterator, Match, Callable
+
+import re
+
+
+class TextPosInfo:
+    line: int
+    start: int
+    end: int
+
+    def __init__(self, line, start, end):
+        self.line = line
+        self.start = start
+        self.end = end
+
+
+shortcode_re_pattern_start = r"(\n)|\{\{[<%]\s*"
+shortcode_re_body = r"(/)?([\w\-]+)\s*(.+?)?"
+shortcode_re_pattern_end = r"\s*[>%]\}\}"
+
+shortcode_re_pattern = (
+    shortcode_re_pattern_start +
+    shortcode_re_body +
+    shortcode_re_pattern_end
+)
+
+
+class ShortcodeTagType(Enum):
+    """Specifies open or close shortcode tag."""
+    OPEN = 1
+    CLOSE = 2
+
+
+class ShortcodeInfo:
+    """Represents the information in a shortcode.
+    """
+    tag_type: ShortcodeTagType
+    tag: str
+    pos_params: list[str]
+    named_params: dict[str, str]
+
+    def parse_params(self, param_str: str):
+        param_re = "|".join([
+            r'"(([^"]|(?<=\\)")*)"',
+            r'((\w+)=([^"\s]+))',
+            r'((\w+)="(([^"]|(?<=\\)")*)")',
+            r'([^"=\s]+)'
+        ])
+
+        for match in re.finditer(param_re, param_str):
+            if match is None:
+                self.pos_params = []
+                self.named_params = {}
+                return
+
+            if match[1]:
+                self.pos_params.append(re.sub(r'\\"', '"', match[1]))
+            elif match[3]:
+                self.named_params[match[4]] = match[5]
+            elif match[6]:
+                self.named_params[match[7]] = re.sub(r'\\"', '"', match[8])
+            elif match[10]:
+                self.pos_params.append(match[10])
+
+    def __init__(
+            self, tag: str,
+            tag_type: ShortcodeTagType,
+            param_text: str = ""
+    ):
+        self.tag = tag
+        self.tag_type = tag_type
+        self.pos_params = []
+        self.named_params = {}
+        self.parse_params(param_text or "")
+
+    def __str__(self) -> str:
+        type_text: str
+
+        if self.tag_type == ShortcodeTagType.OPEN:
+            type_text = "OPEN"
+        else:
+            type_text = "CLOSE"
+
+        result = f"{type_text} {self.tag}"
+
+        if self.pos_params or self.named_params:
+            result += ":"
+
+        for pos_param in self.pos_params:
+            result += f"\n    '{pos_param}'"
+
+        for named_param, named_value in self.named_params.items():
+            result += f"\n    {named_param} = {named_value}"
+
+        return result
+
+
+class ShortcodeIterator:
+    """Iterates through all shortcodes in a string.
+    """
+    re_iterator: Iterator[Match[str]]
+    linenum: int
+    sc_filter: Callable[[ShortcodeInfo], bool]
+
+    def __init__(
+            self,
+            text: str,
+            sc_filter: Callable[[ShortcodeInfo], bool] = lambda x: True
+    ):
+        self.re_iterator = re.finditer(shortcode_re_pattern, text)
+        self.sc_filter = lambda self, x: sc_filter(x)
+        self.linenum = 1
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> tuple[ShortcodeInfo, TextPosInfo]:
+        next_match = self.re_iterator.__next__()
+
+        while True:
+            if next_match[1]:
+                self.linenum += 1
+            elif next_match[2]:
+                result = ShortcodeInfo(
+                    next_match[3], ShortcodeTagType.CLOSE
+                )
+
+                if self.sc_filter(self, result):
+                    return (
+                        result,
+                        TextPosInfo(
+                            self.linenum,
+                            next_match.start(),
+                            next_match.end()
+                        )
+                    )
+            else:
+                result = ShortcodeInfo(
+                    next_match[3],
+                    ShortcodeTagType.OPEN,
+                    next_match[4]
+                )
+
+                if self.sc_filter(self, result):
+                    return (
+                        result,
+                        TextPosInfo(
+                            self.linenum,
+                            next_match.start(),
+                            next_match.end()
+                        )
+                    )
+
+            next_match = self.re_iterator.__next__()

content/integrate/redis-data-integration/architecture.md

@@ -55,7 +55,7 @@ outside the Redis
 Enterprise cluster where the target database is kept. However, RDI keeps
 its state and configuration data and also the change data streams in a Redis database on the same cluster as the target. The following diagram shows the pipeline steps and the path the data takes on its way from the source to the target:
 
-{{< image filename="images/rdi/ingest/ingest-dataflow.png" >}}
+{{< image filename="images/rdi/ingest/ingest-dataflow.webp" >}}
 
 When you first start RDI, the target database is empty and so all
 of the data in the source database is essentially "change" data.
@@ -111,7 +111,7 @@ and to deploy and manage a pipeline. Use the pipeline editor
 diagram below shows the components of the control and management
 planes and the connections between them:
 
-{{< image filename="images/rdi/ingest/ingest-control-plane.png" >}}
+{{< image filename="images/rdi/ingest/ingest-control-plane.webp" >}}
 
 The following sections describe the VM configurations you can use to
 deploy RDI.
@@ -124,7 +124,7 @@ Both the active VM and the standby
 need access to the authentication secrets that RDI uses to encrypt network
 traffic. The diagram below shows this configuration:
 
-{{< image filename="images/rdi/ingest/ingest-active-passive-vms.png" >}}
+{{< image filename="images/rdi/ingest/ingest-active-passive-vms.webp" >}}
 
 See [Install on VMs]({{< relref "/integrate/redis-data-integration/installation/install-vm" >}})
 for more information.

content/integrate/redis-data-integration/data-pipelines/data-denormalization.md

@@ -33,7 +33,7 @@ parent object (the "one") as a JSON document with the children (the "many") nest
 attribute in the parent. The diagram belows shows a nesting with the child objects in a map
 called `InvoiceLineItems`:
 
-{{< image filename="/images/rdi/nest-flow.png" >}}
+{{< image filename="/images/rdi/ingest/nest-flow.webp" width="500px" >}}
 
 You configure normalization with a `nest` block in the child entities' RDI job, as shown in this example:
 

content/integrate/redis-data-integration/data-pipelines/data-pipelines.md

@@ -48,15 +48,15 @@ data in Redis as
 
 The diagram below shows the flow of data through the pipeline:
 
-{{< image filename="/images/rdi/RDIPipeDataflow.drawio.svg" >}}
+{{< image filename="/images/rdi/ingest/RDIPipeDataflow.webp" >}}
 
 ## Pipeline configuration
 
 RDI uses a set of [YAML](https://en.wikipedia.org/wiki/YAML)
 files to configure each pipeline. The following diagram shows the folder
 structure of the configuration:
 
-{{< image filename="images/rdi/ingest/ingest-config-folders.svg" >}}
+{{< image filename="images/rdi/ingest/ingest-config-folders.webp" width="600px" >}}
 
 The main configuration for the pipeline is in the `config.yaml` file.
 This specifies the connection details for the source database (such

content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aur-pgsql.md

@@ -26,7 +26,7 @@ In the [Relational Database Service (RDS) console](https://console.aws.amazon.co
 navigate to **Parameter groups > Create parameter group**. You will see the panel shown
 below:
 
-{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/CreateParamGroup.jpg" alt="Create parameter group panel" >}}
+{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/CreateParamGroup.webp" alt="Create parameter group panel" >}}
 
 Enter the following information:
 
@@ -45,7 +45,7 @@ Select **Create** to create the parameter group.
 Navigate to **Parameter groups** in the console. Select the `rdi-aurora-pg`
 group you have just created and then select **Edit** . You will see this panel:
 
-{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/EditParamGroup.jpg" alt="Edit parameter group panel" >}}
+{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/EditParamGroup.webp" alt="Edit parameter group panel" >}}
 
 Search for the `rds.logical_replication` parameter and set its value to 1. Then,
 select **Save Changes**.
@@ -56,4 +56,4 @@ Go back to your target database on the RDS console, select **Modify** and then
 scroll down to **Additional Configuration**. Set
 the **DB Cluster Parameter Group** to the value `rdi-aurora-pg` that you have just added:
 
-{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/CreateDB6.jpg" alt="Additional Configuration panel" >}}
+{{<image filename="images/rdi/ingest/prepsrc/aurora-pgsql/CreateDB6.webp" alt="Additional Configuration panel" >}}

content/integrate/redis-data-integration/quick-start-guide.md

@@ -26,7 +26,7 @@ In this tutorial you will learn how to install RDI and set up a pipeline to inge
 The following diagram shows the structure of the pipeline we will create (see
 the [architecture overview]({{< relref "/integrate/redis-data-integration/architecture#overview" >}}) to learn how the pipeline works):
 
-{{< image filename="images/rdi/ingest/ingest-qsg.png" >}}
+{{< image filename="images/rdi/ingest/ingest-qsg.webp" >}}
 
 Here, the RDI *collector* tracks changes in PostgreSQL and writes them to streams in the 
 RDI database in Redis. The *stream processor* then reads data records from the RDI