Skip to content

[WD-22501] feat: include and parse markdown files in project trees on content system #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 4, 2025
Merged

[WD-22501] feat: include and parse markdown files in project trees on content system #210

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
59d073a
Parse and include .md files
muhammad-ali-pk Jun 24, 2025
cd1f660
Add lint and format scripts + lint python
muhammad-ali-pk Jun 26, 2025
ab343a0
Open valid github file extensions
muhammad-ali-pk Jul 3, 2025
8d493d0
Merge branch 'main' into WD-22501
muhammad-ali-pk Jul 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
"start": "yarn run build && concurrently --raw 'yarn run watch' 'yarn run serve'",
"test": "vitest",
"watch": "watch -p 'static/client/**/*.{js,jsx,ts,tsx,scss}' -c 'yarn run build'",
"dev": "vite"
"dev": "vite",
"lint-python": "flake8 webapp tests && black --check --line-length 79 webapp tests",
"format-python": "black --line-length 79 webapp tests"
},
"dependencies": {
"@canonical/react-components": "0.60.0",
Expand Down
130 changes: 86 additions & 44 deletions webapp/parse_tree.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,31 @@
"templates/one-column.html",
"_base/base.html",
]
MARKDOWN_TEMPLATES = [
"legal/_base_legal_markdown.html",
"appliance/shared/_base_appliance_index.html",
]
TEMPLATE_PREFIXES = ["base", "_base"]
TAG_MAPPING = {
"title": ["title"],
"description": ["meta_description", "description"],
"link": ["meta_copydoc"],
"link": ["meta_copydoc", "copydoc"],
}

EXCLUDE_PATHS = ["partials"]


def is_index(path):
return path.name == "index.html"
return path.name == "index.html" or path.name == "index.md"


def check_has_index(path):
return (path / "index.html").exists()
if (path / "index.html").exists():
return True, ".html"
elif (path / "index.md").exists():
return True, ".md"
else:
return False, None


def is_template(path):
Expand All @@ -43,7 +52,7 @@ def is_template(path):

def is_partial(path):
"""
Return True if the file name starts with an underscore,
Return True if the file name starts with an underscore,
that indicates it as a partial

Partials are templates that are not meant to be rendered directly, but
Expand Down Expand Up @@ -81,7 +90,7 @@ def extends_base(path, base="templates"):
else:
# extract absolute path from the parent path
absolute_path = str(path)[
0 : str(path).find(base) + len(base)
0: str(path).find(base) + len(base)
]
# check if the file from which the current file
# extends extends from the base template
Expand Down Expand Up @@ -139,7 +148,7 @@ def get_extended_copydoc(path, base):
with base.joinpath(path).open("r") as f:
file_data = f.read()
if match := re.search(
"\{\% block meta_copydoc *\%\}(.*)\{\%( *)endblock", file_data
r"\{\% block meta_copydoc *\%\}(.*)\{\%( *)endblock", file_data
):
return match.group(1)

Expand All @@ -155,6 +164,9 @@ def get_tags_rolling_buffer(path):
# We create a map of the selected variants for each tag
variants_mapping = {v: "" for v in available_tags}

# check if the path is html or md file
is_md = path.suffix == ".md"

with path.open("r") as f:
for tag in available_tags:
buffer = []
Expand All @@ -168,44 +180,63 @@ def get_tags_rolling_buffer(path):
# Return to start of file
f.seek(0)

for line in f.readlines():
if is_buffering:
buffer.append(line)

if not is_buffering and (
match := re.search(f"{{% block {variant}( *)%}}", line)
):
# We remove line contents before the tag
line = line[match.start() :] # noqa: E203

buffer.append(line)
is_buffering = True
variants_mapping[tag] = variant

# We search for the end of the tag in the existing buffer
buffer_string = "".join(buffer)
if is_buffering and re.search(
"(.*){%( *)endblock", buffer_string
):
# We save the buffer contents to the tags dictionary
tags[tag] = buffer_string

# We extract the text within the tags
tags[tag] = extract_text_from_tag(
variants_mapping[tag], tags[tag]
if not is_md:

for line in f.readlines():
if is_buffering:
buffer.append(line)

if not is_buffering and (
match := re.search(
f"{{% block {variant}( *)%}}", line
)
):
# We remove line contents before the tag
line = line[match.start() :] # noqa: E203

buffer.append(line)
is_buffering = True
variants_mapping[tag] = variant

# We search for the end of the tag in the existing
# buffer
buffer_string = "".join(buffer)
if is_buffering and re.search(
"(.*){%( *)endblock", buffer_string
):
# We save the buffer contents to the tags
# dictionary
tags[tag] = buffer_string

# We extract the text within the tags
tags[tag] = extract_text_from_tag(
variants_mapping[tag], tags[tag]
)

# We now reset the buffer
buffer = []
is_buffering = False
tag_found = True
break

else:
for line in f:
match = re.match(
rf"^\s*{variant}\s*:\s*\"?(.+?)\"?\s*$",
line,
re.IGNORECASE,
)

# We now reset the buffer
buffer = []
is_buffering = False
tag_found = True
break
if match:
variants_mapping[tag] = variant
tags[tag] = match.group(1).strip()
tag_found = True
break

if tag_found:
break

# We add the name from the path
raw_name = re.sub(r"(?i)(.html|/index.html)", "", str(path))
raw_name = re.sub(r"(?i)(.html|/index.html|/index.md)", "", str(path))
tags["name"] = raw_name.split("/templates", 1)[-1]

return tags
Expand All @@ -216,11 +247,10 @@ def is_valid_page(path, extended_path, is_index=True):
Determine if path is a valid page. Pages are valid if:
- They contain the same extended path as the index html.
- They extend from the base html.
- Does not live in a shared template directory.
- They are markdown files with a valid wrapper template.
"""

path = Path(path)

if not path.is_file() or is_template(path) or is_partial(path):
if is_template(path):
return False

if not is_index and extended_path:
Expand All @@ -229,6 +259,17 @@ def is_valid_page(path, extended_path, is_index=True):
if match := re.search("{% extends [\"'](.*?)[\"'] %}", line):
if match.group(1) == extended_path:
return True

if "index.md" in str(path):
with path.open("r") as f:
for line in f.readlines():
if match := re.search(
r"wrapper_template:\s*[\"']?(.*?)[\"']?$", line
):
template = match.group(1)
if template in MARKDOWN_TEMPLATES:
return True

# If the file does not share the extended path, check if it extends the
# base html
return extends_base(path)
Expand Down Expand Up @@ -288,9 +329,9 @@ def scan_directory(path_name, base=None):
is_index_page_valid = False

# Check if an index.html file exists in this directory
has_index = check_has_index(node_path)
(has_index, index_type) = check_has_index(node_path)
if has_index:
index_path = node_path / "index.html"
index_path = node_path / ("index" + index_type)
# Get the path extended by the index.html file
extended_path = get_extended_path(index_path)
# If the file is valid, add it as a child
Expand All @@ -299,6 +340,7 @@ def scan_directory(path_name, base=None):
# Get tags, add as child
tags = get_tags_rolling_buffer(index_path)
node = update_tags(node, tags)
node["ext"] = index_type

else:
node["ext"] = ".dir"
Expand Down
Loading