diff --git a/markdownify/__init__.py b/markdownify/__init__.py index c732711..f23a1b3 100644 --- a/markdownify/__init__.py +++ b/markdownify/__init__.py @@ -17,7 +17,9 @@ # Extract (leading_nl, content, trailing_nl) from a string # (functionally equivalent to r'^(\n*)(.*?)(\n*)$', but greedy is faster than reluctant here) -re_extract_newlines = re.compile(r'^(\n*)((?:.*[^\n])?)(\n*)$', flags=re.DOTALL) +re_extract_newlines = re.compile( + r'^(\n*)((?:.*[^\n])?)(\n*)$', + flags=re.DOTALL) # Escape miscellaneous special Markdown characters re_escape_misc_chars = re.compile(r'([]\\&<`[>~=+|])') @@ -78,6 +80,7 @@ def abstract_inline_conversion(markup_fn): the text if it looks like an HTML tag. markup_fn is necessary to allow for references to self.strong_em_symbol etc. """ + def implementation(self, el, text, parent_tags): markup_prefix = markup_fn(self) if markup_prefix.startswith('<') and markup_prefix.endswith('>'): @@ -89,12 +92,14 @@ def implementation(self, el, text, parent_tags): prefix, suffix, text = chomp(text) if not text: return '' - return '%s%s%s%s%s' % (prefix, markup_prefix, text, markup_suffix, suffix) + return '%s%s%s%s%s' % (prefix, markup_prefix, + text, markup_suffix, suffix) return implementation def _todict(obj): - return dict((k, getattr(obj, k)) for k in dir(obj) if not k.startswith('_')) + return dict((k, getattr(obj, k)) + for k in dir(obj) if not k.startswith('_')) def should_remove_whitespace_inside(el): @@ -170,6 +175,8 @@ class DefaultOptions: strip_document = STRIP strong_em_symbol = ASTERISK sub_symbol = '' + preprocess_fn = None + postprocess_fn = None sup_symbol = '' table_infer_header = False wrap = False @@ -184,6 +191,8 @@ def __init__(self, **options): self.options = _todict(self.DefaultOptions) self.options.update(_todict(self.Options)) self.options.update(options) + self.preprocess_fn = self.options['preprocess_fn'] + self.postprocess_fn = self.options['postprocess_fn'] if self.options['strip'] is not None and self.options['convert'] is not None: raise ValueError('You may specify either tags to strip or tags to' ' convert, but not both.') @@ -205,7 +214,8 @@ def process_element(self, node, parent_tags=None): return self.process_tag(node, parent_tags=parent_tags) def process_tag(self, node, parent_tags=None): - # For the top-level element, initialize the parent context with an empty set. + # For the top-level element, initialize the parent context with an + # empty set. if parent_tags is None: parent_tags = set() @@ -226,10 +236,12 @@ def _can_ignore(el): # Non-whitespace text nodes are always processed. return False elif should_remove_inside and (not el.previous_sibling or not el.next_sibling): - # Inside block elements (excluding
), ignore adjacent whitespace elements. + # Inside block elements (excluding), ignore adjacent + # whitespace elements. return True elif should_remove_whitespace_outside(el.previous_sibling) or should_remove_whitespace_outside(el.next_sibling): - # Outside block elements (including), ignore adjacent whitespace elements. + # Outside block elements (including), ignore adjacent + # whitespace elements. return True else: return False @@ -238,21 +250,24 @@ def _can_ignore(el): else: raise ValueError('Unexpected element type: %s' % type(el)) - children_to_convert = [el for el in node.children if not _can_ignore(el)] + children_to_convert = [ + el for el in node.children if not _can_ignore(el)] # Create a copy of this tag's parent context, then update it to include this tag # to propagate down into the children. parent_tags_for_children = set(parent_tags) parent_tags_for_children.add(node.name) - # if this tag is a heading or table cell, add an '_inline' parent pseudo-tag + # if this tag is a heading or table cell, add an '_inline' parent + # pseudo-tag if ( re_html_heading.match(node.name) is not None # headings or node.name in {'td', 'th'} # table cells ): parent_tags_for_children.add('_inline') - # if this tag is a preformatted element, add a '_noformat' parent pseudo-tag + # if this tag is a preformatted element, add a '_noformat' parent + # pseudo-tag if node.name in {'pre', 'code', 'kbd', 'samp'}: parent_tags_for_children.add('_noformat') @@ -274,17 +289,21 @@ def _can_ignore(el): updated_child_strings = [''] # so the first lookback works for child_string in child_strings: # Separate the leading/trailing newlines from the content. - leading_nl, content, trailing_nl = re_extract_newlines.match(child_string).groups() + leading_nl, content, trailing_nl = re_extract_newlines.match( + child_string).groups() # If the last child had trailing newlines and this child has leading newlines, # use the larger newline count, limited to 2. if updated_child_strings[-1] and leading_nl: - prev_trailing_nl = updated_child_strings.pop() # will be replaced by the collapsed value - num_newlines = min(2, max(len(prev_trailing_nl), len(leading_nl))) + # will be replaced by the collapsed value + prev_trailing_nl = updated_child_strings.pop() + num_newlines = min( + 2, max(len(prev_trailing_nl), len(leading_nl))) leading_nl = '\n' * num_newlines # Add the results to the updated child string list. - updated_child_strings.extend([leading_nl, content, trailing_nl]) + updated_child_strings.extend( + [leading_nl, content, trailing_nl]) child_strings = updated_child_strings @@ -292,9 +311,15 @@ def _can_ignore(el): text = ''.join(child_strings) # apply this tag's final conversion function + + if self.preprocess_fn and self.should_convert_tag(node.name): + text = self.preprocess_fn(node, text, parent_tags=parent_tags) + convert_fn = self.get_conv_fn_cached(node.name) if convert_fn is not None: text = convert_fn(node, text, parent_tags=parent_tags) + if self.postprocess_fn and self.should_convert_tag(node.name): + text = self.postprocess_fn(node, text, parent_tags=parent_tags) return text @@ -305,16 +330,20 @@ def convert__document_(self, el, text, parent_tags): elif self.options['strip_document'] == RSTRIP: text = text.rstrip('\n') # remove trailing separation newlines elif self.options['strip_document'] == STRIP: - text = text.strip('\n') # remove leading and trailing separation newlines + # remove leading and trailing separation newlines + text = text.strip('\n') elif self.options['strip_document'] is None: pass # leave leading and trailing separation newlines as-is else: - raise ValueError('Invalid value for strip_document: %s' % self.options['strip_document']) + raise ValueError( + 'Invalid value for strip_document: %s' % + self.options['strip_document']) return text def process_text(self, el, parent_tags=None): - # For the top-level element, initialize the parent context with an empty set. + # For the top-level element, initialize the parent context with an + # empty set. if parent_tags is None: parent_tags = set() @@ -328,7 +357,8 @@ def process_text(self, el, parent_tags=None): text = re_newline_whitespace.sub('\n', text) text = re_whitespace.sub(' ', text) - # escape special characters if we're not inside a preformatted or code element + # escape special characters if we're not inside a preformatted or code + # element if '_noformat' not in parent_tags: text = self.escape(text, parent_tags) @@ -364,7 +394,8 @@ def get_conv_fn(self, tag_name): return None # Look for an explicitly defined conversion function by tag name first - convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub("_", tag_name) + convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub( + "_", tag_name) convert_fn = getattr(self, convert_fn_name, None) if convert_fn: return convert_fn @@ -373,7 +404,8 @@ def get_conv_fn(self, tag_name): match = re_html_heading.match(tag_name) if match: n = int(match.group(1)) # get value of N from- return lambda el, text, parent_tags: self.convert_hN(n, el, text, parent_tags) + return lambda el, text, parent_tags: self.convert_hN( + n, el, text, parent_tags) # No conversion function was found return None @@ -426,9 +458,11 @@ def convert_a(self, el, text, parent_tags): if self.options['default_title'] and not title: title = href title_part = ' "%s"' % title.replace('"', r'\"') if title else '' - return '%s[%s](%s%s)%s' % (prefix, text, href, title_part, suffix) if href else text + return '%s[%s](%s%s)%s' % (prefix, text, href, + title_part, suffix) if href else text - convert_b = abstract_inline_conversion(lambda self: 2 * self.options['strong_em_symbol']) + convert_b = abstract_inline_conversion( + lambda self: 2 * self.options['strong_em_symbol']) def convert_blockquote(self, el, text, parent_tags): # handle some early-exit scenarios @@ -473,7 +507,8 @@ def convert_div(self, el, text, parent_tags): convert_section = convert_div - convert_em = abstract_inline_conversion(lambda self: self.options['strong_em_symbol']) + convert_em = abstract_inline_conversion( + lambda self: self.options['strong_em_symbol']) convert_kbd = convert_code @@ -650,7 +685,8 @@ def convert_pre(self, el, text, parent_tags): code_language = self.options['code_language'] if self.options['code_language_callback']: - code_language = self.options['code_language_callback'](el) or code_language + code_language = self.options['code_language_callback']( + el) or code_language return '\n\n```%s\n%s\n```\n\n' % (code_language, text) @@ -669,9 +705,11 @@ def convert_style(self, el, text, parent_tags): convert_samp = convert_code - convert_sub = abstract_inline_conversion(lambda self: self.options['sub_symbol']) + convert_sub = abstract_inline_conversion( + lambda self: self.options['sub_symbol']) - convert_sup = abstract_inline_conversion(lambda self: self.options['sup_symbol']) + convert_sup = abstract_inline_conversion( + lambda self: self.options['sup_symbol']) def convert_table(self, el, text, parent_tags): return '\n\n' + text.strip() + '\n\n' @@ -704,9 +742,10 @@ def convert_tr(self, el, text, parent_tags): and len(el.parent.find_all('tr')) == 1) ) is_head_row_missing = ( - (is_first_row and not el.parent.name == 'tbody') - or (is_first_row and el.parent.name == 'tbody' and len(el.parent.parent.find_all(['thead'])) < 1) - ) + (is_first_row and not el.parent.name == 'tbody') or ( + is_first_row and el.parent.name == 'tbody' and len( + el.parent.parent.find_all( + ['thead'])) < 1)) overline = '' underline = '' full_colspan = 0 @@ -723,7 +762,8 @@ def convert_tr(self, el, text, parent_tags): # - is headline or # - headline is missing and header inference is enabled # print headline underline - underline += '| ' + ' | '.join(['---'] * full_colspan) + ' |' + '\n' + underline += '| ' + \ + ' | '.join(['---'] * full_colspan) + ' |' + '\n' elif ((is_head_row_missing and not self.options['table_infer_header']) or (is_first_row diff --git a/tests/test_preprocess_postprocess.py b/tests/test_preprocess_postprocess.py new file mode 100644 index 0000000..c05e4b4 --- /dev/null +++ b/tests/test_preprocess_postprocess.py @@ -0,0 +1,128 @@ +from markdownify import markdownify as md + + +def test_preprocess_all_tags(): + + def preprocess(node, text, parent_tags): + alignment = "" + if 'style' in node.attrs and 'text-align' in node.attrs['style']: + style = node.attrs['style'] + alignment = style.split("text-align:")[1].split(";")[0].strip() + + if alignment: + return f"[align={alignment}]{text}[/align]" + return text + + assert md( + ' para
bold', + preprocess_fn=preprocess) == '[align=center]para[/align]\n\n**[align=left]bold[/align]**' + + +def test_postprocess_all_tags(): + + def postprocess(node, text, parent_tags): + alignment = "" + if 'style' in node.attrs and 'text-align' in node.attrs['style']: + style = node.attrs['style'] + alignment = style.split("text-align:")[1].split(";")[0].strip() + + if alignment: + return f"[align={alignment}]{text}[/align]" + return text + b = md( + 'para
bold', + postprocess_fn=postprocess) + print(b) + assert md( + 'para
bold', + postprocess_fn=postprocess) == '[align=center]\n\npara\n\n[/align][align=left]**bold**[/align]' + + +def test_preprocess_runs_before_conversion(): + + def preprocess(node, text, parent_tags): + if node.name == 'b': + return f"PRE_{text}_PRE" + return text + + # Default conversion would make this "**bold**" + # With preprocessing it should become "**PRE_bold_PRE**" + assert md('bold', preprocess_fn=preprocess) == '**PRE_bold_PRE**' + + +def test_postprocess_runs_after_conversion(): + + def postprocess(node, text, parent_tags): + if node.name == 'b': + return f"POST_{text}_POST" + return text + + # Default conversion makes this "**bold**" + # With postprocessing it should become "POST_**bold**_POST" + assert md( + 'bold', + postprocess_fn=postprocess) == 'POST_**bold**_POST' + + +def test_preprocess_doesnt_prevent_conversion(): + + def preprocess(node, text, parent_tags): + return text.upper() # Just modify the text, don't prevent conversion + + # Should still get converted to markdown, just with uppercase content + assert md('bold', preprocess_fn=preprocess) == '**BOLD**' + + +def test_postprocess_doesnt_prevent_conversion(): + + def postprocess(node, text, parent_tags): + return text.upper() # Just modify the result, don't prevent conversion + + # Should get normal markdown conversion but then uppercased + assert md('bold', postprocess_fn=postprocess) == '**BOLD**' + + +def test_combined_pre_and_post_processing(): + + def preprocess(node, text, parent_tags): + print("Running preprocess on", text) + return f"PRE:{text}:PRE" + + def postprocess(node, text, parent_tags): + print("Running postprocess on", text) + return f"POST:{text}:POST" + + # bold normally becomes "**bold**" + # With preprocessing: "(bold)" -> "**(bold)**" + # Then postprocessing: "[**(bold)**]" + assert md('text', + preprocess_fn=preprocess, + postprocess_fn=postprocess) == 'POST:PRE:text:PRE:POST' + + +def test_processing_with_multiple_tags(): + + def preprocess(node, text, parent_tags): + if node.name == 'b': + return f"B:{text}" + elif node.name == 'i': + return f"I:{text}" + return text + + #bold and italic
+ # Should become "**B:bold** and *I:italic*" + assert md('bold and italic
', + preprocess_fn=preprocess) == '**B:bold** and *I:italic*' + + +def test_processing_with_nested_tags(): + + def postprocess(node, text, parent_tags): + if node.name == 'p': + return f"P:{text}" + return text + + #bold text
normally becomes "**bold** text" + # With postprocessing becomes "P:**bold** text" + assert md('bold text
', + postprocess_fn=postprocess) == 'P:\n\n**bold** text'