Merge pull request #2060 from wilzbach/header-gen2

Add header TOC generation
merged-on-behalf-of: Vladimir Panteleev <github@thecybershadow.net>

Author: dlang-bot

ddoc.d

@@ -0,0 +1,180 @@
+#!/usr/bin/env rdmd
+/**
+A wrapper around DDoc to allow custom extensions
+
+Copyright: D Language Foundation 2018
+
+License:   $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
+
+Example usage:
+
+---
+./ddoc --compiler=<path-to-dmd> --output=out.html myfile.dd
+---
+
+Author: Sebastian Wilzbach
+*/
+import std.algorithm, std.array, std.ascii, std.conv, std.file, std.functional,
+        std.meta, std.path, std.range, std.string, std.typecons;
+import std.stdio : writeln, writefln;
+
+struct Config
+{
+    string dmdBinPath = "dmd";
+    string outputFile;
+}
+Config config;
+
+int main(string[] args)
+{
+    import std.getopt;
+    auto helpInformation = getopt(
+        args,
+        "compiler", "Compiler to use", &config.dmdBinPath,
+        "o|output", "Output file", &config.outputFile,
+    );
+
+    assert(config.outputFile, "An output file is required.");
+    assert(args.length > 1, "An input file is required.");
+
+    if (helpInformation.helpWanted)
+    {
+`DDoc wrapper
+./ddoc <file>...
+`.defaultGetoptPrinter(helpInformation.options);
+        return 1;
+    }
+
+    import std.file : readText;
+    auto text = args[$ - 1].readText;
+
+    // transform and extend the ddoc page
+    text = genHeader(text);
+
+    return compile(text, args[1 .. $ - 1]);
+}
+
+auto compile(R)(R buffer, string[] arguments)
+{
+    import std.process : pipeProcess, Redirect, wait;
+    auto args = [config.dmdBinPath, "-c", "-Df"~config.outputFile, "-o-"] ~ arguments;
+    args ~= "-";
+    auto pipes = pipeProcess(args, Redirect.stdin);
+    pipes.stdin.write(buffer);
+    pipes.stdin.close;
+    return wait(pipes.pid);
+}
+
+// replaces the content of a DDoc macro call
+auto updateDdocTag(string fileText, string ddocKey, string newContent)
+{
+    auto pos = fileText.representation.countUntil(ddocKey);
+    if (pos < 0)
+        return fileText;
+    const ddocStartLength = ddocKey.representation.until('(', No.openRight).count;
+    auto len = fileText[pos .. $].representation.drop(ddocStartLength).untilClosingParentheses.walkLength;
+    return fileText.replace(fileText[pos .. pos + len + ddocStartLength + 1], newContent);
+}
+
+// a range until the next ')', nested () are ignored
+auto untilClosingParentheses(R)(R rs)
+{
+    return rs.cumulativeFold!((count, r){
+        switch(r)
+        {
+            case '(':
+                count++;
+                break;
+            case ')':
+                count--;
+                break;
+            default:
+        }
+        return count;
+    })(1).zip(rs).until!(e => e[0] == 0).map!(e => e[1]);
+}
+
+unittest
+{
+    import std.algorithm.comparison : equal;
+    assert("aa $(foo $(bar)foobar)".untilClosingParentheses.equal("aa $(foo $(bar)foobar)"));
+    assert("$(FOO a, b, $(ARGS e, f)))".untilClosingParentheses.equal("$(FOO a, b, $(ARGS e, f))"));
+}
+
+// parse the ddoc file for H2 and H3 items
+// H3 items are listed as subitems
+auto parseToc(string text)
+{
+    alias TocEntry = Tuple!(string, "id", string, "name");
+    alias TocTopEntry = Tuple!(TocEntry, "main", TocEntry[], "children");
+    TocTopEntry[] toc;
+
+    bool isH2 = true;
+    void append(string id, string name)
+    {
+        auto entry = TocEntry(id, name);
+        if (isH2)
+            toc ~= TocTopEntry(entry, null);
+        else
+            toc.back.children ~= entry;
+    }
+    while (!text.empty)
+    {
+        enum needles = AliasSeq!("$(H2 ", "$(SECTION2 ", "$(H3", "$(SECTION3");
+        auto res = text.find(needles);
+        if (res[0].empty)
+            break;
+
+        isH2 = res[1] <= 2;
+        text = res[0].drop(needles.only[res[1] - 1].length);
+        text.skipOver!isWhite;
+
+        enum gname = "$(GNAME ";
+        enum lNameNeedles = AliasSeq!("$(LNAME2", "$(LEGACY_LNAME2");
+        if (text.startsWith(gname))
+        {
+            auto name = text.drop(gname.length).untilClosingParentheses.to!string.strip;
+            append(name, name);
+        }
+        else if (auto idx = text.startsWith(lNameNeedles))
+        {
+            auto arr = text.drop(lNameNeedles.only[idx - 1].length).splitter(",");
+            if (idx == 2)
+                arr.popFront;
+            append(arr.front.strip, arr.dropOne.joiner(",").untilClosingParentheses.to!string.strip);
+        }
+    }
+    return toc;
+}
+
+// Ddoc splits arguments by commas
+auto escapeDdoc(string s)
+{
+    return s.replace(",", "$(COMMA)");
+}
+
+// generated a SPEC_HEADERNAV_TOC Ddoc macro with the parsed H2/H3 entries
+auto genHeader(string fileText)
+{
+    enum ddocKey = "$(SPEC_HEADERNAV_TOC";
+    auto newContent = ddocKey ~ "\n";
+    enum indent = "    ";
+    foreach (entry; fileText.parseToc)
+    {
+        if (entry.children)
+        {
+            newContent ~= "%s$(SPEC_HEADERNAV_SUBITEMS %s, %s,\n".format(indent, entry.main.id, entry.main.name.escapeDdoc);
+            foreach (child; entry.children)
+                newContent ~= "%s$(SPEC_HEADERNAV_ITEM %s, %s)\n".format(indent.repeat(2).joiner, child.id, child.name.escapeDdoc);
+            newContent ~= indent;
+            newContent ~= ")\n";
+        }
+        else
+        {
+            newContent ~= "%s$(SPEC_HEADERNAV_ITEM %s, %s)\n".format(indent, entry.main.id, entry.main.name.escapeDdoc);
+        }
+    }
+    newContent ~= ")";
+    return updateDdocTag(fileText, ddocKey, newContent);
+}
+

ebook.ddoc

@@ -61,4 +61,7 @@ SPEC_RUNNABLE_EXAMPLE=$0
 SPEC_SUBNAV_NEXT=
 SPEC_SUBNAV_PREV_NEXT=
 SPEC_SUBNAV_PREV=
+SPEC_HEADERNAV_TOC=
+SPEC_HEADERNAV_ITEM=
+SPEC_HEADERNAV_SUBITEMS=
 _=

latex.ddoc

@@ -267,10 +267,10 @@ SPEC_S=$(LAYOUT , $1, $(ARGS $+))
 SPEC_SUBNAV_NEXT=
 SPEC_SUBNAV_PREV=
 SPEC_SUBNAV_PREV_NEXT=
+SPEC_HEADERNAV_TOC=
+SPEC_HEADERNAV_ITEM=
+SPEC_HEADERNAV_SUBITEMS=
 SUBSCRIPT=\textsubscript{$0}
-SPEC_SUBNAV_PREV=
-SPEC_SUBNAV_PREV_NEXT=
-SPEC_SUBNAV_NEXT=
 SUP=\textsuperscript{$0}
 _=
 

posix.mak

@@ -277,6 +277,12 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \
 	DOC_OUTPUT_DIR="$W/phobos-prerelease-verbatim" \
 	STDDOC="$(PWD)/verbatim.ddoc"
 
+################################################################################
+# Ddoc binaries
+################################################################################
+
+DDOC_BIN:=$G/ddoc
+
 ################################################################################
 # Resources
 ################################################################################
@@ -460,8 +466,8 @@ $W/changelog/pending.html : changelog/pending.dd $(CHANGELOG_PENDING_DDOC) $(DMD
 $W/changelog/%.html : changelog/%.dd $(CHANGELOG_DDOC) $(DMD)
 	$(DMD) -conf= -c -o- -Df$@ $(CHANGELOG_DDOC) $<
 
-$W/spec/%.html : spec/%.dd $(SPEC_DDOC) $(DMD)
-	$(DMD) -c -o- -Df$@ $(SPEC_DDOC) $<
+$W/spec/%.html : spec/%.dd $(SPEC_DDOC) $(DMD) $(DDOC_BIN)
+	$(DDOC_BIN) --compiler=$(DMD) -o$@ $(SPEC_DDOC) $<
 
 $W/404.html : 404.dd $(DDOC) $(DMD)
 	$(DMD) -conf= -c -o- -Df$@ $(DDOC) errorpage.ddoc $<
@@ -947,4 +953,15 @@ $G/contributors_list.ddoc:  | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
 	echo "D_CONTRIBUTORS=" >> $@
 	cat $G/contributors_list.tmp >> $@
 
+################################################################################
+# Custom DDoc wrapper
+# ------------------
+#
+# This allows extending Ddoc files dynamically on-the-fly.
+# It is currently only used for the specification pages
+################################################################################
+
+$(DDOC_BIN): ddoc.d | $(STABLE_DMD)
+	$(STABLE_DMD) -of$@ $<
+
 .DELETE_ON_ERROR: # GNU Make directive (delete output files on error)

spec/abi.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(SPEC_S Application Binary Interface,
 
+$(SPEC_HEADERNAV_TOC)
+
 	$(P A D implementation that conforms to the D ABI (Application Binary
 	Interface)
 	will be able to generate libraries, DLL's, etc., that can interoperate

spec/arrays.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(SPEC_S Arrays,
 
+$(SPEC_HEADERNAV_TOC)
+
 $(H2 $(LNAME2 array-kinds, Kinds))
 
     $(P There are four kinds of arrays:)

spec/attribute.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(SPEC_S Attributes,
 
+$(SPEC_HEADERNAV_TOC)
+
 $(GRAMMAR
 $(GNAME AttributeSpecifier):
     $(GLINK Attribute) $(D :)

spec/betterc.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(SPEC_S Better C,
 
+$(SPEC_HEADERNAV_TOC)
+
 `-betterC` is a command-line flag for `dmd`,
 which restricts the compiler's support of certain runtime features.
 Notably, D programs or libraries compiled with `betterC` aren't linked with Druntime.

spec/class.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(SPEC_S Classes,
 
+$(SPEC_HEADERNAV_TOC)
+
         $(P The object-oriented features of D all come from classes. The class
         hierarchy
         has as its root the class Object. Object defines a minimum level of functionality

spec/const3.dd

@@ -2,6 +2,8 @@ Ddoc
 
 $(D_S Type Qualifiers,
 
+$(SPEC_HEADERNAV_TOC)
+
 	$(P Type qualifiers modify a type by applying a $(GLINK2 declaration, TypeCtor).
 	$(I TypeCtor)s are: $(D const), $(D immutable),	$(D shared), and $(D inout).
 	Each applies transitively to all subtypes.