From 8267493401bb229b487e04e3e41fa95db188a490 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Tue, 9 Jan 2018 17:13:55 +0100
Subject: [PATCH 01/13] Dynamic writeln magic

---
 posix.mak | 93 ++++++++++++++++++++-----------------------------------
 1 file changed, 34 insertions(+), 59 deletions(-)

diff --git a/posix.mak b/posix.mak
index da7d79be1b..51ba0442a5 100644
--- a/posix.mak
+++ b/posix.mak
@@ -126,6 +126,15 @@
 #  in the build folder `.generated`, s.t. Ddoc can be run on the modified sources.
 #
 #  See also: https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org
+#
+#  Custom DDoc wrapper
+#  -------------------
+#
+#  `ddoc.d` is a wrapper around Ddoc and allows expanding Ddoc macros dynamically
+#  before actually running Ddoc.
+#  Currently this is used for:
+#    - TOC
+#    - assert -> writeln magic
 PWD=$(shell pwd)
 MAKEFILE=$(firstword $(MAKEFILE_LIST))
 SHELL:=/bin/bash
@@ -171,21 +180,13 @@ $(shell [ ! -d $(DMD_DIR) ] && git clone --depth=1 ${GIT_HOME}/dmd $(DMD_DIR))
 $(shell [ ! -d $(DRUNTIME_DIR) ] && git clone --depth=1 ${GIT_HOME}/druntime $(DRUNTIME_DIR))
 
 ################################################################################
-# Automatically generated directories
-PHOBOS_DIR_GENERATED=$(GENERATED)/phobos-prerelease
-PHOBOS_LATEST_DIR_GENERATED=$(GENERATED)/phobos-latest
-# The assert_writeln_magic tool transforms all source files from Phobos. Hence
-# - a temporary folder with a copy of Phobos needs to be generated
-# - a list of all files in Phobos and the temporary copy is needed to setup proper
-#   Makefile dependencies and rules
+# Automatically clone Phobos
 PHOBOS_FILES := $(shell find $(PHOBOS_DIR) -name '*.d' -o -name '*.mak' -o -name '*.ddoc')
-PHOBOS_FILES_GENERATED := $(subst $(PHOBOS_DIR), $(PHOBOS_DIR_GENERATED), $(PHOBOS_FILES))
 ifndef RELEASE
  # TODO: should be replaced by make targets
  $(shell [ ! -d $(PHOBOS_DIR) ] && git clone --depth=1 ${GIT_HOME}/phobos $(PHOBOS_DIR))
  $(shell [ ! -d $(PHOBOS_LATEST_DIR) ] && git clone -b v${LATEST} --depth=1 ${GIT_HOME}/phobos $(PHOBOS_LATEST_DIR))
  PHOBOS_LATEST_FILES := $(shell find $(PHOBOS_LATEST_DIR) -name '*.d' -o -name '*.mak' -o -name '*.ddoc')
- PHOBOS_LATEST_FILES_GENERATED := $(subst $(PHOBOS_LATEST_DIR), $(PHOBOS_LATEST_DIR_GENERATED), $(PHOBOS_LATEST_FILES))
 endif
 ################################################################################
 
@@ -683,21 +684,21 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version
 ################################################################################
 
 .PHONY: phobos-prerelease
-phobos-prerelease : ${PHOBOS_FILES_GENERATED} druntime-target $(STD_DDOC_PRERELEASE)
-	$(MAKE) --directory=$(PHOBOS_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML)
+phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE)
+	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML)
 
-phobos-release : ${PHOBOS_FILES_GENERATED} druntime-target $(STD_DDOC_RELEASE)
-	$(MAKE) --directory=$(PHOBOS_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_RELEASE_HTML)
+phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE)
+	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML)
 
-phobos-latest : ${PHOBOS_LATEST_FILES_GENERATED} druntime-latest-target $(STD_DDOC_LATEST)
-	$(MAKE) --directory=$(PHOBOS_LATEST_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_LATEST_HTML)
+phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST)
+	$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML)
 
-phobos-prerelease-verbatim : ${PHOBOS_FILES_GENERATED} druntime-target \
+phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
 		$W/phobos-prerelease/index.verbatim
 $W/phobos-prerelease/index.verbatim : verbatim.ddoc \
 		$W/phobos-prerelease/object.verbatim \
 		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version
-	${MAKE} --directory=${PHOBOS_DIR_GENERATED} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
+	${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
 	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim
 	$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
 	mv $W/phobos-prerelease-verbatim/* $(dir $@)
@@ -748,7 +749,7 @@ else
 endif
 
 $G/docs-latest.json : ${DMD_LATEST} ${DMD_LATEST_DIR} \
-			${DRUNTIME_LATEST_DIR} ${PHOBOS_LATEST_FILES_GENERATED} | dpl-docs
+			${DRUNTIME_LATEST_DIR} ${PHOBOS_LATEST_FILES} | dpl-docs
 	# remove this after https://github.com/dlang/dmd/pull/7513 has been merged
 	if [ -f $(DMD_LATEST_DIR)/src/*/objc_glue_stubs.d ] ; then \
 	   DMD_EXCLUDE_LATEST_BASH="-e /objc_glue.d/d"; \
@@ -757,17 +758,17 @@ $G/docs-latest.json : ${DMD_LATEST} ${DMD_LATEST_DIR} \
 		sed -e /mscoff/d $${DMD_EXCLUDE_LATEST_BASH} ${DMD_EXCLUDE_LATEST}
 	find ${DRUNTIME_LATEST_DIR}/src -name '*.d' | \
 		sed -e /unittest.d/d -e /gcstub/d >> $G/.latest-files.txt
-	find ${PHOBOS_LATEST_DIR_GENERATED} -name '*.d' | \
+	find ${PHOBOS_LATEST_DIR} -name '*.d' | \
 		sed -e /unittest.d/d | sort >> $G/.latest-files.txt
 	${DMD_LATEST} -J$(DMD_LATEST_DIR)/res -J$(dir $(DMD_LATEST)) -c -o- -version=CoreDdoc \
 		-version=MARS -version=CoreDdoc -version=StdDdoc -Df$G/.latest-dummy.html \
-		-Xf$@ -I${PHOBOS_LATEST_DIR_GENERATED} @$G/.latest-files.txt
+		-Xf$@ -I${PHOBOS_LATEST_DIR} @$G/.latest-files.txt
 	${DPL_DOCS} filter $@ --min-protection=Protected \
 		--only-documented $(MOD_EXCLUDES_LATEST)
 	rm -f $G/.latest-files.txt $G/.latest-dummy.html
 
 $G/docs-prerelease.json : ${DMD} ${DMD_DIR} ${DRUNTIME_DIR} \
-		${PHOBOS_FILES_GENERATED} | dpl-docs
+		${PHOBOS_FILES} | dpl-docs
 	# remove this after https://github.com/dlang/dmd/pull/7513 has been merged
 	if [ -f $(DMD_DIR)/src/*/objc_glue_stubs.d ] ; then \
 	   DMD_EXCLUDE_PRERELEASE="-e /objc_glue.d/d"; \
@@ -776,11 +777,11 @@ $G/docs-prerelease.json : ${DMD} ${DMD_DIR} ${DRUNTIME_DIR} \
 		sed -e /mscoff/d $${DMD_EXCLUDE_PRERELEASE} > $G/.prerelease-files.txt
 	find ${DRUNTIME_DIR}/src -name '*.d' | \
 		sed -e /unittest/d >> $G/.prerelease-files.txt
-	find ${PHOBOS_DIR_GENERATED} -name '*.d' | \
+	find ${PHOBOS_DIR} -name '*.d' | \
 		sed -e /unittest.d/d | sort >> $G/.prerelease-files.txt
 	${DMD} -J$(DMD_DIR)/res -J$(dir $(DMD)) -c -o- -version=MARS -version=CoreDdoc \
 		-version=StdDdoc -Df$G/.prerelease-dummy.html \
-		-Xf$@ -I${PHOBOS_DIR_GENERATED} @$G/.prerelease-files.txt
+		-Xf$@ -I${PHOBOS_DIR} @$G/.prerelease-files.txt
 	${DPL_DOCS} filter $@ --min-protection=Protected \
 		--only-documented $(MOD_EXCLUDES_PRERELEASE)
 	rm -f $G/.prerelease-files.txt $G/.prerelease-dummy.html
@@ -841,42 +842,6 @@ d-release.tag d-tags-release.json : chmgen.d $(STABLE_DMD) $(ALL_FILES) phobos-r
 d-prerelease.tag d-tags-prerelease.json : chmgen.d $(STABLE_DMD) $(ALL_FILES) phobos-prerelease druntime-prerelease chm-nav-prerelease.json
 	$(STABLE_RDMD) chmgen.d --root=$W --target prerelease
 
-################################################################################
-# Assert -> writeln magic
-# -----------------------
-#
-# - This transforms assert(a == b) to writeln(a); // b
-# - It creates a copy of Phobos to apply the transformations
-# - All "d" files are piped through the transformator,
-#   other needed files (e.g. posix.mak) get copied over
-#
-# See also: https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org
-################################################################################
-
-ASSERT_WRITELN_BIN = $(GENERATED)/assert_writeln_magic
-
-$(ASSERT_WRITELN_BIN): assert_writeln_magic.d $(DUB) $(STABLE_DMD)
-	@mkdir -p $(dir $@)
-	$(DUB) -v build --single --compiler=$(STABLE_DMD) $<
-	@mv ./assert_writeln_magic $@
-
-$(ASSERT_WRITELN_BIN)_test: assert_writeln_magic.d $(DUB) $(STABLE_DMD)
-	@mkdir -p $(dir $@)
-	$(DUB) -v build --single --compiler=$(STABLE_DMD) --build=unittest $<
-	@mv ./assert_writeln_magic $@
-
-$(PHOBOS_FILES_GENERATED): $(PHOBOS_DIR_GENERATED)/%: $(PHOBOS_DIR)/% $(DUB) $(ASSERT_WRITELN_BIN)
-	@mkdir -p $(dir $@)
-	@if [ $(subst .,, $(suffix $@)) = "d" ] && [ "$@" != "$(PHOBOS_DIR_GENERATED)/index.d" ] ; then \
-		$(ASSERT_WRITELN_BIN) -i $< -o $@ ; \
-	else cp $< $@ ; fi
-
-$(PHOBOS_LATEST_FILES_GENERATED): $(PHOBOS_LATEST_DIR_GENERATED)/%: $(PHOBOS_LATEST_DIR)/% $(DUB) $(ASSERT_WRITELN_BIN)
-	@mkdir -p $(dir $@)
-	@if [ $(subst .,, $(suffix $@)) = "d" ] && [ "$@" != "$(PHOBOS_LATEST_DIR_GENERATED)/index.d" ] ; then \
-		$(ASSERT_WRITELN_BIN) -i $< -o $@ ; \
-	else cp $< $@ ; fi
-
 ################################################################################
 # Style tests
 ################################################################################
@@ -985,6 +950,16 @@ $G/contributors_list.ddoc:  | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
 #
 # This allows extending Ddoc files dynamically on-the-fly.
 # It is currently only used for the specification pages
+#
+# It does:
+# - dynamic TOC generation
+# - GRAMMAR overview generation
+# - CHANGELOG menu generation
+# - Assert -> writeln magic (https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org)
+#  - This transforms assert(a == b) to writeln(a); // b
+#  - It creates a copy of Phobos to apply the transformations
+#  - All "d" files are piped through the transformator,
+#   other needed files (e.g. posix.mak) get copied over
 ################################################################################
 
 $(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD) $(DMD)

From 7b4d373df950fc7b71574090c1e457333f17099b Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Tue, 9 Jan 2018 20:13:49 +0100
Subject: [PATCH 02/13] Pass Phobos Ddoc files through DDOC_BIN

---
 posix.mak | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/posix.mak b/posix.mak
index 51ba0442a5..7e591d8ec5 100644
--- a/posix.mak
+++ b/posix.mak
@@ -684,14 +684,17 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version
 ################################################################################
 
 .PHONY: phobos-prerelease
-phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE)
-	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML)
+phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD)
+	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) \
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
 
-phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE)
-	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML)
+phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE) $(DDOC_BIN) $(DMD)
+	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) \
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
 
-phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST)
-	$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML)
+phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD)
+	$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML) \
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
 
 phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
 		$W/phobos-prerelease/index.verbatim

From 53ec71d75b07b7afb14ef69a91fd6cf283cc6204 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Sun, 4 Feb 2018 01:53:23 +0100
Subject: [PATCH 03/13] Move files to ddoc

---
 assert_writeln_magic.d => ddoc/source/assert_writeln_magic.d | 0
 ddoc_preprocessor.d => ddoc/source/preprocessor.d            | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename assert_writeln_magic.d => ddoc/source/assert_writeln_magic.d (100%)
 rename ddoc_preprocessor.d => ddoc/source/preprocessor.d (100%)

diff --git a/assert_writeln_magic.d b/ddoc/source/assert_writeln_magic.d
similarity index 100%
rename from assert_writeln_magic.d
rename to ddoc/source/assert_writeln_magic.d
diff --git a/ddoc_preprocessor.d b/ddoc/source/preprocessor.d
similarity index 100%
rename from ddoc_preprocessor.d
rename to ddoc/source/preprocessor.d

From 12cb6593b324b503c7c32d0c4d68fa1da5f877d2 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Sun, 4 Feb 2018 20:37:52 +0100
Subject: [PATCH 04/13] Integrate the assert_writeln_magic in the preprocessor

---
 ddoc/.gitignore                    |   3 +
 ddoc/dub.sdl                       |   4 +
 ddoc/dub.selections.json           |   6 ++
 ddoc/source/assert_writeln_magic.d | 129 +++++------------------------
 ddoc/source/preprocessor.d         |  39 +++++++--
 posix.mak                          |  42 ++++------
 6 files changed, 81 insertions(+), 142 deletions(-)
 create mode 100644 ddoc/.gitignore
 create mode 100644 ddoc/dub.sdl
 create mode 100644 ddoc/dub.selections.json
 mode change 100755 => 100644 ddoc/source/assert_writeln_magic.d
 mode change 100755 => 100644 ddoc/source/preprocessor.d

diff --git a/ddoc/.gitignore b/ddoc/.gitignore
new file mode 100644
index 0000000000..3b53635554
--- /dev/null
+++ b/ddoc/.gitignore
@@ -0,0 +1,3 @@
+/libddoc_preprocessor.a
+/ddoc_preprocessor
+/ddoc_preprocessor-test-application
diff --git a/ddoc/dub.sdl b/ddoc/dub.sdl
new file mode 100644
index 0000000000..8bb4199b5e
--- /dev/null
+++ b/ddoc/dub.sdl
@@ -0,0 +1,4 @@
+name "ddoc_preprocessor"
+description "Preprocesses source code before running Ddoc over it"
+dependency "libdparse" version="0.7.2-alpha.4"
+targetType "executable"
diff --git a/ddoc/dub.selections.json b/ddoc/dub.selections.json
new file mode 100644
index 0000000000..e2138fd3b0
--- /dev/null
+++ b/ddoc/dub.selections.json
@@ -0,0 +1,6 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"libdparse": "0.7.2-alpha.4"
+	}
+}
diff --git a/ddoc/source/assert_writeln_magic.d b/ddoc/source/assert_writeln_magic.d
old mode 100755
new mode 100644
index ccfa2b88e7..957123b45c
--- a/ddoc/source/assert_writeln_magic.d
+++ b/ddoc/source/assert_writeln_magic.d
@@ -177,7 +177,7 @@ private:
     Out fl;
 }
 
-void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
+void parseString(Visitor)(ubyte[] sourceCode, Visitor visitor)
 {
     import dparse.lexer;
     import dparse.parser : parseModule;
@@ -188,89 +188,19 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
     const(Token)[] tokens = getTokensForParser(sourceCode, config, &cache).array;
 
     RollbackAllocator rba;
-    auto m = parseModule(tokens, fileName, &rba);
+    auto m = parseModule(tokens, "magic.d", &rba);
     visitor.visit(m);
 }
 
-void parseFile(string fileName, string destFile)
+auto assertWriteln(string fileText)
 {
-    import std.array : uninitializedArray;
-
-    auto inFile = File(fileName);
-    if (inFile.size == 0)
-        warningf("%s is empty", inFile.name);
-
-    ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
-    if (sourceCode.length == 0)
-        return;
-
-    inFile.rawRead(sourceCode);
-    auto fl = FileLines(fileName, destFile);
+    import std.string : representation;
+    auto fl = FileLines(fileText);
     auto visitor = new TestVisitor!(typeof(fl))(fl);
-    parseString(sourceCode, fileName, visitor);
+    // libdparse doesn't allow to work on immutable source code
+    parseString(cast(ubyte[]) fileText.representation, visitor);
     delete visitor;
-}
-
-// Modify a path under oldBase to a new path with the same subpath under newBase.
-// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar`
-string rebasePath(string path, string oldBase, string newBase)
-{
-    import std.path : absolutePath, buildPath, relativePath;
-    return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath));
-}
-
-version(unittest) { void main(){} } else
-void main(string[] args)
-{
-    import std.file;
-    import std.getopt;
-    import std.path;
-
-    string inputDir, outputDir;
-    string[] ignoredFiles;
-
-    auto helpInfo = getopt(args, config.required,
-            "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
-            "outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir,
-            "ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles);
-
-    if (helpInfo.helpWanted)
-    {
-        return defaultGetoptPrinter(`assert_writeln_magic
-Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls.
-`, helpInfo.options);
-    }
-
-    inputDir = inputDir.asNormalizedPath.array;
-
-    DirEntry[] files;
-
-    // inputDir as default output directory
-    if (!outputDir.length)
-        outputDir = inputDir;
-
-    if (inputDir.isFile)
-    {
-        files = [DirEntry(inputDir)];
-        inputDir = "";
-    }
-    else
-    {
-        files = dirEntries(inputDir, SpanMode.depth).filter!(
-                a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
-    }
-
-    foreach (file; files)
-    {
-        if (!ignoredFiles.any!(x => file.name.canFind(x)))
-        {
-            // single files
-            if (inputDir.length == 0)
-                parseFile(file.name, outputDir);
-            else
-                parseFile(file.name, file.name.rebasePath(inputDir, outputDir));
-        }
-    }
+    return fl.buildLines;
 }
 
 /**
@@ -284,47 +214,26 @@ struct FileLines
 
     string[] lines;
     string destFile;
-    bool overwriteInputFile;
     bool hasWrittenChanges;
 
-    this(string inputFile, string destFile)
+    this(string inputText)
     {
-        stderr.writefln("%s -> %s", inputFile, destFile);
-        this.overwriteInputFile = inputFile == destFile;
-        this.destFile = destFile;
-        lines = File(inputFile).byLineCopy.array;
-
-        destFile.dirName.mkdirRecurse;
-    }
-
-    // dumps all changes
-    ~this()
-    {
-        if (overwriteInputFile)
-        {
-            if (hasWrittenChanges)
-            {
-                auto tmpFile = File(destFile ~ ".tmp", "w");
-                writeLinesToFile(tmpFile);
-                tmpFile.close;
-                tmpFile.name.rename(destFile);
-            }
-        }
-        else
-        {
-            writeLinesToFile(File(destFile, "w"));
-        }
+        lines = inputText.split("\n");
     }
 
     // writes all changes to a random, temporary file
-    void writeLinesToFile(File outFile) {
+    auto buildLines() {
+        auto app = appender!string;
         // dump file
         foreach (line; lines)
-            outFile.writeln(line);
+        {
+            app ~= line;
+            app ~= "\n";
+        }
         // within the docs we automatically inject std.stdio (hence we need to do the same here)
         // writeln needs to be @nogc, @safe, pure and nothrow (we just fake it)
-        outFile.writeln("// \nprivate void writeln(T)(T l) { }");
-        outFile.flush;
+        app ~= "// \nprivate void writeln(T)(T l) { }";
+        return app.data;
     }
 
     string opIndex(size_t i) { return lines[i]; }
@@ -349,7 +258,7 @@ version(unittest)
         import std.string : representation;
         auto mock = FileLinesMock(sourceCode.split("\n"));
         auto visitor = new TestVisitor!(typeof(mock))(mock);
-        parseString(sourceCode.representation.dup, "testmodule", visitor);
+        parseString(sourceCode.representation.dup, visitor);
         delete visitor;
         return mock;
     }
diff --git a/ddoc/source/preprocessor.d b/ddoc/source/preprocessor.d
old mode 100755
new mode 100644
index 2a425b93bb..bbc9440032
--- a/ddoc/source/preprocessor.d
+++ b/ddoc/source/preprocessor.d
@@ -24,12 +24,14 @@ struct Config
 {
     string dmdBinPath = "dmd";
     string outputFile;
-    string cwd = __FILE_FULL_PATH__.dirName;
+    string cwd = __FILE_FULL_PATH__.dirName.dirName.dirName;
 }
 Config config;
 
+version(unittest) {} else
 int main(string[] rootArgs)
 {
+    import assert_writeln_magic;
     import std.getopt;
     auto helpInformation = getopt(
         rootArgs, std.getopt.config.passThrough,
@@ -49,8 +51,11 @@ All unknown options are passed to the compiler.
     assert(pos >= 0, "An input file (.d or .dd) must be provided");
     auto inputFile = args[pos];
     auto text = inputFile.readText;
-    // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
-    args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
+
+    // for now only non-package modules are supported
+    if (!inputFile.endsWith("package.d", "index.d"))
+        // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
+        args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
 
     // transform and extend the ddoc page
     text = genGrammar(text);
@@ -58,24 +63,42 @@ All unknown options are passed to the compiler.
     text = genChangelogVersion(inputFile, text);
     text = genSwitches(text);
 
+    if (inputFile.endsWith(".d"))
+        text = assertWriteln(text);
+
     // inject custom, "dynamic" macros
-    text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    if (inputFile.endsWith(".dd", "index.d"))
+        text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    else if (inputFile.endsWith(".d"))
+        text ~= "/**\nMacros:\n\nSRC_FILENAME=%s\n*/".format(inputFile.buildNormalizedPath);
     return compile(text, args);
 }
 
 auto compile(R)(R buffer, string[] arguments)
 {
+    import core.time : usecs;
+    import core.thread : Thread;
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
-    foreach (arg; ["-c", "-o-", "-"])
+
+    import std.file : write, tempDir;
+    import std.uuid : randomUUID;
+    auto fileName = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ ".d");
+    std.file.write(fileName, buffer);
+    scope(exit) fileName.remove;
+    args = args.replace("-", fileName);
+
+    foreach (arg; ["-c", "-o-"])
     {
         if (!args.canFind(arg))
             args ~= arg;
     }
     auto pipes = pipeProcess(args, Redirect.stdin);
-    pipes.stdin.write(buffer);
-    pipes.stdin.close;
-    return wait(pipes.pid);
+    import std.process : execute;
+    auto ret = execute(args);
+    if (ret.status != 0)
+        stderr.writeln(ret.output);
+    return ret.status;
 }
 
 // replaces the content of a DDoc macro call
diff --git a/posix.mak b/posix.mak
index 7e591d8ec5..b063985161 100644
--- a/posix.mak
+++ b/posix.mak
@@ -134,6 +134,9 @@
 #  before actually running Ddoc.
 #  Currently this is used for:
 #    - TOC
+#    - dynamic TOC generation
+#    - GRAMMAR overview generation
+#    - CHANGELOG menu generation
 #    - assert -> writeln magic
 PWD=$(shell pwd)
 MAKEFILE=$(firstword $(MAKEFILE_LIST))
@@ -284,7 +287,7 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \
 # Ddoc binaries
 ################################################################################
 
-DDOC_BIN:=$G/ddoc
+DDOC_BIN:=$G/ddoc_preprocessor
 DDOC_BIN_DMD:=$(DDOC_BIN) --compiler=$(DMD)
 
 ################################################################################
@@ -684,23 +687,24 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version
 ################################################################################
 
 .PHONY: phobos-prerelease
-phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD)
+phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD) \
+					$G/changelog/next-version
 	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
 
 phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE) $(DDOC_BIN) $(DMD)
 	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
 
-phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD)
+phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD_LATEST)
 	$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD_LATEST))"
 
 phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
 		$W/phobos-prerelease/index.verbatim
 $W/phobos-prerelease/index.verbatim : verbatim.ddoc \
 		$W/phobos-prerelease/object.verbatim \
-		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version
+		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD)
 	${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
 	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim
 	$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
@@ -854,7 +858,7 @@ test_dspec: dspec_tester.d $(DMD) $(PHOBOS_LIB)
 	$(DMD) -run $< --compiler=$(DMD)
 
 .PHONY:
-test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all | $(STABLE_DMD)
+test: $test_dspec test/next_version.sh all | $(STABLE_DMD)
 	@echo "Searching for trailing whitespace"
 	@grep -n '[[:blank:]]$$' $$(find . -type f -name "*.dd" | grep -v .generated) ; test $$? -eq 1
 	@echo "Searching for tabs"
@@ -862,8 +866,8 @@ test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all | $(STABLE_
 	@echo "Checking DDoc's output"
 	$(STABLE_RDMD) -main -unittest check_ddoc.d
 	$(STABLE_RDMD) check_ddoc.d $$(find $W -type f -name "*.html" -not -path "$W/phobos/*")
-	@echo "Executing assert_writeln_magic tests"
-	$<
+	@echo "Executing ddoc_preprocessor tests"
+	dub -C ddoc test
 	@echo "Executing next_version tests"
 	test/next_version.sh
 
@@ -951,22 +955,12 @@ $G/contributors_list.ddoc:  | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
 # Custom DDoc wrapper
 # ------------------
 #
-# This allows extending Ddoc files dynamically on-the-fly.
-# It is currently only used for the specification pages
-#
-# It does:
-# - dynamic TOC generation
-# - GRAMMAR overview generation
-# - CHANGELOG menu generation
-# - Assert -> writeln magic (https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org)
-#  - This transforms assert(a == b) to writeln(a); // b
-#  - It creates a copy of Phobos to apply the transformations
-#  - All "d" files are piped through the transformator,
-#   other needed files (e.g. posix.mak) get copied over
+# This allows extending Ddoc files and D source code files dynamically on-the-fly.
 ################################################################################
 
-$(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD) $(DMD)
-	$(STABLE_RDMD) -version=DdocOptions --build-only -g -of$@ -I$(DMD_DIR)/src $<
+$(DDOC_BIN): ddoc/source/preprocessor.d ddoc/source/assert_writeln_magic.d | $(STABLE_DMD)
+	$(STABLE_DMD_BIN_ROOT)/dub build --compiler=$(STABLE_DMD) --root=ddoc && \
+		mv ddoc/ddoc_preprocessor $@
 
 ################################################################################
 # Build and render the DMD man page

From 1cbf1c843a717c313bfece1866d71947ea7cb950 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 00:22:33 +0100
Subject: [PATCH 05/13] Add main from ddox

---
 dpl-docs/source/app.d       |   2 +-
 dpl-docs/source/ddox_main.d | 470 ++++++++++++++++++++++++++++++++++++
 2 files changed, 471 insertions(+), 1 deletion(-)
 create mode 100644 dpl-docs/source/ddox_main.d

diff --git a/dpl-docs/source/app.d b/dpl-docs/source/app.d
index 66d20e224c..7a5c01b060 100644
--- a/dpl-docs/source/app.d
+++ b/dpl-docs/source/app.d
@@ -1,6 +1,6 @@
 module app;
 
-import ddox.main;
+import ddox_main;
 import std.getopt;
 import std.process;
 import vibe.core.log;
diff --git a/dpl-docs/source/ddox_main.d b/dpl-docs/source/ddox_main.d
new file mode 100644
index 0000000000..eafa98340b
--- /dev/null
+++ b/dpl-docs/source/ddox_main.d
@@ -0,0 +1,470 @@
+// Copied from https://github.com/rejectedsoftware/ddox/blob/master/source/ddox/main.d
+module ddox_main;
+
+import ddox.ddoc;
+import ddox.ddox;
+import ddox.entities;
+import ddox.htmlgenerator;
+import ddox.htmlserver;
+import ddox.parsers.dparse;
+import ddox.parsers.jsonparser;
+
+import vibe.core.core;
+import vibe.core.file;
+import vibe.data.json;
+import vibe.inet.url;
+import vibe.http.fileserver;
+import vibe.http.router;
+import vibe.http.server;
+import vibe.stream.operations;
+import std.array;
+import std.exception : enforce;
+import std.file;
+import std.getopt;
+import std.stdio;
+import std.string;
+
+
+int ddoxMain(string[] args)
+{
+	bool help;
+	getopt(args, config.passThrough, "h|help", &help);
+
+	if( args.length < 2 || help ){
+		showUsage(args);
+		return help ? 0 : 1;
+	}
+
+	if( args[1] == "generate-html" && args.length >= 4 )
+		return cmdGenerateHtml(args);
+	if( args[1] == "serve-html" && args.length >= 3 )
+		return cmdServeHtml(args);
+	if( args[1] == "filter" && args.length >= 3 )
+		return cmdFilterDocs(args);
+	if( args[1] == "serve-test" && args.length >= 3 )
+		return cmdServeTest(args);
+	showUsage(args);
+	return 1;
+}
+
+int cmdGenerateHtml(string[] args)
+{
+	GeneratorSettings gensettings;
+	Package pack;
+	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
+		return ret;
+
+	generateHtmlDocs(Path(args[3]), pack, gensettings);
+	return 0;
+}
+
+int cmdServeHtml(string[] args)
+{
+	string[] webfiledirs;
+	getopt(args,
+		config.passThrough,
+		"web-file-dir", &webfiledirs);
+
+	GeneratorSettings gensettings;
+	Package pack;
+	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
+		return ret;
+
+	// register the api routes and start the server
+	auto router = new URLRouter;
+	registerApiDocs(router, pack, gensettings);
+
+	foreach (dir; webfiledirs)
+		router.get("*", serveStaticFiles(dir));
+
+	writefln("Listening on port 8080...");
+	auto settings = new HTTPServerSettings;
+	settings.port = 8080;
+	listenHTTP(settings, router);
+
+	return runEventLoop();
+}
+
+int cmdServeTest(string[] args)
+{
+	string[] webfiledirs;
+	auto docsettings = new DdoxSettings;
+	auto gensettings = new GeneratorSettings;
+
+	auto pack = parseD(args[2 .. $]);
+
+	processDocs(pack, docsettings);
+
+	// register the api routes and start the server
+	auto router = new URLRouter;
+	registerApiDocs(router, pack, gensettings);
+
+	foreach (dir; webfiledirs)
+		router.get("*", serveStaticFiles(dir));
+
+	writefln("Listening on port 8080...");
+	auto settings = new HTTPServerSettings;
+	settings.port = 8080;
+	listenHTTP(settings, router);
+
+	return runEventLoop();
+}
+
+int setupGeneratorInput(ref string[] args, out GeneratorSettings gensettings, out Package pack)
+{
+	gensettings = new GeneratorSettings;
+	auto docsettings = new DdoxSettings;
+
+	string[] macrofiles;
+	string[] overridemacrofiles;
+	string sitemapurl = "http://127.0.0.1/";
+	bool lowercasenames;
+	bool hyphenate;
+	getopt(args,
+		//config.passThrough,
+		"decl-sort", &docsettings.declSort,
+		"file-name-style", &gensettings.fileNameStyle,
+		"hyphenate", &hyphenate,
+		"lowercase-names", &lowercasenames,
+		"module-sort", &docsettings.moduleSort,
+		"navigation-type", &gensettings.navigationType,
+		"override-macros", &overridemacrofiles,
+		"package-order", &docsettings.packageOrder,
+		"sitemap-url", &sitemapurl,
+		"std-macros", &macrofiles,
+		"enum-member-pages", &gensettings.enumMemberPages,
+		"html-style", &gensettings.htmlOutputStyle,
+		);
+	gensettings.siteUrl = URL(sitemapurl);
+
+	if (lowercasenames) gensettings.fileNameStyle = MethodStyle.lowerCase;
+
+	if( args.length < 3 ){
+		showUsage(args);
+		return 1;
+	}
+
+	setDefaultDdocMacroFiles(macrofiles);
+	setOverrideDdocMacroFiles(overridemacrofiles);
+	if (hyphenate) enableHyphenation();
+
+	// parse the json output file
+	pack = parseDocFile(args[2], docsettings);
+
+	return 0;
+}
+
+int cmdFilterDocs(string[] args)
+{
+	string[] excluded, included;
+	Protection minprot = Protection.Private;
+	bool keeputests = false;
+	bool keepinternals = false;
+	bool unittestexamples = true;
+	bool nounittestexamples = false;
+	bool justdoc = false;
+	getopt(args,
+		//config.passThrough,
+		"ex", &excluded,
+		"in", &included,
+		"min-protection", &minprot,
+		"only-documented", &justdoc,
+		"keep-unittests", &keeputests,
+		"keep-internals", &keepinternals,
+		"unittest-examples", &unittestexamples, // deprecated, kept to not break existing scripts
+		"no-unittest-examples", &nounittestexamples);
+
+	if (keeputests) keepinternals = true;
+	if (nounittestexamples) unittestexamples = false;
+
+	string jsonfile;
+	if( args.length < 3 ){
+		showUsage(args);
+		return 1;
+	}
+
+	Json filterProt(Json json, Json parent, Json last_decl, Json mod)
+	{
+		if (last_decl.type == Json.Type.undefined) last_decl = parent;
+
+		string templateName(Json j){
+			auto n = j["name"].opt!string();
+			auto idx = n.indexOf('(');
+			if( idx >= 0 ) return n[0 .. idx];
+			return n;
+		}
+
+		if( json.type == Json.Type.Object ){
+			auto comment = json["comment"].opt!string;
+			if( justdoc && comment.empty ){
+				if( parent.type != Json.Type.Object || parent["kind"].opt!string() != "template" || templateName(parent) != json["name"].opt!string() )
+					return Json.undefined;
+			}
+
+			Protection prot = Protection.Public;
+			if( auto p = "protection" in json ){
+				switch(p.get!string){
+					default: break;
+					case "private": prot = Protection.Private; break;
+					case "package": prot = Protection.Package; break;
+					case "protected": prot = Protection.Protected; break;
+				}
+			}
+			if( comment.strip == "private" ) prot = Protection.Private;
+			if( prot < minprot ) return Json.undefined;
+
+			auto name = json["name"].opt!string();
+			bool is_internal = name.startsWith("__");
+			bool is_unittest = name.startsWith("__unittest");
+			if (name.startsWith("_staticCtor") || name.startsWith("_staticDtor")) is_internal = true;
+			else if (name.startsWith("_sharedStaticCtor") || name.startsWith("_sharedStaticDtor")) is_internal = true;
+
+			if (unittestexamples && is_unittest && !comment.empty) {
+				assert(last_decl.type == Json.Type.object, "Don't have a last_decl context.");
+				try {
+					string source = extractUnittestSourceCode(json, mod);
+					if (last_decl["comment"].opt!string.empty) {
+						writefln("Warning: Cannot add documented unit test %s to %s, which is not documented.", name, last_decl["name"].opt!string);
+					} else {
+						last_decl["comment"] ~= format("Example:\n%s$(DDOX_UNITTEST_HEADER %s)\n---\n%s\n---\n$(DDOX_UNITTEST_FOOTER %s)\n", comment.strip, name, source, name);
+					}
+				} catch (Exception e) {
+					writefln("Failed to add documented unit test %s:%s as example: %s",
+						mod["file"].get!string(), json["line"].get!long, e.msg);
+					return Json.undefined;
+				}
+			}
+
+			if (!keepinternals && is_internal) return Json.undefined;
+
+			if (!keeputests && is_unittest) return Json.undefined;
+
+			if (auto mem = "members" in json)
+				json["members"] = filterProt(*mem, json, Json.undefined, mod);
+		} else if( json.type == Json.Type.Array ){
+			auto last_child_decl = Json.undefined;
+			Json[] newmem;
+			foreach (m; json) {
+				auto mf = filterProt(m, parent, last_child_decl, mod);
+				if (mf.type == Json.Type.undefined) continue;
+				if (mf.type == Json.Type.object && !mf["name"].opt!string.startsWith("__unittest") && icmp(mf["comment"].opt!string.strip, "ditto") != 0)
+					last_child_decl = mf;
+				newmem ~= mf;
+			}
+			return Json(newmem);
+		}
+		return json;
+	}
+
+	writefln("Reading doc file...");
+	auto text = readText(args[2]);
+	int line = 1;
+	writefln("Parsing JSON...");
+	auto json = parseJson(text, &line);
+
+	writefln("Filtering modules...");
+	Json[] dst;
+	foreach (m; json) {
+		if ("name" !in m) {
+			writefln("No name for module %s - ignoring", m["file"].opt!string);
+			continue;
+		}
+		auto n = m["name"].get!string;
+		bool include = true;
+		foreach (ex; excluded)
+			if (n.startsWith(ex)) {
+				include = false;
+				break;
+			}
+		foreach (inc; included)
+			if (n.startsWith(inc)) {
+				include = true;
+				break;
+			}
+		if (include) {
+			auto doc = filterProt(m, Json.undefined, Json.undefined, m);
+			if (doc.type != Json.Type.undefined)
+				dst ~= doc;
+                }
+	}
+
+	writefln("Writing filtered docs...");
+	auto buf = appender!string();
+	writePrettyJsonString(buf, Json(dst));
+	std.file.write(args[2], buf.data());
+
+	return 0;
+}
+
+Package parseDocFile(string filename, DdoxSettings settings)
+{
+	writefln("Reading doc file...");
+	auto text = readText(filename);
+	int line = 1;
+	writefln("Parsing JSON...");
+	auto json = parseJson(text, &line);
+	writefln("Parsing docs...");
+	Package root;
+	root = parseJsonDocs(json);
+	writefln("Finished parsing docs.");
+
+	processDocs(root, settings);
+	return root;
+}
+
+void showUsage(string[] args)
+{
+	string cmd;
+	if( args.length >= 2 ) cmd = args[1];
+
+	switch(cmd){
+		default:
+			writefln(
+`Usage: %s <COMMAND> [args...]
+
+    <COMMAND> can be one of:
+        generate-html
+        serve-html
+        filter
+
+ -h --help                 Show this help
+
+Use <COMMAND> -h|--help to get detailed usage information for a command.
+`, args[0]);
+			break;
+		case "serve-html":
+			writefln(
+`Usage: %s serve-html <ddocx-input-file>
+    --std-macros=FILE      File containing DDOC macros that will be available
+    --override-macros=FILE File containing DDOC macros that will override local
+                           definitions (Macros: section)
+    --navigation-type=TYPE Change the type of navigation (ModuleList,
+                           ModuleTree (default), DeclarationTree)
+    --package-order=NAME   Causes the specified module to be ordered first. Can
+                           be specified multiple times.
+    --sitemap-url          Specifies the base URL used for sitemap generation
+    --module-sort=MODE     The sort order used for lists of modules
+    --decl-sort=MODE       The sort order used for declaration lists
+    --web-file-dir=DIR     Make files from dir available on the served site
+    --enum-member-pages    Generate a single page per enum member
+    --html-style=STYLE     Sets the HTML output style, either pretty (default)
+                           or compact.
+    --hyphenate            hyphenate text
+ -h --help                 Show this help
+
+The following values can be used as sorting modes: none, name, protectionName,
+protectionInheritanceName
+`, args[0]);
+			break;
+		case "generate-html":
+			writefln(
+`Usage: %s generate-html <ddocx-input-file> <output-dir>
+    --std-macros=FILE      File containing DDOC macros that will be available
+    --override-macros=FILE File containing DDOC macros that will override local
+                           definitions (Macros: section)
+    --navigation-type=TYPE Change the type of navigation (ModuleList,
+                           ModuleTree, DeclarationTree)
+    --package-order=NAME   Causes the specified module to be ordered first. Can
+                           be specified multiple times.
+    --sitemap-url          Specifies the base URL used for sitemap generation
+    --module-sort=MODE     The sort order used for lists of modules
+    --decl-sort=MODE       The sort order used for declaration lists
+    --file-name-style=STY  Sets a translation style for symbol names to file
+                           names. Use this instead of --lowercase-name.
+                           Possible values for STY:
+                             unaltered, camelCase, pascalCase, lowerCase,
+                             upperCase, lowerUnderscored, upperUnderscored
+    --lowercase-names      DEPRECATED: Outputs all file names in lower case.
+                           This option is useful on case insensitive file
+                           systems.
+    --enum-member-pages    Generate a single page per enum member
+    --html-style=STYLE     Sets the HTML output style, either pretty (default)
+                           compact or .
+    --hyphenate            hyphenate text
+ -h --help                 Show this help
+
+The following values can be used as sorting modes: none, name, protectionName,
+protectionInheritanceName
+`, args[0]);
+			break;
+		case "filter":
+			writefln(
+`Usage: %s filter <ddocx-input-file> [options]
+    --ex=PREFIX            Exclude modules with prefix
+    --in=PREFIX            Force include of modules with prefix
+    --min-protection=PROT  Remove items with lower protection level than
+                           specified.
+                           PROT can be: Public, Protected, Package, Private
+    --only-documented      Remove undocumented entities.
+    --keep-unittests       Do not remove unit tests from documentation.
+                           Implies --keep-internals.
+    --keep-internals       Do not remove symbols starting with two underscores.
+    --unittest-examples    Add documented unit tests as examples to the
+                           preceding declaration (deprecated, enabled by
+                           default)
+    --no-unittest-examples Don't convert documented unit tests to examples
+ -h --help                 Show this help
+`, args[0]);
+	}
+	if( args.length < 2 ){
+	} else {
+
+	}
+}
+
+private string extractUnittestSourceCode(Json decl, Json mod)
+{
+	auto filename = mod["file"].get!string();
+	enforce("line" in decl && "endline" in decl, "Missing line/endline fields.");
+	auto from = decl["line"].get!long;
+	auto to = decl["endline"].get!long;
+
+	// read the matching lines out of the file
+	auto app = appender!string();
+	long lc = 1;
+	foreach (str; File(filename).byLine) {
+		if (lc >= from) {
+			app.put(str);
+			app.put('\n');
+		}
+		if (++lc > to) break;
+	}
+	auto ret = app.data;
+
+	// strip the "unittest { .. }" surroundings
+	auto idx = ret.indexOf("unittest");
+	enforce(idx >= 0, format("Missing 'unittest' for unit test at %s:%s.", filename, from));
+	ret = ret[idx .. $];
+
+	idx = ret.indexOf("{");
+	enforce(idx >= 0, format("Missing opening '{' for unit test at %s:%s.", filename, from));
+	ret = ret[idx+1 .. $];
+
+	idx = ret.lastIndexOf("}");
+	enforce(idx >= 0, format("Missing closing '}' for unit test at %s:%s.", filename, from));
+	ret = ret[0 .. idx];
+
+	// unindent lines according to the indentation of the first line
+	app = appender!string();
+	string indent;
+	foreach (i, ln; ret.splitLines) {
+		if (i == 1) {
+			foreach (j; 0 .. ln.length)
+				if (ln[j] != ' ' && ln[j] != '\t') {
+					indent = ln[0 .. j];
+					break;
+				}
+		}
+		if (i > 0 || ln.strip.length > 0) {
+			size_t j = 0;
+			while (j < indent.length && !ln.empty) {
+				if (ln.front != indent[j]) break;
+				ln.popFront();
+				j++;
+			}
+			app.put(ln);
+			app.put('\n');
+		}
+	}
+	return app.data;
+}

From ceeeccb8c4b313b6f7b0127ec51aa9f3e5d8dbce Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 00:23:41 +0100
Subject: [PATCH 06/13] Use assert_writeln_magic in dpl-docs

---
 ddoc/.gitignore                    |  1 +
 ddoc/dub.sdl                       |  7 ++++++-
 ddoc/source/assert_writeln_magic.d | 21 +++++++++++++++++++--
 ddoc/source/preprocessor.d         |  4 ++--
 dpl-docs/dub.json                  |  5 ++++-
 dpl-docs/source/ddox_main.d        |  4 +++-
 posix.mak                          |  4 ++--
 7 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/ddoc/.gitignore b/ddoc/.gitignore
index 3b53635554..646b0fa1e8 100644
--- a/ddoc/.gitignore
+++ b/ddoc/.gitignore
@@ -1,3 +1,4 @@
 /libddoc_preprocessor.a
 /ddoc_preprocessor
 /ddoc_preprocessor-test-application
+/ddoc_preprocessor-test-library
diff --git a/ddoc/dub.sdl b/ddoc/dub.sdl
index 8bb4199b5e..946c7f00d9 100644
--- a/ddoc/dub.sdl
+++ b/ddoc/dub.sdl
@@ -1,4 +1,9 @@
 name "ddoc_preprocessor"
 description "Preprocesses source code before running Ddoc over it"
 dependency "libdparse" version="0.7.2-alpha.4"
-targetType "executable"
+configuration "executable" {
+	versions "IsExecutable"
+	targetType "executable"
+}
+configuration "library" {
+}
diff --git a/ddoc/source/assert_writeln_magic.d b/ddoc/source/assert_writeln_magic.d
index 957123b45c..855c1c8fa8 100644
--- a/ddoc/source/assert_writeln_magic.d
+++ b/ddoc/source/assert_writeln_magic.d
@@ -192,7 +192,7 @@ void parseString(Visitor)(ubyte[] sourceCode, Visitor visitor)
     visitor.visit(m);
 }
 
-auto assertWriteln(string fileText)
+private auto assertWritelnModuleImpl(string fileText)
 {
     import std.string : representation;
     auto fl = FileLines(fileText);
@@ -200,7 +200,24 @@ auto assertWriteln(string fileText)
     // libdparse doesn't allow to work on immutable source code
     parseString(cast(ubyte[]) fileText.representation, visitor);
     delete visitor;
-    return fl.buildLines;
+    return fl;
+}
+
+auto assertWritelnModule(string fileText)
+{
+    return assertWritelnModuleImpl(fileText).buildLines;
+}
+auto assertWritelnBlock(string fileText)
+{
+    auto source = "unittest{\n" ~ fileText ~ "}\n";
+    auto fl = assertWritelnModuleImpl(source);
+    auto app = appender!string;
+    foreach (line; fl.lines[1 .. $ - 2])
+    {
+        app ~= line;
+        app ~= "\n";
+    }
+    return app.data;
 }
 
 /**
diff --git a/ddoc/source/preprocessor.d b/ddoc/source/preprocessor.d
index bbc9440032..2318fa76e4 100644
--- a/ddoc/source/preprocessor.d
+++ b/ddoc/source/preprocessor.d
@@ -28,7 +28,7 @@ struct Config
 }
 Config config;
 
-version(unittest) {} else
+version(IsExecutable)
 int main(string[] rootArgs)
 {
     import assert_writeln_magic;
@@ -64,7 +64,7 @@ All unknown options are passed to the compiler.
     text = genSwitches(text);
 
     if (inputFile.endsWith(".d"))
-        text = assertWriteln(text);
+        text = assertWritelnModule(text);
 
     // inject custom, "dynamic" macros
     if (inputFile.endsWith(".dd", "index.d"))
diff --git a/dpl-docs/dub.json b/dpl-docs/dub.json
index 762ed98877..223c4fd718 100644
--- a/dpl-docs/dub.json
+++ b/dpl-docs/dub.json
@@ -1,7 +1,10 @@
 {
 	"name": "dpl-docs",
 	"dependencies": {
-		"ddox": "~>0.16.7"
+		"ddox": "~>0.16.7",
+		"ddoc_preprocessor": {
+		  "path": "../ddoc"
+		}
 	},
 	"versions": ["VibeNoSSL"],
 	"subConfigurations": { "eventcore": "libasync"}
diff --git a/dpl-docs/source/ddox_main.d b/dpl-docs/source/ddox_main.d
index eafa98340b..787ca1f196 100644
--- a/dpl-docs/source/ddox_main.d
+++ b/dpl-docs/source/ddox_main.d
@@ -226,7 +226,9 @@ int cmdFilterDocs(string[] args)
 					if (last_decl["comment"].opt!string.empty) {
 						writefln("Warning: Cannot add documented unit test %s to %s, which is not documented.", name, last_decl["name"].opt!string);
 					} else {
-						last_decl["comment"] ~= format("Example:\n%s$(DDOX_UNITTEST_HEADER %s)\n---\n%s\n---\n$(DDOX_UNITTEST_FOOTER %s)\n", comment.strip, name, source, name);
+					    import assert_writeln_magic;
+                        auto rewrittenSource = assertWritelnBlock(source);
+                        last_decl["comment"] ~= format("Example:\n%s$(DDOX_UNITTEST_HEADER %s)\n---\n%s\n---\n$(DDOX_UNITTEST_FOOTER %s)\n", comment.strip, name, rewrittenSource, name);
 					}
 				} catch (Exception e) {
 					writefln("Failed to add documented unit test %s:%s as example: %s",
diff --git a/posix.mak b/posix.mak
index b063985161..418eb2645d 100644
--- a/posix.mak
+++ b/posix.mak
@@ -858,7 +858,7 @@ test_dspec: dspec_tester.d $(DMD) $(PHOBOS_LIB)
 	$(DMD) -run $< --compiler=$(DMD)
 
 .PHONY:
-test: $test_dspec test/next_version.sh all | $(STABLE_DMD)
+test: test_dspec test/next_version.sh all | $(STABLE_DMD) $(DUB)
 	@echo "Searching for trailing whitespace"
 	@grep -n '[[:blank:]]$$' $$(find . -type f -name "*.dd" | grep -v .generated) ; test $$? -eq 1
 	@echo "Searching for tabs"
@@ -867,7 +867,7 @@ test: $test_dspec test/next_version.sh all | $(STABLE_DMD)
 	$(STABLE_RDMD) -main -unittest check_ddoc.d
 	$(STABLE_RDMD) check_ddoc.d $$(find $W -type f -name "*.html" -not -path "$W/phobos/*")
 	@echo "Executing ddoc_preprocessor tests"
-	dub -C ddoc test
+	$(DUB) test --compiler=${STABLE_DMD} --root ddoc
 	@echo "Executing next_version tests"
 	test/next_version.sh
 

From f2caa156f35bf48e0ce13738cd2aef1fffde84be Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 08:28:00 +0100
Subject: [PATCH 07/13] Improve auto-macro injection

---
 ddoc/source/preprocessor.d | 39 ++++++++++++++++++++++++--------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/ddoc/source/preprocessor.d b/ddoc/source/preprocessor.d
index 2318fa76e4..5975cf7fc8 100644
--- a/ddoc/source/preprocessor.d
+++ b/ddoc/source/preprocessor.d
@@ -63,30 +63,41 @@ All unknown options are passed to the compiler.
     text = genChangelogVersion(inputFile, text);
     text = genSwitches(text);
 
-    if (inputFile.endsWith(".d"))
+    // Phobos index.d should have been named index.dd
+    if (inputFile.endsWith(".d") && !inputFile.endsWith("index.d"))
         text = assertWritelnModule(text);
 
-    // inject custom, "dynamic" macros
-    if (inputFile.endsWith(".dd", "index.d"))
-        text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
-    else if (inputFile.endsWith(".d"))
-        text ~= "/**\nMacros:\n\nSRC_FILENAME=%s\n*/".format(inputFile.buildNormalizedPath);
-    return compile(text, args);
+    string[string] macros;
+    macros["SRC_FILENAME"] = "%s\n".format(inputFile.buildNormalizedPath);
+    return compile(text, args, macros);
 }
 
-auto compile(R)(R buffer, string[] arguments)
+static auto createTmpFile(string buffer = null, string extension = ".d")
+{
+    import std.uuid : randomUUID;
+    auto name = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ extension);
+    if (buffer !is null)
+        std.file.write(name, buffer);
+    return name;
+}
+
+auto compile(R)(R buffer, string[] arguments, string[string] macros = null)
 {
     import core.time : usecs;
     import core.thread : Thread;
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
 
-    import std.file : write, tempDir;
-    import std.uuid : randomUUID;
-    auto fileName = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ ".d");
-    std.file.write(fileName, buffer);
-    scope(exit) fileName.remove;
-    args = args.replace("-", fileName);
+    string[] tmpFiles = [createTmpFile(buffer, ".d")];
+    args = args.replace("-", tmpFiles[$ - 1]);
+    scope(exit) tmpFiles.each!remove;
+
+    if (macros !is null)
+    {
+        auto macroString = macros.byPair.map!(a => "%s=%s".format(a[0], a[1])).join("\n");
+        args ~= createTmpFile(macroString, ".ddoc");
+        tmpFiles ~= args[$ - 1];
+    }
 
     foreach (arg; ["-c", "-o-"])
     {

From 66114c294b9669eeda0eb638559e1060196300eb Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 17:44:23 +0100
Subject: [PATCH 08/13] Use the assert/writeln pipeline for the verbatim files

---
 posix.mak | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/posix.mak b/posix.mak
index 418eb2645d..e7b7b71a1c 100644
--- a/posix.mak
+++ b/posix.mak
@@ -704,9 +704,9 @@ phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
 		$W/phobos-prerelease/index.verbatim
 $W/phobos-prerelease/index.verbatim : verbatim.ddoc \
 		$W/phobos-prerelease/object.verbatim \
-		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD)
+		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD) $(DDOC_BIN)
 	${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
-	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim
+	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim DMD="$(abspath $(DDOC_BIN))"
 	$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
 	mv $W/phobos-prerelease-verbatim/* $(dir $@)
 	rm -r $W/phobos-prerelease-verbatim

From 7797e5c93226c3dd22827799aa18264e460e248e Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 18:00:19 +0100
Subject: [PATCH 09/13] Imitate package directory layout for package.d files

---
 ddoc/source/preprocessor.d | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/ddoc/source/preprocessor.d b/ddoc/source/preprocessor.d
index 5975cf7fc8..c6b04fe5ef 100644
--- a/ddoc/source/preprocessor.d
+++ b/ddoc/source/preprocessor.d
@@ -53,7 +53,7 @@ All unknown options are passed to the compiler.
     auto text = inputFile.readText;
 
     // for now only non-package modules are supported
-    if (!inputFile.endsWith("package.d", "index.d"))
+    if (!inputFile.endsWith("index.d"))
         // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
         args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
 
@@ -69,34 +69,39 @@ All unknown options are passed to the compiler.
 
     string[string] macros;
     macros["SRC_FILENAME"] = "%s\n".format(inputFile.buildNormalizedPath);
-    return compile(text, args, macros);
+    return compile(text, args, inputFile, macros);
 }
 
-static auto createTmpFile(string buffer = null, string extension = ".d")
+auto createTmpDir()
 {
     import std.uuid : randomUUID;
-    auto name = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ extension);
-    if (buffer !is null)
-        std.file.write(name, buffer);
-    return name;
+    auto dir = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", ""));
+    mkdir(dir);
+    return dir;
 }
 
-auto compile(R)(R buffer, string[] arguments, string[string] macros = null)
+auto compile(R)(R buffer, string[] arguments, string inputFile, string[string] macros = null)
 {
     import core.time : usecs;
     import core.thread : Thread;
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
 
-    string[] tmpFiles = [createTmpFile(buffer, ".d")];
-    args = args.replace("-", tmpFiles[$ - 1]);
-    scope(exit) tmpFiles.each!remove;
+    // Note: ideally we could pass in files directly on stdin.
+    // However, for package.d files, we need to imitate package directory layout to avoid conflicts
+    auto tmpDir = createTmpDir;
+    auto inputTmpFile = tmpDir.buildPath(inputFile);
+    inputTmpFile.dirName.mkdirRecurse;
+    std.file.write(inputTmpFile, buffer);
+    args = args.replace("-", inputTmpFile);
+    scope(exit) tmpDir.rmdirRecurse;
 
     if (macros !is null)
     {
         auto macroString = macros.byPair.map!(a => "%s=%s".format(a[0], a[1])).join("\n");
-        args ~= createTmpFile(macroString, ".ddoc");
-        tmpFiles ~= args[$ - 1];
+        auto macroFile = tmpDir.buildPath("macros.ddoc");
+        std.file.write(macroFile, macroString);
+        args ~= macroFile;
     }
 
     foreach (arg; ["-c", "-o-"])

From e2f21b4571b9c96e3e6023b6ab465feea410b774 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 5 Feb 2018 18:41:33 +0100
Subject: [PATCH 10/13] Temporarily fix broken Ddoc in std.exception.enforceEx

---
 ddoc/source/preprocessor.d | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ddoc/source/preprocessor.d b/ddoc/source/preprocessor.d
index c6b04fe5ef..8c236e6dee 100644
--- a/ddoc/source/preprocessor.d
+++ b/ddoc/source/preprocessor.d
@@ -62,6 +62,7 @@ All unknown options are passed to the compiler.
     text = genHeader(text);
     text = genChangelogVersion(inputFile, text);
     text = genSwitches(text);
+    text = fixDdocBugs(inputFile, text);
 
     // Phobos index.d should have been named index.dd
     if (inputFile.endsWith(".d") && !inputFile.endsWith("index.d"))
@@ -423,3 +424,18 @@ private void highlightSpecialWords(ref string flag, ref string helpText)
             .to!string;
     }
 }
+
+// Fix ddoc bugs
+string fixDdocBugs(string inputFile, string text)
+{
+
+    // https://github.com/dlang/dlang.org/pull/2069#issuecomment-363154934
+    // can be removed once the Phobos PR is in stable and master
+    // https://github.com/dlang/phobos/pull/6126
+    if (inputFile.endsWith("exception.d"))
+    {
+        text = text.replace(`typeof(new E("", __FILE__, __LINE__)`, `typeof(new E("", string.init, size_t.init)`);
+        text = text.replace(`typeof(new E(__FILE__, __LINE__)`, `typeof(new E(string.init, size_t.init)`);
+    }
+    return text;
+}
\ No newline at end of file

From f3f3d6a6fb6db9c22e82efcd2e808edfab257d85 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Thu, 8 Feb 2018 03:54:03 +0100
Subject: [PATCH 11/13] Add dmd as a dependency to the preprocessor (required
 for the CLI switches)

---
 ddoc/dub.sdl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ddoc/dub.sdl b/ddoc/dub.sdl
index 946c7f00d9..aab1479b34 100644
--- a/ddoc/dub.sdl
+++ b/ddoc/dub.sdl
@@ -1,6 +1,8 @@
 name "ddoc_preprocessor"
 description "Preprocesses source code before running Ddoc over it"
 dependency "libdparse" version="0.7.2-alpha.4"
+dependency "dmd" path="../../dmd"
+versions "DdocOptions"
 configuration "executable" {
 	versions "IsExecutable"
 	targetType "executable"

From 1125004a83b57400441d97052c1cc457bde7b3af Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 19 Feb 2018 04:06:11 +0100
Subject: [PATCH 12/13] Use --compiler for the verbatim documentation build

---
 posix.mak | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/posix.mak b/posix.mak
index e7b7b71a1c..90c0ab1ff5 100644
--- a/posix.mak
+++ b/posix.mak
@@ -706,7 +706,7 @@ $W/phobos-prerelease/index.verbatim : verbatim.ddoc \
 		$W/phobos-prerelease/object.verbatim \
 		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD) $(DDOC_BIN)
 	${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
-	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim DMD="$(abspath $(DDOC_BIN))"
+	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
 	$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
 	mv $W/phobos-prerelease-verbatim/* $(dir $@)
 	rm -r $W/phobos-prerelease-verbatim

From 474fb077d1f32e5c6c3ca0ad716234a279e01e42 Mon Sep 17 00:00:00 2001
From: Sebastian Wilzbach <seb@wilzba.ch>
Date: Mon, 19 Feb 2018 05:45:54 +0100
Subject: [PATCH 13/13] Use static import to avoid code duplication from
 ddox.main

---
 dpl-docs/source/ddox_main.d | 229 ++----------------------------------
 1 file changed, 8 insertions(+), 221 deletions(-)

diff --git a/dpl-docs/source/ddox_main.d b/dpl-docs/source/ddox_main.d
index 787ca1f196..02c0f3ac31 100644
--- a/dpl-docs/source/ddox_main.d
+++ b/dpl-docs/source/ddox_main.d
@@ -47,112 +47,14 @@ int ddoxMain(string[] args)
 	return 1;
 }
 
-int cmdGenerateHtml(string[] args)
-{
-	GeneratorSettings gensettings;
-	Package pack;
-	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
-		return ret;
-
-	generateHtmlDocs(Path(args[3]), pack, gensettings);
-	return 0;
-}
-
-int cmdServeHtml(string[] args)
-{
-	string[] webfiledirs;
-	getopt(args,
-		config.passThrough,
-		"web-file-dir", &webfiledirs);
-
-	GeneratorSettings gensettings;
-	Package pack;
-	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
-		return ret;
-
-	// register the api routes and start the server
-	auto router = new URLRouter;
-	registerApiDocs(router, pack, gensettings);
-
-	foreach (dir; webfiledirs)
-		router.get("*", serveStaticFiles(dir));
-
-	writefln("Listening on port 8080...");
-	auto settings = new HTTPServerSettings;
-	settings.port = 8080;
-	listenHTTP(settings, router);
-
-	return runEventLoop();
-}
-
-int cmdServeTest(string[] args)
-{
-	string[] webfiledirs;
-	auto docsettings = new DdoxSettings;
-	auto gensettings = new GeneratorSettings;
+static import ddox.main;
 
-	auto pack = parseD(args[2 .. $]);
-
-	processDocs(pack, docsettings);
-
-	// register the api routes and start the server
-	auto router = new URLRouter;
-	registerApiDocs(router, pack, gensettings);
-
-	foreach (dir; webfiledirs)
-		router.get("*", serveStaticFiles(dir));
-
-	writefln("Listening on port 8080...");
-	auto settings = new HTTPServerSettings;
-	settings.port = 8080;
-	listenHTTP(settings, router);
-
-	return runEventLoop();
-}
-
-int setupGeneratorInput(ref string[] args, out GeneratorSettings gensettings, out Package pack)
-{
-	gensettings = new GeneratorSettings;
-	auto docsettings = new DdoxSettings;
-
-	string[] macrofiles;
-	string[] overridemacrofiles;
-	string sitemapurl = "http://127.0.0.1/";
-	bool lowercasenames;
-	bool hyphenate;
-	getopt(args,
-		//config.passThrough,
-		"decl-sort", &docsettings.declSort,
-		"file-name-style", &gensettings.fileNameStyle,
-		"hyphenate", &hyphenate,
-		"lowercase-names", &lowercasenames,
-		"module-sort", &docsettings.moduleSort,
-		"navigation-type", &gensettings.navigationType,
-		"override-macros", &overridemacrofiles,
-		"package-order", &docsettings.packageOrder,
-		"sitemap-url", &sitemapurl,
-		"std-macros", &macrofiles,
-		"enum-member-pages", &gensettings.enumMemberPages,
-		"html-style", &gensettings.htmlOutputStyle,
-		);
-	gensettings.siteUrl = URL(sitemapurl);
-
-	if (lowercasenames) gensettings.fileNameStyle = MethodStyle.lowerCase;
-
-	if( args.length < 3 ){
-		showUsage(args);
-		return 1;
-	}
-
-	setDefaultDdocMacroFiles(macrofiles);
-	setOverrideDdocMacroFiles(overridemacrofiles);
-	if (hyphenate) enableHyphenation();
-
-	// parse the json output file
-	pack = parseDocFile(args[2], docsettings);
-
-	return 0;
-}
+alias cmdGenerateHtml = ddox.main.cmdGenerateHtml;
+alias cmdServeHtml = ddox.main.cmdServeTest;
+alias cmdServeTest = ddox.main.cmdServeTest;
+alias parseDocFile = ddox.main.parseDocFile;
+alias setupGeneratorInput = ddox.main.setupGeneratorInput;
+alias showUsage = ddox.main.showUsage;
 
 int cmdFilterDocs(string[] args)
 {
@@ -298,122 +200,7 @@ int cmdFilterDocs(string[] args)
 	return 0;
 }
 
-Package parseDocFile(string filename, DdoxSettings settings)
-{
-	writefln("Reading doc file...");
-	auto text = readText(filename);
-	int line = 1;
-	writefln("Parsing JSON...");
-	auto json = parseJson(text, &line);
-	writefln("Parsing docs...");
-	Package root;
-	root = parseJsonDocs(json);
-	writefln("Finished parsing docs.");
-
-	processDocs(root, settings);
-	return root;
-}
-
-void showUsage(string[] args)
-{
-	string cmd;
-	if( args.length >= 2 ) cmd = args[1];
-
-	switch(cmd){
-		default:
-			writefln(
-`Usage: %s <COMMAND> [args...]
-
-    <COMMAND> can be one of:
-        generate-html
-        serve-html
-        filter
-
- -h --help                 Show this help
-
-Use <COMMAND> -h|--help to get detailed usage information for a command.
-`, args[0]);
-			break;
-		case "serve-html":
-			writefln(
-`Usage: %s serve-html <ddocx-input-file>
-    --std-macros=FILE      File containing DDOC macros that will be available
-    --override-macros=FILE File containing DDOC macros that will override local
-                           definitions (Macros: section)
-    --navigation-type=TYPE Change the type of navigation (ModuleList,
-                           ModuleTree (default), DeclarationTree)
-    --package-order=NAME   Causes the specified module to be ordered first. Can
-                           be specified multiple times.
-    --sitemap-url          Specifies the base URL used for sitemap generation
-    --module-sort=MODE     The sort order used for lists of modules
-    --decl-sort=MODE       The sort order used for declaration lists
-    --web-file-dir=DIR     Make files from dir available on the served site
-    --enum-member-pages    Generate a single page per enum member
-    --html-style=STYLE     Sets the HTML output style, either pretty (default)
-                           or compact.
-    --hyphenate            hyphenate text
- -h --help                 Show this help
-
-The following values can be used as sorting modes: none, name, protectionName,
-protectionInheritanceName
-`, args[0]);
-			break;
-		case "generate-html":
-			writefln(
-`Usage: %s generate-html <ddocx-input-file> <output-dir>
-    --std-macros=FILE      File containing DDOC macros that will be available
-    --override-macros=FILE File containing DDOC macros that will override local
-                           definitions (Macros: section)
-    --navigation-type=TYPE Change the type of navigation (ModuleList,
-                           ModuleTree, DeclarationTree)
-    --package-order=NAME   Causes the specified module to be ordered first. Can
-                           be specified multiple times.
-    --sitemap-url          Specifies the base URL used for sitemap generation
-    --module-sort=MODE     The sort order used for lists of modules
-    --decl-sort=MODE       The sort order used for declaration lists
-    --file-name-style=STY  Sets a translation style for symbol names to file
-                           names. Use this instead of --lowercase-name.
-                           Possible values for STY:
-                             unaltered, camelCase, pascalCase, lowerCase,
-                             upperCase, lowerUnderscored, upperUnderscored
-    --lowercase-names      DEPRECATED: Outputs all file names in lower case.
-                           This option is useful on case insensitive file
-                           systems.
-    --enum-member-pages    Generate a single page per enum member
-    --html-style=STYLE     Sets the HTML output style, either pretty (default)
-                           compact or .
-    --hyphenate            hyphenate text
- -h --help                 Show this help
-
-The following values can be used as sorting modes: none, name, protectionName,
-protectionInheritanceName
-`, args[0]);
-			break;
-		case "filter":
-			writefln(
-`Usage: %s filter <ddocx-input-file> [options]
-    --ex=PREFIX            Exclude modules with prefix
-    --in=PREFIX            Force include of modules with prefix
-    --min-protection=PROT  Remove items with lower protection level than
-                           specified.
-                           PROT can be: Public, Protected, Package, Private
-    --only-documented      Remove undocumented entities.
-    --keep-unittests       Do not remove unit tests from documentation.
-                           Implies --keep-internals.
-    --keep-internals       Do not remove symbols starting with two underscores.
-    --unittest-examples    Add documented unit tests as examples to the
-                           preceding declaration (deprecated, enabled by
-                           default)
-    --no-unittest-examples Don't convert documented unit tests to examples
- -h --help                 Show this help
-`, args[0]);
-	}
-	if( args.length < 2 ){
-	} else {
-
-	}
-}
-
+// from ddox
 private string extractUnittestSourceCode(Json decl, Json mod)
 {
 	auto filename = mod["file"].get!string();