diff --git a/ddoc/.gitignore b/ddoc/.gitignore new file mode 100644 index 0000000000..646b0fa1e8 --- /dev/null +++ b/ddoc/.gitignore @@ -0,0 +1,4 @@ +/libddoc_preprocessor.a +/ddoc_preprocessor +/ddoc_preprocessor-test-application +/ddoc_preprocessor-test-library diff --git a/ddoc/dub.sdl b/ddoc/dub.sdl new file mode 100644 index 0000000000..aab1479b34 --- /dev/null +++ b/ddoc/dub.sdl @@ -0,0 +1,11 @@ +name "ddoc_preprocessor" +description "Preprocesses source code before running Ddoc over it" +dependency "libdparse" version="0.7.2-alpha.4" +dependency "dmd" path="../../dmd" +versions "DdocOptions" +configuration "executable" { + versions "IsExecutable" + targetType "executable" +} +configuration "library" { +} diff --git a/ddoc/dub.selections.json b/ddoc/dub.selections.json new file mode 100644 index 0000000000..e2138fd3b0 --- /dev/null +++ b/ddoc/dub.selections.json @@ -0,0 +1,6 @@ +{ + "fileVersion": 1, + "versions": { + "libdparse": "0.7.2-alpha.4" + } +} diff --git a/assert_writeln_magic.d b/ddoc/source/assert_writeln_magic.d old mode 100755 new mode 100644 similarity index 68% rename from assert_writeln_magic.d rename to ddoc/source/assert_writeln_magic.d index ccfa2b88e7..855c1c8fa8 --- a/assert_writeln_magic.d +++ b/ddoc/source/assert_writeln_magic.d @@ -177,7 +177,7 @@ private: Out fl; } -void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor) +void parseString(Visitor)(ubyte[] sourceCode, Visitor visitor) { import dparse.lexer; import dparse.parser : parseModule; @@ -188,89 +188,36 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor) const(Token)[] tokens = getTokensForParser(sourceCode, config, &cache).array; RollbackAllocator rba; - auto m = parseModule(tokens, fileName, &rba); + auto m = parseModule(tokens, "magic.d", &rba); visitor.visit(m); } -void parseFile(string fileName, string destFile) +private auto assertWritelnModuleImpl(string fileText) { - import std.array : uninitializedArray; - - auto inFile = File(fileName); - if (inFile.size == 0) - warningf("%s is empty", inFile.name); - - ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size)); - if (sourceCode.length == 0) - return; - - inFile.rawRead(sourceCode); - auto fl = FileLines(fileName, destFile); + import std.string : representation; + auto fl = FileLines(fileText); auto visitor = new TestVisitor!(typeof(fl))(fl); - parseString(sourceCode, fileName, visitor); + // libdparse doesn't allow to work on immutable source code + parseString(cast(ubyte[]) fileText.representation, visitor); delete visitor; + return fl; } -// Modify a path under oldBase to a new path with the same subpath under newBase. -// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar` -string rebasePath(string path, string oldBase, string newBase) +auto assertWritelnModule(string fileText) { - import std.path : absolutePath, buildPath, relativePath; - return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath)); + return assertWritelnModuleImpl(fileText).buildLines; } - -version(unittest) { void main(){} } else -void main(string[] args) +auto assertWritelnBlock(string fileText) { - import std.file; - import std.getopt; - import std.path; - - string inputDir, outputDir; - string[] ignoredFiles; - - auto helpInfo = getopt(args, config.required, - "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir, - "outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir, - "ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles); - - if (helpInfo.helpWanted) - { - return defaultGetoptPrinter(`assert_writeln_magic -Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls. -`, helpInfo.options); - } - - inputDir = inputDir.asNormalizedPath.array; - - DirEntry[] files; - - // inputDir as default output directory - if (!outputDir.length) - outputDir = inputDir; - - if (inputDir.isFile) - { - files = [DirEntry(inputDir)]; - inputDir = ""; - } - else - { - files = dirEntries(inputDir, SpanMode.depth).filter!( - a => a.name.endsWith(".d") && !a.name.canFind(".git")).array; - } - - foreach (file; files) + auto source = "unittest{\n" ~ fileText ~ "}\n"; + auto fl = assertWritelnModuleImpl(source); + auto app = appender!string; + foreach (line; fl.lines[1 .. $ - 2]) { - if (!ignoredFiles.any!(x => file.name.canFind(x))) - { - // single files - if (inputDir.length == 0) - parseFile(file.name, outputDir); - else - parseFile(file.name, file.name.rebasePath(inputDir, outputDir)); - } + app ~= line; + app ~= "\n"; } + return app.data; } /** @@ -284,47 +231,26 @@ struct FileLines string[] lines; string destFile; - bool overwriteInputFile; bool hasWrittenChanges; - this(string inputFile, string destFile) - { - stderr.writefln("%s -> %s", inputFile, destFile); - this.overwriteInputFile = inputFile == destFile; - this.destFile = destFile; - lines = File(inputFile).byLineCopy.array; - - destFile.dirName.mkdirRecurse; - } - - // dumps all changes - ~this() + this(string inputText) { - if (overwriteInputFile) - { - if (hasWrittenChanges) - { - auto tmpFile = File(destFile ~ ".tmp", "w"); - writeLinesToFile(tmpFile); - tmpFile.close; - tmpFile.name.rename(destFile); - } - } - else - { - writeLinesToFile(File(destFile, "w")); - } + lines = inputText.split("\n"); } // writes all changes to a random, temporary file - void writeLinesToFile(File outFile) { + auto buildLines() { + auto app = appender!string; // dump file foreach (line; lines) - outFile.writeln(line); + { + app ~= line; + app ~= "\n"; + } // within the docs we automatically inject std.stdio (hence we need to do the same here) // writeln needs to be @nogc, @safe, pure and nothrow (we just fake it) - outFile.writeln("// \nprivate void writeln(T)(T l) { }"); - outFile.flush; + app ~= "// \nprivate void writeln(T)(T l) { }"; + return app.data; } string opIndex(size_t i) { return lines[i]; } @@ -349,7 +275,7 @@ version(unittest) import std.string : representation; auto mock = FileLinesMock(sourceCode.split("\n")); auto visitor = new TestVisitor!(typeof(mock))(mock); - parseString(sourceCode.representation.dup, "testmodule", visitor); + parseString(sourceCode.representation.dup, visitor); delete visitor; return mock; } diff --git a/ddoc_preprocessor.d b/ddoc/source/preprocessor.d old mode 100755 new mode 100644 similarity index 82% rename from ddoc_preprocessor.d rename to ddoc/source/preprocessor.d index 2a425b93bb..8c236e6dee --- a/ddoc_preprocessor.d +++ b/ddoc/source/preprocessor.d @@ -24,12 +24,14 @@ struct Config { string dmdBinPath = "dmd"; string outputFile; - string cwd = __FILE_FULL_PATH__.dirName; + string cwd = __FILE_FULL_PATH__.dirName.dirName.dirName; } Config config; +version(IsExecutable) int main(string[] rootArgs) { + import assert_writeln_magic; import std.getopt; auto helpInformation = getopt( rootArgs, std.getopt.config.passThrough, @@ -49,33 +51,71 @@ All unknown options are passed to the compiler. assert(pos >= 0, "An input file (.d or .dd) must be provided"); auto inputFile = args[pos]; auto text = inputFile.readText; - // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017 - args = args[0..pos].chain("-".only, args[pos..$].dropOne).array; + + // for now only non-package modules are supported + if (!inputFile.endsWith("index.d")) + // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017 + args = args[0..pos].chain("-".only, args[pos..$].dropOne).array; // transform and extend the ddoc page text = genGrammar(text); text = genHeader(text); text = genChangelogVersion(inputFile, text); text = genSwitches(text); + text = fixDdocBugs(inputFile, text); + + // Phobos index.d should have been named index.dd + if (inputFile.endsWith(".d") && !inputFile.endsWith("index.d")) + text = assertWritelnModule(text); + + string[string] macros; + macros["SRC_FILENAME"] = "%s\n".format(inputFile.buildNormalizedPath); + return compile(text, args, inputFile, macros); +} - // inject custom, "dynamic" macros - text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath); - return compile(text, args); +auto createTmpDir() +{ + import std.uuid : randomUUID; + auto dir = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "")); + mkdir(dir); + return dir; } -auto compile(R)(R buffer, string[] arguments) +auto compile(R)(R buffer, string[] arguments, string inputFile, string[string] macros = null) { + import core.time : usecs; + import core.thread : Thread; import std.process : pipeProcess, Redirect, wait; auto args = [config.dmdBinPath] ~ arguments; - foreach (arg; ["-c", "-o-", "-"]) + + // Note: ideally we could pass in files directly on stdin. + // However, for package.d files, we need to imitate package directory layout to avoid conflicts + auto tmpDir = createTmpDir; + auto inputTmpFile = tmpDir.buildPath(inputFile); + inputTmpFile.dirName.mkdirRecurse; + std.file.write(inputTmpFile, buffer); + args = args.replace("-", inputTmpFile); + scope(exit) tmpDir.rmdirRecurse; + + if (macros !is null) + { + auto macroString = macros.byPair.map!(a => "%s=%s".format(a[0], a[1])).join("\n"); + auto macroFile = tmpDir.buildPath("macros.ddoc"); + std.file.write(macroFile, macroString); + args ~= macroFile; + } + + foreach (arg; ["-c", "-o-"]) { if (!args.canFind(arg)) args ~= arg; } auto pipes = pipeProcess(args, Redirect.stdin); - pipes.stdin.write(buffer); - pipes.stdin.close; - return wait(pipes.pid); + import std.process : execute; + auto ret = execute(args); + if (ret.status != 0) + stderr.writeln(ret.output); + return ret.status; } // replaces the content of a DDoc macro call @@ -384,3 +424,18 @@ private void highlightSpecialWords(ref string flag, ref string helpText) .to!string; } } + +// Fix ddoc bugs +string fixDdocBugs(string inputFile, string text) +{ + + // https://github.com/dlang/dlang.org/pull/2069#issuecomment-363154934 + // can be removed once the Phobos PR is in stable and master + // https://github.com/dlang/phobos/pull/6126 + if (inputFile.endsWith("exception.d")) + { + text = text.replace(`typeof(new E("", __FILE__, __LINE__)`, `typeof(new E("", string.init, size_t.init)`); + text = text.replace(`typeof(new E(__FILE__, __LINE__)`, `typeof(new E(string.init, size_t.init)`); + } + return text; +} \ No newline at end of file diff --git a/dpl-docs/dub.json b/dpl-docs/dub.json index 762ed98877..223c4fd718 100644 --- a/dpl-docs/dub.json +++ b/dpl-docs/dub.json @@ -1,7 +1,10 @@ { "name": "dpl-docs", "dependencies": { - "ddox": "~>0.16.7" + "ddox": "~>0.16.7", + "ddoc_preprocessor": { + "path": "../ddoc" + } }, "versions": ["VibeNoSSL"], "subConfigurations": { "eventcore": "libasync"} diff --git a/dpl-docs/source/app.d b/dpl-docs/source/app.d index 66d20e224c..7a5c01b060 100644 --- a/dpl-docs/source/app.d +++ b/dpl-docs/source/app.d @@ -1,6 +1,6 @@ module app; -import ddox.main; +import ddox_main; import std.getopt; import std.process; import vibe.core.log; diff --git a/dpl-docs/source/ddox_main.d b/dpl-docs/source/ddox_main.d new file mode 100644 index 0000000000..02c0f3ac31 --- /dev/null +++ b/dpl-docs/source/ddox_main.d @@ -0,0 +1,259 @@ +// Copied from https://github.com/rejectedsoftware/ddox/blob/master/source/ddox/main.d +module ddox_main; + +import ddox.ddoc; +import ddox.ddox; +import ddox.entities; +import ddox.htmlgenerator; +import ddox.htmlserver; +import ddox.parsers.dparse; +import ddox.parsers.jsonparser; + +import vibe.core.core; +import vibe.core.file; +import vibe.data.json; +import vibe.inet.url; +import vibe.http.fileserver; +import vibe.http.router; +import vibe.http.server; +import vibe.stream.operations; +import std.array; +import std.exception : enforce; +import std.file; +import std.getopt; +import std.stdio; +import std.string; + + +int ddoxMain(string[] args) +{ + bool help; + getopt(args, config.passThrough, "h|help", &help); + + if( args.length < 2 || help ){ + showUsage(args); + return help ? 0 : 1; + } + + if( args[1] == "generate-html" && args.length >= 4 ) + return cmdGenerateHtml(args); + if( args[1] == "serve-html" && args.length >= 3 ) + return cmdServeHtml(args); + if( args[1] == "filter" && args.length >= 3 ) + return cmdFilterDocs(args); + if( args[1] == "serve-test" && args.length >= 3 ) + return cmdServeTest(args); + showUsage(args); + return 1; +} + +static import ddox.main; + +alias cmdGenerateHtml = ddox.main.cmdGenerateHtml; +alias cmdServeHtml = ddox.main.cmdServeTest; +alias cmdServeTest = ddox.main.cmdServeTest; +alias parseDocFile = ddox.main.parseDocFile; +alias setupGeneratorInput = ddox.main.setupGeneratorInput; +alias showUsage = ddox.main.showUsage; + +int cmdFilterDocs(string[] args) +{ + string[] excluded, included; + Protection minprot = Protection.Private; + bool keeputests = false; + bool keepinternals = false; + bool unittestexamples = true; + bool nounittestexamples = false; + bool justdoc = false; + getopt(args, + //config.passThrough, + "ex", &excluded, + "in", &included, + "min-protection", &minprot, + "only-documented", &justdoc, + "keep-unittests", &keeputests, + "keep-internals", &keepinternals, + "unittest-examples", &unittestexamples, // deprecated, kept to not break existing scripts + "no-unittest-examples", &nounittestexamples); + + if (keeputests) keepinternals = true; + if (nounittestexamples) unittestexamples = false; + + string jsonfile; + if( args.length < 3 ){ + showUsage(args); + return 1; + } + + Json filterProt(Json json, Json parent, Json last_decl, Json mod) + { + if (last_decl.type == Json.Type.undefined) last_decl = parent; + + string templateName(Json j){ + auto n = j["name"].opt!string(); + auto idx = n.indexOf('('); + if( idx >= 0 ) return n[0 .. idx]; + return n; + } + + if( json.type == Json.Type.Object ){ + auto comment = json["comment"].opt!string; + if( justdoc && comment.empty ){ + if( parent.type != Json.Type.Object || parent["kind"].opt!string() != "template" || templateName(parent) != json["name"].opt!string() ) + return Json.undefined; + } + + Protection prot = Protection.Public; + if( auto p = "protection" in json ){ + switch(p.get!string){ + default: break; + case "private": prot = Protection.Private; break; + case "package": prot = Protection.Package; break; + case "protected": prot = Protection.Protected; break; + } + } + if( comment.strip == "private" ) prot = Protection.Private; + if( prot < minprot ) return Json.undefined; + + auto name = json["name"].opt!string(); + bool is_internal = name.startsWith("__"); + bool is_unittest = name.startsWith("__unittest"); + if (name.startsWith("_staticCtor") || name.startsWith("_staticDtor")) is_internal = true; + else if (name.startsWith("_sharedStaticCtor") || name.startsWith("_sharedStaticDtor")) is_internal = true; + + if (unittestexamples && is_unittest && !comment.empty) { + assert(last_decl.type == Json.Type.object, "Don't have a last_decl context."); + try { + string source = extractUnittestSourceCode(json, mod); + if (last_decl["comment"].opt!string.empty) { + writefln("Warning: Cannot add documented unit test %s to %s, which is not documented.", name, last_decl["name"].opt!string); + } else { + import assert_writeln_magic; + auto rewrittenSource = assertWritelnBlock(source); + last_decl["comment"] ~= format("Example:\n%s$(DDOX_UNITTEST_HEADER %s)\n---\n%s\n---\n$(DDOX_UNITTEST_FOOTER %s)\n", comment.strip, name, rewrittenSource, name); + } + } catch (Exception e) { + writefln("Failed to add documented unit test %s:%s as example: %s", + mod["file"].get!string(), json["line"].get!long, e.msg); + return Json.undefined; + } + } + + if (!keepinternals && is_internal) return Json.undefined; + + if (!keeputests && is_unittest) return Json.undefined; + + if (auto mem = "members" in json) + json["members"] = filterProt(*mem, json, Json.undefined, mod); + } else if( json.type == Json.Type.Array ){ + auto last_child_decl = Json.undefined; + Json[] newmem; + foreach (m; json) { + auto mf = filterProt(m, parent, last_child_decl, mod); + if (mf.type == Json.Type.undefined) continue; + if (mf.type == Json.Type.object && !mf["name"].opt!string.startsWith("__unittest") && icmp(mf["comment"].opt!string.strip, "ditto") != 0) + last_child_decl = mf; + newmem ~= mf; + } + return Json(newmem); + } + return json; + } + + writefln("Reading doc file..."); + auto text = readText(args[2]); + int line = 1; + writefln("Parsing JSON..."); + auto json = parseJson(text, &line); + + writefln("Filtering modules..."); + Json[] dst; + foreach (m; json) { + if ("name" !in m) { + writefln("No name for module %s - ignoring", m["file"].opt!string); + continue; + } + auto n = m["name"].get!string; + bool include = true; + foreach (ex; excluded) + if (n.startsWith(ex)) { + include = false; + break; + } + foreach (inc; included) + if (n.startsWith(inc)) { + include = true; + break; + } + if (include) { + auto doc = filterProt(m, Json.undefined, Json.undefined, m); + if (doc.type != Json.Type.undefined) + dst ~= doc; + } + } + + writefln("Writing filtered docs..."); + auto buf = appender!string(); + writePrettyJsonString(buf, Json(dst)); + std.file.write(args[2], buf.data()); + + return 0; +} + +// from ddox +private string extractUnittestSourceCode(Json decl, Json mod) +{ + auto filename = mod["file"].get!string(); + enforce("line" in decl && "endline" in decl, "Missing line/endline fields."); + auto from = decl["line"].get!long; + auto to = decl["endline"].get!long; + + // read the matching lines out of the file + auto app = appender!string(); + long lc = 1; + foreach (str; File(filename).byLine) { + if (lc >= from) { + app.put(str); + app.put('\n'); + } + if (++lc > to) break; + } + auto ret = app.data; + + // strip the "unittest { .. }" surroundings + auto idx = ret.indexOf("unittest"); + enforce(idx >= 0, format("Missing 'unittest' for unit test at %s:%s.", filename, from)); + ret = ret[idx .. $]; + + idx = ret.indexOf("{"); + enforce(idx >= 0, format("Missing opening '{' for unit test at %s:%s.", filename, from)); + ret = ret[idx+1 .. $]; + + idx = ret.lastIndexOf("}"); + enforce(idx >= 0, format("Missing closing '}' for unit test at %s:%s.", filename, from)); + ret = ret[0 .. idx]; + + // unindent lines according to the indentation of the first line + app = appender!string(); + string indent; + foreach (i, ln; ret.splitLines) { + if (i == 1) { + foreach (j; 0 .. ln.length) + if (ln[j] != ' ' && ln[j] != '\t') { + indent = ln[0 .. j]; + break; + } + } + if (i > 0 || ln.strip.length > 0) { + size_t j = 0; + while (j < indent.length && !ln.empty) { + if (ln.front != indent[j]) break; + ln.popFront(); + j++; + } + app.put(ln); + app.put('\n'); + } + } + return app.data; +} diff --git a/posix.mak b/posix.mak index da7d79be1b..90c0ab1ff5 100644 --- a/posix.mak +++ b/posix.mak @@ -126,6 +126,18 @@ # in the build folder `.generated`, s.t. Ddoc can be run on the modified sources. # # See also: https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org +# +# Custom DDoc wrapper +# ------------------- +# +# `ddoc.d` is a wrapper around Ddoc and allows expanding Ddoc macros dynamically +# before actually running Ddoc. +# Currently this is used for: +# - TOC +# - dynamic TOC generation +# - GRAMMAR overview generation +# - CHANGELOG menu generation +# - assert -> writeln magic PWD=$(shell pwd) MAKEFILE=$(firstword $(MAKEFILE_LIST)) SHELL:=/bin/bash @@ -171,21 +183,13 @@ $(shell [ ! -d $(DMD_DIR) ] && git clone --depth=1 ${GIT_HOME}/dmd $(DMD_DIR)) $(shell [ ! -d $(DRUNTIME_DIR) ] && git clone --depth=1 ${GIT_HOME}/druntime $(DRUNTIME_DIR)) ################################################################################ -# Automatically generated directories -PHOBOS_DIR_GENERATED=$(GENERATED)/phobos-prerelease -PHOBOS_LATEST_DIR_GENERATED=$(GENERATED)/phobos-latest -# The assert_writeln_magic tool transforms all source files from Phobos. Hence -# - a temporary folder with a copy of Phobos needs to be generated -# - a list of all files in Phobos and the temporary copy is needed to setup proper -# Makefile dependencies and rules +# Automatically clone Phobos PHOBOS_FILES := $(shell find $(PHOBOS_DIR) -name '*.d' -o -name '*.mak' -o -name '*.ddoc') -PHOBOS_FILES_GENERATED := $(subst $(PHOBOS_DIR), $(PHOBOS_DIR_GENERATED), $(PHOBOS_FILES)) ifndef RELEASE # TODO: should be replaced by make targets $(shell [ ! -d $(PHOBOS_DIR) ] && git clone --depth=1 ${GIT_HOME}/phobos $(PHOBOS_DIR)) $(shell [ ! -d $(PHOBOS_LATEST_DIR) ] && git clone -b v${LATEST} --depth=1 ${GIT_HOME}/phobos $(PHOBOS_LATEST_DIR)) PHOBOS_LATEST_FILES := $(shell find $(PHOBOS_LATEST_DIR) -name '*.d' -o -name '*.mak' -o -name '*.ddoc') - PHOBOS_LATEST_FILES_GENERATED := $(subst $(PHOBOS_LATEST_DIR), $(PHOBOS_LATEST_DIR_GENERATED), $(PHOBOS_LATEST_FILES)) endif ################################################################################ @@ -283,7 +287,7 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \ # Ddoc binaries ################################################################################ -DDOC_BIN:=$G/ddoc +DDOC_BIN:=$G/ddoc_preprocessor DDOC_BIN_DMD:=$(DDOC_BIN) --compiler=$(DMD) ################################################################################ @@ -683,22 +687,26 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version ################################################################################ .PHONY: phobos-prerelease -phobos-prerelease : ${PHOBOS_FILES_GENERATED} druntime-target $(STD_DDOC_PRERELEASE) - $(MAKE) --directory=$(PHOBOS_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) +phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD) \ + $G/changelog/next-version + $(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) \ + DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))" -phobos-release : ${PHOBOS_FILES_GENERATED} druntime-target $(STD_DDOC_RELEASE) - $(MAKE) --directory=$(PHOBOS_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) +phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE) $(DDOC_BIN) $(DMD) + $(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) \ + DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))" -phobos-latest : ${PHOBOS_LATEST_FILES_GENERATED} druntime-latest-target $(STD_DDOC_LATEST) - $(MAKE) --directory=$(PHOBOS_LATEST_DIR_GENERATED) -f posix.mak html $(DDOC_VARS_LATEST_HTML) +phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD_LATEST) + $(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML) \ + DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD_LATEST))" -phobos-prerelease-verbatim : ${PHOBOS_FILES_GENERATED} druntime-target \ +phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \ $W/phobos-prerelease/index.verbatim $W/phobos-prerelease/index.verbatim : verbatim.ddoc \ $W/phobos-prerelease/object.verbatim \ - $W/phobos-prerelease/mars.verbatim $G/changelog/next-version - ${MAKE} --directory=${PHOBOS_DIR_GENERATED} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \ - DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim + $W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD) $(DDOC_BIN) + ${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \ + DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))" $(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim) mv $W/phobos-prerelease-verbatim/* $(dir $@) rm -r $W/phobos-prerelease-verbatim @@ -748,7 +756,7 @@ else endif $G/docs-latest.json : ${DMD_LATEST} ${DMD_LATEST_DIR} \ - ${DRUNTIME_LATEST_DIR} ${PHOBOS_LATEST_FILES_GENERATED} | dpl-docs + ${DRUNTIME_LATEST_DIR} ${PHOBOS_LATEST_FILES} | dpl-docs # remove this after https://github.com/dlang/dmd/pull/7513 has been merged if [ -f $(DMD_LATEST_DIR)/src/*/objc_glue_stubs.d ] ; then \ DMD_EXCLUDE_LATEST_BASH="-e /objc_glue.d/d"; \ @@ -757,17 +765,17 @@ $G/docs-latest.json : ${DMD_LATEST} ${DMD_LATEST_DIR} \ sed -e /mscoff/d $${DMD_EXCLUDE_LATEST_BASH} ${DMD_EXCLUDE_LATEST} find ${DRUNTIME_LATEST_DIR}/src -name '*.d' | \ sed -e /unittest.d/d -e /gcstub/d >> $G/.latest-files.txt - find ${PHOBOS_LATEST_DIR_GENERATED} -name '*.d' | \ + find ${PHOBOS_LATEST_DIR} -name '*.d' | \ sed -e /unittest.d/d | sort >> $G/.latest-files.txt ${DMD_LATEST} -J$(DMD_LATEST_DIR)/res -J$(dir $(DMD_LATEST)) -c -o- -version=CoreDdoc \ -version=MARS -version=CoreDdoc -version=StdDdoc -Df$G/.latest-dummy.html \ - -Xf$@ -I${PHOBOS_LATEST_DIR_GENERATED} @$G/.latest-files.txt + -Xf$@ -I${PHOBOS_LATEST_DIR} @$G/.latest-files.txt ${DPL_DOCS} filter $@ --min-protection=Protected \ --only-documented $(MOD_EXCLUDES_LATEST) rm -f $G/.latest-files.txt $G/.latest-dummy.html $G/docs-prerelease.json : ${DMD} ${DMD_DIR} ${DRUNTIME_DIR} \ - ${PHOBOS_FILES_GENERATED} | dpl-docs + ${PHOBOS_FILES} | dpl-docs # remove this after https://github.com/dlang/dmd/pull/7513 has been merged if [ -f $(DMD_DIR)/src/*/objc_glue_stubs.d ] ; then \ DMD_EXCLUDE_PRERELEASE="-e /objc_glue.d/d"; \ @@ -776,11 +784,11 @@ $G/docs-prerelease.json : ${DMD} ${DMD_DIR} ${DRUNTIME_DIR} \ sed -e /mscoff/d $${DMD_EXCLUDE_PRERELEASE} > $G/.prerelease-files.txt find ${DRUNTIME_DIR}/src -name '*.d' | \ sed -e /unittest/d >> $G/.prerelease-files.txt - find ${PHOBOS_DIR_GENERATED} -name '*.d' | \ + find ${PHOBOS_DIR} -name '*.d' | \ sed -e /unittest.d/d | sort >> $G/.prerelease-files.txt ${DMD} -J$(DMD_DIR)/res -J$(dir $(DMD)) -c -o- -version=MARS -version=CoreDdoc \ -version=StdDdoc -Df$G/.prerelease-dummy.html \ - -Xf$@ -I${PHOBOS_DIR_GENERATED} @$G/.prerelease-files.txt + -Xf$@ -I${PHOBOS_DIR} @$G/.prerelease-files.txt ${DPL_DOCS} filter $@ --min-protection=Protected \ --only-documented $(MOD_EXCLUDES_PRERELEASE) rm -f $G/.prerelease-files.txt $G/.prerelease-dummy.html @@ -841,42 +849,6 @@ d-release.tag d-tags-release.json : chmgen.d $(STABLE_DMD) $(ALL_FILES) phobos-r d-prerelease.tag d-tags-prerelease.json : chmgen.d $(STABLE_DMD) $(ALL_FILES) phobos-prerelease druntime-prerelease chm-nav-prerelease.json $(STABLE_RDMD) chmgen.d --root=$W --target prerelease -################################################################################ -# Assert -> writeln magic -# ----------------------- -# -# - This transforms assert(a == b) to writeln(a); // b -# - It creates a copy of Phobos to apply the transformations -# - All "d" files are piped through the transformator, -# other needed files (e.g. posix.mak) get copied over -# -# See also: https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org -################################################################################ - -ASSERT_WRITELN_BIN = $(GENERATED)/assert_writeln_magic - -$(ASSERT_WRITELN_BIN): assert_writeln_magic.d $(DUB) $(STABLE_DMD) - @mkdir -p $(dir $@) - $(DUB) -v build --single --compiler=$(STABLE_DMD) $< - @mv ./assert_writeln_magic $@ - -$(ASSERT_WRITELN_BIN)_test: assert_writeln_magic.d $(DUB) $(STABLE_DMD) - @mkdir -p $(dir $@) - $(DUB) -v build --single --compiler=$(STABLE_DMD) --build=unittest $< - @mv ./assert_writeln_magic $@ - -$(PHOBOS_FILES_GENERATED): $(PHOBOS_DIR_GENERATED)/%: $(PHOBOS_DIR)/% $(DUB) $(ASSERT_WRITELN_BIN) - @mkdir -p $(dir $@) - @if [ $(subst .,, $(suffix $@)) = "d" ] && [ "$@" != "$(PHOBOS_DIR_GENERATED)/index.d" ] ; then \ - $(ASSERT_WRITELN_BIN) -i $< -o $@ ; \ - else cp $< $@ ; fi - -$(PHOBOS_LATEST_FILES_GENERATED): $(PHOBOS_LATEST_DIR_GENERATED)/%: $(PHOBOS_LATEST_DIR)/% $(DUB) $(ASSERT_WRITELN_BIN) - @mkdir -p $(dir $@) - @if [ $(subst .,, $(suffix $@)) = "d" ] && [ "$@" != "$(PHOBOS_LATEST_DIR_GENERATED)/index.d" ] ; then \ - $(ASSERT_WRITELN_BIN) -i $< -o $@ ; \ - else cp $< $@ ; fi - ################################################################################ # Style tests ################################################################################ @@ -886,7 +858,7 @@ test_dspec: dspec_tester.d $(DMD) $(PHOBOS_LIB) $(DMD) -run $< --compiler=$(DMD) .PHONY: -test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all | $(STABLE_DMD) +test: test_dspec test/next_version.sh all | $(STABLE_DMD) $(DUB) @echo "Searching for trailing whitespace" @grep -n '[[:blank:]]$$' $$(find . -type f -name "*.dd" | grep -v .generated) ; test $$? -eq 1 @echo "Searching for tabs" @@ -894,8 +866,8 @@ test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all | $(STABLE_ @echo "Checking DDoc's output" $(STABLE_RDMD) -main -unittest check_ddoc.d $(STABLE_RDMD) check_ddoc.d $$(find $W -type f -name "*.html" -not -path "$W/phobos/*") - @echo "Executing assert_writeln_magic tests" - $< + @echo "Executing ddoc_preprocessor tests" + $(DUB) test --compiler=${STABLE_DMD} --root ddoc @echo "Executing next_version tests" test/next_version.sh @@ -983,12 +955,12 @@ $G/contributors_list.ddoc: | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR) # Custom DDoc wrapper # ------------------ # -# This allows extending Ddoc files dynamically on-the-fly. -# It is currently only used for the specification pages +# This allows extending Ddoc files and D source code files dynamically on-the-fly. ################################################################################ -$(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD) $(DMD) - $(STABLE_RDMD) -version=DdocOptions --build-only -g -of$@ -I$(DMD_DIR)/src $< +$(DDOC_BIN): ddoc/source/preprocessor.d ddoc/source/assert_writeln_magic.d | $(STABLE_DMD) + $(STABLE_DMD_BIN_ROOT)/dub build --compiler=$(STABLE_DMD) --root=ddoc && \ + mv ddoc/ddoc_preprocessor $@ ################################################################################ # Build and render the DMD man page