Skip to content

Commit e116c9d

Browse files
wilzbachwilzbach
committed
Integrate the assert_writeln_magic in the preprocessor
1 parent a944ad3 commit e116c9d

File tree

6 files changed

+81
-142
lines changed

6 files changed

+81
-142
lines changed

‎ddoc/.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/libddoc_preprocessor.a
2+
/ddoc_preprocessor
3+
/ddoc_preprocessor-test-application

‎ddoc/dub.sdl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
name "ddoc_preprocessor"
2+
description "Preprocesses source code before running Ddoc over it"
3+
dependency "libdparse" version="0.7.2-alpha.4"
4+
targetType "executable"

‎ddoc/dub.selections.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"fileVersion": 1,
3+
"versions": {
4+
"libdparse": "0.7.2-alpha.4"
5+
}
6+
}

‎ddoc/source/assert_writeln_magic.d

100755100644
Lines changed: 19 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ private:
177177
Out fl;
178178
}
179179

180-
void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
180+
void parseString(Visitor)(ubyte[] sourceCode, Visitor visitor)
181181
{
182182
import dparse.lexer;
183183
import dparse.parser : parseModule;
@@ -188,89 +188,19 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
188188
const(Token)[] tokens = getTokensForParser(sourceCode, config, &cache).array;
189189

190190
RollbackAllocator rba;
191-
auto m = parseModule(tokens, fileName, &rba);
191+
auto m = parseModule(tokens, "magic.d", &rba);
192192
visitor.visit(m);
193193
}
194194

195-
void parseFile(string fileName, string destFile)
195+
auto assertWriteln(string fileText)
196196
{
197-
import std.array : uninitializedArray;
198-
199-
auto inFile = File(fileName);
200-
if (inFile.size == 0)
201-
warningf("%s is empty", inFile.name);
202-
203-
ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
204-
if (sourceCode.length == 0)
205-
return;
206-
207-
inFile.rawRead(sourceCode);
208-
auto fl = FileLines(fileName, destFile);
197+
import std.string : representation;
198+
auto fl = FileLines(fileText);
209199
auto visitor = new TestVisitor!(typeof(fl))(fl);
210-
parseString(sourceCode, fileName, visitor);
200+
// libdparse doesn't allow to work on immutable source code
201+
parseString(cast(ubyte[]) fileText.representation, visitor);
211202
delete visitor;
212-
}
213-
214-
// Modify a path under oldBase to a new path with the same subpath under newBase.
215-
// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar`
216-
string rebasePath(string path, string oldBase, string newBase)
217-
{
218-
import std.path : absolutePath, buildPath, relativePath;
219-
return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath));
220-
}
221-
222-
version(unittest) { void main(){} } else
223-
void main(string[] args)
224-
{
225-
import std.file;
226-
import std.getopt;
227-
import std.path;
228-
229-
string inputDir, outputDir;
230-
string[] ignoredFiles;
231-
232-
auto helpInfo = getopt(args, config.required,
233-
"inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
234-
"outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir,
235-
"ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles);
236-
237-
if (helpInfo.helpWanted)
238-
{
239-
return defaultGetoptPrinter(`assert_writeln_magic
240-
Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls.
241-
`, helpInfo.options);
242-
}
243-
244-
inputDir = inputDir.asNormalizedPath.array;
245-
246-
DirEntry[] files;
247-
248-
// inputDir as default output directory
249-
if (!outputDir.length)
250-
outputDir = inputDir;
251-
252-
if (inputDir.isFile)
253-
{
254-
files = [DirEntry(inputDir)];
255-
inputDir = "";
256-
}
257-
else
258-
{
259-
files = dirEntries(inputDir, SpanMode.depth).filter!(
260-
a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
261-
}
262-
263-
foreach (file; files)
264-
{
265-
if (!ignoredFiles.any!(x => file.name.canFind(x)))
266-
{
267-
// single files
268-
if (inputDir.length == 0)
269-
parseFile(file.name, outputDir);
270-
else
271-
parseFile(file.name, file.name.rebasePath(inputDir, outputDir));
272-
}
273-
}
203+
return fl.buildLines;
274204
}
275205

276206
/**
@@ -284,47 +214,26 @@ struct FileLines
284214

285215
string[] lines;
286216
string destFile;
287-
bool overwriteInputFile;
288217
bool hasWrittenChanges;
289218

290-
this(string inputFile, string destFile)
219+
this(string inputText)
291220
{
292-
stderr.writefln("%s -> %s", inputFile, destFile);
293-
this.overwriteInputFile = inputFile == destFile;
294-
this.destFile = destFile;
295-
lines = File(inputFile).byLineCopy.array;
296-
297-
destFile.dirName.mkdirRecurse;
298-
}
299-
300-
// dumps all changes
301-
~this()
302-
{
303-
if (overwriteInputFile)
304-
{
305-
if (hasWrittenChanges)
306-
{
307-
auto tmpFile = File(destFile ~ ".tmp", "w");
308-
writeLinesToFile(tmpFile);
309-
tmpFile.close;
310-
tmpFile.name.rename(destFile);
311-
}
312-
}
313-
else
314-
{
315-
writeLinesToFile(File(destFile, "w"));
316-
}
221+
lines = inputText.split("\n");
317222
}
318223

319224
// writes all changes to a random, temporary file
320-
void writeLinesToFile(File outFile) {
225+
auto buildLines() {
226+
auto app = appender!string;
321227
// dump file
322228
foreach (line; lines)
323-
outFile.writeln(line);
229+
{
230+
app ~= line;
231+
app ~= "\n";
232+
}
324233
// within the docs we automatically inject std.stdio (hence we need to do the same here)
325234
// writeln needs to be @nogc, @safe, pure and nothrow (we just fake it)
326-
outFile.writeln("// \nprivate void writeln(T)(T l) { }");
327-
outFile.flush;
235+
app ~= "// \nprivate void writeln(T)(T l) { }";
236+
return app.data;
328237
}
329238

330239
string opIndex(size_t i) { return lines[i]; }
@@ -349,7 +258,7 @@ version(unittest)
349258
import std.string : representation;
350259
auto mock = FileLinesMock(sourceCode.split("\n"));
351260
auto visitor = new TestVisitor!(typeof(mock))(mock);
352-
parseString(sourceCode.representation.dup, "testmodule", visitor);
261+
parseString(sourceCode.representation.dup, visitor);
353262
delete visitor;
354263
return mock;
355264
}

‎ddoc/source/preprocessor.d

100755100644
Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ struct Config
2222
{
2323
string dmdBinPath = "dmd";
2424
string outputFile;
25-
string cwd = __FILE_FULL_PATH__.dirName;
25+
string cwd = __FILE_FULL_PATH__.dirName.dirName.dirName;
2626
}
2727
Config config;
2828

29+
version(unittest) {} else
2930
int main(string[] rootArgs)
3031
{
32+
import assert_writeln_magic;
3133
import std.getopt;
3234
auto helpInformation = getopt(
3335
rootArgs, std.getopt.config.passThrough,
@@ -46,32 +48,53 @@ All unknown options are passed to the compiler.
4648
assert(pos >= 0, "An input file (.d or .dd) must be provided");
4749
auto inputFile = args[pos];
4850
auto text = inputFile.readText;
49-
// replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
50-
args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
51+
52+
// for now only non-package modules are supported
53+
if (!inputFile.endsWith("package.d", "index.d"))
54+
// replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
55+
args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
5156

5257
// transform and extend the ddoc page
5358
text = genGrammar(text);
5459
text = genHeader(text);
5560
text = genChangelogVersion(inputFile, text);
5661

62+
if (inputFile.endsWith(".d"))
63+
text = assertWriteln(text);
64+
5765
// inject custom, "dynamic" macros
58-
text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
66+
if (inputFile.endsWith(".dd", "index.d"))
67+
text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
68+
else if (inputFile.endsWith(".d"))
69+
text ~= "/**\nMacros:\n\nSRC_FILENAME=%s\n*/".format(inputFile.buildNormalizedPath);
5970
return compile(text, args);
6071
}
6172

6273
auto compile(R)(R buffer, string[] arguments)
6374
{
75+
import core.time : usecs;
76+
import core.thread : Thread;
6477
import std.process : pipeProcess, Redirect, wait;
6578
auto args = [config.dmdBinPath] ~ arguments;
66-
foreach (arg; ["-c", "-o-", "-"])
79+
80+
import std.file : write, tempDir;
81+
import std.uuid : randomUUID;
82+
auto fileName = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ ".d");
83+
std.file.write(fileName, buffer);
84+
scope(exit) fileName.remove;
85+
args = args.replace("-", fileName);
86+
87+
foreach (arg; ["-c", "-o-"])
6788
{
6889
if (!args.canFind(arg))
6990
args ~= arg;
7091
}
7192
auto pipes = pipeProcess(args, Redirect.stdin);
72-
pipes.stdin.write(buffer);
73-
pipes.stdin.close;
74-
return wait(pipes.pid);
93+
import std.process : execute;
94+
auto ret = execute(args);
95+
if (ret.status != 0)
96+
stderr.writeln(ret.output);
97+
return ret.status;
7598
}
7699

77100
// replaces the content of a DDoc macro call

‎posix.mak

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,9 @@
134134
# before actually running Ddoc.
135135
# Currently this is used for:
136136
# - TOC
137+
# - dynamic TOC generation
138+
# - GRAMMAR overview generation
139+
# - CHANGELOG menu generation
137140
# - assert -> writeln magic
138141
PWD=$(shell pwd)
139142
MAKEFILE=$(firstword $(MAKEFILE_LIST))
@@ -282,7 +285,7 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \
282285
# Ddoc binaries
283286
################################################################################
284287

285-
DDOC_BIN:=$G/ddoc
288+
DDOC_BIN:=$G/ddoc_preprocessor
286289

287290
################################################################################
288291
# Resources
@@ -669,23 +672,24 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version
669672
################################################################################
670673

671674
.PHONY: phobos-prerelease
672-
phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD)
675+
phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD) \
676+
$G/changelog/next-version
673677
$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) \
674-
DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
678+
DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
675679

676680
phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE) $(DDOC_BIN) $(DMD)
677681
$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) \
678-
DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
682+
DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
679683

680-
phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD)
684+
phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD_LATEST)
681685
$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML) \
682-
DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
686+
DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD_LATEST))"
683687

684688
phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
685689
$W/phobos-prerelease/index.verbatim
686690
$W/phobos-prerelease/index.verbatim : verbatim.ddoc \
687691
$W/phobos-prerelease/object.verbatim \
688-
$W/phobos-prerelease/mars.verbatim $G/changelog/next-version
692+
$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD)
689693
${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
690694
DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim
691695
$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
@@ -838,7 +842,7 @@ test_dspec: dspec_tester.d $(DMD) $(PHOBOS_LIB)
838842
@echo "Test the D Language specification"
839843
$(DMD) -run $< --compiler=$(DMD)
840844

841-
test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all
845+
test: test_dspec test/next_version.sh all
842846
@echo "Searching for trailing whitespace"
843847
@grep -n '[[:blank:]]$$' $$(find . -type f -name "*.dd" | grep -v .generated) ; test $$? -eq 1
844848
@echo "Searching for tabs"
@@ -847,8 +851,8 @@ test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all
847851
@grep -n "UNDEFINED MACRO" $$(find $W -type f -name "*.html" -not -path "$W/phobos/*") ; test $$? -eq 1
848852
@echo "Searching for undefined ddoc"
849853
@grep -rn '[$$](' $$(find $W/phobos-prerelease -type f -name "*.html") ; test $$? -eq 1
850-
@echo "Executing assert_writeln_magic tests"
851-
$<
854+
@echo "Executing ddoc_preprocessor tests"
855+
dub -C ddoc test
852856
@echo "Executing next_version tests"
853857
test/next_version.sh
854858

@@ -933,22 +937,12 @@ $G/contributors_list.ddoc: | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
933937
# Custom DDoc wrapper
934938
# ------------------
935939
#
936-
# This allows extending Ddoc files dynamically on-the-fly.
937-
# It is currently only used for the specification pages
938-
#
939-
# It does:
940-
# - dynamic TOC generation
941-
# - GRAMMAR overview generation
942-
# - CHANGELOG menu generation
943-
# - Assert -> writeln magic (https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org)
944-
# - This transforms assert(a == b) to writeln(a); // b
945-
# - It creates a copy of Phobos to apply the transformations
946-
# - All "d" files are piped through the transformator,
947-
# other needed files (e.g. posix.mak) get copied over
940+
# This allows extending Ddoc files and D source code files dynamically on-the-fly.
948941
################################################################################
949942

950-
$(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD)
951-
$(STABLE_DMD) -g -of$@ $<
943+
$(DDOC_BIN): ddoc/source/preprocessor.d ddoc/source/assert_writeln_magic.d | $(STABLE_DMD)
944+
$(STABLE_DMD_BIN_ROOT)/dub build --compiler=$(STABLE_DMD) --root=ddoc && \
945+
mv ddoc/ddoc_preprocessor $@
952946

953947
################################################################################
954948
# Build and render the DMD man page

0 commit comments

Comments
 (0)