remove some custom logic in testament around flags, testExec (#18090)

* remove some custom logic in testament around flags, testExec

* remove testExec, custom logic around flags from testament

* fixup

Author: timotheecour

testament/categories.nim

@@ -23,7 +23,6 @@ const
     "debugger",
     "dll",
     "examples",
-    "flags",
     "gc",
     "io",
     "js",
@@ -48,26 +47,6 @@ proc isTestFile*(file: string): bool =
   let (_, name, ext) = splitFile(file)
   result = ext == ".nim" and name.startsWith("t")
 
-# --------------------- flags tests -------------------------------------------
-
-proc flagTests(r: var TResults, cat: Category, options: string) =
-  # --genscript
-  const filename = testsDir/"flags"/"tgenscript"
-  const genopts = " --genscript"
-  let nimcache = nimcacheDir(filename, genopts, targetC)
-  testSpec r, makeTest(filename, genopts, cat)
-
-  when defined(windows):
-    testExec r, makeTest(filename, " cmd /c cd " & nimcache &
-                         " && compile_tgenscript.bat", cat)
-
-  elif defined(posix):
-    testExec r, makeTest(filename, " sh -c \"cd " & nimcache &
-                         " && sh compile_tgenscript.sh\"", cat)
-
-  # Run
-  testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat)
-
 # --------------------- DLL generation tests ----------------------------------
 
 proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) =
@@ -705,8 +684,6 @@ proc processCategory(r: var TResults, cat: Category,
         jsTests(r, cat, options)
     of "dll":
       dllTests(r, cat, options)
-    of "flags":
-      flagTests(r, cat, options)
     of "gc":
       gcTests(r, cat, options)
     of "longgc":

testament/testament.nim

@@ -621,22 +621,6 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) =
     if exitCode != 0: given.err = reExitcodesDiffer
   if given.err == reSuccess: inc(r.passed)
 
-proc testExec(r: var TResults, test: TTest) =
-  # runs executable or script, just goes by exit code
-  if not checkDisabled(r, test): return
-
-  inc(r.total)
-  let (outp, errC) = execCmdEx(test.options.strip())
-  var given: TSpec
-  if errC == 0:
-    given.err = reSuccess
-  else:
-    given.err = reExitcodesDiffer
-    given.msg = outp
-
-  if given.err == reSuccess: inc(r.passed)
-  r.addResult(test, targetC, "", given.msg, given.err)
-
 proc makeTest(test, options: string, cat: Category): TTest =
   result.cat = cat
   result.name = test

tests/flags/tgenscript.nim

@@ -33,16 +33,23 @@ const
 
 proc runNimCmd(file, options = "", rtarg = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
-  doAssert fileabs.fileExists, fileabs
+  # doAssert fileabs.fileExists, fileabs # disabled because this allows passing `nim r --eval:code fakefile`
   let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}"
   result = execCmdEx(cmd)
-  when false:  echo result[0] & "\n" & result[1] # for debugging
+  when false: # for debugging
+    echo cmd
+    echo result[0] & "\n" & $result[1]
 
 proc runNimCmdChk(file, options = "", rtarg = ""): string =
   let (ret, status) = runNimCmd(file, options, rtarg = rtarg)
   doAssert status == 0, $(file, options) & "\n" & ret
   ret
 
+proc genShellCmd(filename: string): string =
+  let filename = filename.quoteShell
+  when defined(windows): "cmd /c " & filename # or "cmd /c " ?
+  else: "sh " & filename
+
 when defined(nimTrunnerFfi):
   block: # mevalffi
     when defined(openbsd):
@@ -71,7 +78,9 @@ foo:0.03:asdf:103:105
 ret=[s1:foobar s2:foobar age:25 pi:3.14]
 """, output
 
-else: # don't run twice the same test
+elif not defined(nimTestsTrunnerDebugging):
+  # don't run twice the same test with `nimTrunnerFfi`
+  # use `-d:nimTestsTrunnerDebugging` for debugging convenience when you want to just run 1 test
   import std/strutils
   import std/json
   template check2(msg) = doAssert msg in output, output
@@ -330,3 +339,21 @@ running: v2
     doAssert "D20210428T161003" in j["defined_symbols"].to(seq[string])
     doAssert j["version"].to(string) == NimVersion
     doAssert j["nimExe"].to(string) == getCurrentCompilerExe()
+
+  block: # genscript
+    const nimcache2 = buildDir / "D20210524T212851"
+    removeDir(nimcache2)
+    let input = "tgenscript_fakefile" # no need for a real file, --eval is good enough
+    let output = runNimCmdChk(input, fmt"""--genscript --nimcache:{nimcache2.quoteShell} --eval:"echo(12345)" """)
+    doAssert output.len == 0, output
+    let ext = when defined(windows): ".bat" else: ".sh"
+    let filename = fmt"compile_{input}{ext}" # synchronize with `generateScript`
+    doAssert fileExists(nimcache2/filename), nimcache2/filename
+    let (outp, status) = execCmdEx(genShellCmd(filename), options = {poStdErrToStdOut}, workingDir = nimcache2)
+    doAssert status == 0, outp
+    let (outp2, status2) = execCmdEx(nimcache2 / input, options = {poStdErrToStdOut})
+    doAssert outp2 == "12345\n", outp2
+    doAssert status2 == 0
+
+else:
+  discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled