Merge remote-tracking branch 'upstream/main' into incomplete-url-string-sanitization

Conflicts:
	config/identical-files.json
	javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.ql
	javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll
	ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll

Author: aibaars

.github/workflows/check-qldoc.yml

@@ -0,0 +1,50 @@
+name: "Check QLdoc coverage"
+
+on:
+  pull_request:
+    paths:
+      - "*/ql/lib/**"
+      - .github/workflows/check-qldoc.yml
+    branches:
+      - main
+      - "rc/*"
+
+jobs:
+  qldoc:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install CodeQL
+        run: |
+          gh extension install github/gh-codeql
+          gh codeql set-channel nightly
+          gh codeql version
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Check QLdoc coverage
+        shell: bash
+        run: |
+          EXIT_CODE=0
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -o '^[a-z]*/ql/lib' || true; } | sort -u)"
+          for pack_dir in ${changed_lib_packs}; do
+            lang="${pack_dir%/ql/lib}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+          done
+          git checkout HEAD^
+          for pack_dir in ${changed_lib_packs}; do
+            lang="${pack_dir%/ql/lib}"
+            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
+            awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
+            UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"
+            if [ -n "$UNDOCUMENTED" ]; then
+              echo "$UNDOCUMENTED" | awk -F, '{gsub(/"/,""); print "::warning file='"${pack_dir}"'/"$1",line="$2"::Missing QLdoc for "$5, $3 }'
+              EXIT_CODE=1
+            fi
+          done
+          exit "${EXIT_CODE}"

config/blame-deprecations.mjs

@@ -0,0 +1,58 @@
+import fs from "fs";
+import path from "path";
+import cp from "child_process";
+function* walk(dir) {
+  for (const file of fs.readdirSync(dir)) {
+    const filePath = path.join(dir, file);
+    if (fs.statSync(filePath).isDirectory()) {
+      yield* walk(filePath);
+    } else {
+      yield filePath;
+    }
+  }
+}
+
+function* deprecatedFiles(dir) {
+  for (const file of walk(dir)) {
+    if (file.endsWith(".ql") || file.endsWith(".qll")) {
+      const contents = fs.readFileSync(file, "utf8");
+      if (/\sdeprecated\s/.test(contents)) {
+        yield file;
+      }
+    }
+  }
+}
+
+const blameRegExp =
+  /^(\^?\w+)\s.+\s+(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (?:\+|-)\d{4})\s+(\d+)\).*$/;
+
+function* deprecationMessages(dir) {
+  for (const file of deprecatedFiles(dir)) {
+    const blame = cp.execFileSync("git", ["blame", "--", file]);
+    const lines = blame.toString().split("\n");
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (line.includes(" deprecated ")) {
+        try {
+          const [_, sha, time, lineNumber] = line.match(blameRegExp);
+          const date = new Date(time);
+          // check if it's within the last 14 months (a year, plus 2 months for safety, in case a PR was delayed)
+          if (date.getTime() >= Date.now() - 14 * 31 * 24 * 60 * 60 * 1000) {
+            continue;
+          }
+          const message = `${file}:${lineNumber} was last updated on ${date.getFullYear()}-${date.getMonth()}-${date.getDate()}`;
+          yield [message, date];
+        } catch (e) {
+          console.log(e);
+          console.log("----");
+          console.log(line);
+          console.log("----");
+          process.exit(0);
+        }
+      }
+    }
+  }
+}
+[...deprecationMessages(".")]
+  .sort((a, b) => a[1].getTime() - b[1].getTime())
+  .forEach((msg) => console.log(msg[0]));

config/identical-files.json

@@ -73,6 +73,14 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
+  "Model as Data Generation Java/C# - Utils": [
+    "java/ql/src/utils/model-generator/ModelGeneratorUtils.qll",
+    "csharp/ql/src/utils/model-generator/ModelGeneratorUtils.qll"
+  ],
+  "Model as Data Generation Java/C# - SummaryModels": [
+    "java/ql/src/utils/model-generator/CaptureSummaryModels.qll",
+    "csharp/ql/src/utils/model-generator/CaptureSummaryModels.qll"
+  ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/Sign.qll"
@@ -426,7 +434,6 @@
     "python/ql/src/Lexical/CommentedOutCodeMetricOverview.inc.qhelp"
   ],
   "FLinesOfDuplicatedCodeCommon.inc.qhelp": [
-    "cpp/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "java/ql/src/Metrics/Files/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "javascript/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp",
     "python/ql/src/Metrics/FLinesOfDuplicatedCodeCommon.inc.qhelp"
@@ -512,5 +519,13 @@
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
     "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"
+  ],
+  "Hostname Regexp queries": [
+    "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
+  ],
+  "ApiGraphModels": [
+    "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
   ]
 }