feat: Ignore the leading UTF-8 byte-order mark (BOM)

Author: prantlf

README.md

@@ -12,6 +12,7 @@ This is a fork of the original project ([zaach/jsonlint](https://github.com/zaac
 * Handles multiple files on the command line (by Greg Inman).
 * Walks directories recursively (by Paul Vollmer).
 * Provides 100% compatible interface to the native `JSON.parse` method.
+* Optionally ignores the leading UTF-8 byte-order mark (BOM).
 * Optionally recognizes JavaScript-style comments (CJSON) and single quoted strings (JSON5).
 * Optionally ignores trailing commas and reports duplicate object keys as an error.
 * Optionally checks that also the expected format matches, including sorted object keys.
@@ -129,6 +130,7 @@ Usage: `jsonlint [options] [<file, directory, pattern> ...]`
     -c, --compact                compact error display
     -M, --mode [mode]            set other parsing flags according to a format
                                   type (default: "json")
+    -B, --bom                    ignore the leading UTF-8 byte-order mark
     -C, --comments               recognize and ignore JavaScript-style comments
     -S, --single-quoted-strings  support single quotes as string delimiters
     -T, --trailing-commas        ignore trailing commas in objects and arrays
@@ -194,6 +196,7 @@ The configuration is an object with the following properties, described above, w
 | indent | |
 | compact | |
 | mode | |
+| bom | |
 | comments | |
 | single-quoted-strings | singleQuotedStrings |
 | trailing-commas | trailingCommas |
@@ -251,6 +254,7 @@ The `parse` method offers more detailed [error information](#error-handling), th
 
 | Option                     | Description                                 |
 | -------------------------- | ------------------------------------------- |
+| `ignoreBOM`                | ignores the leading UTF-8 byte-order mark (boolean) |
 | `ignoreComments`           | ignores single-line and multi-line JavaScript-style comments during parsing as another "whitespace" (boolean) |
 | `ignoreTrailingCommas`     | ignores trailing commas in objects and arrays (boolean)      |
 | `allowSingleQuotedStrings` | accepts strings delimited by single-quotes too (boolean)     |

lib/cli.js

@@ -26,6 +26,7 @@ const commander = require('commander')
   .option('-t, --indent [num|char]', 'number of spaces or specific characters to use for indentation', 2)
   .option('-c, --compact', 'compact error display')
   .option('-M, --mode [mode]', 'set other parsing flags according to a format type', 'json')
+  .option('-B, --bom', 'ignore the leading UTF-8 byte-order mark')
   .option('-C, --comments', 'recognize and ignore JavaScript-style comments')
   .option('-S, --single-quoted-strings', 'support single quotes as string delimiters')
   .option('-T, --trailing-commas', 'ignore trailing commas in objects and arrays')
@@ -111,7 +112,7 @@ function mergeOptions (target, ...sources) {
   return target
 }
 
-function separateBlocks() {
+function separateBlocks () {
   if (reported) {
     console.log()
   } else {
@@ -135,6 +136,7 @@ function processContents (source, file) {
   try {
     parserOptions = {
       mode: options.mode,
+      ignoreBOM: options.bom,
       ignoreComments: options.comments,
       ignoreTrailingCommas: options.trailingCommas,
       allowSingleQuotedStrings: options.singleQuotedStrings,
@@ -249,7 +251,7 @@ function checkContents (file, source, parsed) {
   }
 }
 
-function diffContents(file, source, parsed) {
+function diffContents (file, source, parsed) {
   const { createTwoFilesPatch, structuredPatch } = require('diff')
   const compact = options.quiet || options.compact
   let diff, length

lib/jsonlint.d.ts

@@ -1,6 +1,7 @@
 type ParseMode = 'json' | 'cjson' | 'json5'
 
 interface ParseOptions {
+  ignoreBOM?: boolean
   ignoreComments?: boolean
   ignoreTrailingCommas?: boolean
   allowSingleQuotedStrings?: boolean
@@ -27,6 +28,7 @@ interface ParseOptions {
 declare function parse (input: string, reviverOrOptions?: Function | ParseOptions): object
 
 interface TokenizeOptions {
+  ignoreBOM?: boolean
   ignoreComments?: boolean
   ignoreTrailingCommas?: boolean
   allowSingleQuotedStrings?: boolean
@@ -58,6 +60,7 @@ declare module '@prantlf/jsonlint/lib/validator' {
   type Environment = 'json-schema-draft-04' | 'json-schema-draft-06' | 'json-schema-draft-07'
 
   interface CompileOptions {
+    ignoreBOM?: boolean
     ignoreComments?: boolean
     ignoreTrailingCommas?: boolean
     allowSingleQuotedStrings?: boolean

lib/validator.js

@@ -148,6 +148,7 @@
     const ajv = createAjv(environment)
     const parseOptions = {
       mode: options.mode,
+      ignoreBOM: options.ignoreBOM,
       ignoreComments: options.ignoreComments,
       ignoreTrailingCommas: options.ignoreTrailingCommas,
       allowSingleQuotedStrings: options.allowSingleQuotedStrings,

src/configurable-parser.js

@@ -4,7 +4,7 @@ const isSafari = typeof navigator !== 'undefined' && /Safari/.test(navigator.use
 const oldNode = typeof process !== 'undefined' && process.version.startsWith('v4.')
 
 function needsCustomParser (options) {
-  return options.ignoreComments || options.ignoreTrailingCommas ||
+  return options.ignoreBOM || options.ignoreComments || options.ignoreTrailingCommas ||
   options.allowSingleQuotedStrings || options.allowDuplicateObjectKeys === false ||
   options.mode === 'cjson' || options.mode === 'json5' || isSafari || oldNode
 }

src/custom-parser.js

@@ -16,6 +16,12 @@ function isDecDigit (x) {
   return x >= '0' && x <= '9'
 }
 
+function isBOM (x) {
+  // This catches EFBBBF (UTF-8 BOM) because the buffer-to-string
+  // conversion in `fs.readFileSync()` translates it to FEFF (UTF-16 BOM).
+  return x.charCodeAt(0) === 0xFEFF
+}
+
 const unescapeMap = {
   '\'': '\'',
   '"': '"',
@@ -35,6 +41,7 @@ function parseInternal (input, options) {
   }
 
   const json5 = options.mode === 'json5'
+  const ignoreBOM = options.ignoreBOM
   const ignoreComments = options.ignoreComments || options.mode === 'cjson' || json5
   const ignoreTrailingCommas = options.ignoreTrailingCommas || json5
   const allowSingleQuotedStrings = options.allowSingleQuotedStrings || json5
@@ -221,6 +228,14 @@ function parseInternal (input, options) {
     }
   }
 
+  function skipBOM () {
+    if (isBOM(input)) {
+      startToken && startToken()
+      ++position
+      endToken && endToken('bom')
+    }
+  }
+
   function skipWhiteSpace () {
     let insideWhiteSpace
     function startWhiteSpace () {
@@ -615,6 +630,9 @@ function parseInternal (input, options) {
     fail()
   }
 
+  if (ignoreBOM) {
+    skipBOM()
+  }
   skipWhiteSpace()
   let returnValue = parseGeneric()
   if (returnValue !== undefined || position < inputLength) {

test/fails/bom.json

@@ -0,0 +1,3 @@
+{
+  "bom": "utf8"
+}

test/parse1.js

@@ -255,6 +255,18 @@ exports['test extra brace'] = function () {
   assert.throws(function () { parse(json) }, 'should throw error')
 }
 
+exports['test failing with bom'] = function () {
+  const json = fs.readFileSync(path.join(__dirname, '/fails/bom.json')).toString()
+  assert.throws(function () { parse(json) }, 'should throw error')
+}
+
+if (!nativeParser) {
+  exports['test ignoring bom'] = function () {
+    const json = fs.readFileSync(path.join(__dirname, '/fails/bom.json')).toString()
+    assert.deepEqual(parse(json, { ignoreBOM: true }), { bom: 'utf8' })
+  }
+}
+
 if (!oldNode) {
   exports['test error location with Windows line breaks using the native parser'] = function () {
     const json = '{\r\n"foo": {\r\n      "bar":\r\n    }\r\n  \r\n  }'

test/typings.test.ts

@@ -22,6 +22,7 @@ addTest('parse', () => {
   parse('{}', { mode: 'cjson' })
   parse('{}', { mode: 'json5' })
   parse('{}', {
+    ignoreBOM: true,
     ignoreComments: true,
     ignoreTrailingCommas: true,
     allowSingleQuotedStrings: true,
@@ -45,6 +46,7 @@ addTest('compile', () => {
   compile('{}', { environment: 'json-schema-draft-06' })
   compile('{}', { environment: 'json-schema-draft-07' })
   compile('{}', {
+    ignoreBOM: true,
     ignoreComments: true,
     ignoreTrailingCommas: true,
     allowSingleQuotedStrings: true,