Skip to content

Commit cc7b554

Browse files
prantlfprantlf
committed
feat: Add the tokenize method returning tokens instead of the parsed object
Add methods convert an item path in the JSON data to JSON pointer and back.
1 parent c2f0148 commit cc7b554

File tree

4 files changed

+297
-25
lines changed

4 files changed

+297
-25
lines changed

‎scripts/bundle-jsonlint.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ var suffix = `
1515
}
1616
exports.parseNative = parseNative
1717
exports.parseCustom = parseCustom
18+
exports.tokenize = tokenize
19+
exports.getErrorTexts = getTexts
20+
exports.pathToPointer = pathToPointer
21+
exports.pointerToPath = pointerToPath
1822
Object.defineProperty(exports, '__esModule', { value: true })
1923
}));
2024
`

‎src/custom-parser.js

Lines changed: 164 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,7 @@ var unescapeMap = {
2929
'/': '/'
3030
}
3131

32-
function parseCustom (input, options) { // eslint-disable-line no-unused-vars
33-
if (typeof options === 'function') {
34-
options = {
35-
reviver: options
36-
}
37-
} else if (!options) {
38-
options = {}
39-
}
40-
32+
function parseInternal (input, options) {
4133
if (typeof input !== 'string' || !(input instanceof String)) {
4234
input = String(input)
4335
}
@@ -48,6 +40,10 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
4840
var allowSingleQuotedStrings = options.allowSingleQuotedStrings || json5
4941
var allowDuplicateObjectKeys = options.allowDuplicateObjectKeys
5042
var reviver = options.reviver
43+
var tokenize = options.tokenize
44+
var rawTokens = options.rawTokens
45+
var tokenLocations = options.tokenLocations
46+
var tokenPaths = options.tokenPaths
5147

5248
var isLineTerminator = json5 ? Uni.isLineTerminator : Uni.isLineTerminatorJSON
5349
var isWhiteSpace = json5 ? Uni.isWhiteSpace : Uni.isWhiteSpaceJSON
@@ -56,7 +52,50 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
5652
var lineNumber = 0
5753
var lineStart = 0
5854
var position = 0
59-
var stack = []
55+
56+
var startToken
57+
var endToken
58+
var tokenPath
59+
60+
if (tokenize) {
61+
var tokens = []
62+
var tokenOffset = null
63+
var tokenLine
64+
var tokenColumn
65+
startToken = function () {
66+
if (tokenOffset !== null) throw Error('internal error, token overlap')
67+
tokenLine = lineNumber + 1
68+
tokenColumn = position - lineStart + 1
69+
tokenOffset = position
70+
}
71+
endToken = function (type, value) {
72+
if (tokenOffset !== position) {
73+
var token = { type: type }
74+
if (rawTokens) {
75+
token.raw = input.substr(tokenOffset, position - tokenOffset)
76+
}
77+
if (value !== undefined) {
78+
token.value = value
79+
}
80+
if (tokenLocations) {
81+
token.location = {
82+
start: {
83+
column: tokenColumn,
84+
line: tokenLine,
85+
offset: tokenOffset
86+
}
87+
}
88+
}
89+
if (tokenPaths) {
90+
token.path = tokenPath.slice()
91+
}
92+
tokens.push(token)
93+
}
94+
tokenOffset = null
95+
return value
96+
}
97+
tokenPaths && (tokenPath = [])
98+
}
6099

61100
function generateMessage () {
62101
var message
@@ -106,27 +145,38 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
106145

107146
function parseGeneric () {
108147
while (position < inputLength) {
148+
startToken && startToken()
109149
var char = input[position++]
110150
if (char === '"' || (char === '\'' && allowSingleQuotedStrings)) {
111-
return parseString(char)
151+
var string = parseString(char)
152+
endToken && endToken('literal', string)
153+
return string
112154
} else if (char === '{') {
155+
endToken && endToken('symbol', '{')
113156
return parseObject()
114157
} else if (char === '[') {
158+
endToken && endToken('symbol', '[')
115159
return parseArray()
116160
} else if (char === '-' || char === '.' || isDecDigit(char) ||
117161
(json5 && (char === '+' || char === 'I' || char === 'N'))) {
118-
return parseNumber()
162+
var number = parseNumber()
163+
endToken && endToken('literal', number)
164+
return number
119165
} else if (char === 'n') {
120166
parseKeyword('null')
167+
endToken && endToken('literal', null)
121168
return null
122169
} else if (char === 't') {
123170
parseKeyword('true')
171+
endToken && endToken('literal', true)
124172
return true
125173
} else if (char === 'f') {
126174
parseKeyword('false')
175+
endToken && endToken('literal', false)
127176
return false
128177
} else {
129178
--position
179+
endToken && endToken()
130180
return undefined
131181
}
132182
}
@@ -135,47 +185,81 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
135185
function parseKey () {
136186
var result
137187
while (position < inputLength) {
188+
startToken && startToken()
138189
var char = input[position++]
139190
if (char === '"' || (char === '\'' && allowSingleQuotedStrings)) {
140-
return parseString(char)
191+
var string = parseString(char)
192+
endToken && endToken('literal', string)
193+
return string
141194
} else if (char === '{') {
195+
endToken && endToken('symbol', '{')
142196
return parseObject()
143197
} else if (char === '[') {
198+
endToken && endToken('symbol', '[')
144199
return parseArray()
145200
} else if (char === '.' || isDecDigit(char)) {
146-
return parseNumber(true)
201+
var number = parseNumber(true)
202+
endToken && endToken('literal', number)
203+
return number
147204
} else if ((json5 && Uni.isIdentifierStart(char)) ||
148205
(char === '\\' && input[position] === 'u')) {
149206
var rollback = position - 1
150207
result = parseIdentifier()
151208
if (result === undefined) {
152209
position = rollback
210+
endToken && endToken()
153211
return undefined
154212
} else {
213+
endToken && endToken('literal', result)
155214
return result
156215
}
157216
} else {
158217
--position
218+
endToken && endToken()
159219
return undefined
160220
}
161221
}
162222
}
163223

164224
function skipWhiteSpace () {
225+
var insideWhiteSpace
226+
function startWhiteSpace () {
227+
if (!insideWhiteSpace) {
228+
insideWhiteSpace = true
229+
--position
230+
startToken()
231+
++position
232+
}
233+
}
234+
function endWhiteSpace () {
235+
if (insideWhiteSpace) {
236+
insideWhiteSpace = false
237+
endToken('whitespace')
238+
}
239+
}
165240
while (position < inputLength) {
166241
var char = input[position++]
167242
if (isLineTerminator(char)) {
243+
startToken && startWhiteSpace()
168244
newLine(char)
169245
} else if (isWhiteSpace(char)) {
170-
// nothing
246+
startToken && startWhiteSpace()
171247
} else if (char === '/' && ignoreComments &&
172248
(input[position] === '/' || input[position] === '*')) {
249+
if (startToken) {
250+
--position
251+
endWhiteSpace()
252+
startToken()
253+
++position
254+
}
173255
skipComment(input[position++] === '*')
256+
endToken && endToken('comment')
174257
} else {
175258
--position
176259
break
177260
}
178261
}
262+
endToken && endWhiteSpace()
179263
}
180264

181265
function skipComment (multiLine) {
@@ -226,8 +310,9 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
226310
fail('Duplicate key: "' + key + '"')
227311
}
228312
skipWhiteSpace()
229-
313+
startToken && startToken()
230314
var char = input[position++]
315+
endToken && endToken('symbol', char)
231316
if (char === '}' && key === undefined) {
232317
if (!ignoreTrailingCommas && isNotEmpty) {
233318
--position
@@ -236,9 +321,9 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
236321
return result
237322
} else if (char === ':' && key !== undefined) {
238323
skipWhiteSpace()
239-
stack.push(key)
324+
tokenPath && tokenPath.push(key)
240325
var value = parseGeneric()
241-
stack.pop()
326+
tokenPath && tokenPath.pop()
242327

243328
if (value === undefined) fail('No value found for key "' + key + '"')
244329
if (typeof key !== 'string') {
@@ -260,7 +345,9 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
260345
}
261346

262347
skipWhiteSpace()
348+
startToken && startToken()
263349
char = input[position++]
350+
endToken && endToken('symbol', char)
264351
if (char === ',') {
265352
continue
266353
} else if (char === '}') {
@@ -281,12 +368,13 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
281368
var result = []
282369
while (position < inputLength) {
283370
skipWhiteSpace()
284-
stack.push(result.length)
371+
tokenPath && tokenPath.push(result.length)
285372
var item = parseGeneric()
286-
stack.pop()
373+
tokenPath && tokenPath.pop()
287374
skipWhiteSpace()
288-
375+
startToken && startToken()
289376
var char = input[position++]
377+
endToken && endToken('symbol', char)
290378
if (item !== undefined) {
291379
if (reviver) {
292380
item = reviver(String(result.length), item)
@@ -535,7 +623,7 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
535623
if (reviver) {
536624
returnValue = reviver('', returnValue)
537625
}
538-
return returnValue
626+
return tokenize ? tokens : returnValue
539627
} else {
540628
fail()
541629
}
@@ -547,3 +635,57 @@ function parseCustom (input, options) { // eslint-disable-line no-unused-vars
547635
}
548636
}
549637
}
638+
639+
function parseCustom (input, options) { // eslint-disable-line no-unused-vars
640+
if (typeof options === 'function') {
641+
options = {
642+
reviver: options
643+
}
644+
} else if (!options) {
645+
options = {}
646+
}
647+
return parseInternal(input, options)
648+
}
649+
650+
function tokenize (input, options) { // eslint-disable-line no-unused-vars
651+
if (!options) {
652+
options = {}
653+
}
654+
options.tokenize = true
655+
return parseInternal(input, options)
656+
}
657+
658+
function escapePointerToken (token) {
659+
return token
660+
.toString()
661+
.replace(/~/g, '~0')
662+
.replace(/\//g, '~1')
663+
}
664+
665+
function pathToPointer (tokens) { // eslint-disable-line no-unused-vars
666+
if (tokens.length === 0) {
667+
return ''
668+
}
669+
return '/' + tokens
670+
.map(escapePointerToken)
671+
.join('/')
672+
}
673+
674+
function unescapePointerToken (token) {
675+
return token
676+
.replace(/~1/g, '/')
677+
.replace(/~0/g, '~')
678+
}
679+
680+
function pointerToPath (pointer) { // eslint-disable-line no-unused-vars
681+
if (pointer === '') {
682+
return []
683+
}
684+
if (pointer[0] !== '/') {
685+
throw new Error('Missing initial "/" in the reference')
686+
}
687+
return pointer
688+
.substr(1)
689+
.split('/')
690+
.map(unescapePointerToken)
691+
}

‎test/parse1.js

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,9 +331,7 @@ exports['test schema validation failure'] = function () {
331331
environment: 'json-schema-draft-04'
332332
})
333333
assert['throws'](function () {
334-
validate(parse(data, {
335-
ignoreComments: !nativeParser
336-
}))
334+
validate(data, { ignoreComments: !nativeParser })
337335
}, 'should throw error')
338336
}
339337

0 commit comments

Comments
 (0)