Skip to content

Commit 29aa86c

Browse files
prashantRaghuprashantRaghu
authored
Merge pull request #747 from postmanlabs/feature/add-support-for-remote-url-refs-first-draft
Added support for handling Remote URL refs in the bundle method
2 parents c5ff50b + 8b045f8 commit 29aa86c

27 files changed

+1614
-51
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## [Unreleased]
44

5+
### Added
6+
7+
- Added support for remote $ref resolution in bundle() API.
8+
59
## [v4.15.0] - 2023-06-27
610

711
### Added
@@ -41,7 +45,7 @@
4145
- Fixed an issue where definition validation was not considering multiple white space characters.
4246
- Fixed issue [#708](https://github.com/postmanlabs/openapi-to-postman/issues/708) where if string is defined for required field, conversion was failing.
4347
- Fixed issue where for certain path segments, collection generation failed.
44-
- Fixed TypeError occurring while checking typeof bodyContent in getXmlVersionContent.
48+
- Fixed TypeError occurring while checking typeof bodyContent in getXmlVersionContent.
4549

4650
## [v4.12.0] - 2023-05-04
4751

@@ -295,7 +299,7 @@ Newer releases follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0
295299
- Added support for internal $ref resolution in validation flows.
296300
- Fixed issue where parameter resolution was "schema" when "example" was specified.
297301
- Add supported formats for schema resolution (deref).
298-
- Fix for [#7643](https://github.com/postmanlabs/postman-app-support/issues/7643), [#7914](https://github.com/postmanlabs/postman-app-support/issues/7914), [#9004](https://github.com/postmanlabs/postman-app-support/issues/9004) - Added support for Auth params in response/example.
302+
- Fix for [#7643](https://github.com/postmanlabs/postman-app-support/issues/7643), [#7914](https://github.com/postmanlabs/postman-app-support/issues/7914), [#9004](https://github.com/postmanlabs/postman-app-support/issues/9004) - Added support for Auth params in response/example.
299303
- Bumped up multiple dependecies and dev-dependencies versions to keep them up-to-date.
300304
- Updated code coverage tool from deprecated istanbul to nyc.
301305

@@ -382,7 +386,7 @@ Newer releases follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0
382386
#### v1.1.13 (April 21, 2020)
383387

384388
- Added support for detailed validation body mismatches with option detailedBlobValidation.
385-
- Fix for [#8098](https://github.com/postmanlabs/postman-app-support/issues/8098) - Unable to validate schema with type array.
389+
- Fix for [#8098](https://github.com/postmanlabs/postman-app-support/issues/8098) - Unable to validate schema with type array.
386390
- Fixed URIError for invalid URI in transaction.
387391
- Fix for [#152](https://github.com/postmanlabs/openapi-to-postman/issues/152) - Path references not resolved due to improver handling of special characters.
388392
- Fix for [#160](https://github.com/postmanlabs/openapi-to-postman/issues/160) - Added handling for variables in local servers not a part of a URL segment. All path servers to be added as collection variables.

‎lib/bundle.js

Lines changed: 225 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
const _ = require('lodash'),
22
{
33
isExtRef,
4+
isExtURLRef,
5+
stringIsAValidUrl,
6+
isExtRemoteRef,
47
getKeyInComponents,
58
getJsonPointerRelationToRoot,
69
removeLocalReferenceFromPath,
710
localPointer,
11+
httpSeparator,
812
jsonPointerLevelSeparator,
913
isLocalRef,
1014
jsonPointerDecodeAndReplace,
@@ -83,14 +87,21 @@ function calculatePath(parentFileName, referencePath) {
8387
* @returns {object} - Detect root files result object
8488
*/
8589
function findNodeFromPath(referencePath, allData) {
86-
const partialComponents = referencePath.split(localPointer);
87-
let isPartial = partialComponents.length > 1,
88-
node = allData.find((node) => {
89-
if (isPartial) {
90-
referencePath = partialComponents[0];
91-
}
92-
return comparePaths(node.fileName, referencePath);
93-
});
90+
const isReferenceRemoteURL = stringIsAValidUrl(referencePath),
91+
partialComponents = referencePath.split(localPointer),
92+
isPartial = partialComponents.length > 1;
93+
94+
let node = allData.find((node) => {
95+
if (isPartial && !isReferenceRemoteURL) {
96+
referencePath = partialComponents[0];
97+
}
98+
99+
if (isReferenceRemoteURL) {
100+
return _.startsWith(node.path, referencePath);
101+
}
102+
103+
return comparePaths(node.fileName, referencePath);
104+
});
94105

95106
return node;
96107
}
@@ -290,13 +301,86 @@ function handleLocalCollisions(trace, initialMainKeys) {
290301
* @param {string} commonPathFromData - The common path in the file's paths
291302
* @param {Array} allData - array of { path, content} objects
292303
* @param {object} globalReferences - The accumulated global references from all nodes
304+
* @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
293305
* @returns {object} - The references in current node and the new content from the node
294306
*/
295-
function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, version, rootMainKeys,
296-
commonPathFromData, allData, globalReferences) {
307+
async function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, version, rootMainKeys,
308+
commonPathFromData, allData, globalReferences, remoteRefResolver) {
297309
let referencesInNode = [],
298310
nodeReferenceDirectory = {},
299-
mainKeys = {};
311+
mainKeys = {},
312+
remoteRefContentMap = new Map(),
313+
remoteRefSet = new Set(),
314+
remoteRefResolutionPromises = [];
315+
316+
remoteRefResolver && traverseUtility(currentNode).forEach(function (property) {
317+
if (property) {
318+
let hasReferenceTypeKey;
319+
320+
hasReferenceTypeKey = Object.keys(property)
321+
.find(
322+
(key) => {
323+
const isExternal = isExtURLRef(property, key),
324+
isReferenciable = isExternal;
325+
326+
return isReferenciable;
327+
}
328+
);
329+
330+
if (hasReferenceTypeKey) {
331+
const tempRef = calculatePath(parentFilename, property.$ref),
332+
isRefEncountered = remoteRefSet.has(tempRef);
333+
334+
if (isRefEncountered) {
335+
return;
336+
}
337+
338+
remoteRefResolutionPromises.push(
339+
new Promise(async (resolveInner) => {
340+
341+
/**
342+
* Converts contents received from remoteRefResolver into stringified JSON
343+
* @param {string | object} content - contents from remoteRefResolver
344+
* @returns {string} Stringified JSON contents
345+
*/
346+
function convertToJSONString (content) {
347+
if (typeof content === 'object') {
348+
return JSON.stringify(content);
349+
}
350+
351+
const parsedFile = parseFile(content);
352+
353+
return JSON.stringify(parsedFile.oasObject);
354+
}
355+
356+
try {
357+
let contentFromRemote = await remoteRefResolver(property.$ref),
358+
nodeTemp = {
359+
fileName: tempRef,
360+
path: tempRef,
361+
content: convertToJSONString(contentFromRemote),
362+
href: property.$ref
363+
};
364+
365+
remoteRefContentMap.set(tempRef, contentFromRemote);
366+
367+
allData.push(nodeTemp);
368+
}
369+
catch (err) {
370+
// swallow the err
371+
}
372+
finally {
373+
resolveInner();
374+
}
375+
})
376+
);
377+
378+
remoteRefSet.add(tempRef);
379+
}
380+
}
381+
});
382+
383+
await Promise.all(remoteRefResolutionPromises);
300384

301385
traverseUtility(currentNode).forEach(function (property) {
302386
if (property) {
@@ -371,6 +455,94 @@ function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, ve
371455
referencesInNode.push({ path: pathSolver(property), keyInComponents: nodeTrace, newValue: this.node });
372456
}
373457
}
458+
459+
const hasRemoteReferenceTypeKey = Object.keys(property)
460+
.find(
461+
(key) => {
462+
const isExternal = isExtURLRef(property, key),
463+
464+
// Only process URL refs if remoteRefResolver is provided and a valid function
465+
isReferenciable = isExternal && _.isFunction(remoteRefResolver);
466+
467+
return isReferenciable;
468+
}
469+
),
470+
handleRemoteURLReference = () => {
471+
const tempRef = calculatePath(parentFilename, property.$ref);
472+
473+
if (remoteRefContentMap.get(tempRef) === undefined) {
474+
return;
475+
}
476+
477+
let nodeTrace = handleLocalCollisions(
478+
getTraceFromParentKeyInComponents(this, tempRef, mainKeys, version, commonPathFromData),
479+
rootMainKeys
480+
),
481+
componentKey = nodeTrace[nodeTrace.length - 1],
482+
referenceInDocument = getJsonPointerRelationToRoot(
483+
tempRef,
484+
nodeTrace,
485+
version
486+
),
487+
traceToParent = [...this.parents.map((item) => {
488+
return item.key;
489+
}).filter((item) => {
490+
return item !== undefined;
491+
}), this.key],
492+
newValue = Object.assign({}, this.node),
493+
[, local] = tempRef.split(localPointer),
494+
nodeFromData,
495+
refHasContent = false,
496+
parseResult,
497+
newRefInDoc,
498+
inline,
499+
contentFromRemote = remoteRefContentMap.get(tempRef),
500+
nodeTemp = {
501+
fileName: tempRef,
502+
path: tempRef,
503+
content: contentFromRemote
504+
};
505+
506+
nodeFromData = nodeTemp;
507+
508+
if (nodeFromData && nodeFromData.content) {
509+
parseResult = parseFile(JSON.stringify(nodeFromData.content));
510+
if (parseResult.result) {
511+
newValue.$ref = referenceInDocument;
512+
refHasContent = true;
513+
nodeFromData.parsed = parseResult;
514+
}
515+
}
516+
this.update({ $ref: tempRef });
517+
518+
if (nodeTrace.length === 0) {
519+
inline = true;
520+
}
521+
522+
if (_.isNil(globalReferences[tempRef])) {
523+
nodeReferenceDirectory[tempRef] = {
524+
local,
525+
keyInComponents: nodeTrace,
526+
node: newValue,
527+
reference: inline ? newRefInDoc : referenceInDocument,
528+
traceToParent,
529+
parentNodeKey: parentFilename,
530+
mainKeyInTrace: nodeTrace[nodeTrace.length - 1],
531+
refHasContent,
532+
inline
533+
};
534+
}
535+
536+
mainKeys[componentKey] = tempRef;
537+
538+
if (!added(property.$ref, referencesInNode)) {
539+
referencesInNode.push({ path: pathSolver(property), keyInComponents: nodeTrace, newValue: this.node });
540+
}
541+
};
542+
543+
if (hasRemoteReferenceTypeKey) {
544+
handleRemoteURLReference();
545+
}
374546
}
375547
});
376548

@@ -386,10 +558,11 @@ function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, ve
386558
* @param {object} rootMainKeys - A dictionary with the component keys in local components object and its mainKeys
387559
* @param {string} commonPathFromData - The common path in the file's paths
388560
* @param {object} globalReferences - The accumulated global refernces from all nodes
561+
* @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
389562
* @returns {object} - Detect root files result object
390563
*/
391-
function getNodeContentAndReferences (currentNode, allData, specRoot, version, rootMainKeys,
392-
commonPathFromData, globalReferences) {
564+
async function getNodeContentAndReferences (currentNode, allData, specRoot, version, rootMainKeys,
565+
commonPathFromData, globalReferences, remoteRefResolver) {
393566
let graphAdj = [],
394567
missingNodes = [],
395568
nodeContent,
@@ -406,7 +579,7 @@ function getNodeContentAndReferences (currentNode, allData, specRoot, version, r
406579
nodeContent = parseResult.oasObject;
407580
}
408581

409-
const { referencesInNode, nodeReferenceDirectory } = getReferences(
582+
const { referencesInNode, nodeReferenceDirectory } = await getReferences(
410583
nodeContent,
411584
currentNode.fileName !== specRoot.fileName,
412585
removeLocalReferenceFromPath,
@@ -415,7 +588,8 @@ function getNodeContentAndReferences (currentNode, allData, specRoot, version, r
415588
rootMainKeys,
416589
commonPathFromData,
417590
allData,
418-
globalReferences
591+
globalReferences,
592+
remoteRefResolver
419593
);
420594

421595
referencesInNode.forEach((reference) => {
@@ -516,9 +690,11 @@ function handleCircularReference(traverseContext, documentContext) {
516690
* @param {function} refTypeResolver - The resolver function to test if node has a reference
517691
* @param {object} components - The global components object
518692
* @param {string} version - The current version
693+
* @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
519694
* @returns {object} The components object related to the file
520695
*/
521-
function generateComponentsObject (documentContext, rootContent, refTypeResolver, components, version) {
696+
function generateComponentsObject(documentContext, rootContent,
697+
refTypeResolver, components, version, remoteRefResolver) {
522698
let notInLine = Object.entries(documentContext.globalReferences).filter(([, value]) => {
523699
return value.keyInComponents.length !== 0;
524700
}),
@@ -555,13 +731,37 @@ function generateComponentsObject (documentContext, rootContent, refTypeResolver
555731
isMissingNode = documentContext.missing.find((missingNode) => {
556732
return missingNode.path === nodeRef;
557733
});
734+
558735
if (isMissingNode) {
559736
refData.nodeContent = refData.node;
560737
refData.local = false;
561738
}
562739
else if (!refData) {
563740
return;
564741
}
742+
else if (!isExtRef(property, '$ref') && isExtURLRef(property, '$ref')) {
743+
let splitPathByHttp = property.$ref.split(httpSeparator),
744+
prefix = splitPathByHttp
745+
.slice(0, splitPathByHttp.length - 1).join(httpSeparator) +
746+
httpSeparator + splitPathByHttp[splitPathByHttp.length - 1]
747+
.split(localPointer)[0],
748+
separatedPaths = [prefix, splitPathByHttp[splitPathByHttp.length - 1].split(localPointer)[1]];
749+
750+
nodeRef = separatedPaths[0];
751+
local = separatedPaths[1];
752+
753+
refData.nodeContent = documentContext.nodeContents[nodeRef];
754+
755+
const isReferenceRemoteURL = stringIsAValidUrl(nodeRef);
756+
757+
if (isReferenceRemoteURL && _.isFunction(remoteRefResolver)) {
758+
Object.keys(documentContext.nodeContents).forEach((key) => {
759+
if (_.startsWith(key, nodeRef) && !key.split(nodeRef)[1].includes(httpSeparator)) {
760+
refData.nodeContent = documentContext.nodeContents[key];
761+
}
762+
});
763+
}
764+
}
565765
else {
566766
refData.nodeContent = documentContext.nodeContents[nodeRef];
567767
}
@@ -697,9 +897,10 @@ module.exports = {
697897
* @param {Array} allData - array of { path, content} objects
698898
* @param {Array} origin - process origin (BROWSER or node)
699899
* @param {string} version - The version we are using
900+
* @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
700901
* @returns {object} - Detect root files result object
701902
*/
702-
getBundleContentAndComponents: function (specRoot, allData, origin, version) {
903+
getBundleContentAndComponents: async function (specRoot, allData, origin, version, remoteRefResolver) {
703904
if (origin === BROWSER) {
704905
path = pathBrowserify;
705906
}
@@ -716,15 +917,16 @@ module.exports = {
716917
commonPathFromData = Utils.findCommonSubpath(allData.map((fileData) => {
717918
return fileData.fileName;
718919
}));
719-
rootContextData = algorithm.traverseAndBundle(specRoot, (currentNode, globalReferences) => {
920+
rootContextData = await algorithm.traverseAndBundle(specRoot, (currentNode, globalReferences) => {
720921
return getNodeContentAndReferences(
721922
currentNode,
722923
allData,
723924
specRoot,
724925
version,
725926
initialMainKeys,
726927
commonPathFromData,
727-
globalReferences
928+
globalReferences,
929+
remoteRefResolver
728930
);
729931
});
730932
components = generateComponentsWrapper(
@@ -735,10 +937,12 @@ module.exports = {
735937
finalElements = generateComponentsObject(
736938
rootContextData,
737939
rootContextData.nodeContents[specRoot.fileName],
738-
isExtRef,
940+
isExtRemoteRef,
739941
components,
740-
version
942+
version,
943+
remoteRefResolver
741944
);
945+
742946
return {
743947
fileContent: finalElements.resRoot,
744948
components: finalElements.newComponents,

0 commit comments

Comments
 (0)