Load JSON-LD from HTML documents.

* Adds options parameter to documentLoader
* Uses xmldom, if loaded.
* Adds util.ParseContentTypeHeader
* Adds documentLoader implementations for xhr and node (still requires tests).

Author: gkellogg

lib/ContextResolver.js

@@ -149,7 +149,7 @@ module.exports = class ContextResolver {
     let remoteDoc;
 
     try {
-      remoteDoc = await documentLoader(url);
+      remoteDoc = await documentLoader(url, {});
       context = remoteDoc.document || null;
       // parse string context as JSON
       if(_isString(context)) {

lib/documentLoaders/node.js

@@ -3,7 +3,11 @@
  */
 'use strict';
 
-const {parseLinkHeader, buildHeaders} = require('../util');
+const {
+  parseLinkHeader,
+  buildHeaders,
+  parseContentTypeHeader
+} = require('../util');
 const {LINK_HEADER_REL} = require('../constants');
 const JsonLdError = require('../JsonLdError');
 const RequestQueue = require('../RequestQueue');
@@ -37,11 +41,11 @@ module.exports = ({
   const http = require('http');
 
   const queue = new RequestQueue();
-  return queue.wrapLoader(function(url) {
-    return loadDocument(url, []);
+  return queue.wrapLoader(function(url, options) {
+    return loadDocument(url, options, []);
   });
 
-  async function loadDocument(url, redirects) {
+  async function loadDocument(url, options, redirects) {
     if(url.indexOf('http:') !== 0 && url.indexOf('https:') !== 0) {
       throw new JsonLdError(
         'URL could not be dereferenced; only "http" and "https" URLs are ' +
@@ -60,6 +64,12 @@ module.exports = ({
       return doc;
     }
 
+    // add any optional requestProfile
+    if(options.requestProfile) {
+      headers.Accept =
+        headers.Accept + ", application/ld+json;profile=${options.requestProfile}";
+    }
+
     let result;
     try {
       result = await _request(request, {
@@ -76,8 +86,17 @@ module.exports = ({
     }
 
     const {res, body} = result;
+    const {contentType, params} = parseContentTypeHeader(res.headers['content-type']);
+
+    doc = {
+      contextUrl: null,
+      documentUrl: url,
+      document: body || null,
+      contentType: contentType,
+      profile: params.profile
+    };
 
-    doc = {contextUrl: null, documentUrl: url, document: body || null};
+    // separate profile from content-type
 
     // handle error
     const statusText = http.STATUS_CODES[res.statusCode];
@@ -93,7 +112,7 @@ module.exports = ({
 
     // handle Link Header
     if(res.headers.link &&
-      res.headers['content-type'] !== 'application/ld+json') {
+      contentType !== 'application/ld+json') {
       // only 1 related link header permitted
       const linkHeader = parseLinkHeader(res.headers.link)[LINK_HEADER_REL];
       if(Array.isArray(linkHeader)) {
@@ -131,7 +150,7 @@ module.exports = ({
           });
       }
       redirects.push(url);
-      return loadDocument(res.headers.location, redirects);
+      return loadDocument(res.headers.location, options, redirects);
     }
 
     // cache for each redirected URL

lib/documentLoaders/xhr.js

@@ -3,7 +3,11 @@
  */
 'use strict';
 
-const {parseLinkHeader, buildHeaders} = require('../util');
+const {
+  parseLinkHeader,
+  buildHeaders,
+  parseContentTypeHeader
+} = require('../util');
 const {LINK_HEADER_REL} = require('../constants');
 const JsonLdError = require('../JsonLdError');
 const RequestQueue = require('../RequestQueue');
@@ -30,7 +34,7 @@ module.exports = ({
   const queue = new RequestQueue();
   return queue.wrapLoader(loader);
 
-  async function loader(url) {
+  async function loader(url, options) {
     if(url.indexOf('http:') !== 0 && url.indexOf('https:') !== 0) {
       throw new JsonLdError(
         'URL could not be dereferenced; only "http" and "https" URLs are ' +
@@ -44,6 +48,12 @@ module.exports = ({
         'jsonld.InvalidUrl', {code: 'loading document failed', url});
     }
 
+    // add any optional requestProfile
+    if(options.requestProfile) {
+      headers.Accept =
+        headers.Accept + ", application/ld+json;profile=${options.requestProfile}";
+    }
+
     let req;
     try {
       req = await _get(xhr, url, headers);
@@ -64,10 +74,18 @@ module.exports = ({
         });
     }
 
-    const doc = {contextUrl: null, documentUrl: url, document: req.response};
+    const {contentType, params} =
+      parseContentTypeHeader(req.getResponseHeader('Content-Type'));
+
+    const doc = {
+      contextUrl: null,
+      documentUrl: url,
+      document: req.response,
+      contentType: contentType,
+      profile: params.profile
+    };
 
     // handle Link Header (avoid unsafe header warning by existence testing)
-    const contentType = req.getResponseHeader('Content-Type');
     let linkHeader;
     if(REGEX_LINK_HEADER.test(req.getAllResponseHeaders())) {
       linkHeader = req.getResponseHeader('Link');

lib/jsonld.js

@@ -42,6 +42,7 @@ const LRU = require('lru-cache');
 const NQuads = require('./NQuads');
 const Rdfa = require('./Rdfa');
 
+const {prependBase: _prependBase} = require ('./url');
 const {expand: _expand} = require('./expand');
 const {flatten: _flatten} = require('./flatten');
 const {fromRDF: _fromRDF} = require('./fromRdf');
@@ -854,6 +855,9 @@ jsonld.documentLoader = async url => {
  * @param url the URL to fetch.
  * @param [options] the options to use:
  *          [documentLoader] the document loader to use.
+ *          [extractAllScripts] concatenates all matching script elements..
+ *          [profile] used when selecting from HTML script elements.
+ *          [requestProfile] one or more profile IRIs to use in the request.
  *
  * @return a Promise that resolves to the retrieved remote document.
  */
@@ -865,7 +869,10 @@ jsonld.get = async function(url, options) {
     load = jsonld.documentLoader;
   }
 
-  const remoteDoc = await load(url);
+  // FIXME: unescape frag?
+  const [reference, frag] = url.split('#', 2);
+
+  const remoteDoc = await load(reference, options);
 
   try {
     if(!remoteDoc.document) {
@@ -874,16 +881,68 @@ jsonld.get = async function(url, options) {
         'jsonld.NullRemoteDocument');
     }
     if(_isString(remoteDoc.document)) {
-      remoteDoc.document = JSON.parse(remoteDoc.document);
+      if(remoteDoc.contentType && remoteDoc.contentType.includes('text/html')) {
+        const domParser = new jsonld.domParser();
+        const dom = domParser.parseFromString(remoteDoc.document);
+
+        // Use any document base
+        const baseElem = dom.getElementsByTagName('base');
+        if(baseElem.length > 0) {
+          const href = baseElem[0].getAttribute('href');
+          options.base = _prependBase(options.base || reference, href);
+        }
+
+        const scripts = dom.getElementsByTagName('script');
+        remoteDoc.document = [];
+
+        for(let i = 0; i < scripts.length; i++) {
+          const script = scripts[i];
+          // only application/ld+json
+          if(!script.getAttribute('type').startsWith('application/ld+json')) {
+            continue;
+          }
+          // If url has a fragment identifier, only matching scripts
+          if(frag && script.getAttribute('id') !== frag) {
+            continue;
+          }
+          try {
+            remoteDoc.document.push(JSON.parse(script.textContent));
+          } catch(e) {
+            throw new JsonLdError(
+              'Illegal script content.',
+              'jsonld.InvalidScriptElement', {
+                code: 'invalid script element',
+                remoteDoc
+              });
+          }
+        }
+        if(frag && remoteDoc.document.length === 0) {
+          throw new JsonLdError(
+            'No script tag found with id=${frag}.',
+            'jsonld.InvalidScriptElement', {
+              code: 'invalid script element',
+              remoteDoc
+            });
+        }
+        if(!options.extractAllScripts) {
+          remoteDoc.document = remoteDoc.document[0];
+        }
+      } else {
+        remoteDoc.document = JSON.parse(remoteDoc.document);
+      }
     }
   } catch(e) {
-    throw new JsonLdError(
-      'Could not retrieve a JSON-LD document from the URL.',
-      'jsonld.LoadDocumentError', {
-        code: 'loading document failed',
-        cause: e,
-        remoteDoc
-      });
+    if(e.name === 'jsonld.InvalidScriptElement') {
+      throw(e)
+    } else {
+      throw new JsonLdError(
+        'Could not retrieve a JSON-LD document from the URL.',
+        'jsonld.LoadDocumentError', {
+          code: 'loading document failed',
+          cause: e,
+          remoteDoc
+        });
+    }
   }
 
   return remoteDoc;
@@ -934,6 +993,20 @@ jsonld.documentLoaders = {};
 jsonld.documentLoaders.node = require('./documentLoaders/node');
 jsonld.documentLoaders.xhr = require('./documentLoaders/xhr');
 
+// Optional DOM parser
+try {
+  jsonld.domParser = require('xmldom').DOMParser;
+} catch(e) {
+  jsonld.domParser = class NoDOMParser {
+    parseFromString() {
+      throw new JsonLdError(
+        'Could not parse HTML document. ' +
+        'HTML parsing not implemented.', 'jsonld.LoadDocumentError',
+        {code: 'loading document failed'});
+    }
+  };
+}
+
 /**
  * Assigns the default document loader for external document URLs to a built-in
  * default. Supported types currently include: 'xhr' and 'node'.

lib/util.js

@@ -15,6 +15,7 @@ const REGEX_LINK_HEADER = /\s*<([^>]*?)>\s*(?:;\s*(.*))?/;
 const REGEX_LINK_HEADER_PARAMS =
   /(.*?)=(?:(?:"([^"]*?)")|([^"]*?))\s*(?:(?:;\s*)|$)/g;
 
+// FIXME: conditinally support text/html
 const DEFAULTS = {
   headers: {
     accept: 'application/ld+json, application/json'
@@ -142,6 +143,34 @@ api.parseLinkHeader = header => {
   return rval;
 };
 
+/**
+ * Parses a content-type header. The results will be key'd by the value of "rel".
+ *
+ * Accept: application/ld+json
+ *
+ * Parses as: ["application/ld+json", {}]
+ *
+ * Accept: application/ld+json;profile=http://www.w3.org/ns/json-ld#context
+ *
+ * Parses as: ["application/ld+json", {profile: "http://www.w3.org/ns/json-ld#context"}]
+ *
+ * If there is more than one 
+ *
+ * @param header the content-type header to parse.
+ */
+api.parseContentTypeHeader = header => {
+  const [type, ...rest] = header.split(';');
+  const params = {};
+  const rval = [type.trim(), params];
+
+  // assign parameters
+  for(const paramString of rest) {
+    const [param, value] = paramString.split('=');
+    params[param.trim().toLowerCase()] = value.trim();
+  }
+  return rval;
+};
+
 /**
  * Throws an exception if the given value is not a valid @type value.
  *