Merge pull request #91 from extractus/dev

v6.2.3

Author: ndaidong

.eslintrc.json

@@ -62,7 +62,7 @@
     "max-lines": [
       "error",
       {
-        "max": 360,
+        "max": 460,
         "skipBlankLines": true,
         "skipComments": false
       }

README.md

@@ -134,6 +134,7 @@ Object with all or several of the following properties:
   - `xmlParserOptions`: Object, used by xml parser, view [fast-xml-parser's docs](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md)
   - `getExtraFeedFields`: Function, to get more fields from feed data
   - `getExtraEntryFields`: Function, to get more fields from feed entry data
+  - `baseUrl`: URL string, to absolutify the links within feed content
 
 For example:
 

dist/cjs/feed-extractor.js

@@ -47,6 +47,11 @@ export interface ReaderOptions {
    * https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md
    */
   xmlParserOptions?: any;
+  /**
+   * fill in the baseurl when it does not exist in the link
+   * default: ''
+   */
+  baseUrl?: string;
   /**
    * merge extra feed fields in result
    */

dist/cjs/index.d.ts

@@ -1,5 +1,5 @@
 {
   "name": "@extractus/feed-extractor",
-  "version": "6.2.2",
+  "version": "6.2.3",
   "main": "./feed-extractor.js"
 }

dist/cjs/package.json

@@ -47,6 +47,11 @@ export interface ReaderOptions {
    * https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md
    */
   xmlParserOptions?: any;
+  /**
+   * fill in the baseurl when it does not exist in the link
+   * default: ''
+   */
+  baseUrl?: string;
   /**
    * merge extra feed fields in result
    */

dist/feed-extractor.esm.js

@@ -1,5 +1,5 @@
 {
-  "version": "6.2.2",
+  "version": "6.2.3",
   "name": "@extractus/feed-extractor",
   "description": "To read and normalize RSS/ATOM/JSON feed data",
   "homepage": "https://extractor-demos.pages.dev",
@@ -36,13 +36,13 @@
   },
   "dependencies": {
     "bellajs": "^11.1.2",
-    "cross-fetch": "^3.1.5",
-    "fast-xml-parser": "^4.2.2",
-    "html-entities": "^2.3.3"
+    "cross-fetch": "^3.1.6",
+    "fast-xml-parser": "^4.2.4",
+    "html-entities": "^2.3.6"
   },
   "devDependencies": {
-    "esbuild": "^0.17.18",
-    "eslint": "^8.40.0",
+    "esbuild": "^0.18.2",
+    "eslint": "^8.42.0",
     "jest": "^29.5.0",
     "nock": "^13.3.1"
   },

index.d.ts

@@ -14,6 +14,7 @@ const getopt = (options = {}) => {
     descriptionMaxLen = 210,
     useISODateFormat = true,
     xmlParserOptions = {},
+    baseUrl = '',
     getExtraFeedFields = () => ({}),
     getExtraEntryFields = () => ({}),
   } = options
@@ -23,6 +24,7 @@ const getopt = (options = {}) => {
     descriptionMaxLen,
     useISODateFormat,
     xmlParserOptions,
+    baseUrl,
     getExtraFeedFields,
     getExtraEntryFields,
   }
@@ -51,6 +53,7 @@ export const extract = async (url, options = {}, fetchOptions = {}) => {
   if (!isValidUrl(url)) {
     throw new Error('Input param must be a valid URL')
   }
+
   const data = await retrieve(url, fetchOptions)
   if (!data.text && !data.json) {
     throw new Error(`Failed to load content from "${url}"`)

package.json

@@ -7,7 +7,7 @@ import nock from 'nock'
 
 import { hasProperty, isString } from 'bellajs'
 
-import { extract, read } from './main.js'
+import { extract, extractFromXml, extractFromJson, read } from './main.js'
 import { isValid as isValidUrl } from './utils/linker.js'
 
 const feedAttrs = 'title link description generator language published entries'.split(' ')
@@ -336,6 +336,65 @@ describe('test extract() without normalization', () => {
   })
 })
 
+describe('test extract with `baseUrl` option', () => {
+  test('extract rss feed with xml', () => {
+    const baseUrl = 'https://huggingface.co'
+    const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
+    const result = extractFromXml(xml, { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(validateProps(result.entries[0])).toBe(true)
+    expect(result.link).toBe(baseUrl + '/blog')
+    expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
+  })
+
+  test('extract rss feed with json', () => {
+    const baseUrl = 'https://www.jsonfeed.org'
+    const json = readFileSync('test-data/json-feed-miss-base-url.json', 'utf8')
+    const result = extractFromJson(JSON.parse(json), { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(result.link).toBe(baseUrl + '/')
+    expect(result.entries[0].link).toBe(baseUrl + '/2020/08/07/json-feed-version.html')
+  })
+
+  test('extract rss feed with url', async () => {
+    const url = 'https://huggingface.co/blog/rss'
+    const xml = readFileSync('test-data/rss-feed-miss-base-url.xml', 'utf8')
+    const { baseUrl, path } = parseUrl(url)
+    nock(baseUrl).get(path).reply(200, xml, {
+      'Content-Type': 'application/xml',
+    })
+    const result = await extract(url, { baseUrl })
+
+    feedAttrs.forEach((k) => {
+      expect(hasProperty(result, k)).toBe(true)
+    })
+
+    entryAttrs.forEach((k) => {
+      expect(hasProperty(result.entries[0], k)).toBe(true)
+    })
+
+    expect(validateProps(result.entries[0])).toBe(true)
+    expect(result.link).toBe(baseUrl + '/blog')
+    expect(result.entries[0].link).toBe(baseUrl + '/blog/intro-graphml')
+  })
+})
+
 describe('check old method read()', () => {
   test('ensure that depricated method read() still works', async () => {
     const url = 'https://realworld-standard-feed.tld/rss'