rename export and provide cjs

Author: aeltorio

"dist/index.cjs.js

@@ -47,7 +47,11 @@ main()
 In the browser, you can use the following code:
 
 ```js
+import { Buffer } from "buffer";
 import { SentencePieceProcessor, cleanText, llama_3_1_tokeniser_b64 } from "@sctg/sentencepiece-js";
+
+// eslint-disable-next-line no-undef
+globalThis.Buffer = Buffer;
 // built in models: llama_3_1_tokeniser_b64, clean_30k_b64, smart_b64
 async function main() {
 
@@ -67,7 +71,8 @@ async function main() {
 main()
 ```
 
-## Note
+See https://github.com/sctg-development/ai-outlook/blob/HEAD/src/aipane/aipane.ts#L11-L23 for an example of how to use this in a react app.  
+Look also at webpack.config.js for the configuration of the webpack bundler.
 
 - devilyouwei updated this repo to make this module support the js `require` keyword and added the using example.
 - 2023-1-10, devilyouwei added `encodePieces`.

README.md

@@ -3,5 +3,5 @@ cd build
 emcmake cmake ../sentencepiece
 emmake make sentencepiece-static
 emcc --bind -o ../src/sentencepiece.js -Wl,--whole-archive src/libsentencepiece.a -Wl,--no-whole-archive ../bindings/sentencepiece.cpp \
--O3 -s EXPORT_ES6=1 -s MODULARIZE=1 -s SINGLE_FILE=1 -s EXPORTED_RUNTIME_METHODS=FS -s ALLOW_MEMORY_GROWTH=1
+-O3 -s EXPORT_ES6=1 -s MODULARIZE=1 -s SINGLE_FILE=1 -s EXPORTED_RUNTIME_METHODS=FS -s ALLOW_MEMORY_GROWTH=1 -s EXPORT_NAME="createSentencePieceModule"
 cd ..

build.sh

@@ -1,8 +1,9 @@
 {
   "name": "@sctg/sentencepiece-js",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Sentencepiece tokenization for natural language processing, JS version.",
   "main": "dist/index.js",
+  "type": "module",
   "exports": {
     "imports": "./dist/index.js",
     "default": "./dist/index.js"
@@ -16,6 +17,7 @@
   },
   "files": [
     "./dist/index.js",
+    "./dist/index.cjs",
     "./dist/index.d.ts"
   ],
   "repository": {
@@ -36,6 +38,7 @@
   "homepage": "https://github.com/sctg-development/sentencepiece-js",
   "devDependencies": {
     "@esm-bundle/chai": "^4.3.4-fix.0",
+    "@rollup/plugin-babel": "^6.0.4",
     "@rollup/plugin-commonjs": "^28.0.0",
     "@rollup/plugin-node-resolve": "^15.3.0",
     "@rollup/plugin-typescript": "^12.1.0",

package.json

@@ -1,24 +1,34 @@
-import { nodeResolve } from '@rollup/plugin-node-resolve';
-import commonjs from '@rollup/plugin-commonjs';
-import { wasm } from '@rollup/plugin-wasm';
-import typescript from '@rollup/plugin-typescript';
-import dts from 'rollup-plugin-dts';
+import { nodeResolve } from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import { babel } from "@rollup/plugin-babel";
+import { wasm } from "@rollup/plugin-wasm";
+import typescript from "@rollup/plugin-typescript";
+import dts from "rollup-plugin-dts";
 
 export default [
-    {
-        input: 'src/index.ts',
-        output: {
-            dir: 'dist',
-            format: 'cjs'
-        },
-        plugins: [nodeResolve({ browser: true }), commonjs(), wasm(), typescript({ target: "es5", downlevelIteration: true })]
+  {
+    input: "src/index.ts",
+    output: [
+      {
+        dir: "dist",
+        format: "es",
+      },
+      { file: "dist/index.cjs", format: "cjs" },
+    ],
+    plugins: [
+      nodeResolve({ browser: true }),
+      commonjs(),
+      babel({ babelHelpers: "bundled" }),
+      wasm(),
+      typescript({ target: "es6", downlevelIteration: true }),
+    ],
+  },
+  {
+    input: "src/index.ts",
+    output: {
+      file: "dist/index.d.ts",
+      format: "es",
     },
-    {
-        input: 'src/index.ts',
-        output: {
-            file: 'dist/index.d.ts',
-            format: 'es'
-        },
-        plugins: [dts()]
-    }
-];
+    plugins: [dts()],
+  },
+];

rollup.config.mjs

@@ -1,4 +1,4 @@
-import Module from "./sentencepiece"
+import createSentencePieceModule from "./sentencepiece.js"
 import * as fs from "fs"
 
 export class SentencePieceProcessor {
@@ -31,7 +31,7 @@ export class SentencePieceProcessor {
     // private function to load model
     private async _loadModel(model: Buffer) {
         const tempName = this.uuidv4() + ".model";
-        this.sentencepiece = await Module();
+        this.sentencepiece = await createSentencePieceModule();
         this.sentencepiece.FS.writeFile(tempName, model);
         const string_view = new this.sentencepiece.StringView(tempName);
         const absl_string_view = string_view.getView();

src/sentencePieceProcessor.ts

@@ -1,5 +1,5 @@
-const { SentencePieceProcessor, cleanText, llama_3_1_tokeniser_b64 } = require("../dist");
-const ROOT = require('app-root-path')
+import { SentencePieceProcessor, cleanText, llama_3_1_tokeniser_b64 } from "../dist/index.js";
+import ROOT from 'app-root-path';
 
 async function main() {