Skip to content
This repository was archived by the owner on Feb 14, 2024. It is now read-only.

Commit 3317661

Browse files
authored
Merge pull request #27 from martinRenou/filesystem
Update jupyterlite and enable file-system access
2 parents 7b7f048 + 7d9288a commit 3317661

File tree

10 files changed

+388
-160
lines changed

10 files changed

+388
-160
lines changed

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@ python_data.js
117117
python_data.data
118118
xpython_wasm.js
119119
xpython_wasm.wasm
120-
xeus_python.worker.js
121120

122121
# Labextension
123122
share

MANIFEST.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ include install.json
77
include ts*.json
88
include yarn.lock
99
include webpack.config.js
10+
include worker.webpack.config.js
1011
include Dockerfile
1112
recursive-include tests *.py
1213

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ The [xeus-python](https://github.com/jupyter-xeus/xeus-python) Python kernel for
1313

1414
## Requirements
1515

16-
- JupyterLite >= 0.1.0a14
16+
- JupyterLite >= 0.1.0b9
1717

1818
## Install
1919

package.json

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,16 @@
2929
"url": "https://github.com/jupyterlite/xeus-python-kernel.git"
3030
},
3131
"scripts": {
32-
"build:dockerimage": "docker build -t mydockerimage . ",
33-
"build:dockerimage_no_cache": "docker build -t mydockerimage --no-cache . ",
34-
"build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) mydockerimage copy_output.sh",
35-
"copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.worker.js src/xpython_wasm.js src/python_data.js src/python_data.data lib",
32+
"build:dockerimage": "docker build -t jupyterlite-xeus-kernel . ",
33+
"build:dockerimage_no_cache": "docker build -t jupyterlite-xeus-kernel --no-cache . ",
34+
"build:emscripten": "echo $(pwd) && docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) jupyterlite-xeus-kernel copy_output.sh",
35+
"copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
3636
"build:wasm": "jlpm run build:dockerimage && jlpm run build:emscripten",
3737
"build:wasm_no_cache": "jlpm run build:dockerimage_no_cache && jlpm run build:emscripten",
38-
"build": "jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension:dev",
39-
"build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension",
38+
"build:worker": "webpack --config worker.webpack.config.js --mode=development",
39+
"build:worker:prod": "webpack --config worker.webpack.config.js --mode=production",
40+
"build": "jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
41+
"build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run build:worker:prod && jlpm run copy-files && jlpm run build:labextension",
4042
"build:labextension": "jupyter labextension build .",
4143
"build:labextension:dev": "jupyter labextension build --development True .",
4244
"build:lib": "tsc",
@@ -56,7 +58,9 @@
5658
"watch:labextension": "jupyter labextension watch ."
5759
},
5860
"dependencies": {
59-
"@jupyterlite/server": "^0.1.0-alpha.14"
61+
"@jupyterlite/contents": "^0.1.0-beta.9",
62+
"@jupyterlite/server": "^0.1.0-beta.9",
63+
"comlink": "^4.3.1"
6064
},
6165
"devDependencies": {
6266
"@jupyterlab/builder": "^3.4.1",
@@ -70,6 +74,7 @@
7074
"npm-run-all": "^4.1.5",
7175
"prettier": "^2.1.1",
7276
"rimraf": "^3.0.2",
77+
"source-map-loader": "^4.0.0",
7378
"typescript": "~4.2.3"
7479
},
7580
"sideEffects": [

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@ build_cmd = "build:prod"
1414
npm = ["jlpm"]
1515

1616
[tool.check-manifest]
17-
ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xeus_python.worker.js", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]
17+
ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "Dockerfile", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "*.sh"]

src/index.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// Distributed under the terms of the Modified BSD License.
44

55
import {
6+
IServiceWorkerRegistrationWrapper,
67
JupyterLiteServer,
78
JupyterLiteServerPlugin
89
} from '@jupyterlite/server';
@@ -18,8 +19,12 @@ import logo64 from '../style/logos/python-logo-64x64.png';
1819
const server_kernel: JupyterLiteServerPlugin<void> = {
1920
id: '@jupyterlite/xeus-kernel-extension:kernel',
2021
autoStart: true,
21-
requires: [IKernelSpecs],
22-
activate: (app: JupyterLiteServer, kernelspecs: IKernelSpecs) => {
22+
requires: [IKernelSpecs, IServiceWorkerRegistrationWrapper],
23+
activate: (
24+
app: JupyterLiteServer,
25+
kernelspecs: IKernelSpecs,
26+
serviceWorkerRegistrationWrapper: IServiceWorkerRegistrationWrapper
27+
) => {
2328
kernelspecs.register({
2429
spec: {
2530
name: 'xeus-python',
@@ -41,7 +46,8 @@ const server_kernel: JupyterLiteServerPlugin<void> = {
4146
},
4247
create: async (options: IKernel.IOptions): Promise<IKernel> => {
4348
return new XeusServerKernel({
44-
...options
49+
...options,
50+
mountDrive: serviceWorkerRegistrationWrapper.enabled
4551
});
4652
}
4753
});

src/worker.ts

Lines changed: 160 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,77 @@
22
// Copyright (c) JupyterLite Contributors
33
// Distributed under the terms of the Modified BSD License.
44

5+
import { expose } from 'comlink';
6+
7+
import {
8+
DriveFS,
9+
DriveFSEmscriptenNodeOps,
10+
IEmscriptenFSNode,
11+
IStats
12+
} from '@jupyterlite/contents';
13+
514
declare function createXeusModule(options: any): any;
615

716
globalThis.Module = {};
817

18+
// TODO Remove this. This is to ensure we always perform node ops on Nodes and
19+
// not Streams, but why is it needed??? Why do we get Streams and not Nodes from
20+
// emscripten in the case of xeus-python???
21+
class StreamNodeOps extends DriveFSEmscriptenNodeOps {
22+
private getNode(nodeOrStream: any) {
23+
if (nodeOrStream['node']) {
24+
return nodeOrStream['node'];
25+
}
26+
return nodeOrStream;
27+
}
28+
29+
lookup(parent: IEmscriptenFSNode, name: string): IEmscriptenFSNode {
30+
return super.lookup(this.getNode(parent), name);
31+
}
32+
33+
getattr(node: IEmscriptenFSNode): IStats {
34+
return super.getattr(this.getNode(node));
35+
}
36+
37+
setattr(node: IEmscriptenFSNode, attr: IStats): void {
38+
super.setattr(this.getNode(node), attr);
39+
}
40+
41+
mknod(
42+
parent: IEmscriptenFSNode,
43+
name: string,
44+
mode: number,
45+
dev: any
46+
): IEmscriptenFSNode {
47+
return super.mknod(this.getNode(parent), name, mode, dev);
48+
}
49+
50+
rename(
51+
oldNode: IEmscriptenFSNode,
52+
newDir: IEmscriptenFSNode,
53+
newName: string
54+
): void {
55+
super.rename(this.getNode(oldNode), this.getNode(newDir), newName);
56+
}
57+
58+
rmdir(parent: IEmscriptenFSNode, name: string): void {
59+
super.rmdir(this.getNode(parent), name);
60+
}
61+
62+
readdir(node: IEmscriptenFSNode): string[] {
63+
return super.readdir(this.getNode(node));
64+
}
65+
}
66+
67+
// TODO Remove this when we don't need StreamNodeOps anymore
68+
class LoggingDrive extends DriveFS {
69+
constructor(options: DriveFS.IOptions) {
70+
super(options);
71+
72+
this.node_ops = new StreamNodeOps(this);
73+
}
74+
}
75+
976
// when a toplevel cell uses an await, the cell is implicitly
1077
// wrapped in a async function. Since the webloop - eventloop
1178
// implementation does not support `eventloop.run_until_complete(f)`
@@ -17,27 +84,7 @@ globalThis.Module = {};
1784
globalThis.toplevel_promise = null;
1885
globalThis.toplevel_promise_py_proxy = null;
1986

20-
// We alias self to ctx and give it our newly created type
21-
const ctx: Worker = self as any;
22-
let raw_xkernel: any;
23-
let raw_xserver: any;
24-
25-
async function waitRunDependency() {
26-
const promise = new Promise((r: any) => {
27-
globalThis.Module.monitorRunDependencies = (n: number) => {
28-
if (n === 0) {
29-
console.log('all `RunDependencies` loaded');
30-
r();
31-
}
32-
};
33-
});
34-
// If there are no pending dependencies left, monitorRunDependencies will
35-
// never be called. Since we can't check the number of dependencies,
36-
// manually trigger a call.
37-
globalThis.Module.addRunDependency('dummy');
38-
globalThis.Module.removeRunDependency('dummy');
39-
return promise;
40-
}
87+
let resolveInputReply: any;
4188

4289
async function get_stdin() {
4390
const replyPromise = new Promise(resolve => {
@@ -46,49 +93,107 @@ async function get_stdin() {
4693
return replyPromise;
4794
}
4895

49-
// eslint-disable-next-line
50-
// @ts-ignore: breaks typedoc
51-
ctx.get_stdin = get_stdin;
96+
(self as any).get_stdin = get_stdin;
5297

53-
let resolveInputReply: any;
98+
class XeusPythonKernel {
99+
constructor() {
100+
this._ready = new Promise(resolve => {
101+
this.initialize(resolve);
102+
});
103+
}
54104

55-
async function load() {
56-
const options: any = {};
105+
async ready(): Promise<void> {
106+
return await this._ready;
107+
}
57108

58-
importScripts('./xpython_wasm.js');
109+
mount(driveName: string, mountpoint: string, baseUrl: string): void {
110+
const { FS, PATH, ERRNO_CODES } = globalThis.Module;
111+
112+
this._drive = new LoggingDrive({
113+
FS,
114+
PATH,
115+
ERRNO_CODES,
116+
baseUrl,
117+
driveName,
118+
mountpoint
119+
});
120+
121+
FS.mkdir(mountpoint);
122+
FS.mount(this._drive, {}, mountpoint);
123+
FS.chdir(mountpoint);
124+
}
59125

60-
globalThis.Module = await createXeusModule(options);
126+
cd(path: string) {
127+
if (!path) {
128+
return;
129+
}
61130

62-
importScripts('./python_data.js');
131+
globalThis.Module.FS.chdir(path);
132+
}
63133

64-
await waitRunDependency();
65-
raw_xkernel = new globalThis.Module.xkernel();
66-
raw_xserver = raw_xkernel.get_server();
67-
raw_xkernel!.start();
68-
}
134+
async processMessage(event: any): Promise<void> {
135+
await this._ready;
136+
137+
if (
138+
globalThis.toplevel_promise !== null &&
139+
globalThis.toplevel_promise_py_proxy !== null
140+
) {
141+
await globalThis.toplevel_promise;
142+
globalThis.toplevel_promise_py_proxy.delete();
143+
globalThis.toplevel_promise_py_proxy = null;
144+
globalThis.toplevel_promise = null;
145+
}
146+
147+
const msg_type = event.msg.header.msg_type;
148+
149+
if (msg_type === 'input_reply') {
150+
resolveInputReply(event.msg);
151+
} else {
152+
this._raw_xserver.notify_listener(event.msg);
153+
}
154+
}
69155

70-
const loadCppModulePromise = load();
156+
private async initialize(resolve: () => void) {
157+
importScripts('./xpython_wasm.js');
71158

72-
ctx.onmessage = async (event: MessageEvent): Promise<void> => {
73-
await loadCppModulePromise;
159+
globalThis.Module = await createXeusModule({});
74160

75-
if (
76-
globalThis.toplevel_promise !== null &&
77-
globalThis.toplevel_promise_py_proxy !== null
78-
) {
79-
await globalThis.toplevel_promise;
80-
globalThis.toplevel_promise_py_proxy.delete();
81-
globalThis.toplevel_promise_py_proxy = null;
82-
globalThis.toplevel_promise = null;
83-
}
161+
importScripts('./python_data.js');
162+
163+
await this.waitRunDependency();
164+
165+
this._raw_xkernel = new globalThis.Module.xkernel();
166+
this._raw_xserver = this._raw_xkernel.get_server();
167+
168+
if (!this._raw_xkernel) {
169+
console.error('Failed to start kernel!');
170+
}
84171

85-
const data = event.data;
86-
const msg = data.msg;
87-
const msg_type = msg.header.msg_type;
172+
this._raw_xkernel.start();
88173

89-
if (msg_type === 'input_reply') {
90-
resolveInputReply(msg);
91-
} else {
92-
raw_xserver!.notify_listener(msg);
174+
resolve();
93175
}
94-
};
176+
177+
private async waitRunDependency() {
178+
const promise = new Promise<void>(resolve => {
179+
globalThis.Module.monitorRunDependencies = (n: number) => {
180+
if (n === 0) {
181+
resolve();
182+
}
183+
};
184+
});
185+
// If there are no pending dependencies left, monitorRunDependencies will
186+
// never be called. Since we can't check the number of dependencies,
187+
// manually trigger a call.
188+
globalThis.Module.addRunDependency('dummy');
189+
globalThis.Module.removeRunDependency('dummy');
190+
return promise;
191+
}
192+
193+
private _raw_xkernel: any;
194+
private _raw_xserver: any;
195+
private _drive: DriveFS | null = null;
196+
private _ready: PromiseLike<void>;
197+
}
198+
199+
expose(new XeusPythonKernel());

0 commit comments

Comments
 (0)