Skip to content

Commit 9585d47

Browse files
mzgoddardmzgoddard
authored
feat: add ParallelModulePlugin (#415)
* feat: add parallel module building * feat: test module parallelization * feat(parallel): move parallelization to build from create * chore: organize TransformNormalModulePlugin for webpack 4 * chore: organize worker start and stop in parallel plugin * chore: iterate forking cli * fixup! feat: test module parallelization * chore: add parallel log messages * chore: clean up parallelization for real use * chore: extract support: use factoryMeta in place of build meta * fixup! chore: organize TransformNormalModulePlugin for webpack 4 * chore: export getter for ParallelModulePlugin * chore: standardize test timeouts at 30s * chore: lint * chore: add ParallelModulePlugin to README
1 parent 33b306b commit 9585d47

File tree

34 files changed

+748
-175
lines changed

34 files changed

+748
-175
lines changed

‎README.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,11 @@ Some further configuration is possible through provided plugins.
6262
```js
6363
plugins: [
6464
new HardSourceWebpackPlugin(),
65+
```
66+
67+
### ExcludeModulePlugin
6568

69+
```js
6670
// You can optionally exclude items that may not be working with HardSource
6771
// or items with custom loaders while you are actively developing the
6872
// loader.
@@ -80,6 +84,30 @@ Some further configuration is possible through provided plugins.
8084
include: path.join(__dirname, 'vendor'),
8185
},
8286
]),
87+
```
88+
89+
### ParallelModulePlugin
90+
91+
```js
92+
// HardSource includes an experimental plugin for parallelizing webpack
93+
// across multiple processes. It requires that the extra processes have the
94+
// same configuration. `mode` must be set in the config. Making standard
95+
// use with webpack-dev-server or webpack-serve is difficult. Using it with
96+
// webpack-dev-server or webpack-serve means disabling any automatic
97+
// configuration and configuring hot module replacement support manually.
98+
new HardSourceWebpackPlugin.ParallelModulePlugin({
99+
// How to launch the extra processes. Default:
100+
fork: (fork, compiler, webpackBin) => fork(
101+
webpackBin(),
102+
['--config', __filename], {
103+
silent: true,
104+
}
105+
),
106+
// Number of workers to spawn. Default:
107+
numWorkers: () => require('os').cpus().length,
108+
// Number of modules built before launching parallel building. Default:
109+
minModules: 10,
110+
}),
83111
]
84112
```
85113

‎index.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,3 +577,9 @@ HardSourceWebpackPlugin.SerializerAppend2Plugin = SerializerAppend2Plugin;
577577
HardSourceWebpackPlugin.SerializerAppendPlugin = SerializerAppendPlugin;
578578
HardSourceWebpackPlugin.SerializerCacachePlugin = SerializerCacachePlugin;
579579
HardSourceWebpackPlugin.SerializerJsonPlugin = SerializerJsonPlugin;
580+
581+
Object.defineProperty(HardSourceWebpackPlugin, 'ParallelModulePlugin', {
582+
get() {
583+
return require('./lib/ParallelModulePlugin');
584+
},
585+
});

‎lib/ParallelLauncherPlugin.js

Whitespace-only changes.

‎lib/ParallelModulePlugin.js

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
const { fork: cpFork } = require('child_process');
2+
const { cpus } = require('os');
3+
const { resolve } = require('path');
4+
5+
const logMessages = require('./util/log-messages');
6+
const pluginCompat = require('./util/plugin-compat');
7+
8+
const webpackBin = () => {
9+
try {
10+
return require.resolve('webpack-cli');
11+
} catch (e) {}
12+
try {
13+
return require.resolve('webpack-command');
14+
} catch (e) {}
15+
throw new Error('webpack cli tool not installed or discoverable');
16+
};
17+
18+
const configPath = compiler => {
19+
try {
20+
return require.resolve(
21+
resolve(compiler.options.context || process.cwd(), 'webpack.config'),
22+
);
23+
} catch (e) {}
24+
try {
25+
return require.resolve(resolve(process.cwd(), 'webpack.config'));
26+
} catch (e) {}
27+
throw new Error('config not in obvious location');
28+
};
29+
30+
class ParallelModulePlugin {
31+
constructor(options) {
32+
this.options = options;
33+
}
34+
35+
apply(compiler) {
36+
try {
37+
require('webpack/lib/JavascriptGenerator');
38+
} catch (e) {
39+
logMessages.parallelRequireWebpack4(compiler);
40+
return;
41+
}
42+
43+
const options = this.options || {};
44+
const fork =
45+
options.fork ||
46+
((fork, compiler, webpackBin) =>
47+
fork(webpackBin(compiler), ['--config', configPath(compiler)], {
48+
silent: true,
49+
}));
50+
const numWorkers = options.numWorkers
51+
? typeof options.numWorkers === 'function'
52+
? options.numWorkers
53+
: () => options.numWorkers
54+
: () => cpus().length;
55+
const minModules =
56+
typeof options.minModules === 'number' ? options.minModules : 10;
57+
58+
const compilerHooks = pluginCompat.hooks(compiler);
59+
60+
let freeze, thaw;
61+
62+
compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => {
63+
freeze = methods.freeze;
64+
thaw = methods.thaw;
65+
});
66+
67+
compilerHooks.thisCompilation.tap(
68+
'ParallelModulePlugin',
69+
(compilation, params) => {
70+
const compilationHooks = pluginCompat.hooks(compilation);
71+
const nmfHooks = pluginCompat.hooks(params.normalModuleFactory);
72+
73+
const doMaster = () => {
74+
const jobs = {};
75+
const readyJobs = {};
76+
const workers = [];
77+
78+
let nextWorkerIndex = 0;
79+
80+
let start = 0;
81+
let started = false;
82+
let configMismatch = false;
83+
84+
let modules = 0;
85+
86+
const startWorkers = () => {
87+
const _numWorkers = numWorkers();
88+
logMessages.parallelStartWorkers(compiler, {
89+
numWorkers: _numWorkers,
90+
});
91+
92+
for (let i = 0; i < _numWorkers; i++) {
93+
const worker = fork(cpFork, compiler, webpackBin);
94+
workers.push(worker);
95+
worker.on('message', _result => {
96+
if (configMismatch) {
97+
return;
98+
}
99+
100+
if (_result.startsWith('ready:')) {
101+
const configHash = _result.split(':')[1];
102+
if (configHash !== compiler.__hardSource_configHash) {
103+
logMessages.parallelConfigMismatch(compiler, {
104+
outHash: compiler.__hardSource_configHash,
105+
theirHash: configHash,
106+
});
107+
108+
configMismatch = true;
109+
killWorkers();
110+
for (const id in jobs) {
111+
jobs[id].cb({ error: true });
112+
delete readyJobs[id];
113+
delete jobs[id];
114+
}
115+
return;
116+
}
117+
}
118+
119+
if (Object.values(readyJobs).length) {
120+
const id = Object.keys(readyJobs)[0];
121+
worker.send(
122+
JSON.stringify({
123+
id,
124+
data: readyJobs[id].data,
125+
}),
126+
);
127+
delete readyJobs[id];
128+
} else {
129+
worker.ready = true;
130+
}
131+
132+
if (_result.startsWith('ready:')) {
133+
start = Date.now();
134+
return;
135+
}
136+
137+
const result = JSON.parse(_result);
138+
jobs[result.id].cb(result);
139+
delete [result.id];
140+
});
141+
}
142+
};
143+
144+
const killWorkers = () => {
145+
Object.values(workers).forEach(worker => worker.kill());
146+
};
147+
148+
const doJob = (module, cb) => {
149+
if (configMismatch) {
150+
cb({ error: new Error('config mismatch') });
151+
return;
152+
}
153+
154+
const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () =>
155+
Math.random()
156+
.toString(16)
157+
.substring(2, 3),
158+
);
159+
jobs[id] = {
160+
id,
161+
data: freeze('Module', null, module, {
162+
id: module.identifier(),
163+
compilation,
164+
}),
165+
cb,
166+
};
167+
168+
const worker = Object.values(workers).find(worker => worker.ready);
169+
if (worker) {
170+
worker.ready = false;
171+
worker.send(
172+
JSON.stringify({
173+
id,
174+
data: jobs[id].data,
175+
}),
176+
);
177+
} else {
178+
readyJobs[id] = jobs[id];
179+
}
180+
181+
if (!started) {
182+
started = true;
183+
startWorkers();
184+
}
185+
};
186+
187+
const _create = params.normalModuleFactory.create;
188+
params.normalModuleFactory.create = (data, cb) => {
189+
_create.call(params.normalModuleFactory, data, (err, module) => {
190+
if (err) {
191+
return cb(err);
192+
}
193+
if (module.constructor.name === 'NormalModule') {
194+
const build = module.build;
195+
module.build = (
196+
options,
197+
compilation,
198+
resolver,
199+
fs,
200+
callback,
201+
) => {
202+
if (modules < minModules) {
203+
build.call(
204+
module,
205+
options,
206+
compilation,
207+
resolver,
208+
fs,
209+
callback,
210+
);
211+
modules += 1;
212+
return;
213+
}
214+
215+
try {
216+
doJob(module, result => {
217+
if (result.error) {
218+
build.call(
219+
module,
220+
options,
221+
compilation,
222+
resolver,
223+
fs,
224+
callback,
225+
);
226+
} else {
227+
thaw('Module', module, result.module, {
228+
compilation,
229+
normalModuleFactory: params.normalModuleFactory,
230+
contextModuleFactory: params.contextModuleFactory,
231+
});
232+
callback();
233+
}
234+
});
235+
} catch (e) {
236+
logMessages.parallelErrorSendingJob(compiler, e);
237+
build.call(
238+
module,
239+
options,
240+
compilation,
241+
resolver,
242+
fs,
243+
callback,
244+
);
245+
}
246+
};
247+
cb(null, module);
248+
} else {
249+
cb(err, module);
250+
}
251+
});
252+
};
253+
254+
compilationHooks.seal.tap('ParallelModulePlugin', () => {
255+
killWorkers();
256+
});
257+
};
258+
259+
const doChild = () => {
260+
const _create = params.normalModuleFactory.create;
261+
params.normalModuleFactory.create = (data, cb) => {};
262+
263+
process.send('ready:' + compiler.__hardSource_configHash);
264+
265+
process.on('message', _job => {
266+
const job = JSON.parse(_job);
267+
const module = thaw('Module', null, job.data, {
268+
compilation,
269+
normalModuleFactory: params.normalModuleFactory,
270+
contextModuleFactory: params.contextModuleFactory,
271+
});
272+
273+
module.build(
274+
compilation.options,
275+
compilation,
276+
compilation.resolverFactory.get('normal', module.resolveOptions),
277+
compilation.inputFileSystem,
278+
error => {
279+
process.send(
280+
JSON.stringify({
281+
id: job.id,
282+
error: error,
283+
module:
284+
module &&
285+
freeze('Module', null, module, {
286+
id: module.identifier(),
287+
compilation,
288+
}),
289+
}),
290+
);
291+
},
292+
);
293+
});
294+
};
295+
296+
if (!process.send) {
297+
doMaster();
298+
} else {
299+
doChild();
300+
}
301+
},
302+
);
303+
}
304+
}
305+
306+
module.exports = ParallelModulePlugin;

‎lib/SupportExtractTextPlugin.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class SupportExtractTextPlugin {
2020
// that assets get built.
2121
if (
2222
module[extractTextNS] ||
23-
(!module.buildMeta && module.meta && module.meta[extractTextNS])
23+
(!module.factoryMeta && module.meta && module.meta[extractTextNS])
2424
) {
2525
return null;
2626
}

0 commit comments

Comments
 (0)