[pve-devel] [PATCH v2 3/3] use worker_threads for linting

[Dominik Csapak]

instead linting all files in the main thread, use worker threads
for that (4 by default) and add the '-t' switch to able to control that

a basic benchmark of eslint of pve-manager showed some performance
gains:

Benchmark #1: Current
  Time (mean ± σ):      6.468 s ±  0.116 s    [User: 9.803 s, System: 0.333 s]
  Range (min … max):    6.264 s …  6.647 s    10 runs

Benchmark #2: 2Threads
  Time (mean ± σ):      4.509 s ±  0.106 s    [User: 12.706 s, System: 0.530 s]
  Range (min … max):    4.335 s …  4.674 s    10 runs

Benchmark #3: 4Threads
  Time (mean ± σ):      3.471 s ±  0.033 s    [User: 16.390 s, System: 0.630 s]
  Range (min … max):    3.431 s …  3.542 s    10 runs

Benchmark #4: 8Threads
  Time (mean ± σ):      2.880 s ±  0.044 s    [User: 22.454 s, System: 0.938 s]
  Range (min … max):    2.813 s …  2.964 s    10 runs

Summary
  '8Threads' ran
    1.21 ± 0.02 times faster than '4Threads'
    1.57 ± 0.04 times faster than '2Threads'
    2.25 ± 0.05 times faster than 'Current'

after 8 threads, there were no real performance benefits since the
overhead to load the module seems to be the biggest factor.

Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
---
 src/bin/app.js    | 35 +++++++++++++++++++++++++++++------
 src/index.js      |  2 ++
 src/lib/worker.js | 27 +++++++++++++++++++++++++++
 src/package.json  |  3 ++-
 4 files changed, 60 insertions(+), 7 deletions(-)
 create mode 100644 src/lib/worker.js

diff --git a/src/bin/app.js b/src/bin/app.js
index 8a28923..10e7e6a 100644
--- a/src/bin/app.js
+++ b/src/bin/app.js
@@ -6,6 +6,7 @@
 const path = require('path');
 const color = require('colors');
 const program = require('commander');
+const worker = require('worker_threads');
 const eslint = require('pve-eslint');
 
 program
@@ -14,6 +15,7 @@ program
     .option('-e, --extend <configfile>', 'uses <configfile> ontop of default eslint config.')
     .option('-f, --fix', 'if set, fixes will be applied.')
     .option('-s, --strict', 'if set, also exit uncleanly on warnings')
+    .option('-t, --threads <threads>', 'how many worker_threads should be used (default=4)')
     .option('--output-config', 'if set, only output the config as JSON and exit.')
     ;
 
@@ -42,6 +44,11 @@ if (!paths.length) {
     paths = [process.cwd()];
 }
 
+let threadCount = 4;
+if (program.threads) {
+    threadCount = program.threads;
+}
+
 const defaultConfig = {
     parserOptions: {
 	ecmaVersion: 2020,
@@ -283,20 +290,36 @@ if (program.outputConfig) {
     process.exit(0);
 }
 
-const cli = new eslint.CLIEngine({
+const cliOptions = {
     baseConfig: config,
     useEslintrc: true,
     fix: !!program.fix,
     cwd: process.cwd(),
-});
+};
+
+let promises = [];
+let filesPerThread = Math.round(paths.length / threadCount);
+for (let i = 0; i < (threadCount - 1); i++) {
+    let files = paths.splice(0, filesPerThread);
+    promises.push(eslint.createWorker({
+	cliOptions,
+	files
+    }));
+}
+
+// the remaining paths
+promises.push(eslint.createWorker({
+    cliOptions,
+    files: paths
+}));
 
-const report = cli.executeOnFiles(paths);
+let results = (await Promise.all(promises)).map(res => res.results).flat(1);
 
 let exitcode = 0;
 let files_err = [], files_warn = [], files_ok = [];
 let fixes = 0;
 console.log('------------------------------------------------------------');
-report.results.forEach(function(result) {
+results.forEach(function(result) {
     let filename = path.relative(process.cwd(), result.filePath);
     let msgs = result.messages;
     let max_sev = 0;
@@ -348,7 +371,7 @@ report.results.forEach(function(result) {
     console.log('------------------------------------------------------------');
 });
 
-if (report.results.length > 1) {
+if (results.length > 1) {
     console.log(`${color.bold(files_ok.length + files_err.length)} files:`);
     if (files_err.length > 0) {
 	console.log(color.red(` ${color.bold(files_err.length)} files have Errors`));
@@ -367,7 +390,7 @@ console.log('------------------------------------------------------------');
 if (program.fix) {
     if (fixes > 0) {
 	console.log(`Writing ${color.bold(fixes)} fixed files...`);
-	eslint.CLIEngine.outputFixes(report);
+	eslint.CLIEngine.outputFixes({ results });
 	console.log('Done');
     } else {
 	console.log("No fixable Errors/Warnings found.");
diff --git a/src/index.js b/src/index.js
index 01b9a1d..311ae38 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,3 +1,5 @@
 const eslint = require('./lib/eslint.js');
+const createWorker = require('./lib/worker.js');
 
 module.exports = eslint;
+module.exports.createWorker = createWorker;
diff --git a/src/lib/worker.js b/src/lib/worker.js
new file mode 100644
index 0000000..9a8c955
--- /dev/null
+++ b/src/lib/worker.js
@@ -0,0 +1,27 @@
+'use strict';
+
+const worker = require('worker_threads');
+
+if (!worker.isMainThread) {
+    const eslint = require('pve-eslint');
+    const data = worker.workerData;
+    const cli = new eslint.CLIEngine(data.cliOptions);
+    const report = cli.executeOnFiles(data.files);
+    worker.parentPort.postMessage(report);
+} else {
+    module.exports = async function createWorker(workerData) {
+	return new Promise((resolve, reject) => {
+	    const child = new worker.Worker(__filename,
+		{
+		    workerData,
+		},
+	    );
+	    child.on('message', resolve);
+	    child.on('error', reject);
+	    child.on('exit', (code) => {
+		if (code !== 0) {reject(new Error(`Worker stopped with exit code ${code}`));}
+	    });
+	});
+    }
+}
+
diff --git a/src/package.json b/src/package.json
index b08184b..e912069 100644
--- a/src/package.json
+++ b/src/package.json
@@ -4,6 +4,7 @@
     "files": [
 	"index.js",
 	"bin/app.js",
-	"lib/eslint.js"
+	"lib/eslint.js",
+	"lib/worker.js"
     ]
 }
-- 
2.30.2