perf(ngcc): reduce maximum worker count (#38840)

Recent optimizations to ngcc have significantly reduced the total time
it takes to process `node_modules`, to such extend that sharding across
multiple processes has become less effective. Previously, running
ngcc asynchronously would allow for up to 8 workers to be allocated,
however these workers have to repeat work that could otherwise be shared.
Because ngcc is now able to reuse more shared computations, the overhead
of multiple workers is increased and therefore becomes less effective.
As an additional benefit, having fewer workers requires less memory and
less startup time.

To give an idea, using the following test setup:

```bash
npx @angular/cli new perf-test
cd perf-test
yarn ng add @angular/material
./node_modules/.bin/ngcc --properties es2015 module main \
  --first-only --create-ivy-entry-points
```

We observe the following figures on CI:

|                   | 10.1.1    | PR #38840 |
| ----------------- | --------- | --------- |
| Sync              | 85s       | 25s       |
| Async (8 workers) | 22s       | 16s       |
| Async (4 workers) | -         | 11s       |

In addition to changing the default number of workers, ngcc will now
use the environment variable `NGCC_MAX_WORKERS` that may be configured
to either reduce or increase the number of workers.

PR Close #38840
This commit is contained in:
JoostK
2020-09-14 14:19:30 +02:00
committed by Andrew Kushnir
parent f0688b4d18
commit fd44d84a33
3 changed files with 96 additions and 13 deletions
@@ -5,12 +5,13 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as os from 'os';
import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '../../src/ngtsc/file_system';
import {runInEachFileSystem} from '../../src/ngtsc/file_system/testing';
import {MockLogger} from '../../src/ngtsc/logging/testing';
import {clearTsConfigCache, getSharedSetup, NgccOptions} from '../src/ngcc_options';
import {clearTsConfigCache, getMaxNumberOfWorkers, getSharedSetup, NgccOptions} from '../src/ngcc_options';
@@ -100,6 +101,67 @@ runInEachFileSystem(() => {
});
});
describe('getMaxNumberOfWorkers', () => {
let processEnv: NodeJS.ProcessEnv;
let cpuSpy: jasmine.Spy;
beforeEach(() => {
processEnv = process.env;
process.env = {...process.env};
cpuSpy = spyOn(os, 'cpus');
});
afterEach(() => {
process.env = processEnv;
});
it('should use NGCC_MAX_WORKERS environment variable if set', () => {
process.env.NGCC_MAX_WORKERS = '16';
expect(getMaxNumberOfWorkers()).toBe(16);
process.env.NGCC_MAX_WORKERS = '8';
expect(getMaxNumberOfWorkers()).toBe(8);
process.env.NGCC_MAX_WORKERS = ' 8 ';
expect(getMaxNumberOfWorkers()).toBe(8);
});
it('should throw an error if NGCC_MAX_WORKERS is less than 1', () => {
process.env.NGCC_MAX_WORKERS = '0';
expect(() => getMaxNumberOfWorkers())
.toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.'));
process.env.NGCC_MAX_WORKERS = '-1';
expect(() => getMaxNumberOfWorkers())
.toThrow(new Error('NGCC_MAX_WORKERS should be at least 1.'));
});
it('should throw an error if NGCC_MAX_WORKERS is not an integer', () => {
process.env.NGCC_MAX_WORKERS = 'a';
expect(() => getMaxNumberOfWorkers())
.toThrow(new Error('NGCC_MAX_WORKERS should be an integer.'));
process.env.NGCC_MAX_WORKERS = '1.5';
expect(() => getMaxNumberOfWorkers())
.toThrow(new Error('NGCC_MAX_WORKERS should be an integer.'));
process.env.NGCC_MAX_WORKERS = '-';
expect(() => getMaxNumberOfWorkers())
.toThrow(new Error('NGCC_MAX_WORKERS should be an integer.'));
});
it('should fallback to the number of cpus, minus one (for the master process), with a maximum of 4 workers',
() => {
simulateNumberOfCpus(1);
expect(getMaxNumberOfWorkers()).toBe(1);
simulateNumberOfCpus(2);
expect(getMaxNumberOfWorkers()).toBe(1);
simulateNumberOfCpus(4);
expect(getMaxNumberOfWorkers()).toBe(3);
simulateNumberOfCpus(6);
expect(getMaxNumberOfWorkers()).toBe(4);
simulateNumberOfCpus(8);
expect(getMaxNumberOfWorkers()).toBe(4);
});
function simulateNumberOfCpus(cpus: number): void {
cpuSpy.and.returnValue(new Array(cpus).fill({model: 'Mock CPU'} as any));
}
});
/**
* This function creates an object that contains the minimal required properties for NgccOptions.
*/