Skip to content

Commit

Permalink
De-Couple Dashboards linux building process (#808)
Browse files Browse the repository at this point in the history 
When running yarn build --skip-os-packages it will build 4 tarballs
for Dashboards (2x linux, 1x macOS, 1x windows) and takes 10+min to
do so.

In this PR, we break the building process to allow single linux to
build. If run `yarn build-platform --linux-x64` only linux x64 tarball
is created. Same for linux arm64 and darwin x64. You could run `yarn
build-platform --linux-arm64` and `yarn build-platform darwin-x64`.

partially solved:
#473

Backport PR:
#795
Signed-off-by: Anan Zhuang <ananzh@amazon.com>

support multiple CLI options and fix comments

Signed-off-by: AnanZ <ananzh@amazon.com>

fix wording to artifacts

Signed-off-by: AnanZ <ananzh@amazon.com>

Co-authored-by: Ubuntu <ubuntu@ip-172-31-50-84.us-west-2.compute.internal>
  • Loading branch information
@ananzh
ananzh and Ubuntu authored Sep 21, 2021
1 parent a472979 commit ac3f309
Show file tree
Hide file tree
Showing 17 changed files with 354 additions and 20 deletions.
16 changes: 16 additions & 0 deletions DEVELOPER_GUIDE.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,22 @@ $ yarn start
When the server is up and ready, click on the link displayed in your terminal to
access it.

### Building the artifacts

To build the archives for each platform, run the following:

```
yarn build --skip-os-packages
```

If you want to build a specific platform, pass the platform flag after `yarn build-platform`. For example, to build darwin x64, run the following:

```
yarn build-platform --darwin
```

You could pass one or multiple flags. If you don't pass any flag, `yarn build-platform` will use your local environment. Currenly we only support `darwin` (darwin x64), `linux` (linux x64) and `linux-arm` (linux arm64).

### Building the Docker Image

To build the Docker image, run the following:
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
"test:ftr:runner": "node scripts/functional_test_runner",
"test:coverage": "grunt test:coverage",
"checkLicenses": "node scripts/check_licenses --dev",
"build-platform": "node scripts/build",
"build": "node scripts/build --all-platforms",
"start": "node scripts/opensearch_dashboards --dev",
"debug": "node --nolazy --inspect scripts/opensearch_dashboards --dev",
Expand Down
108 changes: 108 additions & 0 deletions src/dev/build/args.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,89 @@ it('build dist for current platform, without packages, by default', () => {
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": false,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
"showHelp": false,
"unknownFlags": Array [],
}
`);
});

it('build dist for linux x64 platform, without packages, if --linux-x64 is passed', () => {
expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(`
Object {
"buildOptions": Object {
"createArchives": true,
"createDebPackage": false,
"createDockerPackage": false,
"createDockerUbiPackage": false,
"createRpmPackage": false,
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": false,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
"showHelp": false,
"unknownFlags": Array [],
}
`);
});

it('build dist for linux x64 platform, without packages, if --linux-arm64 is passed', () => {
expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(`
Object {
"buildOptions": Object {
"createArchives": true,
"createDebPackage": false,
"createDockerPackage": false,
"createDockerUbiPackage": false,
"createRpmPackage": false,
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": false,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
"showHelp": false,
"unknownFlags": Array [],
}
`);
});

it('build dist for linux x64 platform, without packages, if --darwin-x64 is passed', () => {
expect(readCliArgs(['node', 'scripts/build-platform'])).toMatchInlineSnapshot(`
Object {
"buildOptions": Object {
"createArchives": true,
"createDebPackage": false,
"createDockerPackage": false,
"createDockerUbiPackage": false,
"createRpmPackage": false,
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": false,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand All @@ -74,6 +157,11 @@ it('builds packages if --all-platforms is passed', () => {
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": true,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand All @@ -95,6 +183,11 @@ it('limits packages if --rpm passed with --all-platforms', () => {
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": true,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand All @@ -116,6 +209,11 @@ it('limits packages if --deb passed with --all-platforms', () => {
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": true,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand All @@ -138,6 +236,11 @@ it('limits packages if --docker passed with --all-platforms', () => {
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": true,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand All @@ -160,6 +263,11 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
"downloadFreshNode": true,
"isRelease": false,
"targetAllPlatforms": true,
"targetPlatforms": Object {
"darwin": false,
"linux": false,
"linuxArm": false,
},
"versionQualifier": "",
},
"log": <ToolingLog>,
Expand Down
8 changes: 8 additions & 0 deletions src/dev/build/args.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ export function readCliArgs(argv: string[]) {
'verbose',
'debug',
'all-platforms',
'darwin',
'linux',
'linux-arm',
'verbose',
'quiet',
'silent',
Expand Down Expand Up @@ -111,6 +114,11 @@ export function readCliArgs(argv: string[]) {
createDebPackage: isOsPackageDesired('deb'),
createDockerPackage: isOsPackageDesired('docker'),
createDockerUbiPackage: isOsPackageDesired('docker') && !Boolean(flags['skip-docker-ubi']),
targetPlatforms: {
darwin: Boolean(flags.darwin),
linux: Boolean(flags.linux),
linuxArm: Boolean(flags['linux-arm']),
},
targetAllPlatforms: Boolean(flags['all-platforms']),
};

Expand Down
3 changes: 2 additions & 1 deletion src/dev/build/build_distributables.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

import { ToolingLog } from '@osd/dev-utils';

import { Config, createRunner } from './lib';
import { Config, createRunner, TargetPlatforms } from './lib';
import * as Tasks from './tasks';

export interface BuildOptions {
Expand All @@ -40,6 +40,7 @@ export interface BuildOptions {
createDockerUbiPackage: boolean;
versionQualifier: string | undefined;
targetAllPlatforms: boolean;
targetPlatforms: TargetPlatforms;
}

export async function buildDistributables(log: ToolingLog, options: BuildOptions) {
Expand Down
3 changes: 3 additions & 0 deletions src/dev/build/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ if (showHelp) {
--skip-archives {dim Don't produce tar/zip archives}
--skip-os-packages {dim Don't produce rpm/deb/docker packages}
--all-platforms {dim Produce archives for all platforms, not just this one}
--linux-x64 {dim Produce archives for only linux x64 platform}
--linux-arm64 {dim Produce archives for only linux arm64 platform}
--darwin-x64 {dim Produce archives for only darwin x64 platform}
--rpm {dim Only build the rpm package}
--deb {dim Only build the deb package}
--docker {dim Only build the docker image}
Expand Down
17 changes: 11 additions & 6 deletions src/dev/build/lib/build.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,12 @@ expect.addSnapshotSerializer(createAbsolutePathSerializer());
const config = new Config(
true,
{
version: '8.0.0',
darwin: false,
linux: false,
linuxArm: false,
},
{
version: '1.0.0',
engines: {
node: '*',
},
Expand All @@ -49,7 +54,7 @@ const config = new Config(
{
buildNumber: 1234,
buildSha: 'abcd1234',
buildVersion: '8.0.0',
buildVersion: '1.0.0',
},
true
);
Expand Down Expand Up @@ -93,21 +98,21 @@ describe('#resolvePath()', () => {
describe('#resolvePathForPlatform()', () => {
it('uses config.resolveFromRepo(), config.getBuildVersion(), and platform.getBuildName() to create path', () => {
expect(build.resolvePathForPlatform(linuxPlatform, 'foo', 'bar')).toMatchInlineSnapshot(
`<absolute path>/build/opensearch-dashboards-8.0.0-linux-x64/foo/bar`
`<absolute path>/build/opensearch-dashboards-1.0.0-linux-x64/foo/bar`
);
});
});

describe('#getPlatformArchivePath()', () => {
it('creates correct path for different platforms', () => {
expect(build.getPlatformArchivePath(linuxPlatform)).toMatchInlineSnapshot(
`<absolute path>/target/opensearch-dashboards-8.0.0-linux-x64.tar.gz`
`<absolute path>/target/opensearch-dashboards-1.0.0-linux-x64.tar.gz`
);
expect(build.getPlatformArchivePath(linuxArmPlatform)).toMatchInlineSnapshot(
`<absolute path>/target/opensearch-dashboards-8.0.0-linux-arm64.tar.gz`
`<absolute path>/target/opensearch-dashboards-1.0.0-linux-arm64.tar.gz`
);
expect(build.getPlatformArchivePath(windowsPlatform)).toMatchInlineSnapshot(
`<absolute path>/target/opensearch-dashboards-8.0.0-windows-x64.zip`
`<absolute path>/target/opensearch-dashboards-1.0.0-windows-x64.zip`
);
});
});
Loading

0 comments on commit ac3f309

Please sign in to comment.