From 59410449da4c90eab617f2c86ec9181976c3d179 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 1 Mar 2022 23:00:22 -0800 Subject: [PATCH] [Build] Build ARM64 for deb and rpm (#1285) * [Build] Build ARM64 for deb and rpm Build ARM64 for deb with the following commands: yarn build --deb-arm --skip-archives yarn build --deb-arm --skip-archives --release Build ARM64 for rpm with the following commands: yarn build --rpm-arm --skip-archives yarn build --rpm-arm --skip-archives --release Issue partially resolved: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1259 Signed-off-by: Kawika Avilla --- DEVELOPER_GUIDE.md | 33 ++- src/dev/build/args.test.ts | 18 ++ src/dev/build/args.ts | 14 +- src/dev/build/build_distributables.ts | 10 + src/dev/build/lib/runner.test.ts | 5 + .../nodejs/download_node_builds_task.test.ts | 5 + .../nodejs/extract_node_builds_task.test.ts | 5 + .../verify_existing_node_builds_task.test.ts | 5 + .../create_os_package_tasks.test.ts | 212 ++++++++++++++++++ .../os_packages/create_os_package_tasks.ts | 30 ++- .../package_scripts/pre_install.sh | 4 +- .../os_packages/package_scripts/pre_remove.sh | 4 +- src/dev/build/tasks/os_packages/run_fpm.ts | 8 +- 13 files changed, 342 insertions(+), 11 deletions(-) create mode 100644 src/dev/build/tasks/os_packages/create_os_package_tasks.test.ts diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3edfbcee464a..1c33af417390 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -74,7 +74,7 @@ access it. ### Building the artifacts -To build the artifacts for all supported platforms, run the following: +To build the artifacts for all supported platforms, run the following: ``` yarn build --skip-os-packages @@ -86,7 +86,36 @@ If you want to build a specific platform, pass the platform flag after `yarn bui yarn build-platform --darwin ``` -You could pass one or multiple flags. If you don't pass any flag, `yarn build-platform` will use your local environment. Currenly we only support `darwin` (darwin x64), `linux` (linux x64) and `linux-arm` (linux arm64). +You could pass one or multiple flags. If you don't pass any flag, `yarn build-platform` will build an artifact based on your local environment. + +Currently, the supported flags for this script are: +* `darwin` (builds Darwin x64) +* `linux` (builds Linux x64) +* `linux-arm` (builds Linux ARM64). + +If you would like to build only a DEB x64 artifact, run the following: + +``` +yarn build --deb --skip-archives +``` + +If you would like to build only a DEB ARM64 artifact, run the following: + +``` +yarn build --deb-arm --skip-archives +``` + +If you would like to build only a RPM x64 artifact, run the following: + +``` +yarn build --rpm --skip-archives +``` + +If you would like to build only a RPM ARM64 artifact, run the following: + +``` +yarn build --rpm-arm --skip-archives +``` ### Building the Docker Image diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index d232e8944a14..5eedbfade448 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -46,9 +46,11 @@ it('build dist for current platform, without packages, by default', () => { Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -72,9 +74,11 @@ it('build dist for linux x64 platform, without packages, if --linux is passed', Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -98,9 +102,11 @@ it('build dist for linux arm64 platform, without packages, if --linux-arm is pas Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -124,9 +130,11 @@ it('build dist for darwin x64 platform, without packages, if --darwin is passed' Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -150,9 +158,11 @@ it('builds packages if --all-platforms is passed', () => { Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": true, "createDebPackage": true, "createDockerPackage": true, "createDockerUbiPackage": true, + "createRpmArmPackage": true, "createRpmPackage": true, "downloadFreshNode": true, "isRelease": false, @@ -176,9 +186,11 @@ it('limits packages if --rpm passed with --all-platforms', () => { Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": true, "downloadFreshNode": true, "isRelease": false, @@ -202,9 +214,11 @@ it('limits packages if --deb passed with --all-platforms', () => { Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": true, "createDockerPackage": false, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -229,9 +243,11 @@ it('limits packages if --docker passed with --all-platforms', () => { Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": true, "createDockerUbiPackage": true, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, @@ -256,9 +272,11 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform Object { "buildOptions": Object { "createArchives": true, + "createDebArmPackage": false, "createDebPackage": false, "createDockerPackage": true, "createDockerUbiPackage": false, + "createRpmArmPackage": false, "createRpmPackage": false, "downloadFreshNode": true, "isRelease": false, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index df7de0477c40..c00ccec07e4d 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -37,7 +37,9 @@ export function readCliArgs(argv: string[]) { 'skip-archives', 'skip-os-packages', 'rpm', + 'rpm-arm', 'deb', + 'deb-arm', 'docker', 'skip-docker-ubi', 'release', @@ -61,7 +63,9 @@ export function readCliArgs(argv: string[]) { default: { debug: true, rpm: null, + 'rpm-arm': null, deb: null, + 'deb-arm': null, docker: null, 'version-qualifier': '', }, @@ -98,7 +102,13 @@ export function readCliArgs(argv: string[]) { } // build all if no flags specified - if (flags.rpm === null && flags.deb === null && flags.docker === null) { + if ( + flags.rpm === null && + flags['rpm-arm'] === null && + flags.deb === null && + flags['deb-arm'] === null && + flags.docker === null + ) { return true; } @@ -111,7 +121,9 @@ export function readCliArgs(argv: string[]) { downloadFreshNode: !Boolean(flags['skip-node-download']), createArchives: !Boolean(flags['skip-archives']), createRpmPackage: isOsPackageDesired('rpm'), + createRpmArmPackage: isOsPackageDesired('rpm-arm'), createDebPackage: isOsPackageDesired('deb'), + createDebArmPackage: isOsPackageDesired('deb-arm'), createDockerPackage: isOsPackageDesired('docker'), createDockerUbiPackage: isOsPackageDesired('docker') && !Boolean(flags['skip-docker-ubi']), targetPlatforms: { diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 39983aec1ce3..8429c5aa16bd 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -35,7 +35,9 @@ export interface BuildOptions { downloadFreshNode: boolean; createArchives: boolean; createRpmPackage: boolean; + createRpmArmPackage: boolean; createDebPackage: boolean; + createDebArmPackage: boolean; createDockerPackage: boolean; createDockerUbiPackage: boolean; versionQualifier: string | undefined; @@ -107,10 +109,18 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions // control w/ --deb or --skip-os-packages await run(Tasks.CreateDebPackage); } + if (options.createDebArmPackage) { + // control w/ --deb-arm or --skip-os-packages + await run(Tasks.CreateDebArmPackage); + } if (options.createRpmPackage) { // control w/ --rpm or --skip-os-packages await run(Tasks.CreateRpmPackage); } + if (options.createRpmArmPackage) { + // control w/ --rpm-arm or --skip-os-packages + await run(Tasks.CreateRpmArmPackage); + } if (options.createDockerPackage) { // control w/ --docker or --skip-docker-ubi or --skip-os-packages await run(Tasks.CreateDockerPackage); diff --git a/src/dev/build/lib/runner.test.ts b/src/dev/build/lib/runner.test.ts index a1c3884ec84c..4cb6b6c38d7f 100644 --- a/src/dev/build/lib/runner.test.ts +++ b/src/dev/build/lib/runner.test.ts @@ -69,6 +69,11 @@ const setup = async () => { isRelease: true, targetAllPlatforms: true, versionQualifier: '-SNAPSHOT', + targetPlatforms: { + linux: false, + linuxArm: false, + darwin: false, + }, }); const run = createRunner({ diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts index 2633a407eef3..a96bcd86a4dd 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts @@ -58,6 +58,11 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + targetPlatforms: { + linux: false, + linuxArm: false, + darwin: false, + }, }); getNodeDownloadInfo.mockImplementation((_: Config, platform: Platform) => { diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts index 5badf05afb49..4ba942f9fb18 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts @@ -62,6 +62,11 @@ async function setup() { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + targetPlatforms: { + linux: false, + linuxArm: false, + darwin: false, + }, }); return { config }; diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts index 3a798c3aa47d..44c26b20faef 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts @@ -67,6 +67,11 @@ async function setup(actualShaSums?: Record) { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + targetPlatforms: { + linux: false, + linuxArm: false, + darwin: false, + }, }); getNodeShasums.mockReturnValue( diff --git a/src/dev/build/tasks/os_packages/create_os_package_tasks.test.ts b/src/dev/build/tasks/os_packages/create_os_package_tasks.test.ts new file mode 100644 index 000000000000..0f9b57b4c753 --- /dev/null +++ b/src/dev/build/tasks/os_packages/create_os_package_tasks.test.ts @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import { + ToolingLog, + ToolingLogCollectingWriter, + createAnyInstanceSerializer, +} from '@osd/dev-utils'; +import { CreateDebArmPackage, CreateDebPackage, CreateRpmPackage, CreateRpmArmPackage } from '.'; + +import { Build, Config } from '../../lib'; + +jest.mock('./run_fpm'); +jest.mock('./docker_generator'); + +const { runFpm } = jest.requireMock('./run_fpm'); + +const log = new ToolingLog(); +const testWriter = new ToolingLogCollectingWriter(); +log.setWriters([testWriter]); + +expect.addSnapshotSerializer(createAnyInstanceSerializer(Config)); +expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog)); + +async function setup() { + const config = await Config.create({ + isRelease: true, + targetAllPlatforms: true, + targetPlatforms: { + linux: false, + linuxArm: false, + darwin: false, + }, + }); + + const build = new Build(config); + + runFpm.mockImplementation( + ( + _: Config, + __: ToolingLog, + ___: Build, + type: string, + arch: string, + pkgSpecificFlags: string[] + ) => { + return { + package: `${type}:${arch}:${pkgSpecificFlags}`, + }; + } + ); + + return { config, build }; +} + +beforeEach(() => { + testWriter.messages.length = 0; + jest.clearAllMocks(); +}); + +it('runs task with the expected values for DEB x64', async () => { + const { config, build } = await setup(); + + await CreateDebPackage.run(config, log, build); + + expect(runFpm).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + , + , + Build { + "config": , + "name": "opensearch-dashboards", + }, + "deb", + "x64", + Array [ + "--architecture", + "amd64", + "--deb-priority", + "optional", + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Object { + "package": "deb:x64:--architecture,amd64,--deb-priority,optional", + }, + }, + ], + } + `); +}); + +it('runs task with the expected values for DEB ARM64', async () => { + const { config, build } = await setup(); + + await CreateDebArmPackage.run(config, log, build); + + expect(runFpm).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + , + , + Build { + "config": , + "name": "opensearch-dashboards", + }, + "deb", + "arm64", + Array [ + "--architecture", + "arm64", + "--deb-priority", + "optional", + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Object { + "package": "deb:arm64:--architecture,arm64,--deb-priority,optional", + }, + }, + ], + } + `); +}); + +it('runs task with the expected values for RPM x64', async () => { + const { config, build } = await setup(); + + await CreateRpmPackage.run(config, log, build); + + expect(runFpm).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + , + , + Build { + "config": , + "name": "opensearch-dashboards", + }, + "rpm", + "x64", + Array [ + "--architecture", + "x64", + "--rpm-os", + "linux", + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Object { + "package": "rpm:x64:--architecture,x64,--rpm-os,linux", + }, + }, + ], + } + `); +}); + +it('runs task with the expected values for RPM ARM64', async () => { + const { config, build } = await setup(); + + await CreateRpmArmPackage.run(config, log, build); + + expect(runFpm).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + , + , + Build { + "config": , + "name": "opensearch-dashboards", + }, + "rpm", + "arm64", + Array [ + "--architecture", + "arm64", + "--rpm-os", + "linux", + ], + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Object { + "package": "rpm:arm64:--architecture,arm64,--rpm-os,linux", + }, + }, + ], + } + `); +}); diff --git a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts index 24a7ee91313e..d7ac5eb39798 100644 --- a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts +++ b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts @@ -33,7 +33,7 @@ export const CreateDebPackage: Task = { description: 'Creating deb package', async run(config, log, build) { - await runFpm(config, log, build, 'deb', [ + await runFpm(config, log, build, 'deb', 'x64', [ '--architecture', 'amd64', '--deb-priority', @@ -42,11 +42,37 @@ export const CreateDebPackage: Task = { }, }; +export const CreateDebArmPackage: Task = { + description: 'Creating deb-arm package', + + async run(config, log, build) { + await runFpm(config, log, build, 'deb', 'arm64', [ + '--architecture', + 'arm64', + '--deb-priority', + 'optional', + ]); + }, +}; + export const CreateRpmPackage: Task = { description: 'Creating rpm package', async run(config, log, build) { - await runFpm(config, log, build, 'rpm', ['--architecture', 'x64', '--rpm-os', 'linux']); + await runFpm(config, log, build, 'rpm', 'x64', ['--architecture', 'x64', '--rpm-os', 'linux']); + }, +}; + +export const CreateRpmArmPackage: Task = { + description: 'Creating rpm-arm package', + + async run(config, log, build) { + await runFpm(config, log, build, 'rpm', 'arm64', [ + '--architecture', + 'arm64', + '--rpm-os', + 'linux', + ]); }, }; diff --git a/src/dev/build/tasks/os_packages/package_scripts/pre_install.sh b/src/dev/build/tasks/os_packages/package_scripts/pre_install.sh index 2b43962c7c8a..d019b40d492d 100644 --- a/src/dev/build/tasks/os_packages/package_scripts/pre_install.sh +++ b/src/dev/build/tasks/os_packages/package_scripts/pre_install.sh @@ -1,8 +1,8 @@ #!/bin/sh set -e -if command -v systemctl >/dev/null && systemctl is-active opensearch_dashboards.service >/dev/null; then - systemctl --no-reload stop opensearch_dashboards.service +if command -v systemctl >/dev/null && systemctl is-active opensearch-dashboards.service >/dev/null; then + systemctl --no-reload stop opensearch-dashboards.service elif [ -x /etc/init.d/opensearch-dashboards ]; then if command -v invoke-rc.d >/dev/null; then invoke-rc.d opensearch-dashboards stop diff --git a/src/dev/build/tasks/os_packages/package_scripts/pre_remove.sh b/src/dev/build/tasks/os_packages/package_scripts/pre_remove.sh index 081c0f258c3c..d359f1159282 100644 --- a/src/dev/build/tasks/os_packages/package_scripts/pre_remove.sh +++ b/src/dev/build/tasks/os_packages/package_scripts/pre_remove.sh @@ -2,8 +2,8 @@ set -e echo -n "Stopping OpenSearch Dashboards service..." -if command -v systemctl >/dev/null && systemctl is-active opensearch_dashboards.service >/dev/null; then - systemctl --no-reload stop opensearch_dashboards.service +if command -v systemctl >/dev/null && systemctl is-active opensearch-dashboards.service >/dev/null; then + systemctl --no-reload stop opensearch-dashboards.service elif [ -x /etc/init.d/opensearch-dashboards ]; then if command -v invoke-rc.d >/dev/null; then invoke-rc.d opensearch-dashboards stop diff --git a/src/dev/build/tasks/os_packages/run_fpm.ts b/src/dev/build/tasks/os_packages/run_fpm.ts index 1b25f7bdc065..e3227b39691d 100644 --- a/src/dev/build/tasks/os_packages/run_fpm.ts +++ b/src/dev/build/tasks/os_packages/run_fpm.ts @@ -36,10 +36,14 @@ export async function runFpm( log: ToolingLog, build: Build, type: 'rpm' | 'deb', + arch: 'x64' | 'arm64', pkgSpecificFlags: string[] ) { - const linux = config.getPlatform('linux', 'x64'); + const linux = config.getPlatform('linux', arch); const version = config.getBuildVersion(); + const fileName = config.resolveFromTarget( + arch === 'arm64' ? `NAME-${version}-arm64.${type}` : `NAME-${version}-ARCH.TYPE` + ); const resolveWithTrailingSlash = (...paths: string[]) => `${resolve(...paths)}/`; @@ -61,7 +65,7 @@ export async function runFpm( // the filtered package version, which would have dashes replaced with // underscores '--package', - config.resolveFromTarget(`NAME-${version}-ARCH.TYPE`), + fileName, // input type '-s',