diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0aa3d7a7382..9b46c44d96c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,19 +6,24 @@ on: - "[0-9]+_[0-9]+_X" pull_request: -# TODO: Can we split things up in a more intutive way? -# i.e. an "Android" job which does the android/java linting and build -# an iOS job which does the swift/ios linting and build -# a JS/CLI job which does the js linting (and cli tests?) -# a packaging job which generates the combined SDK from ios/Android? - jobs: android: - runs-on: ubuntu-latest + # Use Ubuntu 18.04 as later versions do not maintain i386 repositories. + # This is required to run 32-bit mksnapshot binaries for snapshot generation. + runs-on: ubuntu-18.04 name: Android + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + USE_CCACHE: 1 steps: - uses: actions/checkout@v2 + - name: Dependencies including 32-bit support + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update -y -qq + sudo apt-get install -y -qq gperf libc6:i386 libncurses5:i386 libstdc++6:i386 + - name: Use Node.js 12.x uses: actions/setup-node@v1 with: @@ -51,13 +56,46 @@ jobs: - run: npm run lint:android name: Lint -# TODO: Build the android half here and stash/upload the built stuff: -# - dist/android/titanium.bindings.json -# - android/titanium/build/outputs -# - android/runtime/v8/generated -# - dist/android/libv8 -# Or do we run build *and* publish and then copy over the resulting stuff from each platform and piece them together? -# Because I assume there's some intermediate files from gradle/cmake/ndk that'd be missing to separet the build/publish halves + - name: Install ccache + run: brew install ccache + + - name: Retrieve ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-ccache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-ccache- + + - name: Cache V8 + uses: actions/cache@v2 + with: + path: dist/android/libv8 + key: libv8-${{ hashFiles('dist/android/libv8/**') }} + restore-keys: | + libv8- + + - run: npm run build:android + name: Build + + - name: Show summary of ccache configuration and statistics counters + run: ccache --show-stats + + - name: 'Package build output' + run: > + tar -czvf android-build.tar.gz + android/kroll-apt/build/libs/kroll-apt.jar + android/runtime/v8/generated/ + android/titanium/build/outputs/ + dist/android/libv8/ + dist/android/titanium.bindings.json + dist/tmp/common + + - uses: actions/upload-artifact@v2 + with: + name: android-build + retention-days: 1 + path: android-build.tar.gz - name: Cleanup Gradle Cache # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions. @@ -72,7 +110,80 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache USE_CCACHE: 1 + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: '12.x' + + - name: Cache Node.js modules + id: node-cache + uses: actions/cache@v2 + with: + path: node_modules + key: ${{ runner.OS }}-node-modules-${{ hashFiles('package-lock.json') }} + restore-keys: | + ${{ runner.OS }}-node-modules- + ${{ runner.OS }}- + + - run: npm ci + name: Install dependencies + if: steps.node-cache.outputs.cache-hit != 'true' + + - run: npm run lint:ios + name: Lint + + - run: npm run build:ios + name: Build + + - name: 'Package build output' + run: > + tar -czvf ios-build.tar.gz + dist/tmp/common + iphone/TitaniumKit/build/TitaniumKit.xcframework + + - uses: actions/upload-artifact@v2 + with: + name: ios-build + retention-days: 1 + path: ios-build.tar.gz + + js: + runs-on: ubuntu-latest + name: JavaScript + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: '12.x' + + - name: Cache Node.js modules + id: node-cache + uses: actions/cache@v2 + with: + path: node_modules + key: ${{ runner.OS }}-node-modules-${{ hashFiles('package-lock.json') }} + restore-keys: | + ${{ runner.OS }}-node-modules- + ${{ runner.OS }}- + + - run: npm ci + name: Install dependencies + if: steps.node-cache.outputs.cache-hit != 'true' + + - run: npm run lint:js + name: Lint + + package: + runs-on: macos-latest + name: Package + env: SDK_BUILD_CACHE_DIR: ${{ github.workspace }}/.native-modules + needs: [android, ios, js] steps: - uses: actions/checkout@v2 @@ -112,33 +223,19 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - run: npm run lint:ios - name: Lint - - - name: Install ccache - run: brew install ccache - - - name: Retrieve ccache - uses: actions/cache@v2 + - uses: actions/download-artifact@v2 with: - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-ccache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-ccache- + name: android-build - - name: Cache V8 - uses: actions/cache@v2 - with: - path: dist/android/libv8 - key: libv8-${{ hashFiles('dist/android/libv8/**') }} - restore-keys: | - libv8- + - name: 'Extract Android build output' + run: tar -xzvf android-build.tar.gz - - run: npm run build -- --all - name: Build + - uses: actions/download-artifact@v2 + with: + name: ios-build - - name: Show summary of ccache configuration and statistics counters - run: ccache --show-stats + - name: 'Extract iOS build output' + run: tar -xzvf ios-build.tar.gz - name: Cache Native Modules uses: actions/cache@v2 @@ -179,30 +276,8 @@ jobs: rm -f ~/.gradle/caches/modules-2/modules-2.lock rm -f ~/.gradle/caches/modules-2/gc.properties - js: - runs-on: ubuntu-latest - name: JavaScript - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: '12.x' - - - name: Cache Node.js modules - id: node-cache - uses: actions/cache@v2 + - uses: geekyeggo/delete-artifact@v1 with: - path: node_modules - key: ${{ runner.OS }}-node-modules-${{ hashFiles('package-lock.json') }} - restore-keys: | - ${{ runner.OS }}-node-modules- - ${{ runner.OS }}- - - - run: npm ci - name: Install dependencies - if: steps.node-cache.outputs.cache-hit != 'true' - - - run: npm run lint:js - name: Lint \ No newline at end of file + name: | + android-build + ios-build diff --git a/android/titanium/libv8-services.js b/android/titanium/libv8-services.js index feb57604c11..45974e49873 100644 --- a/android/titanium/libv8-services.js +++ b/android/titanium/libv8-services.js @@ -21,7 +21,8 @@ const path = require('path'); const request = require('request-promise-native'); // Determine if we're running on a Windows machine. -const isWindows = (process.platform === 'win32'); +const isWindows = process.platform === 'win32'; +const isLinux = process.platform === 'linux'; /** * Double quotes given path and escapes double quote characters in file/directory names. @@ -47,11 +48,16 @@ async function loadPackageJson() { } /** - * Debug snapshot generation locally. + * Generate snapshots locally. * @param {String} v8SnapshotHeaderFilePath The path to save generated snapshot header. * @param {String} rollupFileContent Javascript content to store in snapshot. */ -async function debugGenerateSnapshot(v8SnapshotHeaderFilePath, rollupFileContent) { +async function generateSnapshot(v8SnapshotHeaderFilePath, rollupFileContent) { + + if (!isLinux) { + console.warn('Snapshot generation only available on linux, skipping...'); + return; + } const distTmpPath = path.join(__dirname, '..', '..', 'dist', 'tmp'); const startupPath = path.join(distTmpPath, 'startup.js'); @@ -117,11 +123,16 @@ async function debugGenerateSnapshot(v8SnapshotHeaderFilePath, rollupFileContent blobs[arch] = Buffer.from(await fs.readFile(blobPath, 'binary'), 'binary'); console.log(`Generated ${arch} snapshot blob.`); } + + // Delete snapshot blob. + await fs.unlink(blobPath); } // Generate 'V8Snapshots.h' from template const template = await util.promisify(ejs.renderFile)('V8Snapshots.h.ejs', { blobs }, {}); await fs.writeFile(v8SnapshotHeaderFilePath, template); + + console.log('Generated snapshot header.'); } /** @@ -151,69 +162,77 @@ async function createSnapshot() { const v8SnapshotHeaderFilePath = path.join(cppOutputDirPath, 'V8Snapshots.h'); await fs.ensureDir(cppOutputDirPath); - // DEBUG: Generate snapshots locally. - // await debugGenerateSnapshot(v8SnapshotHeaderFilePath, rollupFileContent); - // return; - - // Requests our server to create snapshot of rolled-up "ti.main" in a C++ header file. - let wasSuccessful = false; - try { - // Post rolled-up "ti.main" script to server and obtain a snapshot ID as a response. - // We will send an HTTP request for the snapshot code later. - console.log('Attempting to request snapshot...'); - const snapshotUrl = 'https://v8-snapshot.appcelerator.com'; - const packageJsonData = await loadPackageJson(); - const requestOptions = { - body: { - v8: packageJsonData.v8.version, - script: rollupFileContent - }, - json: true - }; - const snapshotId = await request.post(snapshotUrl, requestOptions); - - // Request generated snapshot from server using `snapshotId` obtained from server above. - const MAX_ATTEMPTS = 20; // Time-out after two minutes. - let attempts; - for (attempts = 1; attempts <= MAX_ATTEMPTS; attempts++) { - const response = await request.get(`${snapshotUrl}/snapshot/${snapshotId}`, { - simple: false, - resolveWithFullResponse: true - }); - if (response.statusCode === 200) { - // Server has finished creating a C++ header file containing all V8 snapshots. - // Write it to file and flag that we're done. - console.log('Writing snapshot...'); - await fs.writeFile(v8SnapshotHeaderFilePath, response.body); - wasSuccessful = true; - break; - } else if (response.statusCode === 202) { - // Snapshot server is still building. We need to retry later. - console.log('Waiting for snapshot generation...'); - await new Promise(resolve => setTimeout(resolve, 6000)); - } else { - // Give up if received an unexpected response. - console.error('Could not generate snapshot, skipping...'); - break; + if (isLinux) { + + // Generate snapshots locally. + await generateSnapshot(v8SnapshotHeaderFilePath, rollupFileContent); + } else { + + // Requests our server to create snapshot of rolled-up "ti.main" in a C++ header file. + let wasSuccessful = false; + try { + // Post rolled-up "ti.main" script to server and obtain a snapshot ID as a response. + // We will send an HTTP request for the snapshot code later. + console.log('Attempting to request snapshot...'); + const snapshotUrl = 'https://v8-snapshot.appcelerator.com'; + const packageJsonData = await loadPackageJson(); + const requestOptions = { + body: { + v8: packageJsonData.v8.version, + script: rollupFileContent + }, + json: true + }; + const snapshotId = await request.post(snapshotUrl, requestOptions); + + // Request generated snapshot from server using `snapshotId` obtained from server above. + const MAX_ATTEMPTS = 20; // Time-out after two minutes. + let attempts; + for (attempts = 1; attempts <= MAX_ATTEMPTS; attempts++) { + const response = await request.get(`${snapshotUrl}/snapshot/${snapshotId}`, { + simple: false, + resolveWithFullResponse: true + }); + + if (response.statusCode === 200) { + + // Server has finished creating a C++ header file containing all V8 snapshots. + // Write it to file and flag that we're done. + console.log('Writing snapshot...'); + await fs.writeFile(v8SnapshotHeaderFilePath, response.body); + wasSuccessful = true; + break; + } else if (response.statusCode === 202) { + + // Snapshot server is still building. We need to retry later. + console.log('Waiting for snapshot generation...'); + await new Promise(resolve => setTimeout(resolve, 6000)); + } else { + + // Give up if received an unexpected response. + console.error('Could not generate snapshot, skipping...'); + break; + } } + if (attempts > MAX_ATTEMPTS) { + console.error('Max retries exceeded fetching snapshot from server, skipping...'); + } + } catch (err) { + console.error(`Failed to request snapshot: ${err}`); } - if (attempts > MAX_ATTEMPTS) { - console.error('Max retries exceeded fetching snapshot from server, skipping...'); - } - } catch (err) { - console.error(`Failed to request snapshot: ${err}`); - } - // Do the following if we've failed to generate snapshot header file above. - // Note: The C++ build will fail if file is missing. This is because it is #included in our code. - if (!wasSuccessful) { - // Trigger a build failure if snapshots are required. The "titanium_mobile/build" SDK build scripts set this. - if (process.env.TI_SDK_BUILD_REQUIRES_V8_SNAPSHOTS === '1') { - process.exit(1); - } + // Do the following if we've failed to generate snapshot header file above. + // Note: The C++ build will fail if file is missing. This is because it is #included in our code. + if (!wasSuccessful) { - // Generaet an empty C++ header. Allows build to succeed and app will load "ti.main.js" normally instead. - await fs.writeFile(v8SnapshotHeaderFilePath, '// Failed to build V8 snapshots. See build log.'); + // Trigger a build failure if snapshots are required. The "titanium_mobile/build" SDK build scripts set this. + if (process.env.TI_SDK_BUILD_REQUIRES_V8_SNAPSHOTS === '1') { + process.exit(1); + } + + // Generaet an empty C++ header. Allows build to succeed and app will load "ti.main.js" normally instead. + await fs.writeFile(v8SnapshotHeaderFilePath, '// Failed to generate V8 snapshots. See build log.'); + } } } diff --git a/android/titanium/prebuild.js b/android/titanium/prebuild.js index e71b6cd81d2..f6c5ee16245 100644 --- a/android/titanium/prebuild.js +++ b/android/titanium/prebuild.js @@ -17,7 +17,7 @@ const generateBootstrap = require('./genBootstrap'); const runtimeV8DirPath = path.join(__dirname, '..', 'runtime', 'v8'); // Determine if we're running on a Windows machine. -const isWindows = (process.platform === 'win32'); +const isWindows = process.platform === 'win32'; /** * Double quotes given path and escapes double quote characters in file/directory names. diff --git a/build/lib/builder.js b/build/lib/builder.js index a185e00fff6..2798ce9af05 100644 --- a/build/lib/builder.js +++ b/build/lib/builder.js @@ -16,8 +16,10 @@ const ROOT_DIR = path.join(__dirname, '..', '..'); const DIST_DIR = path.join(ROOT_DIR, 'dist'); const TMP_DIR = path.join(DIST_DIR, 'tmp'); -// platforms/OS mappings -const ALL_OSES = [ 'win32', 'linux', 'osx' ]; +// Platforms/OS mappings +// NOTE: 'linux' could be added, but is not officially supported. +// Specifying --all will only produce a linux output on a linux host. +const ALL_OSES = Array.from(new Set([ 'win32', 'osx', thisOS() ])); const ALL_PLATFORMS = [ 'ios', 'android' ]; const OS_TO_PLATFORMS = { win32: [ 'android' ], diff --git a/package.json b/package.json index 4b8839f303a..21b6224eb8a 100644 --- a/package.json +++ b/package.json @@ -17,43 +17,43 @@ ], "author": "Appcelerator, Inc. ", "scripts": { - "android": "./build/scons cleanbuild android", - "build": "./build/scons build", + "android": "node ./build/scons cleanbuild android", + "build": "node ./build/scons build", "build:android": "npm run build -- android", "build:changelog": "conventional-changelog -n changelog/config.js -i CHANGELOG.md -s -p angular", "build:docs": "docgen apidoc -o ./dist", "build:ios": "npm run build -- ios", - "clean": "./build/scons clean", + "clean": "node ./build/scons clean", "clean:android": "npm run clean -- android", "clean:ios": "npm run clean -- ios", - "clean:modules": "./build/scons clean-modules", - "clean:sdks": "./build/scons clean-sdks", - "cleanbuild": "./build/scons cleanbuild", + "clean:modules": "node ./build/scons clean-modules", + "clean:sdks": "node ./build/scons clean-sdks", + "cleanbuild": "node ./build/scons cleanbuild", "cleanbuild:android": "npm run cleanbuild -- android", "cleanbuild:ios": "npm run cleanbuild -- ios", "commit": "git-cz", - "deploy": "./build/scons install", + "deploy": "node ./build/scons install", "deprecations": "npm run docs:deprecated", - "docs:deprecated": "./build/scons deprecations", - "docs:removed": "./build/scons removals 7.0.0", + "docs:deprecated": "node ./build/scons deprecations", + "docs:removed": "node ./build/scons removals 7.0.0", "format": "npm-run-all --parallel format:!\\(ios\\|android\\)", "format:android": "echo Formatting Android code is not supported.", "format:ios": "npm-run-all --parallel format:objc format:swift", "format:objc": "npm run lint:objc -- --fix", "format:swift": "npm run lint:swift -- autocorrect", "format:js": "npm run lint:js -- --fix", - "ios": "./build/scons cleanbuild ios", - "ios-sanity-check": "./build/scons check-ios-toplevel", + "ios": "node ./build/scons cleanbuild ios", + "ios-sanity-check": "node ./build/scons check-ios-toplevel", "link": "npm run deploy -- --symlink", "lint": "npm-run-all --parallel lint:!\\(ios\\)", - "lint:android": "./build/scons gradlew checkJavaStyle", + "lint:android": "node ./build/scons gradlew checkJavaStyle", "lint:docs": "tdoc-validate ./apidoc", "lint:ios": "npm-run-all --parallel lint:objc lint:swift", "lint:objc": "clang-format-lint $npm_package_config_format_objc", "lint:swift": "swiftlint", "lint:js": "eslint .", - "lint:lockfile": "./build/scons check-lockfile && lockfile-lint --path package-lock.json --type npm --allowed-hosts npm yarn --validate-https", - "package": "./build/scons package", + "lint:lockfile": "node ./build/scons check-lockfile && lockfile-lint --path package-lock.json --type npm --allowed-hosts npm yarn --validate-https", + "package": "node ./build/scons package", "package:android": "npm run package -- android", "package:ios": "npm run package -- ios", "postinstall": "husky install", @@ -64,7 +64,7 @@ "test:iphone": "npm run cleanbuild:ios -- --skip-zip --no-docs --symlink && ti sdk select $npm_package_version && npm run test:integration -- ios -F iphone", "test:ipad": "npm run cleanbuild:ios -- --skip-zip --no-docs --symlink && ti sdk select $npm_package_version && npm run test:integration -- ios -F ipad", "test:mac": "npm run cleanbuild:ios -- --skip-zip --no-docs --symlink && ti sdk select $npm_package_version && npm run test:integration -- ios -T macos", - "test:integration": "./build/scons test" + "test:integration": "node ./build/scons test" }, "commitlint": { "extends": [