-
Notifications
You must be signed in to change notification settings - Fork 373
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* feat: implement parallel operations (#2067) * feat: implement parallel operations * add more parallel operations * add header to test file * update import of fs/promises * fix pathing on windows, fix mocking of fs promises * add jsdoc headers to class and uploadMulti * add jsdoc comments to remaining functions * update comment wording * add experimental jsdoc tags * feat: add directory generator to performance test framework * clarify variable names and comments * capitalization * wip: transfer manager performance tests * feat: merged in application performance tests (#2100) * fix: fixed many bugs (#2102) * fix: cleaning up bugs * fix: fixed many bugs * fix: more work on transfer manager perf metrics (#2103) * fix: more work on transfer manager perf metrics * fix unit tests for tm * fix: performance test refactoring, comments (#2104) * refactor: refactor constants (#2105) * refactor: refactor constants * bug fixes * bug fixes * bug fixes * linter fixes, download to disk for performance test * rename transfer manager functions * remove callbacks from transfer manager * add more experimental tags, update comments * change signature of downloadManyFiles to accept array of strings or a folder name * linter fix * add transfer manager samples and samples tests Co-authored-by: Sameena Shaffeeullah <shaffeeullah@google.com> * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add crc32c validation option to chunked download, cleanup naming (#2110) * feat: add crc32c validation option to chunked download, cleanup naming in perf tests * close file in finally block Co-authored-by: Sameena Shaffeeullah <shaffeeullah@google.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
- Loading branch information
3 people
authored
Dec 7, 2022
1 parent
2474615
commit ce15b5e
Showing
16 changed files
with
1,787 additions
and
128 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| /*! | ||
| * Copyright 2022 Google LLC. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| import yargs from 'yargs'; | ||
| import {promises as fsp, rmSync} from 'fs'; | ||
| import {Bucket, DownloadOptions, DownloadResponse, UploadOptions} from '../src'; | ||
| import {performance} from 'perf_hooks'; | ||
| // eslint-disable-next-line node/no-unsupported-features/node-builtins | ||
| import {parentPort} from 'worker_threads'; | ||
| import { | ||
| BLOCK_SIZE_IN_BYTES, | ||
| DEFAULT_PROJECT_ID, | ||
| DEFAULT_NUMBER_OF_OBJECTS, | ||
| DEFAULT_SMALL_FILE_SIZE_BYTES, | ||
| DEFAULT_LARGE_FILE_SIZE_BYTES, | ||
| NODE_DEFAULT_HIGHWATER_MARK_BYTES, | ||
| generateRandomDirectoryStructure, | ||
| getValidationType, | ||
| performanceTestSetup, | ||
| TestResult, | ||
| } from './performanceUtils'; | ||
| import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; | ||
|
|
||
| const TEST_NAME_STRING = 'nodejs-perf-metrics-application'; | ||
| const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics-shaffeeullah'; | ||
|
|
||
| let bucket: Bucket; | ||
|
|
||
| const checkType = getValidationType(); | ||
|
|
||
| const argv = yargs(process.argv.slice(2)) | ||
| .options({ | ||
| bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, | ||
| small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, | ||
| large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, | ||
| projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, | ||
| numobjects: {type: 'number', default: DEFAULT_NUMBER_OF_OBJECTS}, | ||
| }) | ||
| .parseSync(); | ||
|
|
||
| /** | ||
| * Main entry point. This function performs a test iteration and posts the message back | ||
| * to the parent thread. | ||
| */ | ||
| async function main() { | ||
| let result: TestResult = { | ||
| op: '', | ||
| objectSize: 0, | ||
| appBufferSize: 0, | ||
| libBufferSize: 0, | ||
| crc32Enabled: false, | ||
| md5Enabled: false, | ||
| apiName: 'JSON', | ||
| elapsedTimeUs: 0, | ||
| cpuTimeUs: 0, | ||
| status: '[OK]', | ||
| }; | ||
|
|
||
| ({bucket} = await performanceTestSetup(argv.projectid, argv.bucket)); | ||
|
|
||
| switch (argv.testtype) { | ||
| case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: | ||
| result = await performWriteTest(); | ||
| break; | ||
| case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: | ||
| result = await performReadTest(); | ||
| break; | ||
| // case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD: | ||
| // result = await performLargeReadTest(); | ||
| // break; | ||
| default: | ||
| break; | ||
| } | ||
| parentPort?.postMessage(result); | ||
| } | ||
|
|
||
| async function uploadInParallel( | ||
| bucket: Bucket, | ||
| paths: string[], | ||
| options: UploadOptions | ||
| ) { | ||
| const promises = []; | ||
| for (const index in paths) { | ||
| const path = paths[index]; | ||
| const stat = await fsp.lstat(path); | ||
| if (stat.isDirectory()) { | ||
| continue; | ||
| } | ||
| options.destination = path; | ||
| promises.push(bucket.upload(path, options)); | ||
| } | ||
| await Promise.all(promises).catch(console.error); | ||
| } | ||
|
|
||
| async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { | ||
| const promises: Promise<DownloadResponse>[] = []; | ||
| const [files] = await bucket.getFiles(); | ||
| files.forEach(file => { | ||
| promises.push(file.download(options)); | ||
| }); | ||
| await Promise.all(promises).catch(console.error); | ||
| } | ||
|
|
||
| /** | ||
| * Performs an iteration of the Write multiple objects test. | ||
| * | ||
| * @returns {Promise<TestResult>} Promise that resolves to a test result of an iteration. | ||
| */ | ||
| async function performWriteTest(): Promise<TestResult> { | ||
| await bucket.deleteFiles(); //start clean | ||
|
|
||
| const creationInfo = generateRandomDirectoryStructure( | ||
| argv.numobjects, | ||
| TEST_NAME_STRING, | ||
| argv.small, | ||
| argv.large | ||
| ); | ||
|
|
||
| const start = performance.now(); | ||
| await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); | ||
| const end = performance.now(); | ||
|
|
||
| await bucket.deleteFiles(); //cleanup files | ||
| rmSync(TEST_NAME_STRING, {recursive: true, force: true}); | ||
|
|
||
| const result: TestResult = { | ||
| op: 'WRITE', | ||
| objectSize: creationInfo.totalSizeInBytes, | ||
| appBufferSize: BLOCK_SIZE_IN_BYTES, | ||
| libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, | ||
| crc32Enabled: checkType === 'crc32c', | ||
| md5Enabled: checkType === 'md5', | ||
| apiName: 'JSON', | ||
| elapsedTimeUs: Math.round((end - start) * 1000), | ||
| cpuTimeUs: -1, | ||
| status: '[OK]', | ||
| }; | ||
| return result; | ||
| } | ||
|
|
||
| /** | ||
| * Performs an iteration of the read multiple objects test. | ||
| * | ||
| * @returns {Promise<TestResult>} Promise that resolves to an array of test results for the iteration. | ||
| */ | ||
| async function performReadTest(): Promise<TestResult> { | ||
| await bucket.deleteFiles(); // start clean | ||
| const creationInfo = generateRandomDirectoryStructure( | ||
| argv.numobjects, | ||
| TEST_NAME_STRING, | ||
| argv.small, | ||
| argv.large | ||
| ); | ||
| await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); | ||
|
|
||
| const start = performance.now(); | ||
| await downloadInParallel(bucket, {validation: checkType}); | ||
| const end = performance.now(); | ||
|
|
||
| const result: TestResult = { | ||
| op: 'READ', | ||
| objectSize: creationInfo.totalSizeInBytes, | ||
| appBufferSize: BLOCK_SIZE_IN_BYTES, | ||
| libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, | ||
| crc32Enabled: checkType === 'crc32c', | ||
| md5Enabled: checkType === 'md5', | ||
| apiName: 'JSON', | ||
| elapsedTimeUs: Math.round((end - start) * 1000), | ||
| cpuTimeUs: -1, | ||
| status: '[OK]', | ||
| }; | ||
|
|
||
| rmSync(TEST_NAME_STRING, {recursive: true, force: true}); | ||
| await bucket.deleteFiles(); //cleanup | ||
| return result; | ||
| } | ||
|
|
||
| main(); |
Oops, something went wrong.