Skip to content

Commit 72fd15d

Browse files
blefebvreblefebvre
committed
feat: Implement standalone upload command
1 parent d0900ea commit 72fd15d

File tree

7 files changed

+152
-39
lines changed

7 files changed

+152
-39
lines changed

‎src/bin.js

+2
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ import {hideBin} from 'yargs/helpers';
1616
import {importCommand} from './cmd/import.js';
1717
import {rulesCommand} from './cmd/rules.js';
1818
import {bundleCommand} from './cmd/bundle.js';
19+
import {uploadCommand} from './cmd/upload.js';
1920

2021
const argv = yargs(hideBin(process.argv));
2122

2223
importCommand(argv);
24+
uploadCommand(argv);
2325
rulesCommand(argv);
2426
bundleCommand(argv);
2527

‎src/cmd/import.js

+3-6
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212

1313
import fs from 'fs';
1414
import chalk from 'chalk';
15-
import runImportJobAndPoll from '../import/import-helper.js';
15+
import { runImportJobAndPoll } from '../import/import-helper.js';
16+
import { checkEnvironment } from '../utils/env-utils.js';
1617

1718
export function importCommand(yargs) {
1819
yargs.command({
@@ -51,11 +52,7 @@ export function importCommand(yargs) {
5152
stage
5253
} = argv;
5354

54-
// Ensure required environment variable is set
55-
if (!process.env.AEM_IMPORT_API_KEY) {
56-
console.error(chalk.red('Error: Ensure the AEM_IMPORT_API_KEY environment variable is set.'));
57-
process.exit(1);
58-
}
55+
checkEnvironment(process.env);
5956

6057
// Read URLs from the file
6158
const urls = fs.readFileSync(urlsPath, 'utf8').split('\n').filter(Boolean);

‎src/cmd/upload.js

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2024 Adobe. All rights reserved.
3+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License. You may obtain a copy
5+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under
8+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9+
* OF ANY KIND, either express or implied. See the License for the specific language
10+
* governing permissions and limitations under the License.
11+
*/
12+
13+
import fs from 'fs';
14+
import chalk from 'chalk';
15+
import { uploadJobResult } from '../import/import-helper.js';
16+
import { checkEnvironment } from '../utils/env-utils.js';
17+
18+
export function uploadCommand(yargs) {
19+
yargs.command({
20+
command: 'upload',
21+
describe: 'Upload the result of an import job',
22+
builder: (yargs) => {
23+
return yargs
24+
.option('jobid', {
25+
describe: 'ID of the job to upload',
26+
type: 'string'
27+
})
28+
.option('sharepointurl', {
29+
describe: 'SharePoint URL to upload imported files to',
30+
type: 'string'
31+
})
32+
.option('stage', {
33+
describe: 'use stage endpoint',
34+
type: 'boolean'
35+
});
36+
},
37+
handler: (argv) => {
38+
const {
39+
jobid: jobId,
40+
sharepointurl: sharePointUploadUrl,
41+
stage
42+
} = argv;
43+
44+
checkEnvironment(process.env);
45+
46+
// Process the upload request
47+
uploadJobResult({ jobId, sharePointUploadUrl, stage })
48+
.then(() => {
49+
console.log(chalk.green('Done.'));
50+
}).catch((error) => {
51+
console.error(chalk.red(`Error: ${error.message}`));
52+
process.exit(1);
53+
});
54+
}
55+
});
56+
}

‎src/import/import-helper.js

+20-30
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,16 @@ import { URL } from 'url';
1717
import prepareImportScript from './bundler.js';
1818
import chalk from 'chalk';
1919
import { uploadZipFromS3ToSharePoint } from './sharepoint-uploader.js';
20+
import { makeRequest } from '../utils/http-utils.js';
21+
22+
function getApiBaseUrl(stage) {
23+
const alias = stage ? 'ci' : 'v1';
24+
return `https://spacecat.experiencecloud.live/api/${alias}/tools/import/jobs`;
25+
}
26+
27+
async function getJobResult(jobId, stage) {
28+
return makeRequest(`${getApiBaseUrl(stage)}/${jobId}/result`, 'POST');
29+
}
2030

2131
/**
2232
* Run the import job and begin polling for the result. Logs progress & result to the console.
@@ -27,46 +37,20 @@ import { uploadZipFromS3ToSharePoint } from './sharepoint-uploader.js';
2737
* @param {boolean} stage - Set to true if stage APIs should be used
2838
* @returns {Promise<void>}
2939
*/
30-
async function runImportJobAndPoll( {
40+
export async function runImportJobAndPoll( {
3141
urls,
3242
importJsPath,
3343
options,
3444
sharePointUploadUrl,
3545
stage = false
3646
} ) {
3747
// Determine the base URL
38-
const baseURL = stage
39-
? 'https://spacecat.experiencecloud.live/api/ci/tools/import/jobs'
40-
: 'https://spacecat.experiencecloud.live/api/v1/tools/import/jobs';
48+
const baseURL = getApiBaseUrl(stage);
4149

4250
function hasProvidedSharePointUrl() {
4351
return typeof sharePointUploadUrl === 'string';
4452
}
4553

46-
// Function to make HTTP requests
47-
async function makeRequest(url, method, data) {
48-
const parsedUrl = new URL(url);
49-
const headers = new Headers({
50-
'Content-Type': data ? 'application/json' : '',
51-
'x-api-key': process.env.AEM_IMPORT_API_KEY,
52-
});
53-
if (data instanceof FormData) {
54-
headers.delete('Content-Type');
55-
}
56-
const res = await fetch(parsedUrl, {
57-
method,
58-
headers,
59-
body: data,
60-
});
61-
if (res.ok) {
62-
return res.json();
63-
}
64-
const body = await res.text();
65-
throw new Error(`Request failed with status code ${res.status}. `
66-
+ `x-error header: ${res.headers.get('x-error')}, x-invocation-id: ${res.headers.get('x-invocation-id')}, `
67-
+ `Body: ${body}`);
68-
}
69-
7054
// Function to poll job status
7155
async function pollJobStatus(jobId) {
7256
const url = `${baseURL}/${jobId}`;
@@ -80,7 +64,7 @@ async function runImportJobAndPoll( {
8064
console.log(chalk.green('Job completed:'), jobStatus);
8165

8266
// Print the job result's downloadUrl
83-
const jobResult = await makeRequest(`${url}/result`, 'POST');
67+
const jobResult = await getJobResult(jobId, stage);
8468
console.log(chalk.green('Download the import archive:'), jobResult.downloadUrl);
8569

8670
if (hasProvidedSharePointUrl()) {
@@ -132,4 +116,10 @@ async function runImportJobAndPoll( {
132116
return startJob();
133117
}
134118

135-
export default runImportJobAndPoll;
119+
export async function uploadJobResult({ jobId, sharePointUploadUrl, stage = false }) {
120+
// Fetch the job result
121+
const jobResult = await getJobResult(jobId, stage);
122+
123+
// Upload the import archive to SharePoint
124+
await uploadZipFromS3ToSharePoint(jobResult.downloadUrl, sharePointUploadUrl);
125+
}

‎src/import/sharepoint-uploader.js

+10-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ const downloadDirDefault = `extracted-files-${Date.now()}`;
3030
* @returns {Promise<void>}
3131
*/
3232
async function downloadAndExtractZip(s3PresignedUrl, downloadDirPath) {
33-
console.log(chalk.green('Downloading job archive...'));
3433
try {
3534
// Download the ZIP file using fetch
3635
const response = await fetch(s3PresignedUrl);
@@ -81,9 +80,10 @@ function parseSharePointUrl(sharepointUrl) {
8180
* Includes a basic retry mechanism which will re-attempt to upload a file up to UPLOAD_RETRY_LIMIT times.
8281
* @param {string} s3PresignedUrl - The S3 presigned URL to download the ZIP file from
8382
* @param {string} sharePointUrl - The SharePoint URL to upload the extracted files to
83+
* @param {boolean} cleanup - Whether to cleanup the tmp download directory after the upload
8484
* @returns {Promise<void>}
8585
*/
86-
export async function uploadZipFromS3ToSharePoint(s3PresignedUrl, sharePointUrl) {
86+
export async function uploadZipFromS3ToSharePoint(s3PresignedUrl, sharePointUrl, cleanup = true) {
8787
const successfulUploads = [];
8888
const failedUploads = [];
8989

@@ -133,15 +133,22 @@ export async function uploadZipFromS3ToSharePoint(s3PresignedUrl, sharePointUrl)
133133
}
134134
}
135135

136-
console.log(chalk.green(`Starting document upload to SharePoint, since you provided a SharePoint URL (${sharePointUrl})`));
136+
console.log(chalk.green(`Starting document upload to SharePoint URL ${sharePointUrl}`));
137137

138138
try {
139139
// Step 1: Download and extract the ZIP file
140+
console.log(chalk.green('Downloading job archive...'));
140141
await downloadAndExtractZip(s3PresignedUrl, downloadDirDefault);
141142

142143
// Step 2: Upload files to SharePoint, preserving the directory structure
144+
console.log(chalk.green(`Uploading job artifacts to SharePoint...`));
143145
await uploadDirectoryToSharePoint(path.join(downloadDirDefault, 'docx'));
144146

147+
// Step 3: cleanup the download directory
148+
if (cleanup) {
149+
fs.rmSync(downloadDirDefault, { recursive: true });
150+
}
151+
145152
console.log(chalk.green(`SharePoint upload operation complete. Successful uploads: ${successfulUploads.length}, failed uploads: ${failedUploads.length}`));
146153
} catch (error) {
147154
console.error('Error during SharePoint upload:', error);

‎src/utils/env-utils.js

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright 2024 Adobe. All rights reserved.
3+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License. You may obtain a copy
5+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under
8+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9+
* OF ANY KIND, either express or implied. See the License for the specific language
10+
* governing permissions and limitations under the License.
11+
*/
12+
13+
import chalk from 'chalk';
14+
15+
export function checkEnvironment(env) {
16+
// Ensure required environment variable is set
17+
if (typeof env.AEM_IMPORT_API_KEY !== 'string') {
18+
console.error(chalk.red('Error: Ensure the AEM_IMPORT_API_KEY environment variable is set.'));
19+
process.exit(1);
20+
}
21+
}

‎src/utils/http-utils.js

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2024 Adobe. All rights reserved.
3+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License. You may obtain a copy
5+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under
8+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9+
* OF ANY KIND, either express or implied. See the License for the specific language
10+
* governing permissions and limitations under the License.
11+
*/
12+
13+
// Function to make HTTP requests
14+
export async function makeRequest(url, method, data) {
15+
const parsedUrl = new URL(url);
16+
const headers = new Headers({
17+
'Content-Type': data ? 'application/json' : '',
18+
'x-api-key': process.env.AEM_IMPORT_API_KEY,
19+
});
20+
21+
// FormData requests set the multipart/form-data header (including boundary) automatically
22+
if (data instanceof FormData) {
23+
headers.delete('Content-Type');
24+
}
25+
26+
const res = await fetch(parsedUrl, {
27+
method,
28+
headers,
29+
body: data,
30+
});
31+
32+
if (res.ok) {
33+
return res.json();
34+
}
35+
36+
const body = await res.text();
37+
throw new Error(`Request failed with status code ${res.status}. `
38+
+ `x-error header: ${res.headers.get('x-error')}, x-invocation-id: ${res.headers.get('x-invocation-id')}, `
39+
+ `Body: ${body}`);
40+
}

0 commit comments

Comments
 (0)