Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DX - 781 - fixed the Semgrep issues, version bump and added deprecation note #51

Merged
merged 12 commits into from
Jul 17, 2024
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
package-lock.json
node_modules
logs
contents
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Contentstack import utility

**Note**: The contentstack-import command-line utility will soon be deprecated. We recommend you to check out our latest [CLI documentation](https://www.contentstack.com/docs/developers/cli) for performing content management activities.

Contentstack is a headless CMS with an API-first approach that puts content at the centre. It is designed to simplify the process of publication by separating code from content.

This tool helps you to import content which is exported using [contentstack-export](https://github.com/contentstack/contentstack-export) utility into another stack.
Expand Down
26 changes: 13 additions & 13 deletions lib/import/assets.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,21 @@ var helper = require('../util/fs');
var log = require('../util/log');

var util = require('../util/');

const sanitizePath = require('../util/utility')
var config = util.getConfig();
// var reqConcurrency = config.concurrency;
var assetsConfig = config.modules.assets;
var assetsFolderPath = path.join(config.data, config.modules.assets.dirName);
var mapperDirPath = path.resolve(config.data, 'mapper', 'assets');
var environmentPath = path.resolve(config.data, 'environments', 'environments.json');
var assetsFolderPath = path.join(sanitizePath(config.data), sanitizePath(config.modules.assets.dirName));
var mapperDirPath = path.resolve(sanitizePath(config.data), 'mapper', 'assets');
var environmentPath = path.resolve(sanitizePath(config.data), 'environments', 'environments.json');

var assetBatchLimit = (assetsConfig.hasOwnProperty('batchLimit') && typeof assetBatchLimit === 'number') ?
assetsConfig.assetBatchLimit : 2;

mkdirp.sync(mapperDirPath);

function importAssets () {
this.assets = helper.readFile(path.join(assetsFolderPath, assetsConfig.fileName));
this.assets = helper.readFile(path.join(sanitizePath(assetsFolderPath), sanitizePath(assetsConfig.fileName)));
this.environment = helper.readFile(environmentPath);
this.requestOptions = {
uri: config.host + config.apis.assets,
Expand All @@ -46,9 +46,9 @@ function importAssets () {
this.uidMapping = {};
this.urlMapping = {};
this.fails = [];
this.uidMapperPath = path.join(mapperDirPath, 'uid-mapping.json');
this.urlMapperPath = path.join(mapperDirPath, 'url-mapping.json');
this.failsPath = path.join(mapperDirPath, 'fail.json');
this.uidMapperPath = path.join(sanitizePath(mapperDirPath), 'uid-mapping.json');
this.urlMapperPath = path.join(sanitizePath(mapperDirPath), 'url-mapping.json');
this.failsPath = path.join(sanitizePath(mapperDirPath), 'fail.json');
if (fs.existsSync(this.uidMapperPath)) {
this.uidMapping = helper.readFile(this.uidMapperPath);
}
Expand Down Expand Up @@ -85,7 +85,7 @@ importAssets.prototype = {
// the asset has been already imported
return;
}
var currentAssetFolderPath = path.join(assetsFolderPath, assetUid);
var currentAssetFolderPath = path.join(sanitizePath(assetsFolderPath), sanitizePath(assetUid));
if (fs.existsSync(currentAssetFolderPath)) {
// if this is true, means, the exported asset data is versioned
// hence, upload each asset with its version
Expand All @@ -96,7 +96,7 @@ importAssets.prototype = {
return;
});
} else {
var assetPath = path.resolve(currentAssetFolderPath, self.assets[assetUid].filename);
var assetPath = path.resolve(sanitizePath(currentAssetFolderPath), sanitizePath(self.assets[assetUid].filename));
var uidContainer = {};
var urlContainer = {};
if(self.assets[assetUid].parent_uid && typeof self.assets[assetUid].parent_uid === 'string') {
Expand Down Expand Up @@ -159,7 +159,7 @@ importAssets.prototype = {
uploadVersionedAssets: function (uid, assetFolderPath) {
var self = this;
return new Promise(function (resolve, reject) {
var versionedAssetMetadata = helper.readFile(path.join(assetFolderPath, '_contentstack_' + uid + '.json'));
var versionedAssetMetadata = helper.readFile(path.join(sanitizePath(assetFolderPath), '_contentstack_' + sanitizePath(uid) + '.json'));
// using last version, find asset's parent

var lastVersion = versionedAssetMetadata[versionedAssetMetadata.length - 1];
Expand All @@ -182,7 +182,7 @@ importAssets.prototype = {
var urlContainer = {};
return Promise.map(versionedAssetMetadata, function () {
var assetMetadata = versionedAssetMetadata[counter];
var assetPath = path.join(assetFolderPath, assetMetadata.filename);
var assetPath = path.join(sanitizePath(assetFolderPath), sanitizePath(assetMetadata.filename));
if (++counter === 1) {
// delete assetMetadata.uid;
return self.uploadAsset(assetPath, assetMetadata, uidContainer, urlContainer).then(function () {
Expand Down Expand Up @@ -298,7 +298,7 @@ importAssets.prototype = {
var mappedFolderPath = path.resolve(config.data, 'mapper', 'assets', 'folder-mapping.json');
self.folderDetails = helper.readFile(path.resolve(assetsFolderPath, 'folders.json'));
if (_.isEmpty(self.folderDetails)) {
log.success('No folders were found at: ' + path.join(assetsFolderPath, 'folders.json'));
log.success('No folders were found at: ' + path.join(sanitizePath(assetsFolderPath), 'folders.json'));
return resolve();
}
var tree = self.buildTree(_.cloneDeep(self.folderDetails));
Expand Down
69 changes: 35 additions & 34 deletions lib/import/entries.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ var lookupReplaceEntries = require('../util/lookupReplaceEntries');
var supress = require('../util/supress-mandatory-fields');
var extension_supress = require('../util/extensionsUidReplace');
var util = require('../util/');
const sanitizePath = require('../util/utility')
var config = util.getConfig();

var reqConcurrency = config.concurrency;
var eConfig = config.modules.entries;
var ePath = path.resolve(config.data, eConfig.dirName);
var ctPath = path.resolve(config.data, config.modules.content_types.dirName);
var lPath = path.resolve(config.data, config.modules.locales.dirName, config.modules.locales.fileName);
var ePath = path.resolve(sanitizePath(config.data), sanitizePath(eConfig.dirName));
var ctPath = path.resolve(sanitizePath(config.data), sanitizePath(config.modules.content_types.dirName));
var lPath = path.resolve(sanitizePath(config.data), sanitizePath(config.modules.locales.dirName), sanitizePath(config.modules.locales.fileName));

var mappedAssetUidPath;
var mappedAssetUrlPath;
Expand All @@ -42,19 +43,19 @@ var entryBatchLimit = eConfig.batchLimit || 16;

function importEntries() {
var self = this;
mappedAssetUidPath = path.resolve(config.data, 'mapper', 'assets', 'uid-mapping.json');
mappedAssetUrlPath = path.resolve(config.data, 'mapper', 'assets', 'url-mapping.json');
mappedAssetUidPath = path.resolve(sanitizePath(config.data), 'mapper', 'assets', 'uid-mapping.json');
mappedAssetUrlPath = path.resolve(sanitizePath(config.data), 'mapper', 'assets', 'url-mapping.json');

entryMapperPath = path.resolve(config.data, 'mapper', 'entries');
environmentPath = path.resolve(config.data, 'environments', 'environments.json');
entryMapperPath = path.resolve(sanitizePath(config.data), 'mapper', 'entries');
environmentPath = path.resolve(sanitizePath(config.data), 'environments', 'environments.json');
mkdirp.sync(entryMapperPath);

entryUidMapperPath = path.join(entryMapperPath, 'uid-mapping.json');
uniqueUidMapperPath = path.join(entryMapperPath, 'unique-mapping.json');
modifiedSchemaPath = path.join(entryMapperPath, 'modified-schemas.json');
entryUidMapperPath = path.join(sanitizePath(entryMapperPath), 'uid-mapping.json');
uniqueUidMapperPath = path.join(sanitizePath(entryMapperPath), 'unique-mapping.json');
modifiedSchemaPath = path.join(sanitizePath(entryMapperPath), 'modified-schemas.json');

createdEntriesWOUidPath = path.join(entryMapperPath, 'created-entries-wo-uid.json');
failedWOPath = path.join(entryMapperPath, 'failedWO.json');
createdEntriesWOUidPath = path.join(sanitizePath(entryMapperPath), 'created-entries-wo-uid.json');
failedWOPath = path.join(sanitizePath(entryMapperPath), 'failedWO.json');
// Object of Schemas, referred to by their content type uid
this.ctSchemas = {};
// Array of content type uids, that have reference fields
Expand All @@ -80,7 +81,7 @@ function importEntries() {
try {
if (skipFiles.indexOf(files[index]) === -1) {
if(files[index] != "field_rules_uid.json") {
var schema = require(path.resolve(path.join(ctPath, files[index])));
var schema = require(path.resolve(path.join(sanitizePath(ctPath), sanitizePath(files[index]))));
self.ctSchemas[schema.uid] = schema;
}
}
Expand Down Expand Up @@ -144,12 +145,12 @@ importEntries.prototype = {
}).then(function() {
return self.unSupressFields().then(function() {
return self.removeBuggedEntries().then(async function() {
var ct_field_visibility_uid = helper.readFile(path.join(ctPath + "/field_rules_uid.json"))
var ct_field_visibility_uid = helper.readFile(path.join(sanitizePath(ctPath) + "/field_rules_uid.json"))
var ct_files = fs.readdirSync(ctPath);
if(ct_field_visibility_uid && ct_field_visibility_uid != "undefined") {
for (var index = 0; index < ct_field_visibility_uid.length; index++) {
if (ct_files.indexOf(ct_field_visibility_uid[index] + ".json") > -1) {
var schema = require(path.resolve(ctPath, ct_field_visibility_uid[index]));
var schema = require(path.resolve(sanitizePath(ctPath), sanitizePath(ct_field_visibility_uid[index])));
await self.field_rules_update(schema)
}
}
Expand All @@ -174,16 +175,16 @@ importEntries.prototype = {
}
self.mappedUids = self.mappedUids || {};
return Promise.map(contentTypeUids, function(ctUid) {
var eLangFolderPath = path.join(entryMapperPath, lang);
var eLogFolderPath = path.join(entryMapperPath, lang, ctUid);
var eLangFolderPath = path.join(sanitizePath(entryMapperPath), sanitizePath(lang));
var eLogFolderPath = path.join(sanitizePath(entryMapperPath), sanitizePath(lang), sanitizePath(ctUid));
mkdirp.sync(eLogFolderPath);
// entry file path
var eFilePath = path.resolve(ePath, ctUid, lang + '.json');
var eFilePath = path.resolve(sanitizePath(ePath), sanitizePath(ctUid), sanitizePath(lang) + '.json');

// log created/updated entries
var successEntryLogPath = path.join(eLogFolderPath, 'success.json');
var failedEntryLogPath = path.join(eLogFolderPath, 'fails.json');
var createdEntriesPath = path.join(eLogFolderPath, 'created-entries.json');
var successEntryLogPath = path.join(sanitizePath(eLogFolderPath), 'success.json');
var failedEntryLogPath = path.join(sanitizePath(eLogFolderPath), 'fails.json');
var createdEntriesPath = path.join(sanitizePath(eLogFolderPath), 'created-entries.json');
var createdEntries = {};
if (fs.existsSync(createdEntriesPath)) {
createdEntries = helper.readFile(createdEntriesPath);
Expand Down Expand Up @@ -373,13 +374,13 @@ importEntries.prototype = {
repostEntries: function(lang) {
var self = this;
return new Promise(function(resolve, reject) {
var _mapped_ = helper.readFile(path.join(entryMapperPath, 'uid-mapping.json'));
var _mapped_ = helper.readFile(path.join(sanitizePath(entryMapperPath), 'uid-mapping.json'));
if (_.isPlainObject(_mapped_)) {
self.mappedUids = _.merge(_mapped_, self.mappedUids);
}
return Promise.map(self.refSchemas, function(ctUid) {
var eFolderPath = path.join(entryMapperPath, lang, ctUid);
var eSuccessFilePath = path.join(eFolderPath, 'success.json');
var eFolderPath = path.join(sanitizePath(entryMapperPath), sanitizePath(lang), sanitizePath(ctUid));
var eSuccessFilePath = path.join(sanitizePath(eFolderPath), 'success.json');
if (!fs.existsSync(eSuccessFilePath)) {
log.error(chalk.blue('Success file was not found at: ' + eSuccessFilePath));
return resolve();
Expand All @@ -393,8 +394,8 @@ importEntries.prototype = {
}

// Keep track of entries that have their references updated
var refsUpdatedUids = helper.readFile(path.join(eFolderPath, 'refsUpdatedUids.json'));
var refsUpdateFailed = helper.readFile(path.join(eFolderPath, 'refsUpdateFailed.json'));
var refsUpdatedUids = helper.readFile(path.join(sanitizePath(eFolderPath), 'refsUpdatedUids.json'));
var refsUpdateFailed = helper.readFile(path.join(sanitizePath(eFolderPath), 'refsUpdateFailed.json'));
var schema = self.ctSchemas[ctUid];

var batches = [];
Expand All @@ -403,7 +404,7 @@ importEntries.prototype = {

// map reference uids @mapper/language/mapped-uids.json
// map failed reference uids @mapper/language/unmapped-uids.json
var refUidMapperPath = path.join(entryMapperPath, lang);
var refUidMapperPath = path.join(sanitizePath(entryMapperPath), sanitizePath(lang));

entries = _.map(entries, function(entry) {
try {
Expand Down Expand Up @@ -481,9 +482,9 @@ importEntries.prototype = {
concurrency: reqConcurrency
}).then(function() {
// batch completed successfully
helper.writeFile(path.join(eFolderPath, 'success.json'), entries);
helper.writeFile(path.join(eFolderPath, 'refsUpdatedUids.json'), refsUpdatedUids);
helper.writeFile(path.join(eFolderPath, 'refsUpdateFailed.json'), refsUpdateFailed);
helper.writeFile(path.join(sanitizePath(eFolderPath), 'success.json'), entries);
helper.writeFile(path.join(sanitizePath(eFolderPath), 'refsUpdatedUids.json'), refsUpdatedUids);
helper.writeFile(path.join(sanitizePath(eFolderPath), 'refsUpdateFailed.json'), refsUpdateFailed);
log.success(chalk.blue('Completed batch no: ' + (index + 1) + ' successfully!'));
return;
}).catch(function(error) {
Expand Down Expand Up @@ -600,7 +601,7 @@ importEntries.prototype = {
return resolve();
}
self.mappedUids[query.entry.uid] = response.body.entries[0].uid;
var _ePath = path.join(entryMapperPath, query.locale, query.content_type, 'success.json');
var _ePath = path.join(sanitizePath(entryMapperPath), sanitizePath(query.locale), sanitizePath(query.content_type), 'success.json');
var entries = helper.readFile(_ePath);
entries.push(query.entry);
helper.writeFile(_ePath, entries);
Expand Down Expand Up @@ -713,8 +714,8 @@ importEntries.prototype = {
}
}

helper.writeFile(path.join(entryMapperPath, 'removed-uids.json'), removed);
helper.writeFile(path.join(entryMapperPath, 'pending-uids.json'), bugged);
helper.writeFile(path.join(sanitizePath(entryMapperPath), 'removed-uids.json'), removed);
helper.writeFile(path.join(sanitizePath(entryMapperPath), 'pending-uids.json'), bugged);

log.success(chalk.green('The stack has been eradicated from bugged entries!'));
return resolve();
Expand All @@ -737,7 +738,7 @@ importEntries.prototype = {
var updated_value = [];
for (var j = 0; j < field_rules_array.length; j++) {
var splited_field_rules_value = field_rules_array[j];
var old_uid = helper.readFile(path.join(entryUidMapperPath));
var old_uid = helper.readFile(path.join(sanitizePath(entryUidMapperPath)));
if (old_uid.hasOwnProperty(splited_field_rules_value)) {
updated_value.push(old_uid[splited_field_rules_value])
} else {
Expand Down
9 changes: 5 additions & 4 deletions lib/util/extensionsUidReplace.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ var path = require('path');

var helper = require('./fs');
var util = require('../util/');
const sanitizePath = require('../util/utility')
var config = util.getConfig();
var extensionPath = path.resolve(config.data, 'mapper/extensions', 'uid-mapping.json');
var globalfieldsPath = path.resolve(config.data, 'mapper/globalfields', 'uid-mapping.json');
var extensionPath = path.resolve(sanitizePath(config.data), 'mapper/extensions', 'uid-mapping.json');
var globalfieldsPath = path.resolve(sanitizePath(config.data), 'mapper/globalfields', 'uid-mapping.json');


var extension_uid_Replace = module.exports = function (schema, preserveStackVersion) {
Expand All @@ -35,14 +36,14 @@ var extension_uid_Replace = module.exports = function (schema, preserveStackVers
}
} else if(schema[i].data_type === 'global_field') {
var global_fields_key_value = schema[i]['reference_to'];
var global_fields_data = helper.readFile(path.join(globalfieldsPath));
var global_fields_data = helper.readFile(path.join(sanitizePath(globalfieldsPath)));
if(global_fields_data && global_fields_data.hasOwnProperty(global_fields_key_value)) {
schema[i]['reference_to'] = global_fields_data[global_fields_key_value];
}
} else {
if(schema[i].hasOwnProperty('extension_uid')) {
var extension_key_value = schema[i]['extension_uid'];
var data = helper.readFile(path.join(extensionPath));
var data = helper.readFile(path.join(sanitizePath(extensionPath)));
if(data && data.hasOwnProperty(extension_key_value)) {
schema[i]['extension_uid'] = data[extension_key_value];
}
Expand Down
19 changes: 10 additions & 9 deletions lib/util/fs.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,29 @@
* MIT Licensed
*/

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var fs = require("fs");
var path = require("path");
var mkdirp = require("mkdirp");
const sanitizePath = require('./utility');

exports.readFile = function (filePath, parse) {
var data;
parse = (typeof parse === 'undefined') ? true : parse;
filePath = path.resolve(filePath);
parse = typeof parse === "undefined" ? true : parse;
filePath = path.resolve(sanitizePath(filePath));
if (fs.existsSync(filePath)) {
data = (parse) ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data;
data = parse ? JSON.parse(fs.readFileSync(filePath, "utf-8")) : data;
}
return data;
};

exports.writeFile = function (filePath, data) {
data = (typeof data === 'object') ? JSON.stringify(data) : data || '{}';
data = typeof data === "object" ? JSON.stringify(data) : data || "{}";
fs.writeFileSync(filePath, data);
};

exports.makeDirectory = function () {
for (var key in arguments) {
var dirname = path.resolve(arguments[key]);
var dirname = path.resolve(sanitizePath(arguments[key]));
if (!fs.existsSync(dirname)) {
mkdirp.sync(dirname);
}
Expand All @@ -38,4 +39,4 @@ exports.readdir = function (dirPath) {
} else {
return [];
}
};
};
Loading
Loading