From 67467f04704249fae06f95ca6c9de548795249bb Mon Sep 17 00:00:00 2001 From: Daniel Pollithy Date: Sat, 21 Oct 2017 18:05:23 +0200 Subject: [PATCH] Recover database from files --- docs/sphinx/sources/articles/electron.rst | 12 +- src/database.js | 8 +- src/export.js | 227 ++++++++++++++++++++++ src/server.js | 25 +++ src/views/db_logout.pug | 21 +- 5 files changed, 286 insertions(+), 7 deletions(-) diff --git a/docs/sphinx/sources/articles/electron.rst b/docs/sphinx/sources/articles/electron.rst index ade53c9..8765a1a 100644 --- a/docs/sphinx/sources/articles/electron.rst +++ b/docs/sphinx/sources/articles/electron.rst @@ -1,8 +1,8 @@ Electron for cross platform applications ---------------------------------------- -Operating systems differ. Especially when it comes to writing graphical interfaces. -That is the reason why so many "cross platform frameworks" exist. +Operating systems differ. Especially when it comes to writing graphical user interfaces. +That is the reason why "cross platform frameworks" exist. They try to provide general functions that work on every major operating system. If you want to build a desktop application you first have to decide whether you want to use a @@ -11,7 +11,13 @@ If you want to build a desktop application you first have to decide whether you Overview of cross platform frameworks ..................................... -First of all you have to decide whether your application should work on mobile devices, too. +Qt and wx have been the best known frameworks. But since 20XY the electron framework is there. +Developed by the company behind Github and driven by their own need as they were writing the +code editor Atom. + +The basics of their new framework are not new. They package a light-weight browser (in particular +a version of Google Chrome) with the runtime environment for Node.JS execution. +The last important thing they provide Re-usability diff --git a/src/database.js b/src/database.js index 637e021..4eb3df5 100644 --- a/src/database.js +++ b/src/database.js @@ -475,16 +475,18 @@ Database.remove_image_by_id = function (id_) { * @param fragment_name The identifier for the fragment * @return {Promise} */ -Database.add_fragment = function(image_id, fragment_name) { +Database.add_fragment = function(image_id, fragment_name, comment, completed) { + if (comment == undefined) { comment = '' } + if (completed == undefined) { completed = false } else { completed = true} var session = this._get_session(); var d = new Date(); var upload_date = Math.round(d.getTime()); return session.run("MATCH (i:Image) " + "WHERE ID(i) = toInteger({image_id}) " + "WITH i " + - "CREATE (f:Fragment {fragment_name: {fragment_name}, upload_date: {upload_date}, completed:false, comment:{comment}})-[:image]->(i) " + + "CREATE (f:Fragment {fragment_name: {fragment_name}, upload_date: {upload_date}, completed:{completed}, comment:{comment}})-[:image]->(i) " + "RETURN ID(f) as ident, ID(i) AS image_ident;", - {fragment_name: fragment_name, upload_date: upload_date, image_id: Number(image_id), comment: ''}) + {fragment_name: fragment_name, upload_date: upload_date, image_id: Number(image_id), comment: comment, completed: completed}) .then(function(result){ session.close(); var records = []; diff --git a/src/export.js b/src/export.js index 6f6e186..86f61c0 100644 --- a/src/export.js +++ b/src/export.js @@ -54,6 +54,11 @@ var path = require('path'); var zlib = require('zlib'); var archiver = require('archiver'); var database = require('./database'); +var readline = require('readline'); +var exif_utils = require('./exif_utils'); +var sizeOf = require('image-size'); +var codec = require('./codec'); + @@ -602,4 +607,226 @@ Export.to_csv = function () { }); }; +Export._get_images_and_fragments_from_relations_csv = function(relations_fn) { + return new Promise(function(resolve, reject){ + var images = {}; + var lineReader = readline.createInterface({ + input: fs.createReadStream(relations_fn) + }); + var line_counter = 0; + var splitted_line, relation_id, relation_type, source, target; + lineReader.on('line', function (line) { + if (line_counter > 0) { + splitted_line = line.split(','); + relation_id = splitted_line[0]; + relation_type = splitted_line[1].substring(1, splitted_line[1].length-1); + source = splitted_line[2]; + target = splitted_line[3]; + if (relation_type == 'image') { + // the node is an image + if (images[target] === undefined) { + images[target] = {}; + } + images[target][source] = {}; + } + } + line_counter++; + }); + lineReader.on('close', function(){ + resolve(images); + }); + lineReader.on('error', function(err){ + reject(err); + }); + }) +}; + +Export._get_properties_from_csv = function(node_props_fn, relations_fn) { + return new Promise(function(resolve, reject) { + Export._get_images_and_fragments_from_relations_csv(relations_fn) + .then(function(images){ + var lineReader = readline.createInterface({ + input: fs.createReadStream(node_props_fn) + }); + var line_counter = 0; + var splitted_line, id_, key, value, image_id; + lineReader.on('line', function (line) { + if (line_counter > 0) { + splitted_line = line.split(','); + id_ = splitted_line[0]; + key = splitted_line[1].substring(1, splitted_line[1].length-1); + value = splitted_line[2].substring(1, splitted_line[2].length-1); + if (key == 'file_path') { + // the node is an image + images[id_]['file_path'] = decodeURIComponent(value); + } + if (key == 'fragment_name' || key == 'comment' || key == 'completed') { + // the node is a fragment + image_id = null; + Object.keys(images).forEach(function(image){ + Object.keys(images[image]).forEach(function(fragment_id){ + if (fragment_id == id_) { + image_id = image; + } + }); + }); + if (image_id) { + images[image_id][id_][key] = decodeURIComponent(value); + } + } + } + line_counter++; + }); + lineReader.on('close', function(){ + return resolve(images); + }); + lineReader.on('error', function(err){ + return reject(err); + }); + }); + }); +}; + +Export._rebuild_image = function(image_path) { + return new Promise(function(resolve, reject){ + var new_image_path = path.join(__dirname, '..', 'media', 'uploaded_images', image_path); + exif_utils.get_exif_from_image(new_image_path, function (exif_err, exif_data) { + if (exif_err) { + log.warn('REBUILD: There was an error reading exif from image: ' + new_image_path); + exif_data = null; + } + if (!exif_data) { + sizeOf(new_image_path, function (err, dimensions) { + if (err) { + log.error("REBUILD: "+err); + return reject(err); + } + + var alternative_meta = { + 'exif': { + 'ExifImageWidth': dimensions.width, + 'ExifImageHeight': dimensions.height + } + }; + database.add_image(image_path, alternative_meta).then(function (record) { + log.info('REBUILD: Added image: ' + new_image_path); + var image_id = record.get('ident').toString(); + return resolve(image_id); + + }, function (err) { + log.error('REBUILD: Error on adding an image to the database: ' + new_image_path); + return reject(err); + }); + }) + } else { + database.add_image(image_path, exif_data).then(function (record) { + if (record.name == "Neo4jError") { + return reject(record.message); + } + log.info('REBUILD: Added image: ' + new_image_path); + var image_id = record.get('ident').toString(); + return resolve(image_id); + }, function (err) { + log.error('REBUILD: Error on adding an image to the database: ' + new_image_path); + return reject(err); + }); + } + }); + }); +}; + +Export._copy_xmls = function(fragment_mapping) { + // fragment_mapping is old_fragment_id -> new_fragment_id + var all_promises = []; + Object.keys(fragment_mapping).forEach(function(old_fragment_id) { + var new_fragment_id = fragment_mapping[old_fragment_id]; + var old_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.xml'); + var old_path_backup = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.backup.xml'); + var new_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', new_fragment_id+'.xml'); + + all_promises.push(new Promise(function(resolve, reject){ + var read = fs.createReadStream(old_path); + var write = fs.createWriteStream(old_path_backup); + write.on('error', reject); + write.on('close', resolve); + read.pipe(write); + })); + + all_promises.push(new Promise(function(resolve, reject){ + var read = fs.createReadStream(old_path); + var write = fs.createWriteStream(new_path); + write.on('error', reject); + write.on('close', resolve); + read.pipe(write); + })); + + }); + return Promise.all(all_promises); +}; + +Export.rebuild_database = function(relations_fn, node_props_fn) { + return Export._get_properties_from_csv(node_props_fn, relations_fn) + .then(function(images){ + // fragment_mapping: old_fragment_id -> new_fragment_id + var fragment_mapping = {}; + var image_promises = []; + Object.keys(images).forEach(function(image_id){ + var image = images[image_id]; + var file_path = image['file_path']; + if (file_path != undefined) { + var p = Export._rebuild_image(file_path) + .then(function(db_image_id){ + // create all fragments + var fragment_promises = []; + Object.keys(image).forEach(function(fragment_id){ + if (fragment_id != 'file_path') { + var fragment_name = images[image_id][fragment_id]['fragment_name']; + var comment = images[image_id][fragment_id]['comment']; + if (comment == 'UL') {comment = ''} + var completed = images[image_id][fragment_id]['completed']; + // Attention: is NULL stands in the cell it is shortened to a UL + if (completed == "UL") {completed = undefined} + if (fragment_name == undefined) { + log.error('REBUILD: Missing fragment_name of fragment_id='+fragment_id); + fragment_name = ''; + } + var fp = database.add_fragment(db_image_id, fragment_name, comment, completed) + .then(function(record){ + var db_fragment_id = record.get('ident').toString(); + var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml'; + return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) { + fragment_mapping[fragment_id] = db_fragment_id; + return Promise.resolve(); + }).catch(function(err){ + log.error(err); + return Promise.reject(err); + }); + }).catch(function(err) { + log.error('REBUILD: Error creating fragment fragment_id='+fragment_id); + log.error('REBUILD: ' + err); + }); + fragment_promises.push(fp); + } + }); + return Promise.all(fragment_promises); + }).catch(function(err){ + log.error('REBUILD: image ' + image_id + ' error creating: ' + err); + }); + image_promises.push(p); + } else { + log.error('REBUILD: image ' + image_id + ' has no file_path'); + } + }); + return Promise.all(image_promises) + .then(function(){ + log.info('REBUILD: images were added successfully.'); + return Export._copy_xmls(fragment_mapping); + }) + .catch(function(err){ + log.error('REBUILD: there was an error adding the images and fragment.'); + log.error(err); + }); + }) +}; + module.exports = Export; \ No newline at end of file diff --git a/src/server.js b/src/server.js index 18761d2..35826c8 100644 --- a/src/server.js +++ b/src/server.js @@ -199,6 +199,31 @@ app.post('/', function (req, res) { }); }); +app.post('/rebuild-database', function (req, res) { + if (!req.files || !req.files.edges || !req.files.node_props) { + log.warn('Tried to rebuild database without uploading the dump files'); + return res.redirect('/?e=' + encodeURIComponent('You have to attach the exported files.')) + } + + var edges = req.files.edges; + var node_props = req.files.node_props; + + var edges_fn = path.join(__dirname, '..', 'media', 'export', 'recover_relations.csv'); + var node_props_fn = path.join(__dirname, '..', 'media', 'export', 'recover_node_props.csv'); + + var promises = [edges.mv(edges_fn), node_props.mv(node_props_fn)]; + + Promise.all(promises).then(function(){ + exp.rebuild_database(edges_fn, node_props_fn).then(function(){ + res.redirect('/?i=' + encodeURIComponent('Good luck with the recovered database.')) + }).catch(function(err){ + res.redirect('/?e=' + encodeURIComponent(err)) + }) + }).catch(function(err){ + return res.redirect('/?e=' + encodeURIComponent(err)); + }); +}); + app.get('/autocomplete/:token_type/values', function (req, res) { var token_type = utils.token_type_mapping(req.params.token_type); if (!token_type) { diff --git a/src/views/db_logout.pug b/src/views/db_logout.pug index 45d2a27..5338bcd 100644 --- a/src/views/db_logout.pug +++ b/src/views/db_logout.pug @@ -9,4 +9,23 @@ block content p= 'You are connected. Do you want to drop the connection?' b= '' + message form(method='post', action='/db-drop') - input(type='submit', value='drop connection') \ No newline at end of file + input(type='submit', value='drop connection') + br + br + h1= 'Rebuild data from filesystem' + p= 'If your database got corrupted (for example because you used a sync client).' + p= 'Step 1) Export your data as CSV. Backup the /resources/app/media/uploaded_xmls/ folder.' + p= 'Step 2) Stop GIAnT, stop Neo4j, create a new database for neo4j, start it and restart GIAnT' + p= 'Step 3) Your data is located here: /resources/app/media/' + p= 'Make sure uploaded_xmls/ contains your xmls, uploaded_images/ your images and settings/ your settings' + p= 'If that\'s not the case copy them to these locations.' + + form(method='post', action='rebuild-database', enctype='multipart/form-data') + p= 'Step 4) Upload the dumped CSV files' + p= 'Relations: csv-relations-*.csv' + input(type='file', name='edges') + br + p= 'Node properties: csv-nodeprops-*.csv' + input(type='file', name='node_props') + p= 'Attention: This script will overwrite the current active neo4j database. Ok?' + input(type='submit', value='Rebuild database from filesystem')