Recover database from files

docs/sphinx/sources/articles/electron.rst

@@ -1,8 +1,8 @@
 Electron for cross platform applications
 ----------------------------------------
 
-Operating systems differ. Especially when it comes to writing graphical interfaces.
-That is the reason why so many "cross platform frameworks" exist.
+Operating systems differ. Especially when it comes to writing graphical user interfaces.
+That is the reason why "cross platform frameworks" exist.
 They try to provide general functions that work on every major operating system.
 
 If you want to build a desktop application you first have to decide whether you want to use a
@@ -11,7 +11,13 @@ If you want to build a desktop application you first have to decide whether you
 Overview of cross platform frameworks
 .....................................
 
-First of all you have to decide whether your application should work on mobile devices, too.
+Qt and wx have been the best known frameworks. But since 20XY the electron framework is there.
+Developed by the company behind Github and driven by their own need as they were writing the
+code editor Atom.
+
+The basics of their new framework are not new. They package a light-weight browser (in particular
+a version of Google Chrome) with the runtime environment for Node.JS execution.
+The last important thing they provide
 
 
 Re-usability

src/database.js

@@ -475,16 +475,18 @@ Database.remove_image_by_id = function (id_) {
  * @param fragment_name The identifier for the fragment
  * @return {Promise}
  */
-Database.add_fragment = function(image_id, fragment_name) {
+Database.add_fragment = function(image_id, fragment_name, comment, completed) {
+    if (comment == undefined) { comment = '' }
+    if (completed == undefined) { completed = false } else { completed = true}
     var session = this._get_session();
     var d = new Date();
     var upload_date = Math.round(d.getTime());
     return session.run("MATCH (i:Image) " +
         "WHERE ID(i) = toInteger({image_id}) " +
         "WITH i " +
-        "CREATE (f:Fragment {fragment_name: {fragment_name}, upload_date: {upload_date}, completed:false, comment:{comment}})-[:image]->(i) " +
+        "CREATE (f:Fragment {fragment_name: {fragment_name}, upload_date: {upload_date}, completed:{completed}, comment:{comment}})-[:image]->(i) " +
         "RETURN ID(f) as ident, ID(i) AS image_ident;",
-        {fragment_name: fragment_name, upload_date: upload_date, image_id: Number(image_id), comment: ''})
+        {fragment_name: fragment_name, upload_date: upload_date, image_id: Number(image_id), comment: comment, completed: completed})
         .then(function(result){
             session.close();
             var records = [];

src/export.js

@@ -54,6 +54,11 @@ var path = require('path');
 var zlib = require('zlib');
 var archiver = require('archiver');
 var database = require('./database');
+var readline = require('readline');
+var exif_utils = require('./exif_utils');
+var sizeOf = require('image-size');
+var codec = require('./codec');
+
 
 
 
@@ -602,4 +607,226 @@ Export.to_csv = function () {
     });
 };
 
+Export._get_images_and_fragments_from_relations_csv = function(relations_fn) {
+    return new Promise(function(resolve, reject){
+        var images = {};
+        var lineReader = readline.createInterface({
+          input: fs.createReadStream(relations_fn)
+        });
+        var line_counter = 0;
+        var splitted_line, relation_id, relation_type, source, target;
+        lineReader.on('line', function (line) {
+            if (line_counter > 0) {
+                splitted_line = line.split(',');
+                relation_id = splitted_line[0];
+                relation_type = splitted_line[1].substring(1, splitted_line[1].length-1);
+                source = splitted_line[2];
+                target = splitted_line[3];
+                if (relation_type == 'image') {
+                    // the node is an image
+                    if (images[target] === undefined) {
+                        images[target] = {};
+                    }
+                    images[target][source] = {};
+                }
+            }
+            line_counter++;
+        });
+        lineReader.on('close', function(){
+            resolve(images);
+        });
+        lineReader.on('error', function(err){
+            reject(err);
+        });
+    })
+};
+
+Export._get_properties_from_csv = function(node_props_fn, relations_fn) {
+    return new Promise(function(resolve, reject) {
+        Export._get_images_and_fragments_from_relations_csv(relations_fn)
+            .then(function(images){
+                var lineReader = readline.createInterface({
+                    input: fs.createReadStream(node_props_fn)
+                });
+                var line_counter = 0;
+                var splitted_line, id_, key, value, image_id;
+                lineReader.on('line', function (line) {
+                    if (line_counter > 0) {
+                        splitted_line = line.split(',');
+                        id_ = splitted_line[0];
+                        key = splitted_line[1].substring(1, splitted_line[1].length-1);
+                        value = splitted_line[2].substring(1, splitted_line[2].length-1);
+                        if (key == 'file_path') {
+                            // the node is an image
+                            images[id_]['file_path'] = decodeURIComponent(value);
+                        }
+                        if (key == 'fragment_name' || key == 'comment' || key == 'completed') {
+                            // the node is a fragment
+                            image_id = null;
+                            Object.keys(images).forEach(function(image){
+                                Object.keys(images[image]).forEach(function(fragment_id){
+                                    if (fragment_id == id_) {
+                                        image_id = image;
+                                    }
+                                });
+                            });
+                            if (image_id) {
+                                images[image_id][id_][key] = decodeURIComponent(value);
+                            }
+                        }
+                    }
+                    line_counter++;
+                });
+                lineReader.on('close', function(){
+                    return resolve(images);
+                });
+                lineReader.on('error', function(err){
+                    return reject(err);
+                });
+            });
+    });
+};
+
+Export._rebuild_image = function(image_path) {
+    return new Promise(function(resolve, reject){
+        var new_image_path = path.join(__dirname, '..', 'media', 'uploaded_images', image_path);
+        exif_utils.get_exif_from_image(new_image_path, function (exif_err, exif_data) {
+                if (exif_err) {
+                    log.warn('REBUILD: There was an error reading exif from image: ' + new_image_path);
+                    exif_data = null;
+                }
+                if (!exif_data) {
+                    sizeOf(new_image_path, function (err, dimensions) {
+                        if (err) {
+                            log.error("REBUILD: "+err);
+                            return reject(err);
+                        }
+
+                        var alternative_meta = {
+                            'exif': {
+                                'ExifImageWidth': dimensions.width,
+                                'ExifImageHeight': dimensions.height
+                            }
+                        };
+                        database.add_image(image_path, alternative_meta).then(function (record) {
+                            log.info('REBUILD: Added image: ' + new_image_path);
+                            var image_id = record.get('ident').toString();
+                            return resolve(image_id);
+
+                        }, function (err) {
+                            log.error('REBUILD: Error on adding an image to the database: ' + new_image_path);
+                            return reject(err);
+                        });
+                    })
+                } else {
+                    database.add_image(image_path, exif_data).then(function (record) {
+                        if (record.name == "Neo4jError") {
+                            return reject(record.message);
+                        }
+                        log.info('REBUILD: Added image: ' + new_image_path);
+                        var image_id = record.get('ident').toString();
+                        return resolve(image_id);
+                    }, function (err) {
+                        log.error('REBUILD: Error on adding an image to the database: ' + new_image_path);
+                        return reject(err);
+                    });
+                }
+            });
+        });
+};
+
+Export._copy_xmls = function(fragment_mapping) {
+    // fragment_mapping is old_fragment_id -> new_fragment_id
+    var all_promises = [];
+    Object.keys(fragment_mapping).forEach(function(old_fragment_id) {
+        var new_fragment_id = fragment_mapping[old_fragment_id];
+        var old_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.xml');
+        var old_path_backup = path.join(__dirname, '..', 'media', 'uploaded_xmls', old_fragment_id+'.backup.xml');
+        var new_path = path.join(__dirname, '..', 'media', 'uploaded_xmls', new_fragment_id+'.xml');
+
+        all_promises.push(new Promise(function(resolve, reject){
+            var read = fs.createReadStream(old_path);
+            var write = fs.createWriteStream(old_path_backup);
+            write.on('error', reject);
+            write.on('close', resolve);
+            read.pipe(write);
+        }));
+
+        all_promises.push(new Promise(function(resolve, reject){
+            var read = fs.createReadStream(old_path);
+            var write = fs.createWriteStream(new_path);
+            write.on('error', reject);
+            write.on('close', resolve);
+            read.pipe(write);
+        }));
+
+    });
+    return Promise.all(all_promises);
+};
+
+Export.rebuild_database = function(relations_fn, node_props_fn) {
+    return Export._get_properties_from_csv(node_props_fn, relations_fn)
+        .then(function(images){
+            // fragment_mapping: old_fragment_id -> new_fragment_id
+            var fragment_mapping = {};
+            var image_promises = [];
+            Object.keys(images).forEach(function(image_id){
+                var image = images[image_id];
+                var file_path = image['file_path'];
+                if (file_path != undefined) {
+                    var p = Export._rebuild_image(file_path)
+                        .then(function(db_image_id){
+                            // create all fragments
+                            var fragment_promises = [];
+                            Object.keys(image).forEach(function(fragment_id){
+                                if (fragment_id != 'file_path') {
+                                    var fragment_name = images[image_id][fragment_id]['fragment_name'];
+                                    var comment = images[image_id][fragment_id]['comment'];
+                                    if (comment == 'UL') {comment = ''}
+                                    var completed = images[image_id][fragment_id]['completed'];
+                                    // Attention: is NULL stands in the cell it is shortened to a UL
+                                    if (completed == "UL") {completed = undefined}
+                                    if (fragment_name == undefined) {
+                                        log.error('REBUILD: Missing fragment_name of fragment_id='+fragment_id);
+                                        fragment_name = '';
+                                    }
+                                    var fp = database.add_fragment(db_image_id, fragment_name, comment, completed)
+                                        .then(function(record){
+                                            var db_fragment_id = record.get('ident').toString();
+                                            var overwrite_xml = '../media/uploaded_xmls/' + fragment_id + '.xml';
+                                            return codec.mxgraph_to_neo4j(db_image_id, db_fragment_id, overwrite_xml).then(function (data) {
+                                                fragment_mapping[fragment_id] = db_fragment_id;
+                                                return Promise.resolve();
+                                            }).catch(function(err){
+                                                log.error(err);
+                                                return Promise.reject(err);
+                                            });
+                                        }).catch(function(err) {
+                                            log.error('REBUILD: Error creating fragment fragment_id='+fragment_id);
+                                            log.error('REBUILD: ' + err);
+                                        });
+                                    fragment_promises.push(fp);
+                                }
+                            });
+                            return Promise.all(fragment_promises);
+                        }).catch(function(err){
+                            log.error('REBUILD: image ' + image_id + ' error creating: ' + err);
+                        });
+                    image_promises.push(p);
+                } else {
+                    log.error('REBUILD: image ' + image_id + ' has no file_path');
+                }
+            });
+            return Promise.all(image_promises)
+                .then(function(){
+                    log.info('REBUILD: images were added successfully.');
+                    return Export._copy_xmls(fragment_mapping);
+                })
+                .catch(function(err){
+                    log.error('REBUILD: there was an error adding the images and fragment.');
+                    log.error(err);
+                });
+        })
+};
+
 module.exports = Export;

src/server.js

@@ -199,6 +199,31 @@ app.post('/', function (req, res) {
     });
 });
 
+app.post('/rebuild-database', function (req, res) {
+    if (!req.files || !req.files.edges  || !req.files.node_props) {
+        log.warn('Tried to rebuild database without uploading the dump files');
+        return res.redirect('/?e=' + encodeURIComponent('You have to attach the exported files.'))
+    }
+
+    var edges = req.files.edges;
+    var node_props = req.files.node_props;
+
+    var edges_fn = path.join(__dirname, '..', 'media', 'export', 'recover_relations.csv');
+    var node_props_fn = path.join(__dirname, '..', 'media', 'export', 'recover_node_props.csv');
+
+    var promises = [edges.mv(edges_fn), node_props.mv(node_props_fn)];
+
+    Promise.all(promises).then(function(){
+        exp.rebuild_database(edges_fn, node_props_fn).then(function(){
+            res.redirect('/?i=' + encodeURIComponent('Good luck with the recovered database.'))
+        }).catch(function(err){
+            res.redirect('/?e=' + encodeURIComponent(err))
+        })
+    }).catch(function(err){
+        return res.redirect('/?e=' + encodeURIComponent(err));
+    });
+});
+
 app.get('/autocomplete/:token_type/values', function (req, res) {
     var token_type = utils.token_type_mapping(req.params.token_type);
     if (!token_type) {

src/views/db_logout.pug

@@ -9,4 +9,23 @@ block content
     p= 'You are connected. Do you want to drop the connection?'
     b= '' + message
     form(method='post', action='/db-drop')
-      input(type='submit', value='drop connection')
+      input(type='submit', value='drop connection')
+    br
+    br
+    h1= 'Rebuild data from filesystem'
+    p= 'If your database got corrupted (for example because you used a sync client).'
+    p= 'Step 1) Export your data as CSV. Backup the <Giant-App>/resources/app/media/uploaded_xmls/ folder.'
+    p= 'Step 2) Stop GIAnT, stop Neo4j, create a new database for neo4j, start it and restart GIAnT'
+    p= 'Step 3) Your data is located here: <Giant-App>/resources/app/media/'
+    p= 'Make sure uploaded_xmls/ contains your xmls, uploaded_images/ your images and settings/ your settings'
+    p= 'If that\'s not the case copy them to these locations.'
+
+    form(method='post', action='rebuild-database', enctype='multipart/form-data')
+        p= 'Step 4) Upload the dumped CSV files'
+        p= 'Relations: csv-relations-*.csv'
+        input(type='file', name='edges')
+        br
+        p= 'Node properties: csv-nodeprops-*.csv'
+        input(type='file', name='node_props')
+        p= 'Attention: This script will overwrite the current active neo4j database. Ok?'
+        input(type='submit', value='Rebuild database from filesystem')