Merge pull request #743 from giovannipizzi/add_remote_listdir

Add remote listdir

aiida/cmdline/commands/__init__.py

@@ -24,3 +24,4 @@ def work():
 @verdi.group()
 def user():
     pass
+

aiida/cmdline/commands/data.py

@@ -9,6 +9,8 @@
 ###########################################################################
 import sys
 
+import click
+
 from aiida.backends.utils import load_dbenv, is_dbenv_loaded
 from aiida.cmdline import delayed_load_node as load_node
 from aiida.cmdline.baseclass import (
@@ -40,6 +42,7 @@ def __init__(self):
             'parameter': _Parameter,
             'array': _Array,
             'label': _Label,
+            'remote': _Remote,
             'description': _Description,
         }
 
@@ -1982,3 +1985,146 @@ def _show_json_date(self, exec_name, node_list):
             for arrayname in node.arraynames():
                 the_dict[arrayname] = node.get_array(arrayname).tolist()
             print_dictionary(the_dict, 'json+date')
+
+class _Remote(VerdiCommandWithSubcommands):
+    """
+    Manage RemoteData objects
+    """
+
+    def __init__(self):
+        self.valid_subcommands = {
+            'ls': (self.do_listdir, self.complete_none),
+            'cat': (self.do_cat, self.complete_none),
+            'show': (self.do_show, self.complete_none),
+        }
+
+    def do_listdir(self, *args):
+        """
+        List directory content on remote RemoteData objects.
+        """
+        import argparse
+        import datetime
+        from aiida.backends.utils import load_dbenv, is_dbenv_loaded
+        from aiida.common.utils import get_mode_string
+
+        parser = argparse.ArgumentParser(
+            prog=self.get_full_command_name(),
+            description='List directory content on remote RemoteData objects.')
+
+        parser.add_argument('-l', '--long', action='store_true',
+                      help="Display also file metadata")
+        parser.add_argument('pk', type=int, help="PK of the node")
+        parser.add_argument('path', nargs='?', default='.', help="The folder to list")
+
+        args = list(args)
+        parsed_args = parser.parse_args(args)
+
+        if not is_dbenv_loaded():
+            load_dbenv()
+
+        try:
+            n = load_node(parsed_args.pk)
+        except Exception as e:
+            click.echo(e.message, err=True)
+            sys.exit(1)
+        try:
+            content = n.listdir_withattributes(path=parsed_args.path)
+        except (IOError, OSError) as e:
+            click.echo("Unable to access the remote folder or file, check if it exists.", err=True)
+            click.echo("Original error: {}".format(str(e)), err=True)
+            sys.exit(1)
+        for metadata in content:
+            if parsed_args.long:
+                mtime = datetime.datetime.fromtimestamp(
+                    metadata['attributes'].st_mtime)
+                pre_line = '{} {:10}  {}  '.format(
+                    get_mode_string(metadata['attributes'].st_mode),
+                    metadata['attributes'].st_size,
+                    mtime.strftime("%d %b %Y %H:%M")
+                    )
+                click.echo(pre_line, nl=False)
+            if metadata['isdir']:
+                click.echo(click.style(metadata['name'], fg='blue'))
+            else:
+                click.echo(metadata['name'])
+
+    def do_cat(self, *args):
+        """
+        Show the content of remote files in RemoteData objects.
+        """
+        # Note: the implementation is not very efficient: if first downloads the full file on a file on the disk,
+        # then prints it and finally deletes the file.
+        # TODO: change it to open the file and stream it; it requires to add an openfile() method to the transport
+        import argparse
+        import datetime
+        from aiida.backends.utils import load_dbenv, is_dbenv_loaded
+        import tempfile
+        import os
+
+        parser = argparse.ArgumentParser(
+            prog=self.get_full_command_name(),
+            description='Show the content of remote files in RemoteData objects.')
+
+        parser.add_argument('pk', type=int, help="PK of the node")
+        parser.add_argument('path', type=str, help="The (relative) path to the file to show")
+
+        args = list(args)
+        parsed_args = parser.parse_args(args)
+
+        if not is_dbenv_loaded():
+            load_dbenv()
+
+        try:
+            n = load_node(parsed_args.pk)
+        except Exception as e:
+            click.echo(e.message, err=True)
+            sys.exit(1)
+
+        try:
+            with tempfile.NamedTemporaryFile(delete=False) as f:
+                f.close()
+                n.getfile(parsed_args.path, f.name)
+                with open(f.name) as fobj:
+                    sys.stdout.write(fobj.read())
+        except IOError as e:
+            click.echo("ERROR {}: {}".format(e.errno, str(e)), err=True)
+            sys.exit(1)
+
+        try:
+            os.remove(f.name)
+        except OSError:
+            # If you cannot delete, ignore (maybe I didn't manage to create it in the first place
+            pass
+
+    def do_show(self, *args):
+        """
+        Show information on a RemoteData object.
+        """
+        import argparse
+        import datetime
+        from aiida.backends.utils import load_dbenv, is_dbenv_loaded
+        import tempfile
+        import os
+
+        parser = argparse.ArgumentParser(
+            prog=self.get_full_command_name(),
+            description='Show information on a RemoteData object.')
+
+        parser.add_argument('pk', type=int, help="PK of the node")
+
+        args = list(args)
+        parsed_args = parser.parse_args(args)
+
+        if not is_dbenv_loaded():
+            load_dbenv()
+
+        try:
+            n = load_node(parsed_args.pk)
+        except Exception as e:
+            click.echo(e.message, err=True)
+            sys.exit(1)
+
+        click.echo("- Remote computer name:")
+        click.echo("  {}".format(n.get_computer_name()))
+        click.echo("- Remote folder full path:")
+        click.echo("  {}".format(n.get_remote_path()))

aiida/common/utils.py

@@ -1183,3 +1183,79 @@ def join_labels(labels, join_symbol="|", threshold=1.e-6):
         new_labels = []
 
     return new_labels
+
+def get_mode_string(mode):
+    """
+    Convert a file's mode to a string of the form '-rwxrwxrwx'.
+    Taken (simplified) from cpython 3.3 stat module: https://hg.python.org/cpython/file/3.3/Lib/stat.py
+    """
+    # Constants used as S_IFMT() for various file types
+    # (not all are implemented on all systems)
+
+    S_IFDIR = 0o040000  # directory
+    S_IFCHR = 0o020000  # character device
+    S_IFBLK = 0o060000  # block device
+    S_IFREG = 0o100000  # regular file
+    S_IFIFO = 0o010000  # fifo (named pipe)
+    S_IFLNK = 0o120000  # symbolic link
+    S_IFSOCK = 0o140000  # socket file
+
+    # Names for permission bits
+
+    S_ISUID = 0o4000  # set UID bit
+    S_ISGID = 0o2000  # set GID bit
+    S_ENFMT = S_ISGID  # file locking enforcement
+    S_ISVTX = 0o1000  # sticky bit
+    S_IREAD = 0o0400  # Unix V7 synonym for S_IRUSR
+    S_IWRITE = 0o0200  # Unix V7 synonym for S_IWUSR
+    S_IEXEC = 0o0100  # Unix V7 synonym for S_IXUSR
+    S_IRWXU = 0o0700  # mask for owner permissions
+    S_IRUSR = 0o0400  # read by owner
+    S_IWUSR = 0o0200  # write by owner
+    S_IXUSR = 0o0100  # execute by owner
+    S_IRWXG = 0o0070  # mask for group permissions
+    S_IRGRP = 0o0040  # read by group
+    S_IWGRP = 0o0020  # write by group
+    S_IXGRP = 0o0010  # execute by group
+    S_IRWXO = 0o0007  # mask for others (not in group) permissions
+    S_IROTH = 0o0004  # read by others
+    S_IWOTH = 0o0002  # write by others
+    S_IXOTH = 0o0001  # execute by others
+
+    _filemode_table = (
+        ((S_IFLNK, "l"),
+         (S_IFREG, "-"),
+         (S_IFBLK, "b"),
+         (S_IFDIR, "d"),
+         (S_IFCHR, "c"),
+         (S_IFIFO, "p")),
+
+        ((S_IRUSR, "r"),),
+        ((S_IWUSR, "w"),),
+        ((S_IXUSR | S_ISUID, "s"),
+         (S_ISUID, "S"),
+         (S_IXUSR, "x")),
+
+        ((S_IRGRP, "r"),),
+        ((S_IWGRP, "w"),),
+        ((S_IXGRP | S_ISGID, "s"),
+         (S_ISGID, "S"),
+         (S_IXGRP, "x")),
+
+        ((S_IROTH, "r"),),
+        ((S_IWOTH, "w"),),
+        ((S_IXOTH | S_ISVTX, "t"),
+         (S_ISVTX, "T"),
+         (S_IXOTH, "x"))
+    )
+
+
+    perm = []
+    for table in _filemode_table:
+        for bit, char in table:
+            if mode & bit == bit:
+                perm.append(char)
+                break
+        else:
+            perm.append("-")
+    return "".join(perm)

aiida/orm/data/remote.py

@@ -8,7 +8,7 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 from aiida.orm import Data
-
+import os
 
 
 
@@ -22,6 +22,9 @@ class RemoteData(Data):
     def get_dbcomputer(self):
         return self.dbnode.dbcomputer
 
+    def get_computer_name(self):
+        return self.get_computer().name
+
     def get_remote_path(self):
         return self.get_attr('remote_path')
 
@@ -54,6 +57,115 @@ def is_empty(self):
                     return True  # is indeed empty, i.e. unusable
             return not t.listdir()
 
+    def getfile(self, relpath, destpath):
+        """
+        Connects to the remote folder and gets a string with the (full) content of the file.
+
+        :param relpath: The relative path of the file to show.
+        :param destpath: A path on the local computer to get the file
+        :return: a string with the file content
+        """
+        from aiida.backends.utils import get_authinfo
+
+        authinfo = get_authinfo(computer=self.get_computer(),
+                                aiidauser=self.get_user())
+        t = authinfo.get_transport()
+
+        with t:
+            try:
+                full_path = os.path.join(self.get_remote_path(), relpath)
+                t.getfile(full_path, destpath)
+            except IOError as e:
+                if e.errno == 2:  # file not existing
+                    raise IOError("The required remote file {} on {} does not exist or has been deleted.".format(
+                        full_path, self.get_computer().name
+                    ))
+                else:
+                    raise
+
+            return t.listdir()
+
+
+    def listdir(self, relpath="."):
+        """
+        Connects to the remote folder and lists the directory content.
+
+        :param relpath: If 'relpath' is specified, lists the content of the given subfolder.
+        :return: a flat list of file/directory names (as strings).
+        """
+        from aiida.backends.utils import get_authinfo
+
+        authinfo = get_authinfo(computer=self.get_computer(),
+                                aiidauser=self.get_user())
+        t = authinfo.get_transport()
+
+        with t:
+            try:
+                full_path = os.path.join(self.get_remote_path(), relpath)
+                t.chdir(full_path)
+            except IOError as e:
+                if e.errno == 2 or e.errno == 20:  # directory not existing or not a directory
+                    exc = IOError("The required remote folder {} on {} does not exist, is not a directory or has been deleted.".format(
+                        full_path, self.get_computer().name
+                    ))
+                    exc.errno = e.errno
+                    raise exc
+                else:
+                    raise
+
+            try:
+                return t.listdir()
+            except IOError as e:
+                if e.errno == 2 or e.errno == 20:  # directory not existing or not a directory
+                    exc = IOError(
+                        "The required remote folder {} on {} does not exist, is not a directory or has been deleted.".format(
+                            full_path, self.get_computer().name
+                        ))
+                    exc.errno = e.errno
+                    raise exc
+                else:
+                    raise
+
+    def listdir_withattributes(self, path="."):
+        """
+        Connects to the remote folder and lists the directory content.
+
+        :param relpath: If 'relpath' is specified, lists the content of the given subfolder.
+        :return: a list of dictionaries, where the documentation is in :py:class:Transport.listdir_withattributes.
+        """
+        from aiida.backends.utils import get_authinfo
+
+        authinfo = get_authinfo(computer=self.get_computer(),
+                                aiidauser=self.get_user())
+        t = authinfo.get_transport()
+
+        with t:
+            try:
+                full_path = os.path.join(self.get_remote_path(), path)
+                t.chdir(full_path)
+            except IOError as e:
+                if e.errno == 2 or e.errno == 20:  # directory not existing or not a directory
+                    exc = IOError("The required remote folder {} on {} does not exist, is not a directory or has been deleted.".format(
+                        full_path, self.get_computer().name
+                    ))
+                    exc.errno = e.errno
+                    raise exc
+                else:
+                    raise
+
+            try:
+                return t.listdir_withattributes()
+            except IOError as e:
+                if e.errno == 2 or e.errno == 20:  # directory not existing or not a directory
+                    exc = IOError("The required remote folder {} on {} does not exist, is not a directory or has been deleted.".format(
+                        full_path, self.get_computer().name
+                    ))
+                    exc.errno = e.errno
+                    raise exc
+                else:
+                    raise
+
+
     def _clean(self):
         """
         Remove all content of the remote folder on the remote computer