Support cloud storage file transfers

apps/dashboard/app/controllers/transfers_controller.rb

@@ -1,3 +1,5 @@
+require "rclone_util"
+
 class TransfersController < ApplicationController
 
   def show
@@ -17,7 +19,18 @@ def show
 
   def create
     body_params = JSON.parse(request.body.read).symbolize_keys
-    @transfer = Transfer.build(action: body_params[:command], files: body_params[:files])
+
+    from_fs = body_params.fetch(:from_fs, RcloneUtil::LOCAL_FS_NAME)
+    to_fs = body_params.fetch(:to_fs, RcloneUtil::LOCAL_FS_NAME)
+
+    if from_fs == RcloneUtil::LOCAL_FS_NAME && to_fs == RcloneUtil::LOCAL_FS_NAME
+      @transfer = PosixTransfer.build(action: body_params[:command], files: body_params[:files])
+    elsif ::Configuration.files_app_remote_files?
+      @transfer = RemoteTransfer.build(action: body_params[:command], files: body_params[:files], src_remote: from_fs, dest_remote: to_fs)
+    else
+      render json: { error_message: "Remote file support is not enabled" }
+    end
+
     if ! @transfer.valid?
       # error
       render json: { error_message: @transfer.errors.full_messages.join('. ') }

apps/dashboard/app/javascript/packs/files/clip_board.js

@@ -90,6 +90,7 @@ class ClipBoard {
     } else {
       let clipboardData = {
         from: history.state.currentDirectory,
+        from_fs: history.state.currentFilesystem,
         files: selection.toArray().map((f) => {
           return { directory: f.type == 'd', name: f.name };
         })
@@ -116,6 +117,7 @@ class ClipBoard {
       let clipboard = JSON.parse(localStorage.getItem('filesClipboard') || 'null');
       if (clipboard) {
         clipboard.to = history.state.currentDirectory;
+        clipboard.to_fs = history.state.currentFilesystem;
 
         if (clipboard.from == clipboard.to) {
           // No files are changed, so we just have to clear and update the clipboard
@@ -130,7 +132,9 @@ class ClipBoard {
 
           const eventData = {
             'files': files,
-            'token': csrf_token
+            'token': csrf_token,
+            'from_fs': clipboard.from_fs,
+            'to_fs': clipboard.to_fs,
           };
 
           $(CONTENTID).trigger(FILEOPS_EVENTNAME.moveFile, eventData);
@@ -147,9 +151,11 @@ class ClipBoard {
 
       if (clipboard) {
         clipboard.to = history.state.currentDirectory;
-
+        clipboard.to_fs = history.state.currentFilesystem;
+
         // files is a hashmap with keys of file current path and value as the corresponding files desired path
         let files = {};
+
         if (clipboard.from == clipboard.to) {
           const currentFilenames = history.state.currentFilenames;
           clipboard.files.forEach((f) => {
@@ -187,7 +193,9 @@ class ClipBoard {
 
         const eventData = {
           'files': files,
-          'token': csrf_token
+          'token': csrf_token,
+          'from_fs': clipboard.from_fs,
+          'to_fs': clipboard.to_fs,
         };
 
         $(CONTENTID).trigger(FILEOPS_EVENTNAME.copyFile, eventData);

apps/dashboard/app/javascript/packs/files/data_table.js

@@ -361,6 +361,7 @@ class DataTable {
                         currentDirectoryUrl: data.url,
                         currentFilesPath: data.files_path,
                         currentFilesUploadPath: data.files_upload_path,
+                        currentFilesystem: data.filesystem,
                         currentFilenames: Array.from(data.files, x => x.name)
                     }, data.name, data.url);
                 }      

apps/dashboard/app/javascript/packs/files/file_ops.js

@@ -179,15 +179,15 @@ jQuery(function() {
   });
 
   $(CONTENTID).on(EVENTNAME.deleteFile, function (e, options) {    
-    fileOps.delete(options.files);
+    fileOps.delete(options.files, options.from_fs);
   });
 
   $(CONTENTID).on(EVENTNAME.moveFile, function (e, options) {
-    fileOps.move(options.files, options.token);
+    fileOps.move(options.files, options.token, options.from_fs, options.to_fs);
   });
 
   $(CONTENTID).on(EVENTNAME.copyFile, function (e, options) {
-    fileOps.copy(options.files, options.token);
+    fileOps.copy(options.files, options.token, options.from_fs, options.to_fs);
   });
 
   $(CONTENTID).on(EVENTNAME.changeDirectoryPrompt, function () {
@@ -272,13 +272,13 @@ class FileOps {
 
 
   removeFiles(files) {
-    this.transferFiles(files, "rm", "remove files")
+    this.transferFiles(files, "rm", "remove files", history.state.currentFilesystem)
   } 
 
   renameFile(fileName, newFileName) {
     let files = {};
     files[`${history.state.currentDirectory}/${fileName}`] = `${history.state.currentDirectory}/${newFileName}`;
-    this.transferFiles(files, "mv", "rename file")
+    this.transferFiles(files, "mv", "rename file", history.state.currentFilesystem, history.state.currentFilesystem)
   }
 
   renameFilePrompt(fileName) {
@@ -464,7 +464,7 @@ class FileOps {
     this.removeFiles(files.map(f => [history.state.currentDirectory, f].join('/')), csrf_token);
   }
 
-  transferFiles(files, action, summary){
+  transferFiles(files, action, summary, from_fs, to_fs){
 
     this._failures = 0;
 
@@ -474,7 +474,9 @@ class FileOps {
       method: 'post',
       body: JSON.stringify({
         command: action,
-        files: files
+        files: files,
+        from_fs: from_fs,
+        to_fs: to_fs,
       }),
       headers: { 'X-CSRF-Token': csrf_token }
     })
@@ -566,12 +568,12 @@ class FileOps {
     this.poll(data);
   } 
 
-  move(files, token) {
-    this.transferFiles(files, 'mv', 'move files');
+  move(files, token, from_fs, to_fs) {
+    this.transferFiles(files, 'mv', 'move files', from_fs, to_fs);
   }
 
-  copy(files, token) {
-    this.transferFiles(files, 'cp', 'copy files');
+  copy(files, token, from_fs, to_fs) {
+    this.transferFiles(files, 'cp', 'copy files', from_fs, to_fs);
   }
 
   alertError(title, message) {

apps/dashboard/app/models/local_transfer.rb → apps/dashboard/app/models/posix_transfer.rb

@@ -1,4 +1,4 @@
-class LocalTransfer < Transfer
+class PosixTransfer < Transfer
 
   validates_each :files do |record, attr, files|
     if record.action == 'mv' || record.action == 'cp'
@@ -18,6 +18,18 @@ def transfers
       # all transfers stored in the Transfer class
       Transfer.transfers
     end
+
+    def build(action:, files:)
+      if files.is_a?(Array)
+        # rm action will want to provide an array of files
+        # so if it is an Array we convert it to a hash:
+        #
+        # convert [a1, a2, a3] to {a1 => nil, a2 => nil, a3 => nil}
+        files = Hash[files.map { |f| [f, nil] }].with_indifferent_access
+      end
+
+      self.new(action: action, files: files)
+    end
   end
 
   # number of files to copy, move or delete

apps/dashboard/app/models/remote_transfer.rb

@@ -0,0 +1,157 @@
+class RemoteTransfer < Transfer
+
+  validates_each :src_remote, :dest_remote do |record, _, remote|
+    remote_type = RcloneUtil.remote_type(remote)
+    if remote_type.nil? && remote != RcloneUtil::LOCAL_FS_NAME
+      record.errors.add :base, "Remote #{remote} does not exist"
+    elsif ::Configuration.allowlist_paths.present? && (remote_type == 'local' || remote_type == 'alias')
+      record.errors.add :base, "Remotes of type #{remote_type} are not allowed due to ALLOWLIST_PATH"
+    end
+  end
+
+  validates_each :files do |record, _, files|
+    files.each do |k, v|
+      # Validate paths in the same was as PosixTransfer for the local filesystem (fs)
+      if record.src_remote == RcloneUtil::LOCAL_FS_NAME
+        record.errors.add :base, "#{k} is not included under ALLOWLIST_PATH" unless AllowlistPolicy.default.permitted?(k.to_s)
+      end
+      if record.dest_remote == RcloneUtil::LOCAL_FS_NAME
+        # rm commands are [{ k => nil}] - nil values
+        record.errors.add :base, "#{v} is not included under ALLOWLIST_PATH" if !v.nil? && !AllowlistPolicy.default.permitted?(v.to_s)
+      end
+    end
+
+    if record.action == 'mv' || record.action == 'cp'
+      # local filesystem
+      if record.dest_remote == RcloneUtil::LOCAL_FS_NAME
+        conflicts = files.values.select { |f| File.exist?(f) }
+        record.errors.add :base, "These files already exist: #{conflicts.join(', ')}" if conflicts.present?
+      else
+        # remote
+        begin
+          existing = RcloneUtil.lsf(record.dest_remote, record.to).map { |file| File.join(record.to, file) }
+          conflicts = files.values.intersection(existing)
+          record.errors.add :base, "These files already exist: #{conflicts.join(', ')}" if conflicts.present?
+        rescue RcloneError => e
+          if e.exitstatus != 3
+            # Rclone will return status 3 if directory doesn't exist (ok), other errors are unexpected
+            record.errors.add :base, "Error checking existing files in destination: #{e}"
+          end
+        end
+      end
+    end
+  end
+
+  attr_accessor :src_remote, :dest_remote, :filesizes, :transferred
+
+  class << self
+    def transfers
+      # all transfers stored in the Transfer class
+      Transfer.transfers
+    end
+
+    def build(action:, files:, src_remote:, dest_remote:)
+      if files.is_a?(Array)
+        # rm action will want to provide an array of files
+        # so if it is an Array we convert it to a hash:
+        #
+        # convert [a1, a2, a3] to {a1 => nil, a2 => nil, a3 => nil}
+        files = Hash[files.map { |f| [f, nil] }].with_indifferent_access
+      end
+
+      self.new(action: action, files: files, src_remote: src_remote, dest_remote: dest_remote)
+    end
+  end
+
+  # total number of bytes
+  def steps
+    return @steps if @steps
+
+    @filesizes = {}.with_indifferent_access
+
+    # Get info from `rclone size` (will not work on Google Drive and Google Photos)
+    total_size = files.keys.map do |file|
+      size = RcloneUtil.size(src_remote, file).fetch('bytes', 0)
+      @filesizes[file] = { :size => size, :transferred => 0 }
+      size
+    end.sum
+
+    @steps = total_size
+  end
+
+  def command_str
+    ''
+  end
+
+  def increment_transferred(amount)
+    @transferred = transferred.to_i + amount
+  end
+
+  # Updates the total progress with progress of one file since last update
+  def update_progress(file, percent_done)
+    file_info = filesizes[file]
+
+    current_bytes = (percent_done * file_info[:size]) / 100
+    since_last = current_bytes - file_info[:transferred]
+    filesizes[file][:transferred] = current_bytes
+
+    increment_transferred(since_last)
+    update_percent(transferred)
+  end
+
+  def perform
+    self.status = OodCore::Job::Status.new(state: :running)
+    self.started_at = Time.now.to_i
+
+    # Store info about sizes of files to transfer for tracking progress
+    steps
+
+    # Transfer each file/directory indiviually
+    files.each do |src, dst|
+      if action == 'mv'
+        RcloneUtil.moveto_with_progress(src_remote, dest_remote, src, dst) do |p|
+          update_progress(src, p)
+        end
+      elsif action == 'cp'
+        RcloneUtil.copyto_with_progress(src_remote, dest_remote, src, dst) do |p|
+          update_progress(src, p)
+        end
+      elsif action == 'rm'
+        RcloneUtil.remove_with_progress(src_remote, src) do |p|
+          update_progress(src, p)
+        end
+      else
+        raise StandardError, "Unknown action: #{action.inspect}"
+      end
+    rescue RcloneError => e
+      # TODO: catch more rclone specific errors here, i.e. if the access keys are invalid it would make
+      # sense to not attempt to transfer the rest of the files
+      errors.add :base, "Error when transferring #{src}: #{e.message}"
+    end
+  rescue => e
+    errors.add :base, e.message
+  ensure
+    self.status = OodCore::Job::Status.new(state: :completed)
+  end
+
+  def from
+    File.dirname(files.keys.first) if files.keys.first
+  end
+
+  def to
+    File.dirname(files.values.first) if files.values.first
+  end
+
+  def target_dir
+    # directory where files are being moved/copied to OR removed from
+    if action == 'rm'
+      Pathname.new(from).cleanpath if from
+    else
+      Pathname.new(to).cleanpath if to
+    end
+  end
+
+  def synchronous?
+    false
+  end
+end

apps/dashboard/app/models/transfer.rb

@@ -26,18 +26,6 @@ def transfers
     def find(id)
       transfers.find {|t| t.id == id }
     end
-
-    def build(action:, files:)
-      if files.is_a?(Array)
-        # rm action will want to provide an array of files
-        # so if it is an Array we convert it to a hash:
-        #
-        # convert [a1, a2, a3] to {a1 => nil, a2 => nil, a3 => nil}
-        files = Hash[files.map { |f| [f, nil] }]
-      end
-
-      LocalTransfer.new(action: action, files: files)
-    end
   end
 
   def bootstrap_class

apps/dashboard/app/views/files/_inline_js.html.erb

@@ -10,6 +10,7 @@ history.replaceState({
   currentDirectoryUpdatedAt: '<%= Time.now.to_i %>',
   currentFilesPath: '<%= files_path(@filesystem, '/') %>',
   currentFilesUploadPath: '<%= url_for(fs: @filesystem, action: 'upload') %>',
+  currentFilesystem: '<%= @filesystem %>'
 }, null);
 
 </script>

apps/dashboard/app/views/files/index.json.jbuilder

@@ -4,6 +4,7 @@ json.url files_path(@filesystem, @path).to_s
 json.shell_url OodAppkit.shell.url(path: @path.to_s).to_s
 json.files_path files_path(@filesystem, '/')
 json.files_upload_path url_for(fs: @filesystem, action: 'upload')
+json.filesystem @filesystem
 
 json.files @files do |f|
   json.id f[:id]