feat(exports): Add ability to generate data exports for sponsors (#199)

Fixes #133

app/controllers/manage/application_controller.rb

@@ -8,6 +8,10 @@ def logged_in
     authenticate_user!
   end
 
+  def require_full_admin
+    return redirect_to root_path unless current_user.try(:admin?)
+  end
+
   def require_admin_or_limited_admin
     return redirect_to root_path unless current_user.try(:admin?) || current_user.try(:admin_limited_access?)
   end

app/controllers/manage/data_exports_controller.rb

@@ -0,0 +1,62 @@
+class Manage::DataExportsController < Manage::ApplicationController
+  skip_before_action :require_admin_or_limited_admin
+  before_action :require_full_admin
+
+  before_action :set_data_export, only: [:destroy]
+
+  respond_to :html, :json
+
+  # GET /manage/data_exports
+  def index
+    @data_exports = DataExport.all.order(created_at: :desc)
+    @params = {}
+    if params[:export_type]
+      @params = params.require(:data_export).permit(:export_type).reject { |_, v| v.blank? }
+      @data_exports = @data_exports.where(@params)
+    end
+    respond_with(:manage, @data_exports)
+  end
+
+  # GET /manage/data_exports/new
+  def new
+    export_type = params[:export_type]
+    @data_export = DataExport.new(export_type: export_type)
+    respond_with(:manage, @data_export)
+  end
+
+  # POST /manage/data_exports
+  def create
+    @data_export = DataExport.new(data_export_params)
+
+    if @data_export.save
+      @data_export.enqueue!
+      respond_to do |format|
+        format.html { redirect_to manage_data_exports_path, notice: "Data export was successfully created." }
+        format.json { render json: @data_export }
+      end
+    else
+      response_view_or_errors :new, @data_export
+    end
+  end
+
+  # DELETE /manage/data_exports/1
+  def destroy
+    @data_export.destroy
+    respond_to do |format|
+      format.html { redirect_to manage_data_exports_path, notice: "Data export was successfully destroyed." }
+      format.json { render json: @data_export }
+    end
+  end
+
+  private
+
+  # Use callbacks to share common setup or constraints between actions.
+  def set_data_export
+    @data_export = DataExport.find(params[:id])
+  end
+
+  # Only allow a trusted parameter "white list" through.
+  def data_export_params
+    params.require(:data_export).permit(:export_type)
+  end
+end

app/jobs/generate_data_export_job.rb

@@ -0,0 +1,113 @@
+require "zip"
+
+class GenerateDataExportJob < ApplicationJob
+  queue_as :default
+
+  def perform(*args)
+    data_export = args[0]
+    # Prevent an already-started or already-completed job from running again
+    return unless data_export.status == "queued"
+
+    data_export.update_attribute(:started_at, Time.now)
+
+    begin
+      case data_export.export_type
+      when "sponsor_dump_rsvp_confirmed"
+        generate__sponsor_dump(data_export, "rsvp_confirmed")
+      when "sponsor_dump_checked_in"
+        generate__sponsor_dump(data_export, "checked_in")
+      else
+        raise "Unknown export type: #{data_export.export_type}"
+      end
+
+      data_export.update_attribute(:finished_at, Time.now)
+    rescue => ex
+      data_export.update_attribute(:started_at, nil)
+      data_export.update_attribute(:finished_at, nil)
+      # Re-raise the original exception
+      raise
+    end
+  end
+
+  private
+
+  def generate__sponsor_dump(data_export, attendee_type)
+    print data_export.file.name
+
+    case attendee_type
+    when "rsvp_confirmed"
+      questionnaires = Questionnaire.where(acc_status: "rsvp_confirmed", can_share_info: true)
+    when "checked_in"
+      questionnaires = Questionnaire.where("checked_in_at > 0", can_share_info: true)
+    else
+      raise "Unknown attendee type: #{attendee_type}"
+    end
+
+    Dir.mktmpdir("data-export") do |dir|
+      folder_path = File.join(dir, data_export.file_basename)
+      Dir.mkdir(folder_path)
+      zipfile_name = "#{data_export.file_basename}.zip"
+      zipfile_path = File.join(dir, zipfile_name)
+
+      # Download all of the resumes & generate CSV
+      csv_data = []
+      resume_paths = []
+      questionnaires.each do |q|
+        csv_row = [
+          q.first_name,
+          q.last_name,
+          q.school_name,
+          q.email,
+          q.vcs_url,
+          q.portfolio_url,
+        ]
+
+        if q.resume.attached?
+          filename = "#{q.id}-#{q.resume.filename.sanitized}"
+          puts "--> Downloading #{q.id} resume, filename '#{filename}'"
+          path = File.join(folder_path, filename)
+          File.open(path, "wb") do |file|
+            file.write(q.resume.download)
+          end
+          resume_paths << { path: path, filename: filename }
+          csv_row << filename
+        else
+          csv_row << "" # No resume file
+        end
+
+        csv_data << csv_row
+      end
+
+      csvfile_name = "000-Attendees.csv"
+      csvfile_path = File.join(folder_path, csvfile_name)
+      CSV.open(csvfile_path, "wb") do |csv|
+        csv << ["Fist name", "Last name", "School", "Email", "VCS URL", "Portfolio URL", "Resume filename"]
+        csv_data.each do |row|
+          csv << row
+        end
+      end
+
+      # Zip up all of the files
+      Zip::File.open(zipfile_path, Zip::File::CREATE) do |zipfile|
+        # Add the CSV
+        zipfile.add(csvfile_name, csvfile_path)
+        # Add all resume files
+        resume_paths.each do |resume|
+          path = resume[:path]
+          filename = resume[:filename]
+          # Two arguments:
+          # - The name of the file as it will appear in the archive
+          # - The original file, including the path to find it
+          zipfile.add(filename, path)
+        end
+      end
+
+      # Attach the zip file to the record
+      data_export.file.attach(
+        io: File.open(zipfile_path),
+        filename: zipfile_name,
+        content_type: "application/zip",
+      )
+    end
+  end
+end

app/models/data_export.rb

@@ -0,0 +1,54 @@
+class DataExport < ApplicationRecord
+
+  # A DataExport is a generated .zip of data from HackathonManager, such as a .zip of
+  # resumes & attendee data, or a .zip of the entire database is the form of multiple
+  # CSVs.
+  #
+  # These should be generated asynchronously with a background job, and then stored as an
+  # active storage attachment.
+
+  POSSIBLE_TYPES = [
+    "sponsor_dump_rsvp_confirmed",
+    "sponsor_dump_checked_in",
+  ].freeze
+
+  validates_presence_of :export_type
+  validates_inclusion_of :export_type, in: POSSIBLE_TYPES
+
+  has_one_attached :file
+
+  strip_attributes
+
+  def file_basename
+    time = created_at.strftime("%r").gsub(":", "-")
+    date = created_at.strftime("%F")
+    "#{export_type} #{date} #{time}"
+  end
+
+  def finished?
+    finished_at.present?
+  end
+
+  def started?
+    started_at.present?
+  end
+
+  def queued?
+    queued_at.present?
+  end
+
+  def enqueue!
+    raise "Data export has already been queued" unless status == "created_not_queued"
+
+    GenerateDataExportJob.perform_later(self)
+    update_attribute(:queued_at, Time.now)
+  end
+
+  def status
+    return "finished" if finished?
+    return "started" if started?
+    return "queued" if queued?
+
+    "created_not_queued"
+  end
+end

app/views/layouts/manage/application.html.haml

@@ -89,6 +89,11 @@
                     Doorkeeper
                     %span.fa.fa-external-link.icon-space-l-half
                     .nav-item-description OAuth2 provider management
+                %li.nav-item
+                  = active_link_to manage_data_exports_path, class: "nav-link" do
+                    .fa.fa-download.fa-fw.icon-space-r-half
+                    Data Exports
+                    .nav-item-description Generate & export data
         %main.col-md-10.ml-sm-auto.px-4{role: "main"}
           = render "layouts/manage/flashes"
           = yield

app/views/manage/data_exports/_form.html.haml

@@ -0,0 +1,11 @@
+.form-container
+  = bs_horizontal_simple_form_for @data_export, url: url_for(action: @data_export.new_record? ? "create" : "update", controller: "data_exports") do |f|
+    = f.error_notification
+
+    .form-inputs
+      = f.input :export_type, as: :select, collection: DataExport::POSSIBLE_TYPES.map { |x| [x.titleize, x] }, include_blank: false
+
+    .form-actions.mb-3.mt-3
+      = f.button :submit, class: 'btn-primary'
+
+.mb-4

app/views/manage/data_exports/index.html.haml

@@ -0,0 +1,39 @@
+= render "layouts/manage/page_title", title: "Data Exports" do
+  = link_to "New Data Export", new_manage_data_export_path, class: "btn btn-sm btn-outline-secondary"
+
+%table.table.table-striped
+  %thead
+    %tr
+      %th Type
+      %th Created
+      %th Timeline
+      %th Download
+      %th Delete
+
+  %tbody
+    - if @data_exports.blank?
+      %tr
+        %td{colspan: 5} No data exports have been generated.
+    - @data_exports.each do |data_export|
+      %tr
+        %td= data_export.export_type.titleize
+        %td= display_datetime(data_export.created_at)
+        %td
+          %span
+            Queued: #{data_export.queued_at || "n/a"}
+          %br
+          %span
+            Started: #{data_export.started_at || "n/a"}
+          %br
+          %span
+            Finished: #{data_export.finished_at || "n/a"}
+        %td
+          - if data_export.finished? && data_export.file.attached?
+            = link_to "Download", rails_blob_path(data_export.file)
+          - else
+            Not available
+            %br
+            %small Please wait for generation to finish
+        %td= link_to 'Delete', manage_data_export_path(data_export), method: :delete, data: { confirm: "Are you sure? The data export \"#{data_export.file_basename}\" will be permanently deleted. This action is irreversible." }, class: 'btn btn-sm btn-outline-secondary'
+
+%br

app/views/manage/data_exports/new.html.haml

@@ -0,0 +1,3 @@
+= render "layouts/manage/page_title", title: "New Data Export"
+
+= render 'form'

config/routes.rb

@@ -93,5 +93,6 @@
     end
     resources :trackable_events
     resources :trackable_tags
+    resources :data_exports
   end
 end

db/migrate/20200205160318_create_data_exports.rb

@@ -0,0 +1,12 @@
+class CreateDataExports < ActiveRecord::Migration[5.2]
+  def change
+    create_table :data_exports do |t|
+      t.string :export_type, null: false
+      t.datetime :queued_at
+      t.datetime :started_at
+      t.datetime :finished_at
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_05_29_190615) do
+ActiveRecord::Schema.define(version: 2020_02_05_160318) do
 
   create_table "active_storage_attachments", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
     t.string "name", null: false
@@ -110,6 +110,15 @@
     t.boolean "needs_bus_captain", default: false
   end
 
+  create_table "data_exports", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.string "export_type", null: false
+    t.datetime "queued_at"
+    t.datetime "started_at"
+    t.datetime "finished_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "fips", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
     t.string "fips_code"
     t.string "city"

test/factories/data_export.rb

@@ -0,0 +1,8 @@
+FactoryBot.define do
+  factory :data_export do
+    export_type { "sponsor_dump_rsvp_confirmed" }
+    queued_at { nil }
+    started_at { nil }
+    finished_at { nil }
+  end
+end

test/jobs/generate_data_export_job_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class GenerateDataExportJobTest < ActiveJob::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end