Merge pull request #179 from blackcandy-org/listen-library-changes

Fix #124, automatically sync for media library changes

Gemfile

@@ -61,6 +61,9 @@ gem "acts_as_list", "~> 1.0.2"
 gem "authlogic", "~> 6.4.1"
 gem "bcrypt", "~> 3.1.11"
 
+# For sync on library changes
+gem "listen", "~> 3.7.1"
+
 # Use Capistrano for deployment
 # gem 'capistrano-rails', group: :development
 

Gemfile.lock

@@ -151,6 +151,9 @@ GEM
       activesupport (>= 5.0.0)
     jsbundling-rails (1.0.0)
       railties (>= 6.0.0)
+    listen (3.7.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     loofah (2.18.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -230,6 +233,9 @@ GEM
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
+    rb-fsevent (0.11.1)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
     redis (4.5.1)
     redis-objects (1.5.1)
       redis (~> 4.2)
@@ -326,6 +332,7 @@ DEPENDENCIES
   httparty (~> 0.17.0)
   jbuilder (~> 2.11.5)
   jsbundling-rails (~> 1.0.0)
+  listen (~> 3.7.1)
   memory_profiler (~> 0.9.13)
   pagy (~> 5.6.6)
   pg (~> 1.3.2)

Procfile.dev

@@ -4,3 +4,4 @@ web: rails s -p 3000
 sidekiq: sidekiq
 css: yarn build:css --watch
 js: yarn build --watch
+listen: rails listen_media_changes

app/jobs/media_sync_job.rb

@@ -3,8 +3,8 @@ class MediaSyncJob < ApplicationJob
   before_enqueue :start_syncing
   after_perform :stop_syncing
 
-  def perform
-    Media.sync
+  def perform(type = :all, file_paths = [])
+    Media.sync(type, file_paths)
   end
 
   private

app/models/media.rb

@@ -16,15 +16,22 @@ def id
   end
 
   class << self
-    def sync
-      media_hashes = MediaFile.file_paths.map do |file_path|
-        @file_info = MediaFile.file_info(file_path)
-        @file_info[:md5_hash] if attach
-      rescue
-        next
-      end.compact
-
-      clean_up(media_hashes)
+    def sync(type = :all, file_paths = [])
+      file_paths = MediaFile.file_paths if type == :all
+      return if file_paths.blank?
+
+      case type
+      when :all
+        file_hashes = add_files(file_paths)
+        clean_up(file_hashes)
+      when :added
+        add_files(file_paths)
+      when :removed
+        remove_files(file_paths)
+      when :modified
+        remove_files(file_paths)
+        add_files(file_paths)
+      end
     end
 
     def syncing?
@@ -38,6 +45,22 @@ def syncing=(is_syncing)
 
     private
 
+    def add_files(file_paths)
+      file_paths.map do |file_path|
+        @file_info = MediaFile.file_info(file_path)
+        @file_info[:md5_hash] if attach
+      rescue
+        next
+      end.compact
+    end
+
+    def remove_files(file_paths)
+      file_path_hashes = file_paths.map { |file_path| MediaFile.get_md5_hash(file_path) }
+      Song.where(file_path_hash: file_path_hashes).destroy_all
+
+      clean_up
+    end
+
     def attach
       artist = Artist.find_or_create_by!(name: @file_info[:artist_name])
 
@@ -57,16 +80,16 @@ def attach
     end
 
     def song_info
-      @file_info.slice(:name, :tracknum, :duration, :file_path)
+      @file_info.slice(:name, :tracknum, :duration, :file_path, :file_path_hash)
     end
 
     def various_artists?
       albumartist = @file_info[:albumartist_name]
       albumartist.present? && (albumartist.casecmp("various artists").zero? || albumartist != @file_info[:artist_name])
     end
 
-    def clean_up(media_hashes)
-      Song.where.not(md5_hash: media_hashes).destroy_all
+    def clean_up(file_hashes = [])
+      Song.where.not(md5_hash: file_hashes).destroy_all if file_hashes.present?
 
       # Clean up no content albums and artist.
       Album.where.missing(:songs).destroy_all

app/models/media_file.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 class MediaFile
-  SUPPORT_FORMATE = WahWah.support_formats.freeze
+  SUPPORTED_FORMATS = WahWah.support_formats.freeze
 
   class << self
     def file_paths
       media_path = File.expand_path(Setting.media_path)
-      Dir.glob("#{media_path}/**/*.{#{SUPPORT_FORMATE.join(",")}}", File::FNM_CASEFOLD)
+      Dir.glob("#{media_path}/**/*.{#{SUPPORTED_FORMATS.join(",")}}", File::FNM_CASEFOLD)
     end
 
     def format(file_path)
@@ -22,7 +22,16 @@ def image(file_path)
 
     def file_info(file_path)
       tag_info = get_tag_info(file_path)
-      tag_info.merge(file_path: file_path, md5_hash: get_md5_hash(file_path))
+      tag_info.merge(
+        file_path: file_path.to_s,
+        file_path_hash: get_md5_hash(file_path),
+        md5_hash: get_md5_hash(file_path, with_mtime: true)
+      )
+    end
+
+    def get_md5_hash(file_path, with_mtime: false)
+      string = "#{file_path}#{File.mtime(file_path) if with_mtime}"
+      Digest::MD5.base64digest(string)
     end
 
     private
@@ -39,9 +48,5 @@ def get_tag_info(file_path)
         duration: tag.duration.round
       }
     end
-
-    def get_md5_hash(file_path)
-      Digest::MD5.base64digest(file_path.to_s + File.mtime(file_path).to_s)
-    end
   end
 end

app/models/song.rb

@@ -3,7 +3,7 @@
 class Song < ApplicationRecord
   include Searchable
 
-  validates :name, :file_path, :md5_hash, presence: true
+  validates :name, :file_path, :file_path_hash, :md5_hash, presence: true
 
   belongs_to :album, touch: true
   belongs_to :artist, touch: true

db/migrate/20220705083238_add_file_path_hash_to_songs.rb

@@ -0,0 +1,15 @@
+class AddFilePathHashToSongs < ActiveRecord::Migration[7.0]
+  def change
+    add_column :songs, :file_path_hash, :string
+    add_index :songs, :file_path_hash
+    add_index :songs, :md5_hash
+
+    reversible do |dir|
+      dir.up do
+        Song.find_each do |song|
+          song.update_column(:file_path_hash, MediaFile.get_md5_hash(song.file_path))
+        end
+      end
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_05_31_070546) do
+ActiveRecord::Schema[7.0].define(version: 2022_07_05_083238) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "hstore"
   enable_extension "pg_trgm"
@@ -68,8 +68,11 @@
     t.datetime "updated_at", null: false
     t.bigint "album_id"
     t.bigint "artist_id"
+    t.string "file_path_hash"
     t.index ["album_id"], name: "index_songs_on_album_id"
     t.index ["artist_id"], name: "index_songs_on_artist_id"
+    t.index ["file_path_hash"], name: "index_songs_on_file_path_hash"
+    t.index ["md5_hash"], name: "index_songs_on_md5_hash"
     t.index ["name"], name: "index_songs_on_name"
   end
 

docker-compose.yml

@@ -42,6 +42,10 @@ services:
     ports:
       - 80:80
     command: nginx -g 'pid /tmp/nginx.pid; daemon off;'
+  listener:
+    container_name: 'blackcandy_listener'
+    <<: *app_base
+    command: bundle exec rails listen_media_changes
 volumes:
   production_db_data:
   production_redis_data:

lib/tasks/listen_media_change.rake

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+task listen_media_changes: :environment do
+  supported_formates = MediaFile::SUPPORTED_FORMATS.map { |formate| %r{\.#{formate}$} }
+
+  listener = Listen.to(File.expand_path(Setting.media_path), only: supported_formates) do |modified, added, removed|
+    MediaSyncJob.perform_later(:modified, modified) if modified.present?
+    MediaSyncJob.perform_later(:added, added) if added.present?
+    MediaSyncJob.perform_later(:removed, removed) if removed.present?
+  end
+
+  listener.start
+
+  sleep
+end

test/controllers/cached_transcoded_stream_controller_test.rb

@@ -17,7 +17,7 @@ class CachedTranscodedStreamControllerTest < ActionDispatch::IntegrationTest
 
   test "should set header for nginx send file" do
     get new_cached_transcoded_stream_url(song_id: songs(:flac_sample).id)
-    assert_equal "/private_cache_media/2/ZmFrZV9tZDU=_128.mp3", @response.get_header("X-Accel-Redirect")
+    assert_equal "/private_cache_media/2/ZmxhY19zYW1wbGVfbWQ1X2hhc2g=_128.mp3", @response.get_header("X-Accel-Redirect")
   end
 
   test "should set correct content type header" do

test/fixtures/songs.yml

@@ -2,7 +2,8 @@ mp3_sample:
   id: 1
   name: 'mp3_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist1_album2.mp3') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'mp3_sample_file_path_hash'
+  md5_hash: 'mp3_sample_md5_hash'
   artist: 'artist1'
   album: 'album2'
   duration: 8.0
@@ -11,7 +12,8 @@ flac_sample:
   id: 2
   name: 'flac_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist1_album1.flac') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'flac_sample_file_path_hash'
+  md5_hash: 'flac_sample_md5_hash'
   artist: 'artist1'
   album: 'album1'
   duration: 8.0
@@ -20,7 +22,8 @@ ogg_sample:
   id: 3
   name: 'ogg_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist2_album3.ogg') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'ogg_sample_file_path_hash'
+  md5_hash: 'ogg_sample_md5_hash'
   artist: 'artist2'
   album: 'album3'
   duration: 8.0
@@ -29,7 +32,8 @@ wav_sample:
   id: 4
   name: 'wav_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist2_album3.wav') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'wav_sample_file_path_hash'
+  md5_hash: 'wav_sample_md5_hash'
   artist: 'artist2'
   album: 'album3'
   duration: 8.0
@@ -38,7 +42,8 @@ opus_sample:
   id: 5
   name: 'opus_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist2_album3.opus') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'opus_sample_file_path_hash'
+  md5_hash: 'opus_sample_md5_hash'
   artist: 'artist2'
   album: 'album3'
   duration: 8.0
@@ -47,7 +52,8 @@ m4a_sample:
   id: 6
   name: 'm4a_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist1_album1.m4a') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'm4a_sample_file_path_hash'
+  md5_hash: 'm4a_sample_md5_hash'
   artist: 'artist1'
   album: 'album1'
   duration: 8.0
@@ -56,7 +62,8 @@ oga_sample:
   id: 7
   name: 'oga_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist2_album3.oga') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'oga_sample_file_path_hash'
+  md5_hash: 'oga_sample_md5_hash'
   artist: 'artist2'
   album: 'album3'
   duration: 8.0
@@ -65,7 +72,8 @@ wma_sample:
   id: 8
   name: 'wma_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'artist2_album3.wma') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'wma_sample_file_path_hash'
+  md5_hash: 'wma_sample_md5_hash'
   artist: 'artist2'
   album: 'album3'
   duration: 8.0
@@ -74,7 +82,8 @@ various_artists_sample:
   id: 9
   name: 'various_artists_sample'
   file_path: <%= Rails.root.join('test', 'fixtures', 'files', 'various_artists.mp3') %>
-  md5_hash: 'fake_md5'
+  file_path_hash: 'various_artists_sample_file_path_hash'
+  md5_hash: 'various_artists_sample_md5_hash'
   artist: 'artist1'
   album: 'album4'
   duration: 8.0

test/jobs/media_sync_job_test.rb

@@ -9,7 +9,7 @@ class MediaSyncJobTest < ActiveJob::TestCase
 
   test "sync media" do
     mock = MiniTest::Mock.new
-    mock.expect(:call, true)
+    mock.expect(:call, true, [:all, []])
 
     Setting.update(media_path: Rails.root.join("test/fixtures/files"))
     assert_not Media.syncing?

test/models/album_test.rb

@@ -18,9 +18,9 @@ class AlbumTest < ActiveSupport::TestCase
 
     album.songs.create!(
       [
-        {name: "test_song_1", file_path: "fake_path", md5_hash: "fake_md5", tracknum: 2, artist: artist},
-        {name: "test_song_2", file_path: "fake_path", md5_hash: "fake_md5", tracknum: 3, artist: artist},
-        {name: "test_song_3", file_path: "fake_path", md5_hash: "fake_md5", tracknum: 1, artist: artist}
+        {name: "test_song_1", file_path: "fake_path", file_path_hash: "fake_path_hash", md5_hash: "fake_md5", tracknum: 2, artist: artist},
+        {name: "test_song_2", file_path: "fake_path", file_path_hash: "fake_path_hash", md5_hash: "fake_md5", tracknum: 3, artist: artist},
+        {name: "test_song_3", file_path: "fake_path", file_path_hash: "fake_path_hash", md5_hash: "fake_md5", tracknum: 1, artist: artist}
       ]
     )