Merge pull request #2622 from ronie/script.cu.lrclyrics-6.6.6

[script.cu.lrclyrics] 6.6.6

script.cu.lrclyrics/addon.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<addon id="script.cu.lrclyrics" name="CU LRC Lyrics" version="6.6.4" provider-name="Taxigps, ronie">
+<addon id="script.cu.lrclyrics" name="CU LRC Lyrics" version="6.6.6" provider-name="Taxigps, ronie">
 	<requires>
 		<import addon="xbmc.python" version="3.0.0"/>
 		<import addon="script.module.beautifulsoup4" version="4.8.2+matrix.1"/>

script.cu.lrclyrics/changelog.txt

@@ -1,3 +1,9 @@
+v6.6.6
+- fixed several scrapers
+
+v6.6.5
+- added "%A - %B - %N - %T" file format
+
 v6.6.4
 - added "%N. %A - %T" file format
 

script.cu.lrclyrics/lib/culrcscrapers/lyricscom/lyricsScraper.py

@@ -10,6 +10,7 @@
 __priority__ = '240'
 __lrc__ = False
 
+UserAgent = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"}
 
 class LyricsFetcher:
     def __init__(self, *args, **kwargs):
@@ -25,7 +26,7 @@ def get_lyrics(self, song):
         lyrics.source = __title__
         lyrics.lrc = __lrc__
         try:
-            request = sess.get(self.url % urllib.parse.quote_plus(song.artist), timeout=10)
+            request = sess.get(self.url % urllib.parse.quote_plus(song.artist), headers=UserAgent, timeout=10)
             response = request.text
         except:
             return
@@ -37,7 +38,7 @@ def get_lyrics(self, song):
                 break
         if url:
             try:
-                req = sess.get(url, timeout=10)
+                req = sess.get(url, headers=UserAgent, timeout=10)
                 resp = req.text
             except:
                 return
@@ -49,7 +50,7 @@ def get_lyrics(self, song):
                     break
             if url:
                 try:
-                    req2 = sess.get(url, timeout=10)
+                    req2 = sess.get(url, headers=UserAgent, timeout=10)
                     resp2 = req2.text
                 except:
                     return

script.cu.lrclyrics/lib/culrcscrapers/lyricsify/lyricsScraper.py

@@ -13,8 +13,9 @@
 __priority__ = '130'
 __lrc__ = True
 
-UserAgent = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"}
+UserAgent = {"Host": "www.lyricsify.com", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br, zstd", "DNT": "1", "Alt-Used": "www.lyricsify.com", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Priority": "u=1"}
 
+# lyricsify uses captcha's & cloudflare protection for the search option, only direct lyrics access works
 
 class LyricsFetcher:
     def __init__(self, *args, **kwargs):
@@ -23,6 +24,30 @@ def __init__(self, *args, **kwargs):
         self.SEARCH_URL = 'https://www.lyricsify.com/lyrics/%s/%s'
         self.LYRIC_URL = 'https://www.lyricsify.com%s'
 
+    def get_lyrics(self, song):
+        log("%s: searching lyrics for %s - %s" % (__title__, song.artist, song.title), debug=self.DEBUG)
+        lyrics = Lyrics(settings=self.settings)
+        lyrics.song = song
+        lyrics.source = __title__
+        lyrics.lrc = __lrc__
+        artist = song.artist.replace("'", '').replace('!', '').replace('?', '').replace('"', '').replace('/', '').replace('.', '').replace('&', '').replace(',', '').replace('(', '').replace(')', '').replace(' ', '-')
+        title = song.title.replace("'", '').replace('!', '').replace('?', '').replace('"', '').replace('/', '').replace('.', '').replace('&', '').replace(',', '').replace('(', '').replace(')', '').replace(' ', '-')
+        url = self.SEARCH_URL % (artist.lower(), title.lower())
+        try:
+            log('%s: search url: %s' % (__title__, url), debug=self.DEBUG)
+            search = requests.get(url, headers=UserAgent, timeout=10)
+            response = search.text
+        except:
+            return None
+        matchcode = re.search('details">(.*?)</div', response, flags=re.DOTALL)
+        if matchcode:
+            lyricscode = (matchcode.group(1))
+            lyr = re.sub('<[^<]+?>', '', lyricscode)
+            lyrics.lyrics = lyr
+            return lyrics
+        return None
+
+'''
     def get_lyrics(self, song):
         log("%s: searching lyrics for %s - %s" % (__title__, song.artist, song.title), debug=self.DEBUG)
         lyrics = Lyrics(settings=self.settings)
@@ -73,3 +98,4 @@ def get_lyrics_from_list(self, link):
             lyricscode = (matchcode.group(1))
             cleanlyrics = re.sub('<[^<]+?>', '', lyricscode)
             return cleanlyrics
+'''

script.cu.lrclyrics/lib/culrcscrapers/megalobiz/lyricsScraper.py

@@ -13,7 +13,7 @@
 from lib.utils import *
 
 __title__ = "Megalobiz"
-__priority__ = '140'
+__priority__ = '150'
 __lrc__ = True
 
 

script.cu.lrclyrics/lib/culrcscrapers/musixmatch/lyricsScraper.py

@@ -10,6 +10,7 @@
 import re
 import random
 import difflib
+import html
 from bs4 import BeautifulSoup
 from lib.utils import *
 
@@ -20,14 +21,40 @@
 headers = {}
 headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0'
 
+# search is not possible as it requires javascript, only direct access to the lyrics work.
 
 class LyricsFetcher:
     def __init__(self, *args, **kwargs):
         self.DEBUG = kwargs['debug']
         self.settings = kwargs['settings']
-        self.SEARCH_URL = 'https://www.musixmatch.com/search/'
-        self.LYRIC_URL = 'https://www.musixmatch.com'
+        self.SEARCH_URL = 'https://www.musixmatch.com/search?query='
+        self.LYRIC_URL = 'https://www.musixmatch.com/lyrics/%s/%s'
 
+    def get_lyrics(self, song):
+        log("%s: searching lyrics for %s - %s" % (__title__, song.artist, song.title), debug=self.DEBUG)
+        lyrics = Lyrics(settings=self.settings)
+        lyrics.song = song
+        lyrics.source = __title__
+        lyrics.lrc = __lrc__
+        artist = song.artist.replace("'", '').replace('!', '').replace('?', '').replace('"', '').replace('/', '').replace('.', '').replace('&', '').replace(',', '').replace('(', '').replace(')', '').replace(' ', '-')
+        title = song.title.replace("'", '').replace('!', '').replace('?', '').replace('"', '').replace('/', '').replace('.', '').replace('&', '').replace(',', '').replace('(', '').replace(')', '').replace(' ', '-')
+        url = self.LYRIC_URL % (artist, title)
+        try:
+            log('%s: search url: %s' % (__title__, url), debug=self.DEBUG)
+            search = requests.get(url, headers=headers, timeout=10)
+            response = search.text
+        except:
+            return None
+        matchcode = re.search('Lyrics of (.*?)Writer\(s\): ', response, flags=re.DOTALL)
+        if matchcode:
+            lyricscode = (matchcode.group(1))
+            lyr = re.sub('<[^<]+?>', '\n', lyricscode)
+            lyr = html.unescape(lyr)
+            lyrics.lyrics = lyr.replace('\n\n\n\n', '\n')
+            return lyrics
+        return None
+
+'''
     def get_lyrics(self, song):
         log("%s: searching lyrics for %s - %s" % (__title__, song.artist, song.title), debug=self.DEBUG)
         lyrics = Lyrics(settings=self.settings)
@@ -84,3 +111,4 @@ def get_lyrics_from_list(self, link):
                 for part in lyr:
                     lyrics = lyrics + part.get_text() + '\n'
                 return lyrics
+'''

script.cu.lrclyrics/lib/culrcscrapers/musixmatchlrc/lyricsScraper.py

@@ -78,7 +78,7 @@ def get_lyrics(self, song):
         search = '%s - %s' % (artist, title)
         try:
             url = self.SEARCH_URL % 'track.search'
-            query = [('q', search), ('page_size', '5'), ('page', '1'), ('s_track_rating', 'desc'), ('quorum_factor', '1.0'), ('app_id', 'web-desktop-app-v1.0'), ('usertoken', self.token), ('t', self.current_time)]
+            query = [('q', search), ('page_size', '5'), ('page', '1'), ('app_id', 'web-desktop-app-v1.0'), ('usertoken', self.token), ('t', self.current_time)]
             response = requests.get(url, params=query, timeout=10)
             result = response.json()
         except:

script.cu.lrclyrics/lib/culrcscrapers/rclyricsband/lyricsScraper.py

@@ -0,0 +1,71 @@
+#-*- coding: UTF-8 -*-
+'''
+Scraper for https://www.rclyricsband.com/
+'''
+
+import requests
+import re
+import difflib
+from bs4 import BeautifulSoup
+from lib.utils import *
+
+__title__ = "RCLyricsBand"
+__priority__ = '140'
+__lrc__ = True
+
+UserAgent = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"}
+
+class LyricsFetcher:
+    def __init__(self, *args, **kwargs):
+        self.DEBUG = kwargs['debug']
+        self.settings = kwargs['settings']
+        self.SEARCH_URL = 'https://rclyricsband.com/?search=%s %s'
+        self.LYRIC_URL = 'https://rclyricsband.com/%s'
+
+
+    def get_lyrics(self, song):
+        log("%s: searching lyrics for %s - %s" % (__title__, song.artist, song.title), debug=self.DEBUG)
+        lyrics = Lyrics(settings=self.settings)
+        lyrics.song = song
+        lyrics.source = __title__
+        lyrics.lrc = __lrc__
+        artist = song.artist
+        title = song.title
+        try:
+            url = self.SEARCH_URL % (artist, title)
+            search = requests.get(url, headers=UserAgent, timeout=10)
+            response = search.text
+        except:
+            return None
+        links = []
+        soup = BeautifulSoup(response, 'html.parser')
+        for link in soup.find_all('a', {'class': 'song_search'}):
+            if link.string:
+                foundsong = link.string.split(' - ')[0]
+                foundartist = link.string.split(' - ')[-1]
+                if (difflib.SequenceMatcher(None, artist.lower(), foundartist.lower()).ratio() > 0.8) and (difflib.SequenceMatcher(None, title.lower(), foundsong.lower()).ratio() > 0.8):
+                    links.append((foundartist + ' - ' + foundsong, self.LYRIC_URL % link.get('href'), foundartist, foundsong))
+        if len(links) == 0:
+            return None
+        elif len(links) > 1:
+            lyrics.list = links
+        for link in links:
+            lyr = self.get_lyrics_from_list(link)
+            if lyr:
+                lyrics.lyrics = lyr
+                return lyrics
+        return None
+
+    def get_lyrics_from_list(self, link):
+        title,url,artist,song = link
+        try:
+            log('%s: search url: %s' % (__title__, url), debug=self.DEBUG)
+            search = requests.get(url, headers=UserAgent, timeout=10)
+            response = search.text
+        except:
+            return None
+        matchcode = re.search("lrc_text_format'>(.*?)</p", response, flags=re.DOTALL)
+        if matchcode:
+            lyricscode = (matchcode.group(1))
+            cleanlyrics = re.sub('<br>', '', lyricscode)
+            return cleanlyrics

script.cu.lrclyrics/lib/scrapertest.py

@@ -12,6 +12,7 @@
 from lib.culrcscrapers.music163 import lyricsScraper as lyricsScraper_music163
 from lib.culrcscrapers.musixmatch import lyricsScraper as lyricsScraper_musixmatch
 from lib.culrcscrapers.musixmatchlrc import lyricsScraper as lyricsScraper_musixmatchlrc
+from lib.culrcscrapers.rclyricsband import lyricsScraper as lyricsScraper_rclyricsband
 from lib.culrcscrapers.supermusic import lyricsScraper as lyricsScraper_supermusic
 
 FAILED = []
@@ -47,7 +48,7 @@ def test_scrapers():
         return
 
     # test darklyrics
-    dialog.update(8, LANGUAGE(32163) % 'darklyrics')
+    dialog.update(7, LANGUAGE(32163) % 'darklyrics')
     log('==================== darklyrics ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Neurosis'
@@ -66,7 +67,7 @@ def test_scrapers():
         return
 
     # test genius
-    dialog.update(16, LANGUAGE(32163) % 'genius')
+    dialog.update(14, LANGUAGE(32163) % 'genius')
     log('==================== genius ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Maren Morris'
@@ -85,7 +86,7 @@ def test_scrapers():
         return
 
     # test lrclib
-    dialog.update(24, LANGUAGE(32163) % 'lrclib')
+    dialog.update(21, LANGUAGE(32163) % 'lrclib')
     log('==================== lrclib ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'CHVRCHES'
@@ -104,7 +105,7 @@ def test_scrapers():
         return
 
     # test lyricscom
-    dialog.update(32, LANGUAGE(32163) % 'lyricscom')
+    dialog.update(28, LANGUAGE(32163) % 'lyricscom')
     log('==================== lyricscom ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Blur'
@@ -123,11 +124,11 @@ def test_scrapers():
         return
 
     # test lyricsify
-    dialog.update(40, LANGUAGE(32163) % 'lyricsify')
+    dialog.update(35, LANGUAGE(32163) % 'lyricsify')
     log('==================== lyricsify ====================', debug=True)
     song = Song(opt=lyricssettings)
-    song.artist = 'Madonna'
-    song.title = 'Crazy For You'
+    song.artist = 'Tears For Fears'
+    song.title = 'Shout'
     st = time.time()
     lyrics = lyricsScraper_lyricsify.LyricsFetcher(settings=lyricssettings, debug=True).get_lyrics(song)
     ft = time.time()
@@ -142,7 +143,7 @@ def test_scrapers():
         return
 
     # test lyricsmode
-    dialog.update(48, LANGUAGE(32163) % 'lyricsmode')
+    dialog.update(42, LANGUAGE(32163) % 'lyricsmode')
     log('==================== lyricsmode ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Maren Morris'
@@ -161,7 +162,7 @@ def test_scrapers():
         return
 
     # test megalobiz
-    dialog.update(56, LANGUAGE(32163) % 'megalobiz')
+    dialog.update(50, LANGUAGE(32163) % 'megalobiz')
     log('==================== megalobiz ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Michael Jackson'
@@ -180,7 +181,7 @@ def test_scrapers():
         return
 
     # test music163
-    dialog.update(64, LANGUAGE(32163) % 'music163')
+    dialog.update(58, LANGUAGE(32163) % 'music163')
     log('==================== music163 ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Madonna'
@@ -199,7 +200,7 @@ def test_scrapers():
         return
 
     # test musixmatch
-    dialog.update(72, LANGUAGE(32163) % 'musixmatch')
+    dialog.update(66, LANGUAGE(32163) % 'musixmatch')
     log('==================== musixmatch ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Kate Bush'
@@ -218,7 +219,7 @@ def test_scrapers():
         return
 
     # test musixmatchlrc
-    dialog.update(80, LANGUAGE(32163) % 'musixmatchlrc')
+    dialog.update(73, LANGUAGE(32163) % 'musixmatchlrc')
     log('==================== musixmatchlrc ====================', debug=True)
     song = Song(opt=lyricssettings)
     song.artist = 'Kate Bush'
@@ -236,6 +237,25 @@ def test_scrapers():
     if dialog.iscanceled():
         return
 
+    # test rclyricsband
+    dialog.update(80, LANGUAGE(32163) % 'rclyricsband')
+    log('==================== rclyricsband ====================', debug=True)
+    song = Song(opt=lyricssettings)
+    song.artist = 'Taylor Swift'
+    song.title = 'The Archer'
+    st = time.time()
+    lyrics = lyricsScraper_rclyricsband.LyricsFetcher(settings=lyricssettings, debug=True).get_lyrics(song)
+    ft = time.time()
+    tt = ft - st
+    TIMINGS.append(['rclyricsband',tt])
+    if lyrics:
+        log(lyrics.lyrics, debug=True)
+    else:
+        FAILED.append('rclyricsband')
+        log('FAILED: rclyricsband', debug=True)
+    if dialog.iscanceled():
+        return
+
     # test supermusic
     dialog.update(88, LANGUAGE(32163) % 'supermusic')
     log('==================== supermusic ====================', debug=True)

script.cu.lrclyrics/lib/utils.py

@@ -76,6 +76,9 @@ def get_artist_from_filename(*args, **kwargs):
         elif SETTING_READ_FILENAME_FORMAT == 4:
             artist = basename.split('-', 2)[1].strip()
             title = os.path.splitext(basename.split('-', 2)[2].strip())[0]
+        elif SETTING_READ_FILENAME_FORMAT == 6:
+            artist = basename.split('-', 1)[0].strip()
+            title = os.path.splitext(basename.split('-', 3)[3].strip())[0]
     except:
         # invalid format selected
         log('failed to get artist and title from filename', debug=DEBUG)

script.cu.lrclyrics/resources/language/resource.language.af_za/strings.po

@@ -147,6 +147,14 @@ msgctxt "#32119"
 msgid "Search for and save txt lyrics in song folder"
 msgstr ""
 
+msgctxt "#32120"
+msgid "%N. %A - %T"
+msgstr ""
+
+msgctxt "#32121"
+msgid "%A - %B - %N - %T"
+msgstr ""
+
 # empty strings from id 32120 to 32146
 msgctxt "#32147"
 msgid "Search for local .txt lyrics files"