From 87311e76e7b6a4e7a9b5b9e709c1db000c730613 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 17:17:54 -0400 Subject: [PATCH 1/8] Add firebase-admin to requirements --- backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index ecc4f93..cf27c7c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,6 @@ b2sdk==1.14.0 ffmpeg==1.4 +firebase-admin==5.2.0 Flask==2.0.3 Flask-Cors==3.0.10 Flask-SQLAlchemy==2.5.1 From 6332bea4d16fe8fd290ed2fb7ed5a5a9b5d64646 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 17:28:08 -0400 Subject: [PATCH 2/8] Update channels_bp to use firestore --- backend/blueprints/channels.py | 50 ++------------------- backend/functions/databaseCalls/firebase.py | 50 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 backend/functions/databaseCalls/firebase.py diff --git a/backend/blueprints/channels.py b/backend/blueprints/channels.py index 8b9922c..9b2c26d 100644 --- a/backend/blueprints/channels.py +++ b/backend/blueprints/channels.py @@ -1,59 +1,17 @@ -import requests -import os +from flask import Blueprint, request -from flask import Blueprint, Flask, jsonify, request, redirect - -from functions.utils import getCurrentTime -from functions.databaseCalls import channels +from functions.databaseCalls import firebase from functions.downloader import download -from classes.shared import db -from classes import Channels - channels_bp = Blueprint('channels', __name__, url_prefix='/channels') @channels_bp.route('/', methods=['GET']) def root(): - channelList = channels.getAllChannels() - channelListArray = [] - columns = channels.getColumns() - for channel in channelList: - channelListQuery = channels.getChannelById(channel.id) - for result in channelListQuery: - channelListArray.append(dict(zip(columns, result))) - - return(jsonify(channelListArray)) + return(firebase.getAllChannels()) @channels_bp.route('/add', methods=['GET']) def add(): url = request.args['url'] download_results = download(video=url, video_range=1, download_confirm=False) result_latest_upload = download_results['entries'][0]['original_url'] - channel_exists = channels.getChannelByName(download_results['channel']) - - if channel_exists: - print(f'The {download_results["channel"]} is already in the database.') - elif not channel_exists: - requests.get(os.environ.get("API_URL") + '/download/search?url=' + result_latest_upload) - - db_entry = Channels.Channels( - channel_name = download_results['channel'], - channel_follower_count = download_results['channel_follower_count'], - channel_id = download_results['channel_id'], - description = download_results['description'], - original_url = download_results['original_url'], - uploader = download_results['uploader'], - uploader_id = download_results['uploader_id'], - webpage_url = download_results['webpage_url'], - picture_profile = download_results['thumbnails'][18]['url'], - picture_cover = download_results['thumbnails'][15]['url'], - last_updated = getCurrentTime(), - latest_upload = download_results['entries'][0]['original_url'] - ) - - db.session.add(db_entry) - db.session.commit() - else: - print(f'There was another error.') - - return(jsonify(download_results)) \ No newline at end of file + return(firebase.addChannel(download_results)) \ No newline at end of file diff --git a/backend/functions/databaseCalls/firebase.py b/backend/functions/databaseCalls/firebase.py new file mode 100644 index 0000000..765a477 --- /dev/null +++ b/backend/functions/databaseCalls/firebase.py @@ -0,0 +1,50 @@ +import firebase_admin +from firebase_admin import credentials +from firebase_admin import firestore +from flask import jsonify + +from functions.utils import getCurrentTime + +cred = credentials.Certificate("serviceAccountKey.json") +firebase_admin.initialize_app(cred) + +db = firestore.client() + +def addChannel(information): + channels_ref = db.collection(u'channels').document(information['channel_id']) + channels_ref.set({ + 'channel_id' : information['channel_id'], + 'channel_name' : information['channel'], + 'channel_follower_count' : information['channel_follower_count'], + 'description' : information['description'], + 'original_url' : information['original_url'], + 'uploader' : information['uploader'], + 'uploader_id' : information['uploader_id'], + 'webpage_url' : information['webpage_url'], + 'picture_profile' : information['thumbnails'][18]['url'], + 'picture_cover' : information['thumbnails'][15]['url'], + 'last_updated' : getCurrentTime(), + 'latest_upload' : information['entries'][0]['original_url'] + }) + return(jsonify(information)) + +def getChannel(information): + channels_ref = db.collection(u'channels').document(information) + + doc = channels_ref.get() + if doc.exists: + print(f'Document data: {doc.to_dict()}') + return(jsonify(doc.to_dict())) + else: + print(u'No such document!') + return('No such document!') + +def getAllChannels(): + channels_ref = db.collection(u'channels') + query = channels_ref.order_by('channel_name') + ordered_channels = query.stream() + result = [] + for channel in ordered_channels: + result.append(channel.to_dict()) + + return(jsonify(result)) \ No newline at end of file From ceda7103be7a1448e0a376fb197c3f6b17ea24e2 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 17:28:36 -0400 Subject: [PATCH 3/8] Add serviceAccountKey.json to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b8db43f..289e7ce 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ backend/youtube.sqlite3 backend/blueprints/__pycache__/* backend/classes/__pycache__/* backend/functions/databaseCalls/__pycache__/* +backend/serviceAccountKey.json \ No newline at end of file From 4f05ebe4b97d3d429f0168ac789532ddb4ccd2a0 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 17:57:21 -0400 Subject: [PATCH 4/8] Move video functions to firebase --- backend/blueprints/download.py | 31 +------------- backend/blueprints/search.py | 12 ++---- backend/blueprints/videos.py | 11 +---- backend/functions/databaseCalls/firebase.py | 46 ++++++++++++++++++++- frontend/pages/videos.js | 2 +- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/backend/blueprints/download.py b/backend/blueprints/download.py index ef33150..a92ac38 100644 --- a/backend/blueprints/download.py +++ b/backend/blueprints/download.py @@ -5,6 +5,7 @@ from functions.databaseCalls import videos from functions.databaseCalls import channels +from functions.databaseCalls import firebase from functions.downloader import download from functions.utils import getCurrentTime @@ -17,35 +18,7 @@ def root(): url = request.args['url'] result_download_url = download(video=url, video_range=1, download_confirm=False) - query_video = videos.getVideoByYouTubeId(result_download_url['id']) - - if query_video: - print('The video already in database.') - elif not query_video: - db_entry = Videos.Videos( - channel = result_download_url['channel'], - channel_id = result_download_url['channel_id'], - description = result_download_url['description'], - duration = result_download_url['duration'], - duration_string = result_download_url['duration_string'], - fulltitle = result_download_url['fulltitle'], - video_id = result_download_url['id'], - like_count = result_download_url['like_count'], - view_count = result_download_url['view_count'], - original_url = result_download_url['original_url'], - thumbnail = result_download_url['thumbnail'], - title = result_download_url['title'], - upload_date = result_download_url['upload_date'], - webpage_url = result_download_url['webpage_url'], - downloaded = False, - downloaded_date = getCurrentTime() - ) - db.session.add(db_entry) - db.session.commit() - else: - print('There was another error.') - return(jsonify(result_download_url)) - + return(firebase.addVideo(result_download_url)) @download_bp.route('/latest', methods=['GET']) def latest(): diff --git a/backend/blueprints/search.py b/backend/blueprints/search.py index 53fc369..e8223cb 100644 --- a/backend/blueprints/search.py +++ b/backend/blueprints/search.py @@ -1,14 +1,10 @@ from flask import Blueprint, Flask, jsonify, request, redirect +from functions.databaseCalls import firebase from functions.databaseCalls import videos from functions.databaseCalls import channels search_bp = Blueprint('search', __name__, url_prefix='/search') -@search_bp.route('/videos/channel/', methods=['GET']) -def root(channelName): - videoList = videos.getVideoByChannelName(channelName) - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListArray.append(dict(zip(columns, video))) - return(jsonify(videoListArray)) \ No newline at end of file +@search_bp.route('/videos/channel/', methods=['GET']) +def root(channel_id): + return(jsonify(firebase.getAllVideosByChannel(channel_id))) \ No newline at end of file diff --git a/backend/blueprints/videos.py b/backend/blueprints/videos.py index ceb0451..75a1a02 100644 --- a/backend/blueprints/videos.py +++ b/backend/blueprints/videos.py @@ -1,4 +1,5 @@ from flask import Blueprint, Flask, jsonify, request, redirect +from functions.databaseCalls import firebase from functions.databaseCalls import videos from functions.databaseCalls import channels @@ -6,15 +7,7 @@ @videos_bp.route('/', methods=['GET']) def root(): - videoList = videos.getAllVideos() - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListQuery = videos.getVideoById(video.id) - for result in videoListQuery: - videoListArray.append(dict(zip(columns, result))) - - return(jsonify(videoListArray)) + return(firebase.getAllVideos()) @videos_bp.route('/unique/channel-name', methods=['GET']) def channelName(): diff --git a/backend/functions/databaseCalls/firebase.py b/backend/functions/databaseCalls/firebase.py index 765a477..961a346 100644 --- a/backend/functions/databaseCalls/firebase.py +++ b/backend/functions/databaseCalls/firebase.py @@ -2,6 +2,7 @@ from firebase_admin import credentials from firebase_admin import firestore from flask import jsonify +import json from functions.utils import getCurrentTime @@ -47,4 +48,47 @@ def getAllChannels(): for channel in ordered_channels: result.append(channel.to_dict()) - return(jsonify(result)) \ No newline at end of file + return(jsonify(result)) + +def addVideo(information): + channel_ref = db.collection(u'channels').document(information['channel_id']) + video_ref = channel_ref.collection('videos').document(information['id']) + video_ref.set({ + 'channel_id' : information['channel_id'], + 'channel' : information['channel'], + 'description' : information['description'], + 'duration' : information['duration'], + 'duration_string' : information['duration_string'], + 'fulltitle' : information['fulltitle'], + 'video_id' : information['id'], + 'like_count' : information['like_count'], + 'view_count' : information['view_count'], + 'original_url' : information['original_url'], + 'thumbnail' : information['thumbnail'], + 'title' : information['title'], + 'upload_date' : information['upload_date'], + 'webpage_url' : information['webpage_url'], + 'downloaded' : False, + 'downloaded_date' : getCurrentTime() + }) + return(jsonify(information)) + +def getAllVideosByChannel(channel_id): + channels_ref = db.collection(u'channels').document(channel_id) + video_ref = channels_ref.collection('videos') + query = video_ref.order_by('upload_date') + ordered_videos = query.stream() + all_videos = [] + for video in ordered_videos: + all_videos.append(video.to_dict()) + return(all_videos) + +def getAllVideos(): + channel_ref = getAllChannels() + channel_json = json.loads(channel_ref.data) + all_videos = [] + for channel in channel_json: + query = getAllVideosByChannel(channel['channel_id']) + for q in query: + all_videos.append(q) + return(jsonify(all_videos)) \ No newline at end of file diff --git a/frontend/pages/videos.js b/frontend/pages/videos.js index d598b53..34e115e 100644 --- a/frontend/pages/videos.js +++ b/frontend/pages/videos.js @@ -29,7 +29,7 @@ export default function videos({ results, result_all_channels }) { const handleDropdownClick = async (event) => { setDropdownName(channel) - const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channel) + const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channelID) const new_results = await request_channel_videos.json() setNewResults(new_results) } From 40aae65514689d156c11ff4ffb26f357b6adbed7 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 18:00:45 -0400 Subject: [PATCH 5/8] Update filter-dropdown to use firebase --- backend/blueprints/videos.py | 11 +---------- frontend/pages/videos.js | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/backend/blueprints/videos.py b/backend/blueprints/videos.py index 75a1a02..b142374 100644 --- a/backend/blueprints/videos.py +++ b/backend/blueprints/videos.py @@ -11,16 +11,7 @@ def root(): @videos_bp.route('/unique/channel-name', methods=['GET']) def channelName(): - videoList = videos.getAllVideos(distinct=True) - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListArray.append({ - 'channel' : video.channel, - 'channel_id' : video.channel_id - }) - - return(jsonify(videoListArray)) + return(firebase.getAllChannels()) @videos_bp.route('/sync-channels', methods=['GET']) def syncChannels(): diff --git a/frontend/pages/videos.js b/frontend/pages/videos.js index 34e115e..b8844ed 100644 --- a/frontend/pages/videos.js +++ b/frontend/pages/videos.js @@ -95,7 +95,7 @@ export default function videos({ results, result_all_channels }) { } } > - {channel.channel} + {channel.channel_name} {channel.channel_id} From 1042518f24d2c39fca94688f6c85c19cf1fbb606 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 19:21:38 -0400 Subject: [PATCH 6/8] Add firebase.getMissingVideos() --- backend/blueprints/videos.py | 14 +-- backend/functions/databaseCalls/firebase.py | 105 ++++++++++++-------- 2 files changed, 67 insertions(+), 52 deletions(-) diff --git a/backend/blueprints/videos.py b/backend/blueprints/videos.py index b142374..bd89fab 100644 --- a/backend/blueprints/videos.py +++ b/backend/blueprints/videos.py @@ -15,16 +15,4 @@ def channelName(): @videos_bp.route('/sync-channels', methods=['GET']) def syncChannels(): - videoChannelList = videos.getAllVideos(distinct=True) - columns = videos.getColumns() - - missing_channels = [] - for channel in videoChannelList: - channel_name = channel[1] - query = channels.getChannelByName(channel_name) - if not query: - query2 = videos.getVideoByChannelName(channel_name) - for q in query2: - missing_channels.append(dict(zip(columns, q))) - - return(jsonify(missing_channels)) \ No newline at end of file + return(firebase.getMissingVideos()) \ No newline at end of file diff --git a/backend/functions/databaseCalls/firebase.py b/backend/functions/databaseCalls/firebase.py index 961a346..41bd7ce 100644 --- a/backend/functions/databaseCalls/firebase.py +++ b/backend/functions/databaseCalls/firebase.py @@ -11,22 +11,31 @@ db = firestore.client() -def addChannel(information): +def addChannel(information, empty=False): channels_ref = db.collection(u'channels').document(information['channel_id']) - channels_ref.set({ - 'channel_id' : information['channel_id'], - 'channel_name' : information['channel'], - 'channel_follower_count' : information['channel_follower_count'], - 'description' : information['description'], - 'original_url' : information['original_url'], - 'uploader' : information['uploader'], - 'uploader_id' : information['uploader_id'], - 'webpage_url' : information['webpage_url'], - 'picture_profile' : information['thumbnails'][18]['url'], - 'picture_cover' : information['thumbnails'][15]['url'], - 'last_updated' : getCurrentTime(), - 'latest_upload' : information['entries'][0]['original_url'] - }) + if empty == True: + print('Setting empty') + channels_ref.set({ + 'empty' : True + }) + return(information) + elif empty == False: + channels_ref.set({ + 'channel_id' : information['channel_id'], + 'channel_name' : information['channel'], + 'channel_follower_count' : information['channel_follower_count'], + 'description' : information['description'], + 'original_url' : information['original_url'], + 'uploader' : information['uploader'], + 'uploader_id' : information['uploader_id'], + 'webpage_url' : information['webpage_url'], + 'picture_profile' : information['thumbnails'][18]['url'], + 'picture_cover' : information['thumbnails'][15]['url'], + 'last_updated' : getCurrentTime(), + 'latest_upload' : information['entries'][0]['original_url'] + }) + else: + return(f'There was an error adding {information}.') return(jsonify(information)) def getChannel(information): @@ -37,8 +46,7 @@ def getChannel(information): print(f'Document data: {doc.to_dict()}') return(jsonify(doc.to_dict())) else: - print(u'No such document!') - return('No such document!') + return False def getAllChannels(): channels_ref = db.collection(u'channels') @@ -51,32 +59,41 @@ def getAllChannels(): return(jsonify(result)) def addVideo(information): - channel_ref = db.collection(u'channels').document(information['channel_id']) - video_ref = channel_ref.collection('videos').document(information['id']) - video_ref.set({ - 'channel_id' : information['channel_id'], - 'channel' : information['channel'], - 'description' : information['description'], - 'duration' : information['duration'], - 'duration_string' : information['duration_string'], - 'fulltitle' : information['fulltitle'], - 'video_id' : information['id'], - 'like_count' : information['like_count'], - 'view_count' : information['view_count'], - 'original_url' : information['original_url'], - 'thumbnail' : information['thumbnail'], - 'title' : information['title'], - 'upload_date' : information['upload_date'], - 'webpage_url' : information['webpage_url'], - 'downloaded' : False, - 'downloaded_date' : getCurrentTime() - }) - return(jsonify(information)) + channel_exists = getChannel(information['channel_id']) + + if channel_exists == False: + return(addChannel({'channel_id' : information['channel_id']}, empty=True)) + else: + channel_ref = db.collection(u'channels').document(information['channel_id']) + video_ref = channel_ref.collection('videos').document(information['id']) + video_ref.set({ + 'channel_id' : information['channel_id'], + 'channel' : information['channel'], + 'description' : information['description'], + 'duration' : information['duration'], + 'duration_string' : information['duration_string'], + 'fulltitle' : information['fulltitle'], + 'video_id' : information['id'], + 'like_count' : information['like_count'], + 'view_count' : information['view_count'], + 'original_url' : information['original_url'], + 'thumbnail' : information['thumbnail'], + 'title' : information['title'], + 'upload_date' : information['upload_date'], + 'webpage_url' : information['webpage_url'], + 'downloaded' : False, + 'downloaded_date' : getCurrentTime() + }) + return(jsonify(information)) -def getAllVideosByChannel(channel_id): +def getAllVideosByChannel(channel_id, limit=None): channels_ref = db.collection(u'channels').document(channel_id) video_ref = channels_ref.collection('videos') query = video_ref.order_by('upload_date') + + if limit: + query = query.limit(limit) + ordered_videos = query.stream() all_videos = [] for video in ordered_videos: @@ -91,4 +108,14 @@ def getAllVideos(): query = getAllVideosByChannel(channel['channel_id']) for q in query: all_videos.append(q) + return(jsonify(all_videos)) + +def getMissingVideos(): + channel_ref = db.collection(u'channels').where('empty', '==', True).stream() + all_videos = [] + for channel in channel_ref: + query = getAllVideosByChannel(channel.id) + for q in query: + all_videos.append(q) + return(jsonify(all_videos)) \ No newline at end of file From c77bc13602c6c0530d838266d2b147af631c93c9 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 19:50:35 -0400 Subject: [PATCH 7/8] Update 'download latest' to use firebase --- backend/blueprints/download.py | 26 ++++++++++----------- backend/functions/databaseCalls/firebase.py | 1 - 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/backend/blueprints/download.py b/backend/blueprints/download.py index a92ac38..3071cc7 100644 --- a/backend/blueprints/download.py +++ b/backend/blueprints/download.py @@ -1,3 +1,4 @@ +import json import requests import os @@ -24,24 +25,21 @@ def root(): def latest(): channel_id = request.args['id'] range = int(request.args['range']) + query_channel = [] if channel_id == 'all': - query_channel = channels.getAllChannels() - - elif channel_id: - query_channel = channels.getChannelByYouTubeId(channel_id) - - else: - print('Error') - return('Error') + all_channels = firebase.getAllChannels() + query = json.loads(all_channels.data) + for q in query: + channel_url = q['webpage_url'] + query_channel.append(channel_url) + elif not channel_id == 'all': + single_channel = firebase.getChannel(channel_id) + query = json.loads(single_channel.data) + query_channel.append(query['webpage_url']) for channel in query_channel: - if channel_id == 'all': - channel_url = channel[2] # Channel Original URL - else: - channel_url = channel - - download_results = download(video=channel_url, video_range=range, download_confirm=False) + download_results = download(video=channel, video_range=range, download_confirm=False) for entry in download_results['entries']: video_url = entry['original_url'] requests.get(os.environ.get("API_URL") + '/download/search?url=' + video_url) diff --git a/backend/functions/databaseCalls/firebase.py b/backend/functions/databaseCalls/firebase.py index 41bd7ce..af9322f 100644 --- a/backend/functions/databaseCalls/firebase.py +++ b/backend/functions/databaseCalls/firebase.py @@ -43,7 +43,6 @@ def getChannel(information): doc = channels_ref.get() if doc.exists: - print(f'Document data: {doc.to_dict()}') return(jsonify(doc.to_dict())) else: return False From 338c76fd86e763ea80d94da6ef573dd85947ec85 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Sat, 9 Jul 2022 19:58:08 -0400 Subject: [PATCH 8/8] Update Readme.md --- ReadMe.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ReadMe.md b/ReadMe.md index 7a26546..82126e8 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -2,7 +2,7 @@ This is just a pet project to learn more about [Python Flask](https://flask.palletsprojects.com/en/2.1.x/) and [NextJS](https://nextjs.org/). -The backend is Python Flask and the frontend is NextJS. +The backend is Python Flask and the frontend is NextJS. The database is [Firestore](https://cloud.google.com/firestore/). For now... - This just links the YouTube Video's URL which you can click on. @@ -28,7 +28,7 @@ While YouTubeDL-Material & TubeArchivist are great projects, I wanted something ## To Do -- [ ] Migrate from SQLite to Serverless Database (DynamoDB, Firestore) +- [X] [Using Firebase](https://github.com/hxrsmurf/ytdlp-flask-nextjs/pull/5) ~~Migrate from SQLite to Serverless Database (DynamoDB, Firestore)~~ - [ ] Add download functionality to local storage and s3-compatible (BackBlaze B2) - [ ] Add CDN functionality via CloudFlare or NGINX to cache videos - [ ] Dockerize