diff --git a/.gitignore b/.gitignore index b8db43f..289e7ce 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ backend/youtube.sqlite3 backend/blueprints/__pycache__/* backend/classes/__pycache__/* backend/functions/databaseCalls/__pycache__/* +backend/serviceAccountKey.json \ No newline at end of file diff --git a/ReadMe.md b/ReadMe.md index 7a26546..82126e8 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -2,7 +2,7 @@ This is just a pet project to learn more about [Python Flask](https://flask.palletsprojects.com/en/2.1.x/) and [NextJS](https://nextjs.org/). -The backend is Python Flask and the frontend is NextJS. +The backend is Python Flask and the frontend is NextJS. The database is [Firestore](https://cloud.google.com/firestore/). For now... - This just links the YouTube Video's URL which you can click on. @@ -28,7 +28,7 @@ While YouTubeDL-Material & TubeArchivist are great projects, I wanted something ## To Do -- [ ] Migrate from SQLite to Serverless Database (DynamoDB, Firestore) +- [X] [Using Firebase](https://github.com/hxrsmurf/ytdlp-flask-nextjs/pull/5) ~~Migrate from SQLite to Serverless Database (DynamoDB, Firestore)~~ - [ ] Add download functionality to local storage and s3-compatible (BackBlaze B2) - [ ] Add CDN functionality via CloudFlare or NGINX to cache videos - [ ] Dockerize diff --git a/backend/blueprints/channels.py b/backend/blueprints/channels.py index 8b9922c..9b2c26d 100644 --- a/backend/blueprints/channels.py +++ b/backend/blueprints/channels.py @@ -1,59 +1,17 @@ -import requests -import os +from flask import Blueprint, request -from flask import Blueprint, Flask, jsonify, request, redirect - -from functions.utils import getCurrentTime -from functions.databaseCalls import channels +from functions.databaseCalls import firebase from functions.downloader import download -from classes.shared import db -from classes import Channels - channels_bp = Blueprint('channels', __name__, url_prefix='/channels') @channels_bp.route('/', methods=['GET']) def root(): - channelList = channels.getAllChannels() - channelListArray = [] - columns = channels.getColumns() - for channel in channelList: - channelListQuery = channels.getChannelById(channel.id) - for result in channelListQuery: - channelListArray.append(dict(zip(columns, result))) - - return(jsonify(channelListArray)) + return(firebase.getAllChannels()) @channels_bp.route('/add', methods=['GET']) def add(): url = request.args['url'] download_results = download(video=url, video_range=1, download_confirm=False) result_latest_upload = download_results['entries'][0]['original_url'] - channel_exists = channels.getChannelByName(download_results['channel']) - - if channel_exists: - print(f'The {download_results["channel"]} is already in the database.') - elif not channel_exists: - requests.get(os.environ.get("API_URL") + '/download/search?url=' + result_latest_upload) - - db_entry = Channels.Channels( - channel_name = download_results['channel'], - channel_follower_count = download_results['channel_follower_count'], - channel_id = download_results['channel_id'], - description = download_results['description'], - original_url = download_results['original_url'], - uploader = download_results['uploader'], - uploader_id = download_results['uploader_id'], - webpage_url = download_results['webpage_url'], - picture_profile = download_results['thumbnails'][18]['url'], - picture_cover = download_results['thumbnails'][15]['url'], - last_updated = getCurrentTime(), - latest_upload = download_results['entries'][0]['original_url'] - ) - - db.session.add(db_entry) - db.session.commit() - else: - print(f'There was another error.') - - return(jsonify(download_results)) \ No newline at end of file + return(firebase.addChannel(download_results)) \ No newline at end of file diff --git a/backend/blueprints/download.py b/backend/blueprints/download.py index ef33150..3071cc7 100644 --- a/backend/blueprints/download.py +++ b/backend/blueprints/download.py @@ -1,3 +1,4 @@ +import json import requests import os @@ -5,6 +6,7 @@ from functions.databaseCalls import videos from functions.databaseCalls import channels +from functions.databaseCalls import firebase from functions.downloader import download from functions.utils import getCurrentTime @@ -17,58 +19,27 @@ def root(): url = request.args['url'] result_download_url = download(video=url, video_range=1, download_confirm=False) - query_video = videos.getVideoByYouTubeId(result_download_url['id']) - - if query_video: - print('The video already in database.') - elif not query_video: - db_entry = Videos.Videos( - channel = result_download_url['channel'], - channel_id = result_download_url['channel_id'], - description = result_download_url['description'], - duration = result_download_url['duration'], - duration_string = result_download_url['duration_string'], - fulltitle = result_download_url['fulltitle'], - video_id = result_download_url['id'], - like_count = result_download_url['like_count'], - view_count = result_download_url['view_count'], - original_url = result_download_url['original_url'], - thumbnail = result_download_url['thumbnail'], - title = result_download_url['title'], - upload_date = result_download_url['upload_date'], - webpage_url = result_download_url['webpage_url'], - downloaded = False, - downloaded_date = getCurrentTime() - ) - db.session.add(db_entry) - db.session.commit() - else: - print('There was another error.') - return(jsonify(result_download_url)) - + return(firebase.addVideo(result_download_url)) @download_bp.route('/latest', methods=['GET']) def latest(): channel_id = request.args['id'] range = int(request.args['range']) + query_channel = [] if channel_id == 'all': - query_channel = channels.getAllChannels() - - elif channel_id: - query_channel = channels.getChannelByYouTubeId(channel_id) - - else: - print('Error') - return('Error') + all_channels = firebase.getAllChannels() + query = json.loads(all_channels.data) + for q in query: + channel_url = q['webpage_url'] + query_channel.append(channel_url) + elif not channel_id == 'all': + single_channel = firebase.getChannel(channel_id) + query = json.loads(single_channel.data) + query_channel.append(query['webpage_url']) for channel in query_channel: - if channel_id == 'all': - channel_url = channel[2] # Channel Original URL - else: - channel_url = channel - - download_results = download(video=channel_url, video_range=range, download_confirm=False) + download_results = download(video=channel, video_range=range, download_confirm=False) for entry in download_results['entries']: video_url = entry['original_url'] requests.get(os.environ.get("API_URL") + '/download/search?url=' + video_url) diff --git a/backend/blueprints/search.py b/backend/blueprints/search.py index 53fc369..e8223cb 100644 --- a/backend/blueprints/search.py +++ b/backend/blueprints/search.py @@ -1,14 +1,10 @@ from flask import Blueprint, Flask, jsonify, request, redirect +from functions.databaseCalls import firebase from functions.databaseCalls import videos from functions.databaseCalls import channels search_bp = Blueprint('search', __name__, url_prefix='/search') -@search_bp.route('/videos/channel/', methods=['GET']) -def root(channelName): - videoList = videos.getVideoByChannelName(channelName) - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListArray.append(dict(zip(columns, video))) - return(jsonify(videoListArray)) \ No newline at end of file +@search_bp.route('/videos/channel/', methods=['GET']) +def root(channel_id): + return(jsonify(firebase.getAllVideosByChannel(channel_id))) \ No newline at end of file diff --git a/backend/blueprints/videos.py b/backend/blueprints/videos.py index ceb0451..bd89fab 100644 --- a/backend/blueprints/videos.py +++ b/backend/blueprints/videos.py @@ -1,4 +1,5 @@ from flask import Blueprint, Flask, jsonify, request, redirect +from functions.databaseCalls import firebase from functions.databaseCalls import videos from functions.databaseCalls import channels @@ -6,41 +7,12 @@ @videos_bp.route('/', methods=['GET']) def root(): - videoList = videos.getAllVideos() - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListQuery = videos.getVideoById(video.id) - for result in videoListQuery: - videoListArray.append(dict(zip(columns, result))) - - return(jsonify(videoListArray)) + return(firebase.getAllVideos()) @videos_bp.route('/unique/channel-name', methods=['GET']) def channelName(): - videoList = videos.getAllVideos(distinct=True) - videoListArray = [] - columns = videos.getColumns() - for video in videoList: - videoListArray.append({ - 'channel' : video.channel, - 'channel_id' : video.channel_id - }) - - return(jsonify(videoListArray)) + return(firebase.getAllChannels()) @videos_bp.route('/sync-channels', methods=['GET']) def syncChannels(): - videoChannelList = videos.getAllVideos(distinct=True) - columns = videos.getColumns() - - missing_channels = [] - for channel in videoChannelList: - channel_name = channel[1] - query = channels.getChannelByName(channel_name) - if not query: - query2 = videos.getVideoByChannelName(channel_name) - for q in query2: - missing_channels.append(dict(zip(columns, q))) - - return(jsonify(missing_channels)) \ No newline at end of file + return(firebase.getMissingVideos()) \ No newline at end of file diff --git a/backend/functions/databaseCalls/firebase.py b/backend/functions/databaseCalls/firebase.py new file mode 100644 index 0000000..af9322f --- /dev/null +++ b/backend/functions/databaseCalls/firebase.py @@ -0,0 +1,120 @@ +import firebase_admin +from firebase_admin import credentials +from firebase_admin import firestore +from flask import jsonify +import json + +from functions.utils import getCurrentTime + +cred = credentials.Certificate("serviceAccountKey.json") +firebase_admin.initialize_app(cred) + +db = firestore.client() + +def addChannel(information, empty=False): + channels_ref = db.collection(u'channels').document(information['channel_id']) + if empty == True: + print('Setting empty') + channels_ref.set({ + 'empty' : True + }) + return(information) + elif empty == False: + channels_ref.set({ + 'channel_id' : information['channel_id'], + 'channel_name' : information['channel'], + 'channel_follower_count' : information['channel_follower_count'], + 'description' : information['description'], + 'original_url' : information['original_url'], + 'uploader' : information['uploader'], + 'uploader_id' : information['uploader_id'], + 'webpage_url' : information['webpage_url'], + 'picture_profile' : information['thumbnails'][18]['url'], + 'picture_cover' : information['thumbnails'][15]['url'], + 'last_updated' : getCurrentTime(), + 'latest_upload' : information['entries'][0]['original_url'] + }) + else: + return(f'There was an error adding {information}.') + return(jsonify(information)) + +def getChannel(information): + channels_ref = db.collection(u'channels').document(information) + + doc = channels_ref.get() + if doc.exists: + return(jsonify(doc.to_dict())) + else: + return False + +def getAllChannels(): + channels_ref = db.collection(u'channels') + query = channels_ref.order_by('channel_name') + ordered_channels = query.stream() + result = [] + for channel in ordered_channels: + result.append(channel.to_dict()) + + return(jsonify(result)) + +def addVideo(information): + channel_exists = getChannel(information['channel_id']) + + if channel_exists == False: + return(addChannel({'channel_id' : information['channel_id']}, empty=True)) + else: + channel_ref = db.collection(u'channels').document(information['channel_id']) + video_ref = channel_ref.collection('videos').document(information['id']) + video_ref.set({ + 'channel_id' : information['channel_id'], + 'channel' : information['channel'], + 'description' : information['description'], + 'duration' : information['duration'], + 'duration_string' : information['duration_string'], + 'fulltitle' : information['fulltitle'], + 'video_id' : information['id'], + 'like_count' : information['like_count'], + 'view_count' : information['view_count'], + 'original_url' : information['original_url'], + 'thumbnail' : information['thumbnail'], + 'title' : information['title'], + 'upload_date' : information['upload_date'], + 'webpage_url' : information['webpage_url'], + 'downloaded' : False, + 'downloaded_date' : getCurrentTime() + }) + return(jsonify(information)) + +def getAllVideosByChannel(channel_id, limit=None): + channels_ref = db.collection(u'channels').document(channel_id) + video_ref = channels_ref.collection('videos') + query = video_ref.order_by('upload_date') + + if limit: + query = query.limit(limit) + + ordered_videos = query.stream() + all_videos = [] + for video in ordered_videos: + all_videos.append(video.to_dict()) + return(all_videos) + +def getAllVideos(): + channel_ref = getAllChannels() + channel_json = json.loads(channel_ref.data) + all_videos = [] + for channel in channel_json: + query = getAllVideosByChannel(channel['channel_id']) + for q in query: + all_videos.append(q) + return(jsonify(all_videos)) + +def getMissingVideos(): + channel_ref = db.collection(u'channels').where('empty', '==', True).stream() + all_videos = [] + for channel in channel_ref: + query = getAllVideosByChannel(channel.id) + for q in query: + all_videos.append(q) + + return(jsonify(all_videos)) \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index ecc4f93..cf27c7c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,6 @@ b2sdk==1.14.0 ffmpeg==1.4 +firebase-admin==5.2.0 Flask==2.0.3 Flask-Cors==3.0.10 Flask-SQLAlchemy==2.5.1 diff --git a/frontend/pages/videos.js b/frontend/pages/videos.js index d598b53..b8844ed 100644 --- a/frontend/pages/videos.js +++ b/frontend/pages/videos.js @@ -29,7 +29,7 @@ export default function videos({ results, result_all_channels }) { const handleDropdownClick = async (event) => { setDropdownName(channel) - const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channel) + const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channelID) const new_results = await request_channel_videos.json() setNewResults(new_results) } @@ -95,7 +95,7 @@ export default function videos({ results, result_all_channels }) { } } > - {channel.channel} + {channel.channel_name} {channel.channel_id}