Add Basic Firestore functionality

* Add firebase-admin to requirements
* Update channels_bp to use firestore
* Add serviceAccountKey.json to .gitignore
* Move video functions to firebase
* Update filter-dropdown to use firebase
* Add firebase.getMissingVideos()
* Update 'download latest' to use firebase
* Update Readme.md

.gitignore

@@ -6,3 +6,4 @@ backend/youtube.sqlite3
 backend/blueprints/__pycache__/*
 backend/classes/__pycache__/*
 backend/functions/databaseCalls/__pycache__/*
+backend/serviceAccountKey.json

ReadMe.md

@@ -2,7 +2,7 @@
 
 This is just a pet project to learn more about [Python Flask](https://flask.palletsprojects.com/en/2.1.x/) and [NextJS](https://nextjs.org/).
 
-The backend is Python Flask and the frontend is NextJS.
+The backend is Python Flask and the frontend is NextJS. The database is [Firestore](https://cloud.google.com/firestore/).
 
 For now...
 - This just links the YouTube Video's URL which you can click on.
@@ -28,7 +28,7 @@ While YouTubeDL-Material & TubeArchivist are great projects, I wanted something
 
 ## To Do
 
-- [ ] Migrate from SQLite to Serverless Database (DynamoDB, Firestore)
+- [X] [Using Firebase](https://github.com/hxrsmurf/ytdlp-flask-nextjs/pull/5) ~~Migrate from SQLite to Serverless Database (DynamoDB, Firestore)~~
 - [ ] Add download functionality to local storage and s3-compatible (BackBlaze B2)
 - [ ] Add CDN functionality via CloudFlare or NGINX to cache videos
 - [ ] Dockerize

backend/blueprints/channels.py

@@ -1,59 +1,17 @@
-import requests
-import os
+from flask import Blueprint, request
 
-from flask import Blueprint, Flask, jsonify, request, redirect
-
-from functions.utils import getCurrentTime
-from functions.databaseCalls import channels
+from functions.databaseCalls import firebase
 from functions.downloader import download
 
-from classes.shared import db
-from classes import Channels
-
 channels_bp = Blueprint('channels', __name__, url_prefix='/channels')
 
 @channels_bp.route('/', methods=['GET'])
 def root():
-    channelList = channels.getAllChannels()
-    channelListArray = []
-    columns = channels.getColumns()
-    for channel in channelList:
-        channelListQuery = channels.getChannelById(channel.id)
-        for result in channelListQuery:
-            channelListArray.append(dict(zip(columns, result)))
-
-    return(jsonify(channelListArray))
+    return(firebase.getAllChannels())
 
 @channels_bp.route('/add', methods=['GET'])
 def add():
     url = request.args['url']
     download_results = download(video=url, video_range=1, download_confirm=False)
     result_latest_upload = download_results['entries'][0]['original_url']
-    channel_exists = channels.getChannelByName(download_results['channel'])
-
-    if channel_exists:
-        print(f'The {download_results["channel"]} is already in the database.')
-    elif not channel_exists:
-        requests.get(os.environ.get("API_URL") + '/download/search?url=' + result_latest_upload)
-
-        db_entry = Channels.Channels(
-            channel_name = download_results['channel'],
-            channel_follower_count = download_results['channel_follower_count'],
-            channel_id = download_results['channel_id'],
-            description = download_results['description'],
-            original_url = download_results['original_url'],
-            uploader = download_results['uploader'],
-            uploader_id = download_results['uploader_id'],
-            webpage_url = download_results['webpage_url'],
-            picture_profile = download_results['thumbnails'][18]['url'],
-            picture_cover = download_results['thumbnails'][15]['url'],
-            last_updated = getCurrentTime(),
-            latest_upload = download_results['entries'][0]['original_url']
-        )
-
-        db.session.add(db_entry)
-        db.session.commit()
-    else:
-        print(f'There was another error.')
-
-    return(jsonify(download_results))
+    return(firebase.addChannel(download_results))

backend/blueprints/download.py

@@ -1,10 +1,12 @@
+import json
 import requests
 import os
 
 from flask import Blueprint, Flask, jsonify, request, redirect
 
 from functions.databaseCalls import videos
 from functions.databaseCalls import channels
+from functions.databaseCalls import firebase
 from functions.downloader import download
 from functions.utils import getCurrentTime
 
@@ -17,58 +19,27 @@
 def root():
     url = request.args['url']
     result_download_url = download(video=url, video_range=1, download_confirm=False)
-    query_video = videos.getVideoByYouTubeId(result_download_url['id'])
-
-    if query_video:
-        print('The video already in database.')
-    elif not query_video:
-        db_entry = Videos.Videos(
-            channel = result_download_url['channel'],
-            channel_id = result_download_url['channel_id'],
-            description = result_download_url['description'],
-            duration = result_download_url['duration'],
-            duration_string = result_download_url['duration_string'],
-            fulltitle = result_download_url['fulltitle'],
-            video_id = result_download_url['id'],
-            like_count = result_download_url['like_count'],
-            view_count = result_download_url['view_count'],
-            original_url = result_download_url['original_url'],
-            thumbnail = result_download_url['thumbnail'],
-            title = result_download_url['title'],
-            upload_date = result_download_url['upload_date'],
-            webpage_url = result_download_url['webpage_url'],
-            downloaded = False,
-            downloaded_date = getCurrentTime()
-        )
-        db.session.add(db_entry)
-        db.session.commit()
-    else:
-        print('There was another error.')
-    return(jsonify(result_download_url))
-
+    return(firebase.addVideo(result_download_url))
 
 @download_bp.route('/latest', methods=['GET'])
 def latest():
     channel_id = request.args['id']
     range = int(request.args['range'])
+    query_channel = []
 
     if channel_id == 'all':
-        query_channel = channels.getAllChannels()
-
-    elif channel_id:
-        query_channel = channels.getChannelByYouTubeId(channel_id)
-
-    else:
-        print('Error')
-        return('Error')
+        all_channels = firebase.getAllChannels()
+        query = json.loads(all_channels.data)
+        for q in query:
+            channel_url = q['webpage_url']
+            query_channel.append(channel_url)
+    elif not channel_id == 'all':
+        single_channel = firebase.getChannel(channel_id)
+        query = json.loads(single_channel.data)
+        query_channel.append(query['webpage_url'])
 
     for channel in query_channel:
-        if channel_id == 'all':
-            channel_url = channel[2] # Channel Original URL
-        else:
-            channel_url = channel
-
-        download_results = download(video=channel_url, video_range=range, download_confirm=False)
+        download_results = download(video=channel, video_range=range, download_confirm=False)
         for entry in download_results['entries']:
             video_url = entry['original_url']
             requests.get(os.environ.get("API_URL") + '/download/search?url=' + video_url)

backend/blueprints/search.py

@@ -1,14 +1,10 @@
 from flask import Blueprint, Flask, jsonify, request, redirect
+from functions.databaseCalls import firebase
 from functions.databaseCalls import videos
 from functions.databaseCalls import channels
 
 search_bp = Blueprint('search', __name__, url_prefix='/search')
 
-@search_bp.route('/videos/channel/<string:channelName>', methods=['GET'])
-def root(channelName):
-    videoList = videos.getVideoByChannelName(channelName)
-    videoListArray = []
-    columns = videos.getColumns()
-    for video in videoList:
-        videoListArray.append(dict(zip(columns, video)))
-    return(jsonify(videoListArray))
+@search_bp.route('/videos/channel/<string:channel_id>', methods=['GET'])
+def root(channel_id):
+    return(jsonify(firebase.getAllVideosByChannel(channel_id)))

backend/blueprints/videos.py

@@ -1,46 +1,18 @@
 from flask import Blueprint, Flask, jsonify, request, redirect
+from functions.databaseCalls import firebase
 from functions.databaseCalls import videos
 from functions.databaseCalls import channels
 
 videos_bp = Blueprint('videos', __name__, url_prefix='/videos')
 
 @videos_bp.route('/', methods=['GET'])
 def root():
-    videoList = videos.getAllVideos()
-    videoListArray = []
-    columns = videos.getColumns()
-    for video in videoList:
-        videoListQuery = videos.getVideoById(video.id)
-        for result in videoListQuery:
-            videoListArray.append(dict(zip(columns, result)))
-
-    return(jsonify(videoListArray))
+    return(firebase.getAllVideos())
 
 @videos_bp.route('/unique/channel-name', methods=['GET'])
 def channelName():
-    videoList = videos.getAllVideos(distinct=True)
-    videoListArray = []
-    columns = videos.getColumns()
-    for video in videoList:
-        videoListArray.append({
-            'channel' : video.channel,
-            'channel_id' : video.channel_id
-        })
-
-    return(jsonify(videoListArray))
+    return(firebase.getAllChannels())
 
 @videos_bp.route('/sync-channels', methods=['GET'])
 def syncChannels():
-    videoChannelList = videos.getAllVideos(distinct=True)
-    columns = videos.getColumns()
-
-    missing_channels = []
-    for channel in videoChannelList:
-        channel_name = channel[1]
-        query = channels.getChannelByName(channel_name)
-        if not query:
-            query2 = videos.getVideoByChannelName(channel_name)
-            for q in query2:
-                missing_channels.append(dict(zip(columns, q)))
-
-    return(jsonify(missing_channels))
+    return(firebase.getMissingVideos())

backend/functions/databaseCalls/firebase.py

@@ -0,0 +1,120 @@
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+from flask import jsonify
+import json
+
+from functions.utils import getCurrentTime
+
+cred = credentials.Certificate("serviceAccountKey.json")
+firebase_admin.initialize_app(cred)
+
+db = firestore.client()
+
+def addChannel(information, empty=False):
+    channels_ref = db.collection(u'channels').document(information['channel_id'])
+    if empty == True:
+        print('Setting empty')
+        channels_ref.set({
+            'empty' : True
+        })
+        return(information)
+    elif empty == False:
+        channels_ref.set({
+            'channel_id' : information['channel_id'],
+            'channel_name' : information['channel'],
+            'channel_follower_count' : information['channel_follower_count'],
+            'description' : information['description'],
+            'original_url' : information['original_url'],
+            'uploader' : information['uploader'],
+            'uploader_id' : information['uploader_id'],
+            'webpage_url' : information['webpage_url'],
+            'picture_profile' : information['thumbnails'][18]['url'],
+            'picture_cover' : information['thumbnails'][15]['url'],
+            'last_updated' : getCurrentTime(),
+            'latest_upload' : information['entries'][0]['original_url']
+        })
+    else:
+        return(f'There was an error adding {information}.')
+    return(jsonify(information))
+
+def getChannel(information):
+    channels_ref = db.collection(u'channels').document(information)
+
+    doc = channels_ref.get()
+    if doc.exists:
+        return(jsonify(doc.to_dict()))
+    else:
+        return False
+
+def getAllChannels():
+    channels_ref = db.collection(u'channels')
+    query = channels_ref.order_by('channel_name')
+    ordered_channels = query.stream()
+    result = []
+    for channel in ordered_channels:
+        result.append(channel.to_dict())
+
+    return(jsonify(result))
+
+def addVideo(information):
+    channel_exists = getChannel(information['channel_id'])
+
+    if channel_exists == False:
+        return(addChannel({'channel_id' : information['channel_id']}, empty=True))
+    else:
+        channel_ref = db.collection(u'channels').document(information['channel_id'])
+        video_ref = channel_ref.collection('videos').document(information['id'])
+        video_ref.set({
+            'channel_id' : information['channel_id'],
+            'channel' : information['channel'],
+            'description' : information['description'],
+            'duration' : information['duration'],
+            'duration_string' : information['duration_string'],
+            'fulltitle' : information['fulltitle'],
+            'video_id' : information['id'],
+            'like_count' : information['like_count'],
+            'view_count' : information['view_count'],
+            'original_url' : information['original_url'],
+            'thumbnail' : information['thumbnail'],
+            'title' : information['title'],
+            'upload_date' : information['upload_date'],
+            'webpage_url' : information['webpage_url'],
+            'downloaded' : False,
+            'downloaded_date' : getCurrentTime()
+        })
+        return(jsonify(information))
+
+def getAllVideosByChannel(channel_id, limit=None):
+    channels_ref = db.collection(u'channels').document(channel_id)
+    video_ref = channels_ref.collection('videos')
+    query = video_ref.order_by('upload_date')
+
+    if limit:
+        query = query.limit(limit)
+
+    ordered_videos = query.stream()
+    all_videos = []
+    for video in ordered_videos:
+        all_videos.append(video.to_dict())
+    return(all_videos)
+
+def getAllVideos():
+    channel_ref = getAllChannels()
+    channel_json = json.loads(channel_ref.data)
+    all_videos = []
+    for channel in channel_json:
+        query = getAllVideosByChannel(channel['channel_id'])
+        for q in query:
+            all_videos.append(q)
+    return(jsonify(all_videos))
+
+def getMissingVideos():
+    channel_ref = db.collection(u'channels').where('empty', '==', True).stream()
+    all_videos = []
+    for channel in channel_ref:
+        query = getAllVideosByChannel(channel.id)
+        for q in query:
+            all_videos.append(q)
+
+    return(jsonify(all_videos))

backend/requirements.txt

@@ -1,5 +1,6 @@
 b2sdk==1.14.0
 ffmpeg==1.4
+firebase-admin==5.2.0
 Flask==2.0.3
 Flask-Cors==3.0.10
 Flask-SQLAlchemy==2.5.1

frontend/pages/videos.js

@@ -29,7 +29,7 @@ export default function videos({ results, result_all_channels }) {
 
     const handleDropdownClick = async (event) => {
         setDropdownName(channel)
-        const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channel)
+        const request_channel_videos = await fetch(process.env.NEXT_PUBLIC_BASE_API_URL + '/search/videos/channel/' + channelID)
         const new_results = await request_channel_videos.json()
         setNewResults(new_results)
     }
@@ -95,7 +95,7 @@ export default function videos({ results, result_all_channels }) {
                                                 }
                                                 }
                                             >
-                                                <Col>{channel.channel}</Col>
+                                                <Col>{channel.channel_name}</Col>
                                                 <Col hidden>{channel.channel_id}</Col>
                                             </Row>
                                         </Dropdown.Item>