Merge pull request #234 from ShaneIsrael/develop

Improved video scanning and uploading

app/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fireshare",
-  "version": "1.2.15",
+  "version": "1.2.16",
   "private": true,
   "dependencies": {
     "@emotion/react": "^11.9.0",

app/client/src/components/admin/UploadCard.js

@@ -11,7 +11,7 @@ const Input = styled('input')({
 
 const numberFormat = new Intl.NumberFormat('en-US')
 
-const UploadCard = ({ authenticated, feedView = false, publicUpload = false, cardWidth, handleAlert }) => {
+const UploadCard = ({ authenticated, feedView = false, publicUpload = false, fetchVideos, cardWidth, handleAlert }) => {
   const cardHeight = cardWidth / 1.77 + 32
   const [selectedFile, setSelectedFile] = React.useState()
   const [isSelected, setIsSelected] = React.useState(false)
@@ -58,7 +58,13 @@ const UploadCard = ({ authenticated, feedView = false, publicUpload = false, car
         if (!publicUpload && authenticated) {
           await VideoService.upload(formData, uploadProgress)
         }
-        handleAlert({ type: 'success', message: "Your upload will be available after the next scan.", open: true })
+        handleAlert({
+          type: 'success',
+          message: 'Your upload will be available shortly',
+          autohideDuration: 2500,
+          open: true,
+          onClose: () => fetchVideos(),
+        })
       } catch (err) {
         handleAlert({
           type: 'error',

app/client/src/components/admin/VideoCards.js

@@ -7,7 +7,15 @@ import SensorsIcon from '@mui/icons-material/Sensors'
 import { VideoService } from '../../services'
 import UploadCard from './UploadCard'
 
-const VideoCards = ({ videos, loadingIcon = null, feedView = false, showUploadCard = false, authenticated, size }) => {
+const VideoCards = ({
+  videos,
+  loadingIcon = null,
+  feedView = false,
+  showUploadCard = false,
+  fetchVideos,
+  authenticated,
+  size,
+}) => {
   const [vids, setVideos] = React.useState(videos)
   const [alert, setAlert] = React.useState({ open: false })
   const [videoModal, setVideoModal] = React.useState({
@@ -126,7 +134,12 @@ const VideoCards = ({ videos, loadingIcon = null, feedView = false, showUploadCa
         authenticated={authenticated}
         updateCallback={handleUpdate}
       />
-      <SnackbarAlert severity={alert.type} open={alert.open} setOpen={(open) => setAlert({ ...alert, open })}>
+      <SnackbarAlert
+        severity={alert.type}
+        open={alert.open}
+        onClose={alert.onClose}
+        setOpen={(open) => setAlert({ ...alert, open })}
+      >
         {alert.message}
       </SnackbarAlert>
 
@@ -139,6 +152,7 @@ const VideoCards = ({ videos, loadingIcon = null, feedView = false, showUploadCa
               feedView={feedView}
               cardWidth={size}
               handleAlert={memoizedHandleAlert}
+              fetchVideos={fetchVideos}
               publicUpload={feedView}
             />
           )}

app/client/src/components/alert/SnackbarAlert.js

@@ -2,18 +2,21 @@ import * as React from 'react'
 import Snackbar from '@mui/material/Snackbar'
 import Alert from './Alert'
 
-export default function SnackbarAlert({ severity, children, open, setOpen }) {
+export default function SnackbarAlert({ severity, children, open, autoHideDuration, setOpen, onClose }) {
   const handleClose = (event, reason) => {
     if (reason === 'clickaway') {
       return
     }
     setOpen(false)
+    if (onClose) {
+      onClose()
+    }
   }
 
   return (
     <Snackbar
       open={open}
-      autoHideDuration={5000}
+      autoHideDuration={autoHideDuration || 5000}
       onClose={handleClose}
       anchorOrigin={{
         vertical: 'bottom',

app/client/src/views/Dashboard.js

@@ -25,9 +25,7 @@ const Dashboard = ({ authenticated, searchText, cardSize, listStyle }) => {
   const [selectedFolder, setSelectedFolder] = React.useState(
     getSetting('folder') || { value: 'All Videos', label: 'All Videos' },
   )
-  const [selectedSort, setSelectedSort] = React.useState(
-    getSetting('sortOption') || SORT_OPTIONS[0],
-  )
+  const [selectedSort, setSelectedSort] = React.useState(getSetting('sortOption') || SORT_OPTIONS[0])
 
   const [alert, setAlert] = React.useState({ open: false })
 
@@ -84,12 +82,12 @@ const Dashboard = ({ authenticated, searchText, cardSize, listStyle }) => {
     setSetting('folder', folder)
     setSelectedFolder(folder)
   }
-  
+
   const handleSortSelection = (sortOption) => {
     setSetting('sortOption', sortOption)
     setSelectedSort(sortOption)
   }
-  
+
   return (
     <>
       <SnackbarAlert severity={alert.type} open={alert.open} setOpen={(open) => setAlert({ ...alert, open })}>
@@ -146,6 +144,7 @@ const Dashboard = ({ authenticated, searchText, cardSize, listStyle }) => {
                   loadingIcon={loading ? <LoadingSpinner /> : null}
                   size={cardSize}
                   showUploadCard={selectedFolder.value === 'All Videos'}
+                  fetchVideos={fetchVideos}
                   videos={
                     selectedFolder.value === 'All Videos'
                       ? filteredVideos

app/client/src/views/Feed.js

@@ -38,9 +38,7 @@ const Feed = ({ authenticated, searchText, cardSize, listStyle }) => {
       ? { value: category, label: category }
       : getSetting('folder') || { value: 'All Videos', label: 'All Videos' },
   )
-  const [selectedSort, setSelectedSort] = React.useState(
-    getSetting('sortOption') || SORT_OPTIONS[0],
-  )
+  const [selectedSort, setSelectedSort] = React.useState(getSetting('sortOption') || SORT_OPTIONS[0])
 
   const [alert, setAlert] = React.useState({ open: false })
 
@@ -102,12 +100,12 @@ const Feed = ({ authenticated, searchText, cardSize, listStyle }) => {
       window.history.replaceState({ category: folder.value }, '', `/#/feed?${searchParams.toString()}`)
     }
   }
-  
+
   const handleSortSelection = (sortOption) => {
     setSetting('sortOption', sortOption)
     setSelectedSort(sortOption)
   }
-  
+
   return (
     <>
       <SnackbarAlert severity={alert.type} open={alert.open} setOpen={(open) => setAlert({ ...alert, open })}>
@@ -167,6 +165,7 @@ const Feed = ({ authenticated, searchText, cardSize, listStyle }) => {
                   loadingIcon={loading ? <LoadingSpinner /> : null}
                   feedView={true}
                   size={cardSize}
+                  fetchVideos={fetchVideos}
                   showUploadCard={selectedFolder.value === 'All Videos'}
                   videos={
                     selectedFolder.value === 'All Videos'

app/server/fireshare/api.py

@@ -244,6 +244,8 @@ def public_upload_video():
     if not config['app_config']['allow_public_upload']:
         logging.warn("A public upload attempt was made but public uploading is disabled")
         return Response(status=401)
+
+    upload_folder = config['app_config']['public_upload_folder_name']
 
     if 'file' not in request.files:
         return Response(status=400)
@@ -254,16 +256,16 @@ def public_upload_video():
     filetype = file.filename.split('.')[-1]
     if not filetype in SUPPORTED_FILE_TYPES:
         return Response(status=400)
-    upload_directory = paths['video'] / config['app_config']['public_upload_folder_name']
+    upload_directory = paths['video'] / upload_folder
     if not os.path.exists(upload_directory):
         os.makedirs(upload_directory)
     save_path = os.path.join(upload_directory, filename)
     if (os.path.exists(save_path)):
         name_no_type = ".".join(filename.split('.')[0:-1])
         uid = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(6))
-        save_path = os.path.join(paths['video'], config['app_config']['public_upload_folder_name'], f"{name_no_type}-{uid}.{filetype}")
+        save_path = os.path.join(paths['video'], upload_folder, f"{name_no_type}-{uid}.{filetype}")
     file.save(save_path)
-    Popen("fireshare bulk-import", shell=True)
+    Popen(f"fireshare scan-video --path=\"{save_path}\"", shell=True)
     return Response(status=201)
 
 @api.route('/api/upload', methods=['POST'])
@@ -276,6 +278,9 @@ def upload_video():
         except:
             return Response(status=500, response="Invalid or corrupt config file")
         configfile.close()
+
+    upload_folder = config['app_config']['admin_upload_folder_name']
+
     if 'file' not in request.files:
         return Response(status=400)
     file = request.files['file']
@@ -285,16 +290,16 @@ def upload_video():
     filetype = file.filename.split('.')[-1]
     if not filetype in SUPPORTED_FILE_TYPES:
         return Response(status=400)
-    upload_directory = paths['video'] / config['app_config']['admin_upload_folder_name']
+    upload_directory = paths['video'] / upload_folder
     if not os.path.exists(upload_directory):
         os.makedirs(upload_directory)
     save_path = os.path.join(upload_directory, filename)
     if (os.path.exists(save_path)):
         name_no_type = ".".join(filename.split('.')[0:-1])
         uid = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(6))
-        save_path = os.path.join(paths['video'], config['app_config']['admin_upload_folder_name'], f"{name_no_type}-{uid}.{filetype}")
+        save_path = os.path.join(paths['video'], upload_folder, f"{name_no_type}-{uid}.{filetype}")
     file.save(save_path)
-    Popen("fireshare bulk-import", shell=True)
+    Popen(f"fireshare scan-video --path=\"{save_path}\"", shell=True)
     return Response(status=201)
 
 @api.route('/api/video')

app/server/fireshare/cli.py

@@ -34,10 +34,11 @@ def add_user(username, password):
         click.echo(f"Created user {username}")
 
 @cli.command()
-def scan_videos():
+@click.option("--root", "-r", help="root video path to scan", required=False)
+def scan_videos(root):
     with create_app().app_context():
         paths = current_app.config['PATHS']
-        raw_videos = paths["video"]
+        videos_path = paths["video"]
         video_links = paths["processed"] / "video_links"
 
         config_file = open(paths["data"] / "config.json")
@@ -47,13 +48,13 @@ def scan_videos():
         if not video_links.is_dir():
             video_links.mkdir()
 
-        logger.info(f"Scanning {str(raw_videos)} for {', '.join(SUPPORTED_FILE_EXTENSIONS)} video files")
-        video_files = [f for f in raw_videos.glob('**/*') if f.is_file() and f.suffix.lower() in SUPPORTED_FILE_EXTENSIONS]
+        logger.info(f"Scanning {str(videos_path)} for {', '.join(SUPPORTED_FILE_EXTENSIONS)} video files")
+        video_files = [f for f in (videos_path / root if root else videos_path).glob('**/*') if f.is_file() and f.suffix.lower() in SUPPORTED_FILE_EXTENSIONS]
         video_rows = Video.query.all()
 
         new_videos = []
         for vf in video_files:
-            path = str(vf.relative_to(raw_videos))
+            path = str(vf.relative_to(videos_path)) 
             video_id = util.video_id(vf)
             existing = next((vr for vr in video_rows if vr.video_id == video_id), None)
             duplicate = next((dvr for dvr in new_videos if dvr.video_id == video_id), None)
@@ -64,16 +65,16 @@ def scan_videos():
                     logger.info(f"Updating Video {video_id}, available=True")
                     db.session.query(Video).filter_by(video_id=existing.video_id).update({ "available": True })
                 if not existing.created_at:
-                    created_at = datetime.fromtimestamp(os.path.getctime(f"{raw_videos}/{path}"))
+                    created_at = datetime.fromtimestamp(os.path.getctime(f"{videos_path}/{path}"))
                     logger.info(f"Updating Video {video_id}, created_at={created_at}")
                     db.session.query(Video).filter_by(video_id=existing.video_id).update({ "created_at": created_at })
                 if not existing.updated_at:
-                    updated_at = datetime.fromtimestamp(os.path.getmtime(f"{raw_videos}/{path}"))
+                    updated_at = datetime.fromtimestamp(os.path.getmtime(f"{videos_path}/{path}"))
                     logger.info(f"Updating Video {video_id}, updated_at={updated_at}")
                     db.session.query(Video).filter_by(video_id=existing.video_id).update({ "updated_at": updated_at })
             else:
-                created_at = datetime.fromtimestamp(os.path.getctime(f"{raw_videos}/{path}"))
-                updated_at = datetime.fromtimestamp(os.path.getmtime(f"{raw_videos}/{path}"))
+                created_at = datetime.fromtimestamp(os.path.getctime(f"{videos_path}/{path}"))
+                updated_at = datetime.fromtimestamp(os.path.getmtime(f"{videos_path}/{path}"))
                 v = Video(video_id=video_id, extension=vf.suffix, path=path, available=True, created_at=created_at, updated_at=updated_at)
                 logger.info(f"Adding new Video {video_id} at {str(path)} (created {created_at.isoformat()}, updated {updated_at.isoformat()})")
                 new_videos.append(v)
@@ -112,6 +113,83 @@ def scan_videos():
                 db.session.query(Video).filter_by(video_id=ev.video_id).update({ "available": False})
         db.session.commit()
 
+@cli.command()
+@click.option("--path", "-p", help="path to video to scan", required=False)
+def scan_video(path):
+    with create_app().app_context():
+        paths = current_app.config['PATHS']
+        videos_path = paths["video"]
+        video_links = paths["processed"] / "video_links"
+
+        config_file = open(paths["data"] / "config.json")
+        video_config = json.load(config_file)["app_config"]["video_defaults"]
+        config_file.close()
+
+        if not video_links.is_dir():
+            video_links.mkdir()
+
+        video_file = (videos_path / path) if (videos_path / path).is_file() and (videos_path / path).suffix.lower() in SUPPORTED_FILE_EXTENSIONS else None
+        if video_file:
+            video_rows = Video.query.all()
+            logger.info(f"Scanning {str(video_file)}")
+
+            path = str(video_file.relative_to(videos_path)) 
+            video_id = util.video_id(video_file)
+            existing = next((vr for vr in video_rows if vr.video_id == video_id), None)
+            if existing:
+                if not existing.available:
+                    logger.info(f"Updating Video {video_id}, available=True")
+                    db.session.query(Video).filter_by(video_id=existing.video_id).update({ "available": True })
+                if not existing.created_at:
+                    created_at = datetime.fromtimestamp(os.path.getctime(f"{videos_path}/{path}"))
+                    logger.info(f"Updating Video {video_id}, created_at={created_at}")
+                    db.session.query(Video).filter_by(video_id=existing.video_id).update({ "created_at": created_at })
+                if not existing.updated_at:
+                    updated_at = datetime.fromtimestamp(os.path.getmtime(f"{videos_path}/{path}"))
+                    logger.info(f"Updating Video {video_id}, updated_at={updated_at}")
+                    db.session.query(Video).filter_by(video_id=existing.video_id).update({ "updated_at": updated_at })
+            else:
+                created_at = datetime.fromtimestamp(os.path.getctime(f"{videos_path}/{path}"))
+                updated_at = datetime.fromtimestamp(os.path.getmtime(f"{videos_path}/{path}"))
+                v = Video(video_id=video_id, extension=video_file.suffix, path=path, available=True, created_at=created_at, updated_at=updated_at)
+                logger.info(f"Adding new Video {video_id} at {str(path)} (created {created_at.isoformat()}, updated {updated_at.isoformat()})")
+                db.session.add(v)
+                fd = os.open(str(video_links.absolute()), os.O_DIRECTORY)
+                src = Path((paths["video"] / v.path).absolute())
+                dst = Path(paths["processed"] / "video_links" / (video_id + video_file.suffix))
+                common_root = Path(*os.path.commonprefix([src.parts, dst.parts]))
+                num_up = len(dst.parts)-1 - len(common_root.parts)
+                prefix = "../" * num_up
+                rel_src = Path(prefix + str(src).replace(str(common_root), ''))
+                if not dst.exists():
+                    logger.info(f"Linking {str(rel_src)} --> {str(dst)}")
+                    try:
+                        os.symlink(src, dst, dir_fd=fd)
+                    except FileExistsError:
+                        logger.info(f"{dst} exists already")
+                info = VideoInfo(video_id=v.video_id, title=Path(v.path).stem, private=video_config["private"])
+                db.session.add(info)
+
+                processed_root = Path(current_app.config['PROCESSED_DIRECTORY'])
+                logger.info(f"Checking for videos with missing posters...")
+                derived_path = Path(processed_root, "derived", info.video_id)
+                video_path = Path(processed_root, "video_links", info.video_id + video_file.suffix)
+                if video_path.exists():
+                    poster_path = Path(derived_path, "poster.jpg")
+                    should_create_poster = (not poster_path.exists() or regenerate)
+                    if should_create_poster:
+                        if not derived_path.exists():
+                            derived_path.mkdir(parents=True)
+                        poster_time = 0
+                        util.create_poster(video_path, derived_path / "poster.jpg", poster_time)
+                    else:
+                        logger.debug(f"Skipping creation of poster for video {info.video_id} because it exists at {str(poster_path)}")
+                    db.session.commit()
+                else:
+                    logger.warn(f"Skipping creation of poster for video {info.video_id} because the video at {str(video_path)} does not exist or is not accessible")
+        else:
+            logger.info(f"Invalid video file, unable to scan: {str(videos_path / path)}")
+
 @cli.command()
 def repair_symlinks():
     with create_app().app_context():
@@ -265,7 +343,8 @@ def create_boomerang_posters(regenerate):
 
 @cli.command()
 @click.pass_context
-def bulk_import(ctx):
+@click.option("--root", "-r", help="root video path to scan", required=False)
+def bulk_import(ctx, root):
     with create_app().app_context():
         paths = current_app.config['PATHS']
         if util.lock_exists(paths["data"]):
@@ -275,7 +354,7 @@ def bulk_import(ctx):
 
         timing = {}
         s = time.time()
-        ctx.invoke(scan_videos)
+        ctx.invoke(scan_videos, root=root)
         timing['scan_videos'] = time.time() - s
         s = time.time()
         ctx.invoke(sync_metadata)