Inject thumbnails in DASH

src/components/VideoPlayer/Video.bs

@@ -1,6 +1,8 @@
 import "pkg:/source/services/Invidious.bs"
 import "pkg:/components/Dialog/DialogUtils.bs"
 
+#const DASH_THUMBNAILS = false
+
 function ShowVideoScreen(videoId as string, sender as object)
     m.videoPlayer = CreateObject("roSGNode", "VideoPlayer")
     m.videoPlayer.sender = sender
@@ -93,12 +95,22 @@ function StartVideoDetailsTask(videoId as string)
         metadata = task.output.metadata
         if metadata <> invalid
             contentNode = CreateObject("roSGNode", "ContentNode")
-            contentNode.addFields({ videoId: videoId })
+            contentNode.addFields({
+                videoId: videoId,
+                metadata: metadata
+            })
 
             if metadata.hlsUrl <> invalid
                 contentNode.url = metadata.hlsUrl
             else if metadata.dashUrl <> invalid
-                contentNode.url = metadata.dashUrl
+                #if DASH_THUMBNAILS
+                    ' Redirect to our server so we can inject thumbnails (storyboards) into the DASH manifest
+                    ' TODO: make this optional, in case it breaks the DASH manifest
+                    contentNode.url = `http://${GetLocalIpAddress()}:8888/dash?v=${videoId}`
+                #else
+                    contentNode.url = metadata.dashUrl
+                #end if
+
             else
                 stream = metadata.formatStreams[metadata.formatStreams.Count() - 1]
                 itag = stream.itag

src/components/WebServer/HttpRequest.bs

@@ -160,7 +160,7 @@ namespace Http
                     m.ready_for_response = true
                     return false
                 end if
-                
+
                 m.ready_for_response = m.body.len() = contentLength
                 return m.ready_for_response
             end if

src/components/WebServer/Middleware/DashRouter.bs

@@ -0,0 +1,151 @@
+import "pkg:/source/utils/WebUtils.bs"
+import "pkg:/source/services/InvidiousSettings.bs"
+import "pkg:/source/roku_modules/rokurequests/Requests.brs"
+
+namespace Http
+
+    class DashRouter extends HttpRouter
+
+        function new()
+            super()
+
+            m.Get("/dash", function(context as object) as boolean
+                request = context.request
+                response = context.response
+                router = context.router
+
+                v = request.query.v
+                local = request.query.local
+
+                instance = InvidiousSettings.GetSelectedInstance()
+                metadata = router.GetVideoMetadata(context, v)
+                dashUrl = `${instance}/api/manifest/dash/id/${v}`
+
+                if metadata = invalid and local = invalid
+                    m.Redirect(dashUrl, 302)
+                    return true
+                end if
+
+                if local <> invalid
+                    dashUrl = `${dashUrl}?local=${local}`
+                end if
+
+                resp = Requests().get(dashUrl)
+                text = resp.text ?? ""
+                if resp.statuscode = 200 and local = "true"
+                    text = text.Replace("<BaseURL>/videoplayback", `<BaseURL>${instance}/videoplayback`)
+                end if
+
+                if resp.statuscode = 200 and metadata <> invalid
+                    text = router.InjectStoryBoard(router, text, metadata)
+                end if
+
+                response.http_code = resp.statuscode
+                response.SetBodyDataString(text)
+                response.ContentType("application/dash+xml")
+                response.GenerateHeader(true)
+                response.source = Http.HttpResponseSource.GENERATED
+
+                return true
+            end function)
+        end function
+
+        function GetVideoMetadata(context as object, videoId as string) as object
+            scene = context.server.task.top.getScene()
+            videoPlayer = scene.findNode("VideoPlayer")
+            metadata = videoPlayer?.content?.metadata
+            if metadata <> invalid and metadata.videoId = videoId
+                return metadata
+            end if
+            metadata = Invidious.GetVideoMetadata(videoId)
+            return metadata
+        end function
+
+        function InjectStoryBoard(router as object, dash as string, metadata as object) as string
+            xml = CreateObject ("roXMLElement")
+            if not xml.Parse(dash)
+                return dash
+            end if
+
+            thumb_id = 0
+            period = xml.GetChildNodes()[0]
+            sets = period.GetChildNodes()
+            for each set in sets
+                attributes = set.GetAttributes()
+                id = attributes.id
+                idInt = id.toInt()
+                if idInt >= thumb_id
+                    thumb_id = idInt + 1
+                end if
+            end for
+
+            thumbnails = router.GenerateThumbnailAdaptationSet(router, metadata, thumb_id)
+
+            injectPoint = StringLastIndexOf(dash, "</AdaptationSet>") + "</AdaptationSet>".Len()
+
+            newDash = dash.Left(injectPoint) + thumbnails + dash.Mid(injectPoint)
+
+            return newDash
+        end function
+
+        function GenerateThumbnailAdaptationSet(router as object, metadata as object, id as integer) as string
+            result = ""
+            storyboards = metadata.storyboards
+            for each storyboard in storyboards
+                ' BUG: Broken storyboards have interval = 0 https://github.com/iv-org/invidious/issues/3441
+                if storyboard.interval > 0
+                    ' BUG: storyboardHeight can be wrong https://github.com/iv-org/invidious/issues/3440
+                    ' TODO: this fix/assumption is only if we have storyboard.storyboardCount = 1
+                    storyboardHeight = storyboard.storyboardHeight
+                    if storyboard.storyboardCount = 1
+                        storyboardHeight = storyboard.count \ storyboard.storyboardWidth
+                        if storyboard.count mod storyboard.storyboardWidth > 0
+                            storyboardHeight += 1
+                        end if
+                    end if
+
+                    tilesPerPage = storyboard.storyboardWidth * storyboardHeight
+                    intervalSeconds = (storyboard.interval / 1000)
+
+                    ' If the template changed from known format, abort.
+                    if storyboard.templateUrl.Instr("$M") = -1
+                        return ""
+                    end if
+                    ' Youtube template uses the var $M for tile pages
+                    ' DASH-IF uses $Number$ in the SegmentTemplate
+                    url = WebUtils.EscapeUrlForXml(storyboard.templateUrl.Replace("$M", "$Number$"))
+
+                    tileCount = tilesPerPage
+                    duration = tileCount * intervalSeconds
+
+                    width = storyboard.width * storyboard.storyboardWidth
+                    height = storyboard.height * storyboardHeight
+
+                    ' Bandwidth is kind of a guess...
+                    bandwidth = Int((width * height * 0.5) / duration)
+
+                    result += `<AdaptationSet id="${id}" mimeType="image/jpeg" contentType="image">`
+                    result += `<SegmentTemplate media="${url}" duration="${duration}" startNumber="0" />`
+                    result += `<Representation id="thumbnails_${id}" bandwidth="${bandwidth}" `
+                    result += ` width="${width}" height="${height}">`
+
+                    ' TODO: the last image in the list is usually smaller than the others. For example:
+                    ' Consider a video with 85 thumbnails, in tiles of 5x5
+                    ' 1st tile is 5x5 (25)
+                    ' 2nd tile is 5x5 (50)
+                    ' 3rd tile is 5x5 (75)
+                    ' 4th tile is 5x2 (85)
+                    ' This makes the display always misinterpret the the last tile if it is smaller
+                    result += `<EssentialProperty schemeIdUri="http://dashif.org/guidelines/thumbnail_tile" value="${storyboard.storyboardWidth}x${storyboardHeight}"/>`
+                    result += `</Representation>`
+                    result += `</AdaptationSet>`
+
+                    id += 1
+                end if
+            end for
+            return result
+        end function
+
+    end class
+
+end namespace

src/components/WebServer/Task/WebServerTask.bs

@@ -8,6 +8,7 @@ import "pkg:/components/WebServer/Middleware/StateApiRouter.bs"
 import "pkg:/components/WebServer/Middleware/CommandsApiRouter.bs"
 import "pkg:/components/WebServer/Middleware/CorsMiddleware.bs"
 import "pkg:/components/WebServer/Middleware/BasicAuthMiddleware.bs"
+import "pkg:/components/WebServer/Middleware/DashRouter.bs"
 
 #const WEB_SERVER_BASIC_AUTH = false
 
@@ -21,7 +22,7 @@ function WebServerLoop()
     m.settings = new Http.HttpSettings(m.webServerPort)
     ' Root at www to get http://IP_ADDRESS:PORT/index.html
     m.settings.WwwRoot = "pkg:/www"
-    m.settings.Timeout = 5000
+    m.settings.Timeout = 2000
     m.server = new Http.HttpServer(m.settings, m)
 
     homeRouter = new Http.HttpRouter()
@@ -33,6 +34,7 @@ function WebServerLoop()
 
     m.server.UseRouter(new Http.CorsMiddleware(`http://${GetLocalIpAddress()}:8888`))
     m.server.UseRouter(homeRouter)
+    m.server.UseRouter(new Http.DashRouter())
     m.server.UseRouter(new Http.CommandsApiRouter())
     m.server.UseRouter(new Http.StateApiRouter())
     m.server.UseRouter(new Http.InvidiousRouter())

src/source/utils/StringUtils.bs

@@ -113,3 +113,15 @@ end function
 function Quote() as string
     return chr(34)
 end function
+
+function StringLastIndexOf(str as string, substr as string) as integer
+    index = str.InStr(substr)
+    while index <> -1
+        newIndex = str.InStr(index + 1, substr)
+        if newIndex = -1
+            return index
+        end if
+        index = newIndex
+    end while
+    return index
+end function

src/source/utils/WebUtils.bs

@@ -125,4 +125,18 @@ namespace WebUtils
         return validstr(hcm["n" + Stri(code).trim()])
     end function
 
+    function EscapeUrlForXml(url as string) as string
+        urlEscapeChars = {}
+        urlEscapeChars["&"] = "&"
+        urlEscapeChars["'"] = "'"
+        urlEscapeChars[`"`] = """
+        urlEscapeChars["<"] = "&lt;"
+        urlEscapeChars[">"] = "&gt;"
+
+        for each escapeChar in urlEscapeChars
+            url = url.Replace(escapeChar, urlEscapeChars[escapeChar])
+        end for
+        return url
+    end function
+
 end namespace