Merge pull request #1121 from Svallinn/improved-captions

Overhaul of the captions subsystem

package-lock.json

@@ -18399,11 +18399,6 @@
         }
       }
     },
-    "yt-xml2vtt": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/yt-xml2vtt/-/yt-xml2vtt-1.2.0.tgz",
-      "integrity": "sha512-4ZzqHIUfdPFa0Xb+8M3vsbokXooOhQuFuXa8bw4CJ5V0xWjRA/CPlZ3u0VTYoce4sUmMgoOVN7Xcj8NpUNujXA=="
-    },
     "ytdl-core": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.5.0.tgz",

package.json

@@ -51,7 +51,6 @@
     "yt-comment-scraper": "^4.0.1",
     "yt-dash-manifest-generator": "1.1.0",
     "yt-trending-scraper": "^1.1.1",
-    "yt-xml2vtt": "^1.2.0",
     "ytdl-core": "^4.5.0",
     "ytpl": "^2.0.5",
     "ytsr": "^3.3.1"

src/renderer/components/ft-video-player/ft-video-player.js

@@ -51,7 +51,7 @@ export default Vue.extend({
       type: Array,
       default: null
     },
-    captionList: {
+    captionHybridList: {
       type: Array,
       default: () => { return [] }
     },
@@ -145,16 +145,6 @@ export default Vue.extend({
       return this.$store.getters.getAutoplayVideos
     }
   },
-  watch: {
-    sourceList: function () {
-      this.determineFormatType()
-    },
-    captionList: function () {
-      this.player.caption({
-        data: this.captionList
-      })
-    }
-  },
   mounted: function () {
     this.id = this._uid
 
@@ -196,6 +186,7 @@ export default Vue.extend({
 
         this.player = videojs(videoPlayer, {
           html5: {
+            preloadTextTracks: false,
             vhs: {
               limitRenditionByPlayerDimensions: false,
               smoothQualityChange: false,
@@ -246,6 +237,9 @@ export default Vue.extend({
         this.player.on('ready', function () {
           v.$emit('ready')
           v.checkAspectRatio()
+          if (v.captionHybridList.length !== 0) {
+            v.transformAndInsertCaptions()
+          }
         })
 
         this.player.on('ended', function () {
@@ -716,6 +710,26 @@ export default Vue.extend({
       this.determineDefaultQualityDash()
     },
 
+    transformAndInsertCaptions: async function() {
+      let captionList
+      if (this.captionHybridList[0] instanceof Promise) {
+        captionList = await Promise.all(this.captionHybridList)
+        this.$emit('store-caption-list', captionList)
+      } else {
+        captionList = this.captionHybridList
+      }
+
+      for (const caption of captionList) {
+        this.player.addRemoteTextTrack({
+          kind: 'subtitles',
+          src: caption.baseUrl || caption.url,
+          srclang: caption.languageCode,
+          label: caption.label || caption.name.simpleText,
+          type: caption.type
+        }, true)
+      }
+    },
+
     toggleFullWindow: function() {
       if (!this.player.isFullscreen_) {
         if (this.player.isFullWindow) {

src/renderer/components/ft-video-player/ft-video-player.vue

@@ -18,15 +18,6 @@
         :label="source.qualityLabel"
         :selected="source.qualityLabel === selectedDefaultQuality"
       >
-      <track
-        v-for="(caption, index) in captionList"
-        :key="index + '_caption'"
-        kind="subtitles"
-        :src="caption.baseUrl || caption.url"
-        :srclang="caption.languageCode"
-        :label="caption.label || caption.name.simpleText"
-        :type="caption.type"
-      >
     </video>
   </div>
 </template>

src/renderer/views/Watch/Watch.js

@@ -1,6 +1,5 @@
 import Vue from 'vue'
 import { mapActions } from 'vuex'
-import xml2vtt from 'yt-xml2vtt'
 import $ from 'jquery'
 import fs from 'fs'
 import ytDashGen from 'yt-dash-manifest-generator'
@@ -69,7 +68,7 @@ export default Vue.extend({
       activeSourceList: [],
       videoSourceList: [],
       audioSourceList: [],
-      captionSourceList: [],
+      captionHybridList: [], // [] -> Promise[] -> string[] (URIs)
       recommendedVideos: [],
       downloadLinks: [],
       watchingPlaylist: false,
@@ -156,7 +155,7 @@ export default Vue.extend({
       this.firstLoad = true
       this.activeFormat = this.defaultVideoFormat
       this.videoStoryboardSrc = ''
-      this.captionSourceList = []
+      this.captionHybridList = []
       this.downloadLinks = []
 
       this.checkIfPlaylist()
@@ -293,17 +292,6 @@ export default Vue.extend({
           this.isLiveContent = result.player_response.videoDetails.isLiveContent
           this.isUpcoming = result.player_response.videoDetails.isUpcoming ? result.player_response.videoDetails.isUpcoming : false
 
-          if (!this.isLive && !this.isUpcoming) {
-            const captionTracks =
-              result.player_response.captions &&
-              result.player_response.captions.playerCaptionsTracklistRenderer
-                .captionTracks
-
-            if (typeof captionTracks !== 'undefined') {
-              await this.createCaptionUrls(captionTracks)
-            }
-          }
-
           if (this.videoDislikeCount === null && !this.hideVideoLikesAndDislikes) {
             this.videoDislikeCount = 0
           }
@@ -390,16 +378,32 @@ export default Vue.extend({
 
                 return object
               })
-              let captionLinks = result.player_response.captions
+
-              if (typeof captionLinks !== 'undefined') {
+              const captionTracks =
-                captionLinks = captionLinks.playerCaptionsTracklistRenderer.captionTracks.map((caption) => {
+                result.player_response.captions &&
+                result.player_response.captions.playerCaptionsTracklistRenderer
+                  .captionTracks
+
+              if (typeof captionTracks !== 'undefined') {
+                const standardLocale = localStorage.getItem('locale').replace('_', '-')
+                const noLocaleCaption = !captionTracks.some(track =>
+                  track.languageCode === standardLocale && track.kind !== 'asr'
+                )
+
+                if (!standardLocale.startsWith('en') && noLocaleCaption) {
+                  const baseUrl = result.player_response.captions.playerCaptionsRenderer.baseUrl
+                  this.tryAddingTranslatedLocaleCaption(captionTracks, standardLocale, baseUrl)
+                }
+
+                this.captionHybridList = this.createCaptionPromiseList(captionTracks)
+
+                const captionLinks = captionTracks.map((caption) => {
                   const label = `${caption.name.simpleText} (${caption.languageCode}) - text/vtt`
-                  const object = {
+
+                  return {
                     url: caption.baseUrl,
                     label: label
                   }
-
-                  return object
                 })
 
                 this.downloadLinks = this.downloadLinks.concat(captionLinks)
@@ -528,7 +532,7 @@ export default Vue.extend({
           this.videoDescriptionHtml = result.descriptionHtml
           this.recommendedVideos = result.recommendedVideos
           this.isLive = result.liveNow
-          this.captionSourceList = result.captions.map(caption => {
+          this.captionHybridList = result.captions.map(caption => {
             caption.url = this.invidiousInstance + caption.url
             caption.type = ''
             caption.dataSource = 'invidious'
@@ -1073,29 +1077,80 @@ export default Vue.extend({
       })
     },
 
-    createCaptionUrls: function (captionTracks) {
+    tryAddingTranslatedLocaleCaption: function (captionTracks, locale, baseUrl) {
-      this.captionSourceList = captionTracks.map(caption => {
+      const enCaptionIdx = captionTracks.findIndex(track =>
+        track.languageCode === 'en' && track.kind !== 'asr'
+      )
+
+      const enCaptionExists = enCaptionIdx !== -1
+      const asrEnabled = captionTracks.some(track => track.kind === 'asr')
+
+      if (enCaptionExists || asrEnabled) {
+        let label
+        let url
+
+        if (this.$te('Video.translated from English') && this.$t('Video.translated from English') !== '') {
+          label = `${this.$t('Locale Name')} (${this.$t('Video.translated from English')})`
+        } else {
+          label = `${this.$t('Locale Name')} (translated from English)`
+        }
+
+        if (enCaptionExists) {
+          url = new URL(captionTracks[enCaptionIdx].baseUrl)
+        } else {
+          url = new URL(baseUrl)
+          url.searchParams.set('lang', 'en')
+          url.searchParams.set('kind', 'asr')
+        }
+
+        url.searchParams.set('tlang', locale)
+        captionTracks.unshift({
+          baseUrl: url.toString(),
+          name: { simpleText: label },
+          languageCode: locale
+        })
+      }
+    },
+
+    createCaptionPromiseList: function (captionTracks) {
+      return captionTracks.map(caption => new Promise((resolve, reject) => {
         caption.type = 'text/vtt'
         caption.charset = 'charset=utf-8'
         caption.dataSource = 'local'
 
-        $.get(caption.baseUrl, response => {
+        const url = new URL(caption.baseUrl)
-          xml2vtt
+        url.searchParams.set('fmt', 'vtt')
-            .Parse(new XMLSerializer().serializeToString(response))
+
-            .then(vtt => {
+        $.get(url.toString(), response => {
+          // The character '#' needs to be percent-encoded in a (data) URI
+          // because it signals an identifier, which means anything after it
+          // is automatically removed when the URI is used as a source
+          let vtt = response.replace(/#/g, '%23')
+
+          // A lot of videos have messed up caption positions that need to be removed
+          // This can be either because this format isn't really used by YouTube
+          // or because it's expected for the player to be able to somehow
+          // wrap the captions so that they won't step outside its boundaries
+          //
+          // Auto-generated captions are also all aligned to the start
+          // so those instances must also be removed
+          // In addition, all aligns seem to be fixed to "start" when they do pop up in normal captions
+          // If it's prominent enough that people start to notice, it can be removed then
+          if (caption.kind === 'asr') {
+            vtt = vtt.replace(/ align:start| position:\d{1,3}%/g, '')
+          } else {
+            vtt = vtt.replace(/ position:\d{1,3}%/g, '')
+          }
+
           caption.baseUrl = `data:${caption.type};${caption.charset},${vtt}`
-            })
+          resolve(caption)
-            .catch(err =>
-              console.log(`Error while converting XML to VTT : ${err}`)
-            )
         }).fail((xhr, textStatus, error) => {
           console.log(xhr)
           console.log(textStatus)
           console.log(error)
+          reject(error)
         })
-
+      }))
-        return caption
-      })
     },
 
     getWatchedProgress: function () {

src/renderer/views/Watch/Watch.vue

@@ -18,7 +18,7 @@
           ref="videoPlayer"
           :dash-src="dashSrc"
           :source-list="activeSourceList"
-          :caption-list="captionSourceList"
+          :caption-hybrid-list="captionHybridList"
           :storyboard-src="videoStoryboardSrc"
           :format="activeFormat"
           :thumbnail="thumbnail"
@@ -27,6 +27,7 @@
           @ready="checkIfWatched"
           @ended="handleVideoEnded"
           @error="handleVideoError"
+          @store-caption-list="captionHybridList = $event"
         />
         <div
           v-if="!isLoading && isUpcoming"

static/locales/en-US.yaml

@@ -467,6 +467,7 @@ Video:
   Published on: Published on
   Streamed on: Streamed on
   Started streaming on: Started streaming on
+  translated from English: translated from English
   # $ is replaced with the number and % with the unit (days, hours, minutes...)
   Publicationtemplate: $ % ago
 #& Videos