Merge pull request #674 from GilgusMaximus/development

Comment scraper child process [Help needed]

_scripts/webpack.web.config.js

@@ -111,6 +111,7 @@ const config = {
     fs: 'empty',
     net: 'empty',
     tls: 'empty',
+    child_process: 'empty'
   },
   plugins: [
     // new WriteFilePlugin(),

src/process/comment-module-controller.js

@@ -0,0 +1,23 @@
+const ModuleScraper = require('yt-comment-scraper')
+let scraper = null
+let currentSort = null
+let currentVideoId = null
+
+process.on('message', (msg) => {
+  if (msg === 'end') {
+    process.exit(0)
+  }
+  if (msg.id !== currentVideoId || msg.sortNewest !== currentSort) {
+    if (scraper !== null) {
+      scraper.cleanupStatics()
+    }
+    currentSort = msg.sortNewest
+    currentVideoId = msg.id
+    scraper = new ModuleScraper(true, currentSort)
+  }
+  scraper.scrape_next_page_youtube_comments(currentVideoId).then((comments) => {
+    process.send({ comments: JSON.stringify(comments), error: null })
+  }).catch((error) => {
+    process.send({ comments: null, error: error })
+  })
+})

src/renderer/components/watch-video-comments/watch-video-comments.js

@@ -5,7 +5,10 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue'
 import FtSelect from '../../components/ft-select/ft-select.vue'
 import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue'
 import autolinker from 'autolinker'
-import CommentScraper from 'yt-comment-scraper'
+import { fork } from 'child_process'
+import path from 'path'
+// eslint-disable-next-line
+import commentControllerRelativePath from 'file-loader!../../../process/comment-module-controller.js'
 
 export default Vue.extend({
   name: 'WatchVideoComments',
@@ -32,10 +35,16 @@ export default Vue.extend({
       commentScraper: null,
       nextPageToken: null,
       commentData: [],
-      sortNewest: false
+      sortNewest: false,
+      commentProcess: null,
+      sortingChanged: false
     }
   },
   computed: {
+    isDev: function () {
+      return process.env.NODE_ENV === 'development'
+    },
+
     backendPreference: function () {
       return this.$store.getters.getBackendPreference
     },
@@ -69,6 +78,12 @@ export default Vue.extend({
       return (this.sortNewest) ? 'newest' : 'top'
     }
   },
+
+  beforeDestroy: function () {
+    if (this.commentProcess !== null) {
+      this.commentProcess.send('end')
+    }
+  },
   methods: {
     onTimestamp: function (timestamp) {
       this.$emit('timestamp-event', timestamp)
@@ -78,7 +93,9 @@ export default Vue.extend({
       this.sortNewest = !this.sortNewest
       switch (this.backendPreference) {
         case 'local':
-          this.getCommentDataLocal(true)
+          console.log('In handle')
+          this.sortingChanged = true
+          this.getCommentDataLocal()
           break
         case 'invidious':
           this.isLoading = true
@@ -88,11 +105,11 @@ export default Vue.extend({
       }
     },
 
-    getCommentData: function (sortChanged = false) {
+    getCommentData: function () {
       this.isLoading = true
       switch (this.backendPreference) {
         case 'local':
-          this.getCommentDataLocal(sortChanged)
+          this.getCommentDataLocal()
           break
         case 'invidious':
           this.getCommentDataInvidious(this.nextPageToken)
@@ -126,13 +143,24 @@ export default Vue.extend({
       }
     },
 
-    getCommentDataLocal: function (sortChanged = false) {
-      if (this.commentScraper === null || sortChanged === true) {
-        this.commentScraper = new CommentScraper(false, this.sortNewest)
-        this.commentData = []
+    getCommentDataLocal: function () {
+      // we need the path from the working directory to fork correctly
+      if (this.commentProcess === null) {
+        let modulePath
+        if (this.isDev) {
+          modulePath = '../../../process/comment-module-controller.js'
+        } else {
+          modulePath = commentControllerRelativePath
         }
-      this.commentScraper.scrape_next_page_youtube_comments(this.id).then((response) => {
-        if (response === null) {
+
+        this.commentProcess = fork(path.join(__dirname, modulePath), ['args'], {
+          stdio: ['pipe', 'pipe', 'pipe', 'ipc']
+        })
+
+        this.commentProcess.on('message', (msg) => {
+          if (msg.error === null) {
+            const commentJSON = JSON.parse(msg.comments)
+            if (commentJSON === null) {
               this.showToast({
                 message: this.$t('Comments.No more comments available'),
                 time: 7000,
@@ -140,10 +168,9 @@ export default Vue.extend({
                 }
               })
               this.isLoading = false
-          return
-        }
-        console.log(response)
-        const commentData = response.map((comment) => {
+            } else {
+              // console.log(msg.comments)
+              const commentData = commentJSON.map((comment) => {
                 comment.showReplies = false
                 comment.dataType = 'local'
                 this.toLocalePublicationString({
@@ -169,18 +196,23 @@ export default Vue.extend({
                 })
                 return comment
               })
+              if (this.sortingChanged) {
+                this.commentData = []
+                this.sortingChanged = false
+              }
               this.commentData = this.commentData.concat(commentData)
               this.isLoading = false
               this.showComments = true
               this.nextPageToken = ''
-      }).catch((err) => {
-        console.log(err)
+            }
+          } else {
+            console.log(msg.error)
             const errorMessage = this.$t('Local API Error (Click to copy)')
             this.showToast({
-          message: `${errorMessage}: ${err}`,
+              message: `${errorMessage}: ${msg.error}`,
               time: 10000,
               action: () => {
-            navigator.clipboard.writeText(err)
+                navigator.clipboard.writeText(msg.error)
               }
             })
             if (this.backendFallback && this.backendPreference === 'local') {
@@ -191,7 +223,11 @@ export default Vue.extend({
             } else {
               this.isLoading = false
             }
+          }
         })
+      }
+
+      this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest })
     },
 
     getCommentDataInvidious: function () {