Added implementation of child process for comment scraping
This commit is contained in:
parent
91e10a4716
commit
5b83c45d21
|
@ -0,0 +1,24 @@
|
|||
const ModuleScraper = require('yt-comment-scraper')
|
||||
const fs = require('fs')
|
||||
let scraper = null
|
||||
let currentSort = null
|
||||
let currentVideoId = null
|
||||
fs.writeFileSync('D:/Workspace/textELEC.txt', 'hallololol')
|
||||
process.on('message', (msg) => {
|
||||
if (msg === 'end') {
|
||||
process.exit(0)
|
||||
}
|
||||
if (msg.id !== currentVideoId || msg.sortNewest !== currentSort) {
|
||||
if (scraper !== null) {
|
||||
scraper.cleanupStatics()
|
||||
}
|
||||
currentSort = msg.sortNewest
|
||||
currentVideoId = msg.id
|
||||
scraper = new ModuleScraper(true, currentSort)
|
||||
}
|
||||
scraper.scrape_next_page_youtube_comments(currentVideoId).then((comments) => {
|
||||
process.send({ comments: JSON.stringify(comments), error: null })
|
||||
}).catch((error) => {
|
||||
process.send({ comments: null, error: error })
|
||||
})
|
||||
})
|
|
@ -5,7 +5,8 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue'
|
|||
import FtSelect from '../../components/ft-select/ft-select.vue'
|
||||
import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue'
|
||||
import autolinker from 'autolinker'
|
||||
import CommentScraper from 'yt-comment-scraper'
|
||||
import { fork } from 'child_process'
|
||||
import path from 'path'
|
||||
|
||||
export default Vue.extend({
|
||||
name: 'WatchVideoComments',
|
||||
|
@ -28,7 +29,9 @@ export default Vue.extend({
|
|||
commentScraper: null,
|
||||
nextPageToken: null,
|
||||
commentData: [],
|
||||
sortNewest: false
|
||||
sortNewest: false,
|
||||
commentProcess: null,
|
||||
sortingChanged: false
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
|
@ -65,6 +68,12 @@ export default Vue.extend({
|
|||
return (this.sortNewest) ? 'newest' : 'top'
|
||||
}
|
||||
},
|
||||
|
||||
beforeDestroy: function () {
|
||||
if (this.commentProcess !== null) {
|
||||
this.commentProcess.send('end')
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onTimestamp: function (timestamp) {
|
||||
this.$emit('timestamp-event', timestamp)
|
||||
|
@ -74,7 +83,9 @@ export default Vue.extend({
|
|||
this.sortNewest = !this.sortNewest
|
||||
switch (this.backendPreference) {
|
||||
case 'local':
|
||||
this.getCommentDataLocal(true)
|
||||
console.log('In handle')
|
||||
this.sortingChanged = true
|
||||
this.getCommentDataLocal()
|
||||
break
|
||||
case 'invidious':
|
||||
this.isLoading = true
|
||||
|
@ -84,11 +95,11 @@ export default Vue.extend({
|
|||
}
|
||||
},
|
||||
|
||||
getCommentData: function (sortChanged = false) {
|
||||
getCommentData: function () {
|
||||
this.isLoading = true
|
||||
switch (this.backendPreference) {
|
||||
case 'local':
|
||||
this.getCommentDataLocal(sortChanged)
|
||||
this.getCommentDataLocal()
|
||||
break
|
||||
case 'invidious':
|
||||
this.getCommentDataInvidious(this.nextPageToken)
|
||||
|
@ -122,13 +133,17 @@ export default Vue.extend({
|
|||
}
|
||||
},
|
||||
|
||||
getCommentDataLocal: function (sortChanged = false) {
|
||||
if (this.commentScraper === null || sortChanged === true) {
|
||||
this.commentScraper = new CommentScraper(false, this.sortNewest)
|
||||
this.commentData = []
|
||||
}
|
||||
this.commentScraper.scrape_next_page_youtube_comments(this.id).then((response) => {
|
||||
if (response === null) {
|
||||
getCommentDataLocal: function () {
|
||||
// we need the path from the working directory to fork correctly
|
||||
if (this.commentProcess === null) {
|
||||
this.commentProcess = fork('./src/process/comment-module-controller.js', ['args'], {
|
||||
stdio: 'pipe',
|
||||
cwd: path.join(__dirname, '../../')
|
||||
})
|
||||
this.commentProcess.on('message', (msg) => {
|
||||
if (msg.error === null) {
|
||||
const commentJSON = JSON.parse(msg.comments)
|
||||
if (commentJSON === null) {
|
||||
this.showToast({
|
||||
message: this.$t('Comments.No more comments available'),
|
||||
time: 7000,
|
||||
|
@ -136,10 +151,9 @@ export default Vue.extend({
|
|||
}
|
||||
})
|
||||
this.isLoading = false
|
||||
return
|
||||
}
|
||||
console.log(response)
|
||||
const commentData = response.map((comment) => {
|
||||
} else {
|
||||
// console.log(msg.comments)
|
||||
const commentData = commentJSON.map((comment) => {
|
||||
comment.showReplies = false
|
||||
comment.dataType = 'local'
|
||||
this.toLocalePublicationString({
|
||||
|
@ -165,18 +179,23 @@ export default Vue.extend({
|
|||
})
|
||||
return comment
|
||||
})
|
||||
if (this.sortingChanged) {
|
||||
this.commentData = []
|
||||
this.sortingChanged = false
|
||||
}
|
||||
this.commentData = this.commentData.concat(commentData)
|
||||
this.isLoading = false
|
||||
this.showComments = true
|
||||
this.nextPageToken = ''
|
||||
}).catch((err) => {
|
||||
console.log(err)
|
||||
}
|
||||
} else {
|
||||
console.log(msg.error)
|
||||
const errorMessage = this.$t('Local API Error (Click to copy)')
|
||||
this.showToast({
|
||||
message: `${errorMessage}: ${err}`,
|
||||
message: `${errorMessage}: ${msg.error}`,
|
||||
time: 10000,
|
||||
action: () => {
|
||||
navigator.clipboard.writeText(err)
|
||||
navigator.clipboard.writeText(msg.error)
|
||||
}
|
||||
})
|
||||
if (this.backendFallback && this.backendPreference === 'local') {
|
||||
|
@ -187,7 +206,10 @@ export default Vue.extend({
|
|||
} else {
|
||||
this.isLoading = false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest })
|
||||
},
|
||||
|
||||
getCommentDataInvidious: function () {
|
||||
|
|
Loading…
Reference in New Issue