Update comment scraper logic
This commit is contained in:
Preston
parent
4cec171bc8
commit
b7846493ef
|
|
@ -9014,8 +9014,7 @@
|
|||
"he": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
||||
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
|
||||
"dev": true
|
||||
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
|
||||
},
|
||||
"highlight.js": {
|
||||
"version": "9.18.3",
|
||||
|
|
@ -9167,11 +9166,6 @@
|
|||
"util.promisify": "1.0.0"
|
||||
}
|
||||
},
|
||||
"html2json": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/html2json/-/html2json-1.0.2.tgz",
|
||||
"integrity": "sha1-ydbSAvplQCOGwgKzRc9RvOgO0e8="
|
||||
},
|
||||
"htmlparser2": {
|
||||
"version": "3.10.1",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
|
||||
|
|
@ -12586,6 +12580,14 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node-html-parser": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-2.0.2.tgz",
|
||||
"integrity": "sha512-N2000Ho9dkabvRZcyiwm6zOpdiAzxAxcJ0Z0WNoh/yXHG0YCuiK2WpNQfN+9vheLNY/h/It11Gk7uwT4QTfk9Q==",
|
||||
"requires": {
|
||||
"he": "1.2.0"
|
||||
}
|
||||
},
|
||||
"node-int64": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
|
||||
|
|
@ -19191,12 +19193,12 @@
|
|||
}
|
||||
},
|
||||
"yt-comment-scraper": {
|
||||
"version": "1.3.11",
|
||||
"resolved": "https://registry.npmjs.org/yt-comment-scraper/-/yt-comment-scraper-1.3.11.tgz",
|
||||
"integrity": "sha512-F7a1/BAjASsX3MheAqO1uvY0sPfstLKmkHETiFyDe3Hp+qUMyniuPmAvXrZPOanvBHyb6sH5+d3wVj/S9dD/ug==",
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yt-comment-scraper/-/yt-comment-scraper-2.0.0.tgz",
|
||||
"integrity": "sha512-goa9Z5REPXRKddKO6MJUzaa/cBgdQPj/akIGbyb1R5KnOgl4bLy6d4nD1pVUhLkP0Z8aEqwJAMmYpPBjeXTOIg==",
|
||||
"requires": {
|
||||
"axios": "^0.21.1",
|
||||
"html2json": "^1.0.2"
|
||||
"node-html-parser": "^2.0.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": {
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@
|
|||
"youtube-chat": "^1.1.0",
|
||||
"youtube-suggest": "^1.1.0",
|
||||
"yt-channel-info": "^1.2.0",
|
||||
"yt-comment-scraper": "^1.3.11",
|
||||
"yt-comment-scraper": "^2.0.0",
|
||||
"yt-dash-manifest-generator": "^1.1.0",
|
||||
"yt-trending-scraper": "1.1.0",
|
||||
"yt-xml2vtt": "^1.2.0",
|
||||
|
|
|
|||
|
|
@ -5,10 +5,7 @@ import FtLoader from '../../components/ft-loader/ft-loader.vue'
|
|||
import FtSelect from '../../components/ft-select/ft-select.vue'
|
||||
import FtTimestampCatcher from '../../components/ft-timestamp-catcher/ft-timestamp-catcher.vue'
|
||||
import autolinker from 'autolinker'
|
||||
import { fork } from 'child_process'
|
||||
import path from 'path'
|
||||
// eslint-disable-next-line
|
||||
import commentControllerRelativePath from 'file-loader!../../../process/comment-module-controller.js'
|
||||
import ytcm from 'yt-comment-scraper'
|
||||
|
||||
export default Vue.extend({
|
||||
name: 'WatchVideoComments',
|
||||
|
|
@ -93,8 +90,9 @@ export default Vue.extend({
|
|||
this.sortNewest = !this.sortNewest
|
||||
switch (this.backendPreference) {
|
||||
case 'local':
|
||||
console.log('In handle')
|
||||
this.sortingChanged = true
|
||||
this.isLoading = true
|
||||
this.commentData = []
|
||||
this.nextPageToken = undefined
|
||||
this.getCommentDataLocal()
|
||||
break
|
||||
case 'invidious':
|
||||
|
|
@ -122,7 +120,6 @@ export default Vue.extend({
|
|||
this.showToast({
|
||||
message: this.$t('Comments.There are no more comments for this video')
|
||||
})
|
||||
this.getCommentData()
|
||||
} else {
|
||||
this.getCommentData()
|
||||
}
|
||||
|
|
@ -144,31 +141,16 @@ export default Vue.extend({
|
|||
},
|
||||
|
||||
getCommentDataLocal: function () {
|
||||
// we need the path from the working directory to fork correctly
|
||||
if (this.commentProcess === null) {
|
||||
let modulePath
|
||||
if (this.isDev) {
|
||||
modulePath = '../../../process/comment-module-controller.js'
|
||||
} else {
|
||||
modulePath = commentControllerRelativePath
|
||||
const payload = {
|
||||
videoId: this.id,
|
||||
setCookie: false,
|
||||
sortByNewest: this.sortNewest,
|
||||
continuation: this.nextPageToken ? this.nextPageToken : undefined
|
||||
}
|
||||
|
||||
this.commentProcess = fork(path.join(__dirname, modulePath))
|
||||
|
||||
this.commentProcess.on('message', (msg) => {
|
||||
if (msg.error === null) {
|
||||
const commentJSON = JSON.parse(msg.comments)
|
||||
if (commentJSON === null) {
|
||||
this.showToast({
|
||||
message: this.$t('Comments.No more comments available'),
|
||||
time: 7000,
|
||||
action: () => {
|
||||
}
|
||||
})
|
||||
this.isLoading = false
|
||||
} else {
|
||||
// console.log(msg.comments)
|
||||
const commentData = commentJSON.map((comment) => {
|
||||
ytcm.getComments(payload).then((response) => {
|
||||
console.log(response)
|
||||
const commentData = response.comments.map((comment) => {
|
||||
comment.showReplies = false
|
||||
comment.dataType = 'local'
|
||||
this.toLocalePublicationString({
|
||||
|
|
@ -189,9 +171,13 @@ export default Vue.extend({
|
|||
comment.likes = null
|
||||
}
|
||||
comment.text = autolinker.link(comment.text)
|
||||
|
||||
if (comment.numReplies > 0) {
|
||||
comment.replies.forEach((reply) => {
|
||||
reply.text = autolinker.link(reply.text)
|
||||
})
|
||||
}
|
||||
|
||||
return comment
|
||||
})
|
||||
if (this.sortingChanged) {
|
||||
|
|
@ -201,16 +187,15 @@ export default Vue.extend({
|
|||
this.commentData = this.commentData.concat(commentData)
|
||||
this.isLoading = false
|
||||
this.showComments = true
|
||||
this.nextPageToken = ''
|
||||
}
|
||||
} else {
|
||||
console.log(msg.error)
|
||||
this.nextPageToken = response.continuation
|
||||
}).catch((err) => {
|
||||
console.log(err)
|
||||
const errorMessage = this.$t('Local API Error (Click to copy)')
|
||||
this.showToast({
|
||||
message: `${errorMessage}: ${msg.error}`,
|
||||
message: `${errorMessage}: ${err}`,
|
||||
time: 10000,
|
||||
action: () => {
|
||||
navigator.clipboard.writeText(msg.error)
|
||||
navigator.clipboard.writeText(err)
|
||||
}
|
||||
})
|
||||
if (this.backendFallback && this.backendPreference === 'local') {
|
||||
|
|
@ -221,11 +206,7 @@ export default Vue.extend({
|
|||
} else {
|
||||
this.isLoading = false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
this.commentProcess.send({ id: this.id, sortNewest: this.sortNewest })
|
||||
},
|
||||
|
||||
getCommentDataInvidious: function () {
|
||||
|
|
@ -267,7 +248,6 @@ export default Vue.extend({
|
|||
return comment
|
||||
})
|
||||
|
||||
console.log(commentData)
|
||||
this.commentData = this.commentData.concat(commentData)
|
||||
this.nextPageToken = response.continuation
|
||||
this.isLoading = false
|
||||
|
|
@ -327,7 +307,6 @@ export default Vue.extend({
|
|||
return comment
|
||||
})
|
||||
|
||||
console.log(commentData)
|
||||
this.commentData[index].replies = commentData
|
||||
this.commentData[index].showReplies = true
|
||||
this.isLoading = false
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
<template>
|
||||
<ft-card>
|
||||
<ft-loader
|
||||
v-if="isLoading"
|
||||
/>
|
||||
<h4
|
||||
v-if="commentData.length === 0 && !isLoading"
|
||||
class="getCommentsTitle"
|
||||
|
|
@ -148,6 +145,9 @@
|
|||
>
|
||||
{{ $t("Comments.Load More Comments") }}
|
||||
</h4>
|
||||
<ft-loader
|
||||
v-if="isLoading"
|
||||
/>
|
||||
</ft-card>
|
||||
</template>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue