tesses50
/
freetube
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace rss-parser with fetch and the native DomParser (#2726)

This commit is contained in:
absidue 2022-10-17 18:09:06 +02:00 committed by GitHub
parent b5c486bf1f
commit 294df19f1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 111 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -64,7 +64,6 @@
"nedb-promises": "^6.2.1",
"opml-to-json": "^1.0.1",
"process": "^0.11.10",
"rss-parser": "^3.12.0",
"socks-proxy-agent": "^6.0.0",
"video.js": "7.18.1",
"videojs-contrib-quality-levels": "^2.1.0",

19
src/renderer/App.js
View File

@ -10,7 +10,6 @@ import FtButton from './components/ft-button/ft-button.vue'
import FtToast from './components/ft-toast/ft-toast.vue'
import FtProgressBar from './components/ft-progress-bar/ft-progress-bar.vue'
import { marked } from 'marked'
import Parser from 'rss-parser'
import { IpcChannels } from '../constants'
import packageDetails from '../../package.json'
import { showToast } from './helpers/utils'
@ -229,21 +228,25 @@ export default Vue.extend({
checkForNewBlogPosts: function () {
if (this.checkForBlogPosts) {
const parser = new Parser()
const feedUrl = 'https://write.as/freetube/feed/'
let lastAppWasRunning = localStorage.getItem('lastAppWasRunning')
if (lastAppWasRunning !== null) {
lastAppWasRunning = new Date(lastAppWasRunning)
}
parser.parseURL(feedUrl).then((response) => {
const latestBlog = response.items[0]
const latestPubDate = new Date(latestBlog.pubDate)
fetch('https://write.as/freetube/feed/')
.then(response => response.text())
.then(response => {
const xmlDom = new DOMParser().parseFromString(response, 'application/xml')
const latestBlog = xmlDom.querySelector('item')
const latestPubDate = new Date(latestBlog.querySelector('pubDate').textContent)
if (lastAppWasRunning === null || latestPubDate > lastAppWasRunning) {
this.blogBannerMessage = this.$t('A new blog is now available, {blogTitle}. Click to view more', { blogTitle: latestBlog.title })
this.latestBlogUrl = latestBlog.link
const title = latestBlog.querySelector('title').textContent
this.blogBannerMessage = this.$t('A new blog is now available, {blogTitle}. Click to view more', { blogTitle: title })
this.latestBlogUrl = latestBlog.querySelector('link').textContent
this.showBlogBanner = true
}

143
src/renderer/views/Subscriptions/Subscriptions.js
View File

@ -9,7 +9,6 @@ import FtElementList from '../../components/ft-element-list/ft-element-list.vue'
import FtChannelBubble from '../../components/ft-channel-bubble/ft-channel-bubble.vue'
import ytch from 'yt-channel-info'
import Parser from 'rss-parser'
import { MAIN_PROFILE_ID } from '../../../constants'
import { calculatePublishedDate, showToast } from '../../helpers/utils'
@ -299,62 +298,40 @@ export default Vue.extend({
})
},
getChannelVideosLocalRSS: function (channel, failedAttempts = 0) {
return new Promise((resolve, reject) => {
const parser = new Parser()
getChannelVideosLocalRSS: async function (channel, failedAttempts = 0) {
const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${channel.id}`
parser.parseURL(feedUrl).then(async (feed) => {
const items = await Promise.all(feed.items.map((video) => {
video.authorId = channel.id
video.videoId = video.id.replace('yt:video:', '')
video.type = 'video'
video.lengthSeconds = '0:00'
video.isRSS = true
try {
const response = await fetch(feedUrl)
video.publishedDate = new Date(video.pubDate)
if (video.publishedDate.toString() === 'Invalid Date') {
video.publishedDate = new Date(video.isoDate)
if (response.status === 404) {
this.errorChannels.push(channel)
return []
}
video.publishedText = video.publishedDate.toLocaleString()
return video
}))
resolve(items)
}).catch((err) => {
console.error(err)
if (err.toString().match(/404/)) {
this.errorChannels.push(channel)
resolve([])
} else {
return await this.parseYouTubeRSSFeed(await response.text(), channel.id)
} catch (error) {
console.error(error)
const errorMessage = this.$t('Local API Error (Click to copy)')
showToast(`${errorMessage}: ${err}`, 10000, () => {
this.copyToClipboard({ content: err })
showToast(`${errorMessage}: ${error}`, 10000, () => {
this.copyToClipboard({ content: error })
})
switch (failedAttempts) {
case 0:
resolve(this.getChannelVideosLocalScraper(channel, failedAttempts + 1))
break
return this.getChannelVideosLocalScraper(channel, failedAttempts + 1)
case 1:
if (this.backendFallback) {
showToast(this.$t('Falling back to Invidious API'))
resolve(this.getChannelVideosInvidiousRSS(channel, failedAttempts + 1))
this.showToast(this.$t('Falling back to Invidious API'))
return this.getChannelVideosInvidiousRSS(channel, failedAttempts + 1)
} else {
resolve([])
return []
}
break
case 2:
resolve(this.getChannelVideosLocalScraper(channel, failedAttempts + 1))
break
return this.getChannelVideosLocalScraper(channel, failedAttempts + 1)
default:
resolve([])
return []
}
}
})
})
},
getChannelVideosInvidiousScraper: function (channel, failedAttempts = 0) {
@ -398,54 +375,72 @@ export default Vue.extend({
})
},
getChannelVideosInvidiousRSS: function (channel, failedAttempts = 0) {
return new Promise((resolve, reject) => {
const parser = new Parser()
getChannelVideosInvidiousRSS: async function (channel, failedAttempts = 0) {
const feedUrl = `${this.currentInvidiousInstance}/feed/channel/${channel.id}`
parser.parseURL(feedUrl).then(async (feed) => {
resolve(await Promise.all(feed.items.map((video) => {
video.authorId = channel.id
video.videoId = video.id.replace('yt:video:', '')
video.type = 'video'
video.publishedDate = new Date(video.pubDate)
video.publishedText = video.publishedDate.toLocaleString()
video.lengthSeconds = '0:00'
video.isRSS = true
try {
const response = await fetch(feedUrl)
return video
})))
}).catch((err) => {
console.error(err)
const errorMessage = this.$t('Invidious API Error (Click to copy)')
showToast(`${errorMessage}: ${err}`, 10000, () => {
this.copyToClipboard({ content: err })
})
if (err.toString().match(/500/)) {
if (response.status === 500) {
this.errorChannels.push(channel)
resolve([])
} else {
return []
}
return await this.parseYouTubeRSSFeed(await response.text(), channel.id)
} catch (error) {
console.error(error)
const errorMessage = this.$t('Invidious API Error (Click to copy)')
showToast(`${errorMessage}: ${error}`, 10000, () => {
this.copyToClipboard({ content: error })
})
switch (failedAttempts) {
case 0:
resolve(this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1))
break
return this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1)
case 1:
if (this.backendFallback) {
showToast(this.$t('Falling back to the local API'))
resolve(this.getChannelVideosLocalRSS(channel, failedAttempts + 1))
this.showToast(this.$t('Falling back to the local API'))
return this.getChannelVideosLocalRSS(channel, failedAttempts + 1)
} else {
resolve([])
return []
}
break
case 2:
resolve(this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1))
break
return this.getChannelVideosInvidiousScraper(channel, failedAttempts + 1)
default:
resolve([])
return []
}
}
})
})
},
async parseYouTubeRSSFeed(rssString, channelId) {
const xmlDom = new DOMParser().parseFromString(rssString, 'application/xml')
const channelName = xmlDom.querySelector('author > name').textContent
const entries = xmlDom.querySelectorAll('entry')
const promises = []
for (const entry of entries) {
promises.push(this.parseRSSEntry(entry, channelId, channelName))
}
return await Promise.all(promises)
},
async parseRSSEntry(entry, channelId, channelName) {
const published = new Date(entry.querySelector('published').textContent)
return {
authorId: channelId,
author: channelName,
// querySelector doesn't support xml namespaces so we have to use getElementsByTagName here
videoId: entry.getElementsByTagName('yt:videoId')[0].textContent,
title: entry.querySelector('title').textContent,
publishedDate: published,
publishedText: published.toLocaleString(),
viewCount: entry.getElementsByTagName('media:statistics')[0]?.getAttribute('views') || null,
type: 'video',
lengthSeconds: '0:00',
isRSS: true
}
},
increaseLimit: function () {

25
yarn.lock
View File

@ -3553,7 +3553,7 @@ enquirer@^2.3.5:
dependencies:
ansi-colors "^4.1.1"
entities@^2.0.0, entities@^2.0.3:
entities@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
@ -7027,14 +7027,6 @@ roarr@^2.15.3:
semver-compare "^1.0.0"
sprintf-js "^1.1.2"
rss-parser@^3.12.0:
version "3.12.0"
resolved "https://registry.yarnpkg.com/rss-parser/-/rss-parser-3.12.0.tgz#b8888699ea46304a74363fbd8144671b2997984c"
integrity sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==
dependencies:
entities "^2.0.3"
xml2js "^0.4.19"
run-parallel@^1.1.9:
version "1.2.0"
resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
@ -7104,7 +7096,7 @@ sass@^1.54.9:
immutable "^4.0.0"
source-map-js ">=0.6.2 <2.0.0"
sax@>=0.6.0, sax@^1.1.3, sax@^1.2.4:
sax@^1.1.3, sax@^1.2.4:
version "1.2.4"
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@ -8482,14 +8474,6 @@ xml-name-validator@^4.0.0:
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"
integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==
xml2js@^0.4.19:
version "0.4.23"
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
dependencies:
sax ">=0.6.0"
xmlbuilder "~11.0.0"
xmlbuilder@>=11.0.1:
version "15.1.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5"
@ -8500,11 +8484,6 @@ xmlbuilder@^9.0.7:
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
xmlbuilder@~11.0.0:
version "11.0.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2, xtend@~4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"