From 73b2f98cb51089d028b3d82e85fc2de4bce2d7f9 Mon Sep 17 00:00:00 2001 From: Karl0ss Date: Tue, 24 Jan 2023 12:04:14 +0000 Subject: [PATCH] latest updates --- .gitignore | 2 ++ FeedFilter.js | 12 +++++++----- JDRssDownloader.js | 3 ++- apiFunctions.js | 21 +++++++++++++++++++-- checkDownloadHistory.js | 5 ----- package.json | 2 +- retryCache.js | 21 +++++++++++++++++++++ routes/cache.js | 25 +++++++++++++++++++++++++ routes/root.js | 2 ++ utils.js | 31 +++++++++++++++++++++++++++---- views/feedCache.pug | 26 ++++++++++++++++++++++++++ views/index.pug | 11 +++++++++-- views/retryCache.pug | 26 ++++++++++++++++++++++++++ 13 files changed, 167 insertions(+), 20 deletions(-) create mode 100644 retryCache.js create mode 100644 routes/cache.js create mode 100644 views/feedCache.pug create mode 100644 views/retryCache.pug diff --git a/.gitignore b/.gitignore index 1f8a2b8..dbf8bde 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ dist/jdrssdownloader-win.exe dist/jdrssdownloader-macos downloadHistory.json shows.json +shows.json +cache/retryCache.json diff --git a/FeedFilter.js b/FeedFilter.js index bd88791..45c9196 100644 --- a/FeedFilter.js +++ b/FeedFilter.js @@ -3,12 +3,14 @@ const { linkAdder } = require('./JDLinkAdder'); const { getLinksFromURL } = require('./LinkGrabber') const { checkFileName } = require('./checkFileName') const { checkDownloadHistory } = require('./checkDownloadHistory') +const { retryCache } = require('./retryCache') async function filterFeed() { let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC let myshowlist = JSON.parse(fs.readFileSync('shows.json')) - let feed = JSON.parse(fs.readFileSync('./cache/feedCache.json')); - let retryShowCache = [] + let rssFeed = JSON.parse(fs.readFileSync('./cache/feedCache.json')); + let retryCacheData = retryCache() + let fullFeedToCheck = retryCacheData.concat(rssFeed) let urlsToCheck = [] @@ -16,7 +18,7 @@ async function filterFeed() { try { // Find show on feed - let listFilteredForShow = feed.filter(item => item.title.includes(show.Name)) + let listFilteredForShow = fullFeedToCheck.filter(item => item.title.includes(show.Name)) if (listFilteredForShow.length > 0) { for (let match of listFilteredForShow) { // If show is found get url then return all links on that page @@ -60,7 +62,7 @@ async function filterFeed() { // No HEVC links found log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.') for (let feedItem of listFilteredForShow) { - retryShowCache.push(feedItem) + retryCache(feedItem) } global.linkCheckTime = new Date(); } @@ -75,7 +77,7 @@ async function filterFeed() { } } log.info('Wiping feed cache') - fs.writeFileSync('./cache/feedCache.json', JSON.stringify(retryShowCache)); + fs.writeFileSync('./cache/feedCache.json', JSON.stringify([])); global.linkCheckTime = new Date(); } diff --git a/JDRssDownloader.js b/JDRssDownloader.js index fbf05ad..5ef072b 100644 --- a/JDRssDownloader.js +++ b/JDRssDownloader.js @@ -1,9 +1,9 @@ const fs = require("fs"); config = JSON.parse(fs.readFileSync('config.json')) - const { feedUpdater } = require('./FeedUpdater') const { filterFeed } = require('./FeedFilter') const { telegrambot } = require('./telegramCommunication') +const { create_empty_cache_files } = require('./utils') const express = require('express'); const bodyParser = require('body-parser'); const cors = require('cors'); @@ -52,6 +52,7 @@ async function main() { log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`)) + create_empty_cache_files() setInterval(await feedUpdater, RSSFeedRefreshMins * 60000); setInterval(await filterFeed, JDPostLinksMins * 60000); } diff --git a/apiFunctions.js b/apiFunctions.js index e76809a..fb4a0b1 100644 --- a/apiFunctions.js +++ b/apiFunctions.js @@ -61,7 +61,24 @@ async function editShow(showData) { } } -module.exports = { - addNewShow, removeShow, editShow +async function removeShowFromCache(showData) { + let shows = JSON.parse(fs.readFileSync('./cache/retryCache.json')) + + myArray = shows.filter(function (obj) { + return obj.title !== showData; + }); + + shows = myArray + try { + fs.writeFileSync('./cache/retryCache.json', JSON.stringify(shows)); + log.info(showData + ' Removed from retry cache.') + } catch (err) { + console.error(err); + } +} + + +module.exports = { + addNewShow, removeShow, editShow, removeShowFromCache } diff --git a/checkDownloadHistory.js b/checkDownloadHistory.js index 9496b9e..010f592 100644 --- a/checkDownloadHistory.js +++ b/checkDownloadHistory.js @@ -1,11 +1,6 @@ const fs = require('fs') function checkDownloadHistory(urlObj) { - try { - history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); - } catch (error) { - fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([])); - } history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); if (history.includes(urlObj.fileName)) { return true diff --git a/package.json b/package.json index 5498df0..c2358c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jdrssdownloader", - "version": "1.1.1", + "version": "1.1.2", "description": "", "main": "JDRssDownloader.js", "bin": "JDRssDownloader.js", diff --git a/retryCache.js b/retryCache.js new file mode 100644 index 0000000..c4d2e6e --- /dev/null +++ b/retryCache.js @@ -0,0 +1,21 @@ +const fs = require('fs') + +function retryCache(cache) { + if (cache == null) { + retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json')); + return retryCacheData + } + retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json')); + if (retryCacheData.some(e => e.title === cache.title)) { + log.info(cache.title + ' is already in the retry cache.') + } + else { + cache.retryCount = 5 + retryCacheData.push(cache) + fs.writeFileSync('./cache/retryCache.json', JSON.stringify(retryCacheData)); + log.info(cache.title + ' written to retry cache.') + return retryCacheData + } +} + +module.exports = { retryCache } \ No newline at end of file diff --git a/routes/cache.js b/routes/cache.js new file mode 100644 index 0000000..c4d4641 --- /dev/null +++ b/routes/cache.js @@ -0,0 +1,25 @@ +const fs = require("fs"); +const {removeShowFromCache} = require('../apiFunctions') + +module.exports = function (app) { + app.get("/feedCache", (req, res) => { + feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json')) + res.render("feedCache", { title: "Feed Cache", feedCache: feedCache }); + }); + + app.get("/retryCache", (req, res) => { + retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json')) + for (let index = 0; index < retryCache.length; index++) { + const item = retryCache[index]; + retryCache[index].newtitle = item.title.replace(/ /g, "‡"); + } + res.render("retryCache", { title: "Retry Cache", retryCache: retryCache }); + }); + + app.get('/retryCache/remove', (req, res) => { + const showName = req.query.name.replaceAll("‡", " "); + console.log(req) + removeShowFromCache(showName) + res.redirect("/retryCache"); + }); +} diff --git a/routes/root.js b/routes/root.js index a687c36..f78f57e 100644 --- a/routes/root.js +++ b/routes/root.js @@ -5,6 +5,8 @@ module.exports = function (app) { app.get("/", (req, res) => { showListLength = JSON.parse(fs.readFileSync('shows.json')).length + feedCacheLength = JSON.parse(fs.readFileSync('./cache/feedCache.json')).length + retryCacheLength = JSON.parse(fs.readFileSync('./cache/retryCache.json')).length rssTime = nextRssRefresh() linkCheck = nextLinkCheck() res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: rssTime, linkCheck: linkCheck, lastDownloaded: get_last_downloaded() }); diff --git a/utils.js b/utils.js index 92bc34c..a1731e1 100644 --- a/utils.js +++ b/utils.js @@ -16,13 +16,36 @@ function nextLinkCheck() { return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins) } -function get_last_downloaded(){ +function get_last_downloaded() { history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json')) - last = history.slice(-1)[0] + last = history.slice(-1)[0] return last } -module.exports = { - nextRssRefresh, nextLinkCheck, get_last_downloaded +function create_empty_downloadHistory() { + try { + return JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); + } catch (error) { + fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([])); + return JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); + } +} + +function create_empty_retry_cache() { + try { + return JSON.parse(fs.readFileSync('./cache/retryCache.json')); + } catch (error) { + fs.writeFileSync('./cache/retryCache.json', JSON.stringify([])); + return JSON.parse(fs.readFileSync('./cache/retryCache.json')); + } +} + +function create_empty_cache_files() { + create_empty_downloadHistory() + create_empty_retry_cache() +} + +module.exports = { + nextRssRefresh, nextLinkCheck, get_last_downloaded, create_empty_cache_files } diff --git a/views/feedCache.pug b/views/feedCache.pug new file mode 100644 index 0000000..4cd5b39 --- /dev/null +++ b/views/feedCache.pug @@ -0,0 +1,26 @@ +extends layout + +block layout-content + div.View + h1.Banner Feed Cache + if (feedCache.length==0) + div.Message + h2 No shows in Feed Cache + else + div.Message + table + thead + tr + th Show Name + //- th Remove + tbody + each val, key in feedCache + tr + td + a(href=val.link + target="_blank") #{val.title} + //- td + //- a(href='/retryCache/remove?name=' + val.newtitle) Remove + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/index.pug b/views/index.pug index 6420fa7..c61cf0a 100644 --- a/views/index.pug +++ b/views/index.pug @@ -8,11 +8,18 @@ block layout-content h2 Number of Tracked Shows h3 #{showListLength} h2 Last Downloaded - h3 #{lastDownloaded} + h3 #{lastDownloaded} + div.Message + h2 + a(href='/feedCache') RSS Feed Cache Size + h3 #{feedCacheLength} + h2 + a(href='/retryCache') Retry Cache Size + h3 #{retryCacheLength} h2 Next RSS Refresh h3 #{rssTime} h2 Next Link Check - h3 #{linkCheck} + h3 #{linkCheck} div.NavButtons a(href="/shows") div.NavButton Show List diff --git a/views/retryCache.pug b/views/retryCache.pug new file mode 100644 index 0000000..19cbec3 --- /dev/null +++ b/views/retryCache.pug @@ -0,0 +1,26 @@ +extends layout + +block layout-content + div.View + h1.Banner Retry Cache + if (retryCache.length==0) + div.Message + h2 No shows in Retry Cache + else + div.Message + table + thead + tr + th Show Name + th Remove + tbody + each val, key in retryCache + tr + td + a(href=val.link + target="_blank") #{val.title} + td + a(href='/retryCache/remove?name=' + val.newtitle) Remove + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file