diff --git a/.gitignore b/.gitignore index a75cf9d..dbf8bde 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ dist/jdrssdownloader-linux dist/jdrssdownloader-win.exe dist/jdrssdownloader-macos downloadHistory.json +shows.json +shows.json +cache/retryCache.json diff --git a/FeedFilter.js b/FeedFilter.js index 3bb61d8..45c9196 100644 --- a/FeedFilter.js +++ b/FeedFilter.js @@ -3,12 +3,14 @@ const { linkAdder } = require('./JDLinkAdder'); const { getLinksFromURL } = require('./LinkGrabber') const { checkFileName } = require('./checkFileName') const { checkDownloadHistory } = require('./checkDownloadHistory') +const { retryCache } = require('./retryCache') async function filterFeed() { - let myshowlist = JSON.parse(fs.readFileSync('config.json')).Shows let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC - let feed = JSON.parse(fs.readFileSync('./feedCache.json')); - let retryShowCache = [] + let myshowlist = JSON.parse(fs.readFileSync('shows.json')) + let rssFeed = JSON.parse(fs.readFileSync('./cache/feedCache.json')); + let retryCacheData = retryCache() + let fullFeedToCheck = retryCacheData.concat(rssFeed) let urlsToCheck = [] @@ -16,7 +18,7 @@ async function filterFeed() { try { // Find show on feed - let listFilteredForShow = feed.filter(item => item.title.includes(show.Name)) + let listFilteredForShow = fullFeedToCheck.filter(item => item.title.includes(show.Name)) if (listFilteredForShow.length > 0) { for (let match of listFilteredForShow) { // If show is found get url then return all links on that page @@ -60,7 +62,7 @@ async function filterFeed() { // No HEVC links found log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.') for (let feedItem of listFilteredForShow) { - retryShowCache.push(feedItem) + retryCache(feedItem) } global.linkCheckTime = new Date(); } @@ -75,7 +77,7 @@ async function filterFeed() { } } log.info('Wiping feed cache') - fs.writeFileSync('./feedCache.json', JSON.stringify(retryShowCache)); + fs.writeFileSync('./cache/feedCache.json', JSON.stringify([])); global.linkCheckTime = new Date(); } diff --git a/FeedUpdater.js b/FeedUpdater.js index 6905fb2..8e9268a 100644 --- a/FeedUpdater.js +++ b/FeedUpdater.js @@ -11,15 +11,15 @@ async function feedUpdater() { let items = []; - if (fs.existsSync('./feedCache.json')) { - items = JSON.parse(fs.readFileSync('./feedCache.json')) + if (fs.existsSync('./cache/feedCache.json')) { + items = JSON.parse(fs.readFileSync('./cache/feedCache.json')) } // Compare existing cache and new items and merge differences let updatedArray = lodash.unionBy(feed.items, items, 'title'); // Save the file log.info(updatedArray.length + ' items in file cache') - fs.writeFileSync('./feedCache.json', JSON.stringify(updatedArray)); + fs.writeFileSync('./cache/feedCache.json', JSON.stringify(updatedArray)); global.rssRefreshTime = new Date(); } diff --git a/JDRssDownloader.js b/JDRssDownloader.js index fbf05ad..5ef072b 100644 --- a/JDRssDownloader.js +++ b/JDRssDownloader.js @@ -1,9 +1,9 @@ const fs = require("fs"); config = JSON.parse(fs.readFileSync('config.json')) - const { feedUpdater } = require('./FeedUpdater') const { filterFeed } = require('./FeedFilter') const { telegrambot } = require('./telegramCommunication') +const { create_empty_cache_files } = require('./utils') const express = require('express'); const bodyParser = require('body-parser'); const cors = require('cors'); @@ -52,6 +52,7 @@ async function main() { log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`)) + create_empty_cache_files() setInterval(await feedUpdater, RSSFeedRefreshMins * 60000); setInterval(await filterFeed, JDPostLinksMins * 60000); } diff --git a/apiFunctions.js b/apiFunctions.js index 81e7ac3..fb4a0b1 100644 --- a/apiFunctions.js +++ b/apiFunctions.js @@ -1,10 +1,10 @@ const fs = require('fs'); -const { config } = require('process'); + async function addNewShow(showData) { - let config = JSON.parse(fs.readFileSync('config.json')) + let shows = JSON.parse(fs.readFileSync('shows.json')) let exist = false - for (let show of config.Shows) { + for (let show of shows) { if (show.Name == showData.showName) { exist = true } @@ -12,12 +12,12 @@ async function addNewShow(showData) { if (exist) { log.error(showData.showName + ' Already exists in list and not added') } else { - config.Shows.push({ + shows.push({ "Name": showData.showName, "Quality": showData.quality }) try { - fs.writeFileSync('config.json', JSON.stringify(config)); + fs.writeFileSync('shows.json', JSON.stringify(shows)); log.info(showData.showName + ' Added to the list, checking for ' + showData.quality + 'p') } catch (err) { console.error(err); @@ -26,22 +26,59 @@ async function addNewShow(showData) { } async function removeShow(showData) { - let config = JSON.parse(fs.readFileSync('config.json')) + let shows = JSON.parse(fs.readFileSync('shows.json')) - myArray = config.Shows.filter(function (obj) { + myArray = shows.filter(function (obj) { return obj.Name !== showData.showName; }); - config.Shows = myArray + shows = myArray try { - fs.writeFileSync('config.json', JSON.stringify(config)); + fs.writeFileSync('shows.json', JSON.stringify(shows)); log.info(showData.showName + ' Removed from tracking list.') } catch (err) { console.error(err); } } -module.exports = { - addNewShow, removeShow + +async function editShow(showData) { + let shows = JSON.parse(fs.readFileSync('shows.json')) + for (let index = 0; index < shows.length; index++) { + const element = shows[index]; + if (element.Name == showData.showName) { + shows[index] = { + "Name": showData.showName, + "Quality": showData.quality + } + } + } + try { + fs.writeFileSync('shows.json', JSON.stringify(shows)); + log.info(showData.showName + ' Quality modified to ' + showData.quality + 'p') + } catch (err) { + console.error(err); + } +} + +async function removeShowFromCache(showData) { + let shows = JSON.parse(fs.readFileSync('./cache/retryCache.json')) + + myArray = shows.filter(function (obj) { + return obj.title !== showData; + }); + + shows = myArray + try { + fs.writeFileSync('./cache/retryCache.json', JSON.stringify(shows)); + log.info(showData + ' Removed from retry cache.') + } catch (err) { + console.error(err); + } +} + + +module.exports = { + addNewShow, removeShow, editShow, removeShowFromCache } diff --git a/checkDownloadHistory.js b/checkDownloadHistory.js index e3c923b..010f592 100644 --- a/checkDownloadHistory.js +++ b/checkDownloadHistory.js @@ -1,17 +1,12 @@ const fs = require('fs') function checkDownloadHistory(urlObj) { - try { - history = JSON.parse(fs.readFileSync('./downloadHistory.json')); - } catch (error) { - fs.writeFileSync('./downloadHistory.json', JSON.stringify([])); - } - history = JSON.parse(fs.readFileSync('./downloadHistory.json')); + history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); if (history.includes(urlObj.fileName)) { return true } else { history.push(urlObj.fileName) - fs.writeFileSync('./downloadHistory.json', JSON.stringify(history)); + fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify(history)); return false } } diff --git a/package.json b/package.json index 4b72bf3..c2358c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jdrssdownloader", - "version": "1.1.0", + "version": "1.1.2", "description": "", "main": "JDRssDownloader.js", "bin": "JDRssDownloader.js", @@ -15,6 +15,7 @@ "cors": "^2.8.5", "express": "^4.18.2", "express-basic-auth": "^1.2.1", + "express-validator": "^6.14.2", "jdownloader-client": "^1.0.0", "line-reader": "^0.4.0", "lodash": "^4.17.21", diff --git a/public/style.css b/public/style.css index dd4012f..8b73dba 100644 --- a/public/style.css +++ b/public/style.css @@ -97,9 +97,15 @@ a { .Message { background: white; - padding: 30px; + border-radius: 5px; + padding: 20px; border-bottom: var(--ui-shadow-border); box-shadow: var(--ui-shadow); + overflow-x: hidden; + overflow-y: auto; + /* text-align: center; */ + margin-top:10px; + margin-bottom:10px; } .Message > .Title { diff --git a/retryCache.js b/retryCache.js new file mode 100644 index 0000000..c4d2e6e --- /dev/null +++ b/retryCache.js @@ -0,0 +1,21 @@ +const fs = require('fs') + +function retryCache(cache) { + if (cache == null) { + retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json')); + return retryCacheData + } + retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json')); + if (retryCacheData.some(e => e.title === cache.title)) { + log.info(cache.title + ' is already in the retry cache.') + } + else { + cache.retryCount = 5 + retryCacheData.push(cache) + fs.writeFileSync('./cache/retryCache.json', JSON.stringify(retryCacheData)); + log.info(cache.title + ' written to retry cache.') + return retryCacheData + } +} + +module.exports = { retryCache } \ No newline at end of file diff --git a/routes/cache.js b/routes/cache.js new file mode 100644 index 0000000..c4d4641 --- /dev/null +++ b/routes/cache.js @@ -0,0 +1,25 @@ +const fs = require("fs"); +const {removeShowFromCache} = require('../apiFunctions') + +module.exports = function (app) { + app.get("/feedCache", (req, res) => { + feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json')) + res.render("feedCache", { title: "Feed Cache", feedCache: feedCache }); + }); + + app.get("/retryCache", (req, res) => { + retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json')) + for (let index = 0; index < retryCache.length; index++) { + const item = retryCache[index]; + retryCache[index].newtitle = item.title.replace(/ /g, "‡"); + } + res.render("retryCache", { title: "Retry Cache", retryCache: retryCache }); + }); + + app.get('/retryCache/remove', (req, res) => { + const showName = req.query.name.replaceAll("‡", " "); + console.log(req) + removeShowFromCache(showName) + res.redirect("/retryCache"); + }); +} diff --git a/routes/root.js b/routes/root.js index ec52316..f78f57e 100644 --- a/routes/root.js +++ b/routes/root.js @@ -1,11 +1,14 @@ const fs = require("fs"); -const { nextLinkCheck, nextRssRefresh } = require('.././utils') +const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils') module.exports = function (app) { app.get("/", (req, res) => { - showListLength = JSON.parse(fs.readFileSync('config.json')).Shows.length - a = - res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: nextRssRefresh(), linkCheck: nextLinkCheck() }); + showListLength = JSON.parse(fs.readFileSync('shows.json')).length + feedCacheLength = JSON.parse(fs.readFileSync('./cache/feedCache.json')).length + retryCacheLength = JSON.parse(fs.readFileSync('./cache/retryCache.json')).length + rssTime = nextRssRefresh() + linkCheck = nextLinkCheck() + res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: rssTime, linkCheck: linkCheck, lastDownloaded: get_last_downloaded() }); }); } \ No newline at end of file diff --git a/routes/shows.js b/routes/shows.js index 96369c6..ad05fc0 100644 --- a/routes/shows.js +++ b/routes/shows.js @@ -1,9 +1,11 @@ const fs = require("fs"); -const { addNewShow, removeShow } = require('.././apiFunctions') +const { addNewShow, removeShow, editShow } = require('.././apiFunctions') +const { check, validationResult } = require('express-validator'); + module.exports = function (app) { app.get("/shows", (req, res) => { - showList = JSON.parse(fs.readFileSync('config.json')).Shows + showList = JSON.parse(fs.readFileSync('shows.json')) res.render("shows", { title: "Show List", showList: showList }); }); @@ -12,17 +14,36 @@ module.exports = function (app) { }); app.get("/shows/remove", (req, res) => { - showList = JSON.parse(fs.readFileSync('config.json')).Shows + showList = JSON.parse(fs.readFileSync('shows.json')) res.render("removeshow", { title: "Remove Show", showList: showList }); }); - app.post('/addNewShow', (req, res) => { - addNewShow(req.body) - res.redirect("/shows"); + app.get("/shows/edit", (req, res) => { + showList = JSON.parse(fs.readFileSync('shows.json')) + res.render("editShow", { title: "Edit Show", showList: showList }); }); + app.post('/addNewShow', [ + check('showName') + .isLength({ min: 1 }) + ], (req, res) => { + if (validationResult(req).isEmpty()) { + addNewShow(req.body) + res.redirect("/shows"); + } else { + log.error('You cannot add a show without a name.') + res.redirect("/shows"); + } + } + ); + app.post('/removeShow', (req, res) => { removeShow(req.body) res.redirect("/shows"); }); + + app.post('/editShow', (req, res) => { + editShow(req.body) + res.redirect("/shows"); + }); } diff --git a/shows.json b/shows.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/shows.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/utils.js b/utils.js index 0ceaf32..a1731e1 100644 --- a/utils.js +++ b/utils.js @@ -16,7 +16,36 @@ function nextLinkCheck() { return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins) } -module.exports = { - nextRssRefresh, nextLinkCheck +function get_last_downloaded() { + history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json')) + last = history.slice(-1)[0] + return last +} + +function create_empty_downloadHistory() { + try { + return JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); + } catch (error) { + fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([])); + return JSON.parse(fs.readFileSync('./cache/downloadHistory.json')); + } +} + +function create_empty_retry_cache() { + try { + return JSON.parse(fs.readFileSync('./cache/retryCache.json')); + } catch (error) { + fs.writeFileSync('./cache/retryCache.json', JSON.stringify([])); + return JSON.parse(fs.readFileSync('./cache/retryCache.json')); + } +} + +function create_empty_cache_files() { + create_empty_downloadHistory() + create_empty_retry_cache() +} + +module.exports = { + nextRssRefresh, nextLinkCheck, get_last_downloaded, create_empty_cache_files } diff --git a/views/addshow.pug b/views/addshow.pug index 3bf8c4c..c6b4176 100644 --- a/views/addshow.pug +++ b/views/addshow.pug @@ -11,6 +11,7 @@ block layout-content select(name="quality") option(value='720') #{'720p'} option(value='1080') #{'1080p'} + option(value='2160') #{'2160p'} input(type="submit", value="Add Show") div.NavButtons a(href="/") diff --git a/views/editShow.pug b/views/editShow.pug new file mode 100644 index 0000000..e5d16b1 --- /dev/null +++ b/views/editShow.pug @@ -0,0 +1,20 @@ +extends layout + +block layout-content + div.View + h1.Banner Edit Show + div.Message + form(action="/editShow" method="POST") + p Show Name: + select(name="showName") + each show in showList + option(value=show.Name) #{show.Name} + p Quality: + select(name="quality") + option(value='720') #{'720p'} + option(value='1080') #{'1080p'} + option(value='2160') #{'2160p'} + input(type="submit", value="Edit Show") + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/feedCache.pug b/views/feedCache.pug new file mode 100644 index 0000000..4cd5b39 --- /dev/null +++ b/views/feedCache.pug @@ -0,0 +1,26 @@ +extends layout + +block layout-content + div.View + h1.Banner Feed Cache + if (feedCache.length==0) + div.Message + h2 No shows in Feed Cache + else + div.Message + table + thead + tr + th Show Name + //- th Remove + tbody + each val, key in feedCache + tr + td + a(href=val.link + target="_blank") #{val.title} + //- td + //- a(href='/retryCache/remove?name=' + val.newtitle) Remove + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/index.pug b/views/index.pug index b94462d..c61cf0a 100644 --- a/views/index.pug +++ b/views/index.pug @@ -5,17 +5,28 @@ block layout-content h1.Banner JDRssDownloader #{version} body div.Message - h3 Number of Tracked Shows - h1 #{showListLength} - h3 Next RSS Refresh - h1 #{rssTime} - h3 Next Link Check - h1 #{linkCheck} + h2 Number of Tracked Shows + h3 #{showListLength} + h2 Last Downloaded + h3 #{lastDownloaded} + div.Message + h2 + a(href='/feedCache') RSS Feed Cache Size + h3 #{feedCacheLength} + h2 + a(href='/retryCache') Retry Cache Size + h3 #{retryCacheLength} + h2 Next RSS Refresh + h3 #{rssTime} + h2 Next Link Check + h3 #{linkCheck} div.NavButtons a(href="/shows") div.NavButton Show List a(href="/shows/add") div.NavButton Add New Show + a(href="/shows/edit") + div.NavButton Edit Show a(href="/shows/remove") div.NavButton Remove Show a(href="/logFile") diff --git a/views/layout.pug b/views/layout.pug index 508cfc0..f8d555b 100644 --- a/views/layout.pug +++ b/views/layout.pug @@ -4,7 +4,7 @@ html head meta(charset="utf-8") link(rel="shortcut icon", href="/favicon.ico") - meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no") + meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=yes") meta(name="theme-color", content="#000000") title #{title} | JDRssDownloader link(rel="stylesheet" href="/style.css") diff --git a/views/retryCache.pug b/views/retryCache.pug new file mode 100644 index 0000000..19cbec3 --- /dev/null +++ b/views/retryCache.pug @@ -0,0 +1,26 @@ +extends layout + +block layout-content + div.View + h1.Banner Retry Cache + if (retryCache.length==0) + div.Message + h2 No shows in Retry Cache + else + div.Message + table + thead + tr + th Show Name + th Remove + tbody + each val, key in retryCache + tr + td + a(href=val.link + target="_blank") #{val.title} + td + a(href='/retryCache/remove?name=' + val.newtitle) Remove + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file