latest updates

.gitignore

@@ -10,3 +10,5 @@ dist/jdrssdownloader-win.exe
 dist/jdrssdownloader-macos
 downloadHistory.json
 shows.json
+shows.json
+cache/retryCache.json

FeedFilter.js

@@ -3,12 +3,14 @@ const { linkAdder } = require('./JDLinkAdder');
 const { getLinksFromURL } = require('./LinkGrabber')
 const { checkFileName } = require('./checkFileName')
 const { checkDownloadHistory } = require('./checkDownloadHistory')
+const { retryCache } = require('./retryCache')
 
 async function filterFeed() {
     let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC
     let myshowlist = JSON.parse(fs.readFileSync('shows.json'))
-    let feed = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
-    let retryShowCache = []
+    let rssFeed = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
+    let retryCacheData = retryCache()
+    let fullFeedToCheck = retryCacheData.concat(rssFeed)
     let urlsToCheck = []
 
 
@@ -16,7 +18,7 @@ async function filterFeed() {
 
         try {
             // Find show on feed
-            let listFilteredForShow = feed.filter(item => item.title.includes(show.Name))
+            let listFilteredForShow = fullFeedToCheck.filter(item => item.title.includes(show.Name))
             if (listFilteredForShow.length > 0) {
                 for (let match of listFilteredForShow) {
                     // If show is found get url then return all links on that page
@@ -60,7 +62,7 @@ async function filterFeed() {
                         // No HEVC links found
                         log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.')
                         for (let feedItem of listFilteredForShow) {
-                            retryShowCache.push(feedItem)
+                            retryCache(feedItem)
                         }
                         global.linkCheckTime = new Date();
                     }
@@ -75,7 +77,7 @@ async function filterFeed() {
         }
     }
     log.info('Wiping feed cache')
-    fs.writeFileSync('./cache/feedCache.json', JSON.stringify(retryShowCache));
+    fs.writeFileSync('./cache/feedCache.json', JSON.stringify([]));
     global.linkCheckTime = new Date();
 }
 

JDRssDownloader.js

@@ -1,9 +1,9 @@
 const fs = require("fs");
 config = JSON.parse(fs.readFileSync('config.json'))
-
 const { feedUpdater } = require('./FeedUpdater')
 const { filterFeed } = require('./FeedFilter')
 const { telegrambot } = require('./telegramCommunication')
+const { create_empty_cache_files } = require('./utils')
 const express = require('express');
 const bodyParser = require('body-parser');
 const cors = require('cors');
@@ -52,6 +52,7 @@ async function main() {
     log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes')
     log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes')
     app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`))
+    create_empty_cache_files()
     setInterval(await feedUpdater, RSSFeedRefreshMins * 60000);
     setInterval(await filterFeed, JDPostLinksMins * 60000);
 }

apiFunctions.js

@@ -61,7 +61,24 @@ async function editShow(showData) {
     }
 }
 
-module.exports = {
-    addNewShow, removeShow, editShow
+async function removeShowFromCache(showData) {
+    let shows = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+
+    myArray = shows.filter(function (obj) {
+        return obj.title !== showData;
+    });
+
+    shows = myArray
+    try {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(shows));
+        log.info(showData + ' Removed from retry cache.')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+module.exports = {
+    addNewShow, removeShow, editShow, removeShowFromCache
 }
 

checkDownloadHistory.js

@@ -1,11 +1,6 @@
 const fs = require('fs')
 
 function checkDownloadHistory(urlObj) {
-    try {
-        history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
-    } catch (error) {
-        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([]));
-    }
     history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
     if (history.includes(urlObj.fileName)) {
         return true

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jdrssdownloader",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "",
   "main": "JDRssDownloader.js",
   "bin": "JDRssDownloader.js",

retryCache.js

@@ -0,0 +1,21 @@
+const fs = require('fs')
+
+function retryCache(cache) {
+    if (cache == null) {
+        retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+        return retryCacheData
+    }
+    retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    if (retryCacheData.some(e => e.title === cache.title)) {
+        log.info(cache.title + ' is already in the retry cache.')
+    }
+    else {
+        cache.retryCount = 5
+        retryCacheData.push(cache)
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(retryCacheData));
+        log.info(cache.title + ' written to retry cache.')
+        return retryCacheData
+    }
+}
+
+module.exports = { retryCache }

routes/cache.js

@@ -0,0 +1,25 @@
+const fs = require("fs");
+const {removeShowFromCache} = require('../apiFunctions')
+
+module.exports = function (app) {
+    app.get("/feedCache", (req, res) => {
+        feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json'))
+        res.render("feedCache", { title: "Feed Cache", feedCache: feedCache });
+    });
+
+    app.get("/retryCache", (req, res) => {
+        retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+        for (let index = 0; index < retryCache.length; index++) {
+            const item = retryCache[index];
+            retryCache[index].newtitle = item.title.replace(/ /g, "‡");
+        }
+        res.render("retryCache", { title: "Retry Cache", retryCache: retryCache });
+    });
+
+    app.get('/retryCache/remove', (req, res) => {
+        const showName = req.query.name.replaceAll("‡", " ");
+        console.log(req)
+        removeShowFromCache(showName)
+        res.redirect("/retryCache");
+    });
+}

routes/root.js

@@ -5,6 +5,8 @@ module.exports = function (app) {
 
     app.get("/", (req, res) => {
         showListLength = JSON.parse(fs.readFileSync('shows.json')).length
+        feedCacheLength = JSON.parse(fs.readFileSync('./cache/feedCache.json')).length
+        retryCacheLength = JSON.parse(fs.readFileSync('./cache/retryCache.json')).length
         rssTime = nextRssRefresh()
         linkCheck = nextLinkCheck()
         res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: rssTime, linkCheck: linkCheck, lastDownloaded: get_last_downloaded() });

utils.js

@@ -16,13 +16,36 @@ function nextLinkCheck() {
     return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins)
 }
 
-function get_last_downloaded(){
+function get_last_downloaded() {
     history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'))
     last = history.slice(-1)[0]
     return last
 }
 
-module.exports = {
-    nextRssRefresh, nextLinkCheck, get_last_downloaded
+function create_empty_downloadHistory() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    }
+}
+
+function create_empty_retry_cache() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    }
+}
+
+function create_empty_cache_files() {
+    create_empty_downloadHistory()
+    create_empty_retry_cache()
+}
+
+module.exports = {
+    nextRssRefresh, nextLinkCheck, get_last_downloaded, create_empty_cache_files
 }
 

views/feedCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Feed Cache
+    if (feedCache.length==0)
+      div.Message
+        h2 No shows in Feed Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              //- th Remove
+          tbody
+            each val, key in feedCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                //- td 
+                //-   a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home

views/index.pug

@@ -9,6 +9,13 @@ block layout-content
         h3 #{showListLength}
         h2 Last Downloaded
         h3 #{lastDownloaded}             
+      div.Message
+        h2 
+          a(href='/feedCache') RSS Feed Cache Size 
+        h3 #{feedCacheLength}
+        h2
+          a(href='/retryCache') Retry Cache Size
+        h3 #{retryCacheLength}
         h2 Next RSS Refresh
         h3 #{rssTime}
         h2 Next Link Check

views/retryCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Retry Cache
+    if (retryCache.length==0)
+      div.Message
+        h2 No shows in Retry Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              th Remove
+          tbody
+            each val, key in retryCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                td 
+                  a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home