Merge pull request #8 from karl0ss/split_shows

Split shows

.gitignore

@@ -9,3 +9,6 @@ dist/jdrssdownloader-linux
 dist/jdrssdownloader-win.exe
 dist/jdrssdownloader-macos
 downloadHistory.json
+shows.json
+shows.json
+cache/retryCache.json

FeedFilter.js

@@ -3,12 +3,14 @@ const { linkAdder } = require('./JDLinkAdder');
 const { getLinksFromURL } = require('./LinkGrabber')
 const { checkFileName } = require('./checkFileName')
 const { checkDownloadHistory } = require('./checkDownloadHistory')
+const { retryCache } = require('./retryCache')
 
 async function filterFeed() {
-    let myshowlist = JSON.parse(fs.readFileSync('config.json')).Shows
     let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC
-    let feed = JSON.parse(fs.readFileSync('./feedCache.json'));
-    let retryShowCache = []
+    let myshowlist = JSON.parse(fs.readFileSync('shows.json'))
+    let rssFeed = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
+    let retryCacheData = retryCache()
+    let fullFeedToCheck = retryCacheData.concat(rssFeed)
     let urlsToCheck = []
 
 
@@ -16,7 +18,7 @@ async function filterFeed() {
 
         try {
             // Find show on feed
-            let listFilteredForShow = feed.filter(item => item.title.includes(show.Name))
+            let listFilteredForShow = fullFeedToCheck.filter(item => item.title.includes(show.Name))
             if (listFilteredForShow.length > 0) {
                 for (let match of listFilteredForShow) {
                     // If show is found get url then return all links on that page
@@ -60,7 +62,7 @@ async function filterFeed() {
                         // No HEVC links found
                         log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.')
                         for (let feedItem of listFilteredForShow) {
-                            retryShowCache.push(feedItem)
+                            retryCache(feedItem)
                         }
                         global.linkCheckTime = new Date();
                     }
@@ -75,7 +77,7 @@ async function filterFeed() {
         }
     }
     log.info('Wiping feed cache')
-    fs.writeFileSync('./feedCache.json', JSON.stringify(retryShowCache));
+    fs.writeFileSync('./cache/feedCache.json', JSON.stringify([]));
     global.linkCheckTime = new Date();
 }
 

FeedUpdater.js

@@ -11,15 +11,15 @@ async function feedUpdater() {
 
     let items = [];
 
-    if (fs.existsSync('./feedCache.json')) {
-        items = JSON.parse(fs.readFileSync('./feedCache.json'))
+    if (fs.existsSync('./cache/feedCache.json')) {
+        items = JSON.parse(fs.readFileSync('./cache/feedCache.json'))
     }
     // Compare existing cache and new items and merge differences
     let updatedArray = lodash.unionBy(feed.items, items, 'title');
 
     // Save the file
     log.info(updatedArray.length + ' items in file cache')
-    fs.writeFileSync('./feedCache.json', JSON.stringify(updatedArray));
+    fs.writeFileSync('./cache/feedCache.json', JSON.stringify(updatedArray));
     global.rssRefreshTime = new Date();
 }
 

JDRssDownloader.js

@@ -1,9 +1,9 @@
 const fs = require("fs");
 config = JSON.parse(fs.readFileSync('config.json'))
-
 const { feedUpdater } = require('./FeedUpdater')
 const { filterFeed } = require('./FeedFilter')
 const { telegrambot } = require('./telegramCommunication')
+const { create_empty_cache_files } = require('./utils')
 const express = require('express');
 const bodyParser = require('body-parser');
 const cors = require('cors');
@@ -52,6 +52,7 @@ async function main() {
     log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes')
     log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes')
     app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`))
+    create_empty_cache_files()
     setInterval(await feedUpdater, RSSFeedRefreshMins * 60000);
     setInterval(await filterFeed, JDPostLinksMins * 60000);
 }

apiFunctions.js

@@ -1,10 +1,10 @@
 const fs = require('fs');
-const { config } = require('process');
+
 
 async function addNewShow(showData) {
-    let config = JSON.parse(fs.readFileSync('config.json'))
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
     let exist = false
-    for (let show of config.Shows) {
+    for (let show of shows) {
         if (show.Name == showData.showName) {
             exist = true
         }
@@ -12,12 +12,12 @@ async function addNewShow(showData) {
     if (exist) {
         log.error(showData.showName + ' Already exists in list and not added')
     } else {
-        config.Shows.push({
+        shows.push({
             "Name": showData.showName,
             "Quality": showData.quality
         })
         try {
-            fs.writeFileSync('config.json', JSON.stringify(config));
+            fs.writeFileSync('shows.json', JSON.stringify(shows));
             log.info(showData.showName + ' Added to the list, checking for ' + showData.quality + 'p')
         } catch (err) {
             console.error(err);
@@ -26,22 +26,59 @@ async function addNewShow(showData) {
 }
 
 async function removeShow(showData) {
-    let config = JSON.parse(fs.readFileSync('config.json'))
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
 
-    myArray = config.Shows.filter(function (obj) {
+    myArray = shows.filter(function (obj) {
         return obj.Name !== showData.showName;
     });
 
-    config.Shows = myArray
+    shows = myArray
     try {
-        fs.writeFileSync('config.json', JSON.stringify(config));
+        fs.writeFileSync('shows.json', JSON.stringify(shows));
         log.info(showData.showName + ' Removed from tracking list.')
     } catch (err) {
         console.error(err);
     }
 }
 
-module.exports = {
-    addNewShow, removeShow
+
+async function editShow(showData) {
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
+    for (let index = 0; index < shows.length; index++) {
+        const element = shows[index];
+        if (element.Name == showData.showName) {
+            shows[index] = {
+                "Name": showData.showName,
+                "Quality": showData.quality
+            }
+        }
+    }
+    try {
+        fs.writeFileSync('shows.json', JSON.stringify(shows));
+        log.info(showData.showName + ' Quality modified to ' + showData.quality + 'p')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+async function removeShowFromCache(showData) {
+    let shows = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+
+    myArray = shows.filter(function (obj) {
+        return obj.title !== showData;
+    });
+
+    shows = myArray
+    try {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(shows));
+        log.info(showData + ' Removed from retry cache.')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+module.exports = {
+    addNewShow, removeShow, editShow, removeShowFromCache
 }
 

checkDownloadHistory.js

@@ -1,17 +1,12 @@
 const fs = require('fs')
 
 function checkDownloadHistory(urlObj) {
-    try {
-        history = JSON.parse(fs.readFileSync('./downloadHistory.json'));
-    } catch (error) {
-        fs.writeFileSync('./downloadHistory.json', JSON.stringify([]));
-    }
-    history = JSON.parse(fs.readFileSync('./downloadHistory.json'));
+    history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
     if (history.includes(urlObj.fileName)) {
         return true
     } else {
         history.push(urlObj.fileName)
-        fs.writeFileSync('./downloadHistory.json', JSON.stringify(history));
+        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify(history));
         return false
     }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jdrssdownloader",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "",
   "main": "JDRssDownloader.js",
   "bin": "JDRssDownloader.js",
@@ -15,6 +15,7 @@
     "cors": "^2.8.5",
     "express": "^4.18.2",
     "express-basic-auth": "^1.2.1",
+    "express-validator": "^6.14.2",
     "jdownloader-client": "^1.0.0",
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",

public/style.css

@@ -97,9 +97,15 @@ a {
 
 .Message {
   background: white;
-  padding: 30px;
+  border-radius: 5px;
+  padding: 20px;
   border-bottom: var(--ui-shadow-border);
   box-shadow: var(--ui-shadow);
+  overflow-x: hidden;
+  overflow-y: auto;
+  /* text-align: center; */
+  margin-top:10px;
+  margin-bottom:10px;
 }
 
 .Message > .Title {

retryCache.js

@@ -0,0 +1,21 @@
+const fs = require('fs')
+
+function retryCache(cache) {
+    if (cache == null) {
+        retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+        return retryCacheData
+    }
+    retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    if (retryCacheData.some(e => e.title === cache.title)) {
+        log.info(cache.title + ' is already in the retry cache.')
+    }
+    else {
+        cache.retryCount = 5
+        retryCacheData.push(cache)
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(retryCacheData));
+        log.info(cache.title + ' written to retry cache.')
+        return retryCacheData
+    }
+}
+
+module.exports = { retryCache }

routes/cache.js

@@ -0,0 +1,25 @@
+const fs = require("fs");
+const {removeShowFromCache} = require('../apiFunctions')
+
+module.exports = function (app) {
+    app.get("/feedCache", (req, res) => {
+        feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json'))
+        res.render("feedCache", { title: "Feed Cache", feedCache: feedCache });
+    });
+
+    app.get("/retryCache", (req, res) => {
+        retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+        for (let index = 0; index < retryCache.length; index++) {
+            const item = retryCache[index];
+            retryCache[index].newtitle = item.title.replace(/ /g, "‡");
+        }
+        res.render("retryCache", { title: "Retry Cache", retryCache: retryCache });
+    });
+
+    app.get('/retryCache/remove', (req, res) => {
+        const showName = req.query.name.replaceAll("‡", " ");
+        console.log(req)
+        removeShowFromCache(showName)
+        res.redirect("/retryCache");
+    });
+}

routes/root.js

@@ -1,11 +1,14 @@
 const fs = require("fs");
-const { nextLinkCheck, nextRssRefresh } = require('.././utils')
+const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils')
 
 module.exports = function (app) {
 
     app.get("/", (req, res) => {
-        showListLength = JSON.parse(fs.readFileSync('config.json')).Shows.length
-        a =
-            res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: nextRssRefresh(), linkCheck: nextLinkCheck() });
+        showListLength = JSON.parse(fs.readFileSync('shows.json')).length
+        feedCacheLength = JSON.parse(fs.readFileSync('./cache/feedCache.json')).length
+        retryCacheLength = JSON.parse(fs.readFileSync('./cache/retryCache.json')).length
+        rssTime = nextRssRefresh()
+        linkCheck = nextLinkCheck()
+        res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: rssTime, linkCheck: linkCheck, lastDownloaded: get_last_downloaded() });
     });
 }

routes/shows.js

@@ -1,9 +1,11 @@
 const fs = require("fs");
-const { addNewShow, removeShow } = require('.././apiFunctions')
+const { addNewShow, removeShow, editShow } = require('.././apiFunctions')
+const { check, validationResult } = require('express-validator');
+
 
 module.exports = function (app) {
     app.get("/shows", (req, res) => {
-        showList = JSON.parse(fs.readFileSync('config.json')).Shows
+        showList = JSON.parse(fs.readFileSync('shows.json'))
         res.render("shows", { title: "Show List", showList: showList });
     });
 
@@ -12,17 +14,36 @@ module.exports = function (app) {
     });
 
     app.get("/shows/remove", (req, res) => {
-        showList = JSON.parse(fs.readFileSync('config.json')).Shows
+        showList = JSON.parse(fs.readFileSync('shows.json'))
         res.render("removeshow", { title: "Remove Show", showList: showList });
     });
 
-    app.post('/addNewShow', (req, res) => {
-        addNewShow(req.body)
-        res.redirect("/shows");
+    app.get("/shows/edit", (req, res) => {
+        showList = JSON.parse(fs.readFileSync('shows.json'))
+        res.render("editShow", { title: "Edit Show", showList: showList });
     });
 
+    app.post('/addNewShow', [
+        check('showName')
+            .isLength({ min: 1 })
+    ], (req, res) => {
+        if (validationResult(req).isEmpty()) {
+            addNewShow(req.body)
+            res.redirect("/shows");
+        } else {
+            log.error('You cannot add a show without a name.')
+            res.redirect("/shows");
+        }
+    }
+    );
+
     app.post('/removeShow', (req, res) => {
         removeShow(req.body)
         res.redirect("/shows");
     });
+
+    app.post('/editShow', (req, res) => {
+        editShow(req.body)
+        res.redirect("/shows");
+    });
 }

shows.json

@@ -0,0 +1 @@
+[]

utils.js

@@ -16,7 +16,36 @@ function nextLinkCheck() {
     return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins)
 }
 
-module.exports = {
-    nextRssRefresh, nextLinkCheck
+function get_last_downloaded() {
+    history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'))
+    last = history.slice(-1)[0]
+    return last
+}
+
+function create_empty_downloadHistory() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    }
+}
+
+function create_empty_retry_cache() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    }
+}
+
+function create_empty_cache_files() {
+    create_empty_downloadHistory()
+    create_empty_retry_cache()
+}
+
+module.exports = {
+    nextRssRefresh, nextLinkCheck, get_last_downloaded, create_empty_cache_files
 }
 

views/addshow.pug

@@ -11,6 +11,7 @@ block layout-content
           select(name="quality")
             option(value='720') #{'720p'}
             option(value='1080') #{'1080p'}
+            option(value='2160') #{'2160p'}
         input(type="submit", value="Add Show")
     div.NavButtons
       a(href="/")

views/editShow.pug

@@ -0,0 +1,20 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Edit Show
+    div.Message
+      form(action="/editShow" method="POST")
+        p Show Name: 
+          select(name="showName")
+            each show in showList
+                option(value=show.Name) #{show.Name}
+        p Quality: 
+          select(name="quality")
+            option(value='720') #{'720p'}
+            option(value='1080') #{'1080p'}
+            option(value='2160') #{'2160p'}
+        input(type="submit", value="Edit Show")
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home

views/feedCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Feed Cache
+    if (feedCache.length==0)
+      div.Message
+        h2 No shows in Feed Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              //- th Remove
+          tbody
+            each val, key in feedCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                //- td 
+                //-   a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home

views/index.pug

@@ -5,17 +5,28 @@ block layout-content
     h1.Banner JDRssDownloader #{version}
     body
       div.Message
-        h3 Number of Tracked Shows
-        h1 #{showListLength}
-        h3 Next RSS Refresh
-        h1 #{rssTime}
-        h3 Next Link Check
-        h1 #{linkCheck}        
+        h2 Number of Tracked Shows
+        h3 #{showListLength}
+        h2 Last Downloaded
+        h3 #{lastDownloaded}             
+      div.Message
+        h2 
+          a(href='/feedCache') RSS Feed Cache Size 
+        h3 #{feedCacheLength}
+        h2
+          a(href='/retryCache') Retry Cache Size
+        h3 #{retryCacheLength}
+        h2 Next RSS Refresh
+        h3 #{rssTime}
+        h2 Next Link Check
+        h3 #{linkCheck}   
     div.NavButtons
       a(href="/shows")
           div.NavButton Show List
       a(href="/shows/add")
           div.NavButton Add New Show
+      a(href="/shows/edit")
+          div.NavButton Edit Show          
       a(href="/shows/remove")
           div.NavButton Remove Show
       a(href="/logFile")

views/layout.pug

@@ -4,7 +4,7 @@ html
   head
     meta(charset="utf-8")
     link(rel="shortcut icon", href="/favicon.ico")
-    meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no")
+    meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=yes")
     meta(name="theme-color", content="#000000")
     title #{title} | JDRssDownloader
     link(rel="stylesheet" href="/style.css")

views/retryCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Retry Cache
+    if (retryCache.length==0)
+      div.Message
+        h2 No shows in Retry Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              th Remove
+          tbody
+            each val, key in retryCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                td 
+                  a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home