Merge pull request #11 from karl0ss/karl0ss-patch-1

Update README.md

.gitignore

@@ -8,3 +8,8 @@ feedCache.json
 dist/jdrssdownloader-linux
 dist/jdrssdownloader-win.exe
 dist/jdrssdownloader-macos
+downloadHistory.json
+shows.json
+shows.json
+cache/retryCache.json
+shows.json

FeedFilter.js

@@ -2,59 +2,69 @@ const fs = require('fs')
 const { linkAdder } = require('./JDLinkAdder');
 const { getLinksFromURL } = require('./LinkGrabber')
 const { checkFileName } = require('./checkFileName')
+const { checkDownloadHistory } = require('./checkDownloadHistory')
+const { retryCache } = require('./retryCache')
 
 async function filterFeed() {
-    let myshowlist = JSON.parse(fs.readFileSync('config.json')).Shows
     let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC
-    let feed = JSON.parse(fs.readFileSync('./feedCache.json'));
+    let myshowlist = JSON.parse(fs.readFileSync('shows.json'))
-    let retry_show_cache = []
+    let rssFeed = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
-    let urls_to_check = []
+    let retryCacheData = retryCache()
+    let fullFeedToCheck = retryCacheData.concat(rssFeed)
+    let urlsToCheck = []
 
 
     for (let show of myshowlist) {
 
         try {
             // Find show on feed
-            let list_filtered_for_show = feed.filter(item => item.title.includes(show.Name))
+            let listFilteredForShow = fullFeedToCheck.filter(item => item.title.includes(show.Name))
-            if (list_filtered_for_show.length > 0) {
+            if (listFilteredForShow.length > 0) {
-                for (let match of list_filtered_for_show) {
+                for (let match of listFilteredForShow) {
                     // If show is found get url then return all links on that page
-                    let full_link_list_from_page = await getLinksFromURL(match.link)
+                    let fullLinkListFromPage = await getLinksFromURL(match.link)
                     if (hevcSwitch) {
                         // Only get urls with HEVC in name
-                        urls_to_check = full_link_list_from_page.filter(item => item.includes('HEVC'))
+                        urlsToCheck = fullLinkListFromPage.filter(item => item.includes('HEVC'))
-                        if (urls_to_check.length == 0) {
+                        if (urlsToCheck.length == 0) {
                             // If no urls with HEVC check for H265
-                            urls_to_check = full_link_list_from_page.filter(item => item.includes('H265'))
+                            urlsToCheck = fullLinkListFromPage.filter(item => item.includes('H265'))
                         }
                     } else {
-                        urls_to_check = full_link_list_from_page
+                        urlsToCheck = fullLinkListFromPage
                     }
                     // Only keep urls that match show quality
-                    let urls_with_quality_in_url = urls_to_check.filter(item => item.includes(show.Quality))
+                    let urlsWithQualityInUrl = urlsToCheck.filter(item => item.includes(show.Quality))
                     // Remove any url trying to direct to a torrent site search
-                    let urls_without_torrent_in_url = urls_with_quality_in_url.filter(item => !item.includes('torrent'))
+                    let urlsWithoutTorrentInUrl = urlsWithQualityInUrl.filter(item => !item.includes('torrent'))
                     // Remove any url that doesn't include MeGusta
                     if (hevcSwitch) {
-                        pre_nitroFlare = urls_without_torrent_in_url.filter(item => item.includes('MeGusta'))
+                        preNitroflare = urlsWithoutTorrentInUrl.filter(item => item.includes('MeGusta'))
                     } else {
-                        pre_nitroFlare = urls_without_torrent_in_url
+                        preNitroflare = urlsWithoutTorrentInUrl
                     }
                     // NitroFlare doesn't group with the rest of the links in JD, remove them.
-                    let remove_nitroflare = pre_nitroFlare.filter(item => !item.includes('nitro'))
+                    let removeNitroflare = preNitroflare.filter(item => !item.includes('nitro'))
                     // Do some stuff
-                    urlObj = checkFileName(remove_nitroflare)
+                    urlObj = checkFileName(removeNitroflare)
-                    let download_list = urlObj.urlList
+                    let downloadList = urlObj.urlList
                     // Send Links to JDdownloader
-                    if (download_list.length !== 0) {
+                    if (downloadList.length !== 0) {
-                        log.info(download_list.length + ' links for ' + urlObj.fileName + ' have been sent to JDdownloader')
+                        if (checkDownloadHistory(urlObj)) {
-                        linkAdder(download_list)
+                            log.info(urlObj.fileName + ' already downloaded, skipped.')
+                            break
+                        } else {
+                            log.tele(downloadList.length + ' links for ' + urlObj.fileName + ' have been sent to JDdownloader.')
+                            linkAdder(downloadList)
+                            global.linkCheckTime = new Date();
+                        }
                     } else {
                         // No HEVC links found
-                        log.info(download_list.length + ' links for ' + show.Name + ' have been found, will recheck next time.')
+                        log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.')
-                        for (let feed_item of list_filtered_for_show) {
+                        for (let feedItem of listFilteredForShow) {
-                            retry_show_cache.push(feed_item)
+                            retryCache(feedItem)
                         }
+                        global.linkCheckTime = new Date();
                     }
                 }
             } else {
@@ -63,10 +73,12 @@ async function filterFeed() {
             }
         } catch (error) {
             log.error('Something went wrong ' + error)
+            global.linkCheckTime = new Date();
         }
     }
     log.info('Wiping feed cache')
-    fs.writeFileSync('./feedCache.json', JSON.stringify(retry_show_cache));
+    fs.writeFileSync('./cache/feedCache.json', JSON.stringify([]));
+    global.linkCheckTime = new Date();
 }
 
 module.exports = {

FeedUpdater.js

@@ -11,17 +11,16 @@ async function feedUpdater() {
 
     let items = [];
 
-    if (fs.existsSync('./feedCache.json')) {
+    if (fs.existsSync('./cache/feedCache.json')) {
-        items = JSON.parse(fs.readFileSync('./feedCache.json'))
+        items = JSON.parse(fs.readFileSync('./cache/feedCache.json'))
     }
     // Compare existing cache and new items and merge differences
     let updatedArray = lodash.unionBy(feed.items, items, 'title');
 
     // Save the file
     log.info(updatedArray.length + ' items in file cache')
-    fs.writeFileSync('./feedCache.json', JSON.stringify(updatedArray));
+    fs.writeFileSync('./cache/feedCache.json', JSON.stringify(updatedArray));
-
+    global.rssRefreshTime = new Date();
-
 }
 
 module.exports = {

JDRssDownloader.js

@@ -1,19 +1,58 @@
 const fs = require("fs");
+config = JSON.parse(fs.readFileSync('config.json'))
 const { feedUpdater } = require('./FeedUpdater')
 const { filterFeed } = require('./FeedFilter')
+const { telegrambot } = require('./telegramCommunication')
+const { create_empty_cache_files } = require('./utils')
+const express = require('express');
+const bodyParser = require('body-parser');
+const cors = require('cors');
+const path = require("path");
+const basicAuth = require('express-basic-auth')
+const app = express();
+app.set("views", path.join(__dirname, "views"));
+app.set("view engine", "pug");
+app.use(express.static(path.join(__dirname, "public")));
+app.use(cors());
+app.use(bodyParser.urlencoded({ extended: false }));
+app.use(bodyParser.json());
+require('./routes')(app);
+app.use(basicAuth({
+    users: { 'admin': config.AdminPassword },
+    challenge: true,
+}))
+
+global.rssRefreshTime = new Date();
+global.linkCheckTime = new Date();
+global.version = require('./package.json').version;
+
 global.log = require('simple-node-logger').createSimpleLogger({
     logFilePath: 'jdrssdownloader.log',
     timestampFormat: 'YYYY-MM-DD HH:mm:ss.SSS'
 });
 
+log.tele = function () {
+    var args = Array.prototype.slice.call(arguments),
+        entry = log.log('info', args);
+    process.nextTick(function () {
+        if (config.TelegramBot) {
+            telegrambot(entry.msg[0])
+        }
+    });
+};
+
 async function main() {
-    let RSSFeedRefreshMins = JSON.parse(fs.readFileSync('config.json')).RSSFeedRefreshMins
+    log.tele('Running JDRssDownloader version ' + global.version)
-    let JDPostLinksMins = JSON.parse(fs.readFileSync('config.json')).JDPostLinksMins
+    try {
-
+        RSSFeedRefreshMins = config.RSSFeedRefreshMins
-    log.info('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes')
+        JDPostLinksMins = config.JDPostLinksMins
-    log.info('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes')
+    } catch (error) {
-
+        log.error('config.json file is missing.')
-
+    }
+    log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes')
+    log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes')
+    app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`))
+    create_empty_cache_files()
     setInterval(await feedUpdater, RSSFeedRefreshMins * 60000);
     setInterval(await filterFeed, JDPostLinksMins * 60000);
 }

LinkGrabber.js

@@ -1,13 +1,32 @@
 const axios = require('axios');
 var cheerio = require('cheerio');
 
+async function flareSolverr(url) {
+    var data = JSON.stringify({
+        "cmd": "request.get",
+        "url": url,
+        "maxTimeout": 120000
+    });
+    var config = {
+        method: 'post',
+        url: 'http://127.0.01:8191/v1',
+        headers: {
+            'Content-Type': 'application/json'
+        },
+        data: data
+    };
+    res = await axios(config)
+    return res.data.solution.response
+}
+
 async function getLinksFromURL(url) {
 
     try {
         let links = [];
-        let httpResponse = await axios.get(url);
+        let scrape = await flareSolverr(url)
+        // let httpResponse = await axios.get(url);
 
-        let $ = cheerio.load(httpResponse.data);
+        let $ = cheerio.load(scrape);
         let linkObjects = $('a'); // get all hyperlinks
 
         linkObjects.each((index, element) => {

README.md

@@ -1,80 +1,152 @@
+
 # JDRssDownloader
 
+  
+
 JDownloader 2 is a great tool, but since V1 has been missing a way to automatically download from RSS feeds, and filter downloads to only download what you want, in my case 720p HEVC files, MeGusta rips by preference.
 
+  
+
 I have put together this simple project to allow me to do that, people may find useful.
 
+  
+
 - Automatically check an RSS feed and send to JDownloader
+
 - Uses MyJDownloader API to allow running on separate system
+
 - Local file cache of RSS feed
+
 - Specify time to check RSS feed
+
 - Specify time to check file cache to send links to JDownloader
+
 - Ability to add multiple shows to check for
+
 - Ability to check for different qualities per show you are looking for
+
 - Ability to turn OFF only HEVC search
 
+  
+
 # Configuration
 
+  
+
 There is a `config-sample.json` file that needs to be renamed to `config.json`, after this you can update it with your required settings.
 
+  
+
 - JDUserName - Your MyJDownloader Username
+
 - JDPassword - Your MyJDownloader Password
+
+- AdminPassword - Password to be set for the WebUI
+
+- WebUIPort - Port for the WebUI to run on
+
 - RSSFeed - The url to the rss feed you want to watch (Only tested with - rlsbb)
+
 - RSSFeedRefreshMins - How often to poll your rss feed down to local file cache
+
 - JDPostLinksMins - How often to check your file cache for your shows and send found links to JDownloader
+
 - Autostart - Tells JDownloader to add and start the downloads straight away (true/false)
+
 - OnlyHEVC - If false, this will download any files that it finds on the post that matches the quality (true/false)
-- Shows - This needs to be a comma separated list of json objects of the show and quality you want to check for.
+
+- TelegramBot - Set to true if you wish to have updates sent via telegramBot
+
+- TelegramBotID - Set this to the id you recieve from TheBotFather
+
+- TelegramChatID - Chat or Group ID for the bot to send messages to
+
+  
 
 An example shown below
 
+
 ```
 {
-    "JDUserName": "User",
+"JDUserName": "User",
-    "JDPassword": "Pass",
+"JDPassword": "Pass",
-    "RSSFeed": "https://mypage.com/feed/",
+"AdminPassword":"",
-    "RSSFeedRefreshMins": 10,
+"WebUIPort": 3100,
-    "JDPostLinksMins": 180,
+"RSSFeed": "https://mypage.com/feed/",
-    "Autostart": false,
+"RSSFeedRefreshMins": 10,
-    "OnlyHEVC": true,
+"JDPostLinksMins": 180,
-    "Shows": [
+"Autostart": false,
-        {
+"OnlyHEVC": true,
-            "Name": "Obi-Wan Kenobi",
+"TelegramBot": true,
-            "Quality": "1080"
+"TelegramBotID":"",
-        },
+"TelegramChatID":123456789,
-        {
-            "Name": "Taskmaster",
-            "Quality": "720"
-        }
-    ]
 }
 ```
 
+  
+
 # Running
 
+  
+
+## FlareSolverr
+
+Due to issues with a number of sites I use, I have had to rework the client to now use [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) this allows the tool to bypass/workout Cloudflare site issues, the implication of this is that we need to run a docker image in the background that will actually open the page up in a Chrome window, then return the HTML and get the links as usual, this does slow down the process, as it takes upto 2 mins for it to return HTML from the page, but it works..
+
+So you need to have docker installed and to run this command 
+
+    docker-compose up -d
+
+This will run the container in the background, then you can run as usual, and it will proxy all requests via FlareSolverr, please open an issue if you find problems.  
+
 ## Release Version
 
+  
+
 Either download the version on the releases, as well as the `config-sample.json` and run execute, this is the simplest way, but may not be the latest code, and will not run in the background
 
+  
+
 ## Source Version
 
+  
+
 You will need NodeJS installed, then you can checkout this repo.
 
+  
+
 For basic usage you can just navigate into the folder and run -
 
-- `npm i` to install the requirements.
+  
-- `node JDRssDownloader.js` This will execute the process and add the links if they are found.
 
-My suggestion would be to use pm2 so it can run "in the background"
+-  `npm i` to install the requirements.
+
+-  `node JDRssDownloader.js` This will execute the process and add the links if they are found.
+
+  
+
+My suggestion would be to use [pm2](https://pm2.keymetrics.io/docs/usage/quick-start/) so it can run "in the background"
+
+  
 
 # Issues
 
+  
+
 Not alot of testing has gone into this, and I threw it together in a few hours, and only for my use case, so there are bound to be issues, please open them and let me know if you find any.
 
+  
+
 # Future
 
+  
+
 I have some ideas to make this a bit smarter and I want to add the ability to look at multiple RSS feeds, this seems quite easy, and I will do in the next couple of weeks.
 
+  
+
 # Thanks
 
+  
+
 Thank for all the people who made any of the modules that I used to create this.

apiFunctions.js

@@ -0,0 +1,84 @@
+const fs = require('fs');
+
+
+async function addNewShow(showData) {
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
+    let exist = false
+    for (let show of shows) {
+        if (show.Name == showData.showName) {
+            exist = true
+        }
+    }
+    if (exist) {
+        log.error(showData.showName + ' Already exists in list and not added')
+    } else {
+        shows.push({
+            "Name": showData.showName,
+            "Quality": showData.quality
+        })
+        try {
+            fs.writeFileSync('shows.json', JSON.stringify(shows));
+            log.info(showData.showName + ' Added to the list, checking for ' + showData.quality + 'p')
+        } catch (err) {
+            console.error(err);
+        }
+    }
+}
+
+async function removeShow(showData) {
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
+
+    myArray = shows.filter(function (obj) {
+        return obj.Name !== showData.showName;
+    });
+
+    shows = myArray
+    try {
+        fs.writeFileSync('shows.json', JSON.stringify(shows));
+        log.info(showData.showName + ' Removed from tracking list.')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+async function editShow(showData) {
+    let shows = JSON.parse(fs.readFileSync('shows.json'))
+    for (let index = 0; index < shows.length; index++) {
+        const element = shows[index];
+        if (element.Name == showData.showName) {
+            shows[index] = {
+                "Name": showData.showName,
+                "Quality": showData.quality
+            }
+        }
+    }
+    try {
+        fs.writeFileSync('shows.json', JSON.stringify(shows));
+        log.info(showData.showName + ' Quality modified to ' + showData.quality + 'p')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+async function removeShowFromCache(showData) {
+    let shows = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+
+    myArray = shows.filter(function (obj) {
+        return obj.title !== showData;
+    });
+
+    shows = myArray
+    try {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(shows));
+        log.info(showData + ' Removed from retry cache.')
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+module.exports = {
+    addNewShow, removeShow, editShow, removeShowFromCache
+}
+

checkDownloadHistory.js

@@ -0,0 +1,14 @@
+const fs = require('fs')
+
+function checkDownloadHistory(urlObj) {
+    history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    if (history.includes(urlObj.fileName)) {
+        return true
+    } else {
+        history.push(urlObj.fileName)
+        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify(history));
+        return false
+    }
+}
+
+module.exports = { checkDownloadHistory }

config-sample.json

@@ -1,15 +1,14 @@
 {
     "JDUserName": "",
     "JDPassword": "",
+    "AdminPassword":"",
+    "WebUIPort": 3100,
     "RSSFeed": "",
     "RSSFeedRefreshMins": 10,
     "JDPostLinksMins": 180,
     "Autostart": false,
     "OnlyHEVC": true,
-    "Shows": [
+    "TelegramBot": true,
-        {
+    "TelegramBotID":"",
-            "Name": "",
+    "TelegramChatID":123456789
-            "Quality": "720"
-        }
-    ]
 }

docker-compose.yml

@@ -0,0 +1,15 @@
+---
+version: "2.1"
+services:
+  flaresolverr:
+    # DockerHub mirror flaresolverr/flaresolverr:latest
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - LOG_HTML=${LOG_HTML:-false}
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
+    ports:
+      - "127.0.0.1:${PORT:-8191}:8191"
+    restart: unless-stopped

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jdrssdownloader",
-  "version": "1.0.1",
+  "version": "1.2.0",
   "description": "",
   "main": "JDRssDownloader.js",
   "bin": "JDRssDownloader.js",
@@ -11,9 +11,19 @@
   "license": "ISC",
   "dependencies": {
     "axios": "^0.27.2",
+    "bluebird": "^3.7.2",
+    "body-parser": "^1.20.2",
     "cheerio": "^1.0.0-rc.11",
+    "cors": "^2.8.5",
+    "express": "^4.18.2",
+    "express-basic-auth": "^1.2.1",
+    "express-validator": "^6.14.2",
     "jdownloader-client": "^1.0.0",
+    "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
+    "moment": "^2.29.4",
+    "node-telegram-bot-api": "^0.59.0",
+    "pug": "^3.0.2",
     "rss-parser": "^3.12.0",
     "simple-node-logger": "^21.8.12"
   },

public/style.css

@@ -0,0 +1,158 @@
+@import url("https://fonts.googleapis.com/css?family=Raleway:800|Merriweather+Sans|Share+Tech+Mono");
+
+:root {
+  --ui-shadow: 0 2px 4px -1px rgba(0, 0, 0, 0.06),
+    0 4px 5px 0 rgba(0, 0, 0, 0.06), 0 1px 10px 0 rgba(0, 0, 0, 0.08);
+  fill: rgba(0, 0, 0, 0.54);
+  --ui-shadow-border: 1px solid rgba(0, 0, 0, 0.14);
+}
+
+* {
+  box-sizing: border-box;
+}
+
+html,
+body,
+#root {
+  height: 100%;
+  width: 100%;
+}
+
+body {
+  margin: 0;
+  padding: 0;
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
+    "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  -webkit-tap-highlight-color: rgba(255, 255, 255, 0);
+  background-color: #0e0e0e;
+}
+
+h1,
+h2,
+h3 {
+  font-family: "Raleway", sans-serif;
+  text-transform: uppercase;
+
+  padding: 0;
+  margin: 0;
+
+  color: #2a3747;
+}
+
+h1 {
+  font-size: 40px;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+  cursor: pointer;
+  user-select: none;
+}
+
+#root {
+  display: flex;
+  flex-direction: column;
+}
+
+.View {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  align-items: center;
+
+  height: 100%;
+  width: 100%;
+
+  padding: 20px;
+
+  background-size: cover;
+
+  font-family: "Merriweather Sans", sans-serif;
+}
+
+.Banner {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+
+  width: 100%;
+
+  border-radius: 5px;
+
+  overflow: hidden;
+  background: white;
+  padding: 15px;
+
+  font-family: "Share Tech Mono", monospace;
+
+  border-bottom: var(--ui-shadow-border);
+  box-shadow: var(--ui-shadow);
+}
+
+.Message {
+  background: white;
+  border-radius: 5px;
+  padding: 20px;
+  border-bottom: var(--ui-shadow-border);
+  box-shadow: var(--ui-shadow);
+  overflow-x: hidden;
+  overflow-y: auto;
+  /* text-align: center; */
+  margin-top:10px;
+  margin-bottom:10px;
+}
+
+.Message > .Title {
+  padding-bottom: 20px;
+}
+
+.Message > .Details {
+  display: flex;
+  flex-direction: column;
+  line-height: 1.5em;
+}
+
+.NavButtons {
+  display: flex;
+  width: 100%;
+  justify-content: space-around;
+  align-items: center;
+  padding: 0 20px;
+}
+
+.NavButton {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+
+  height: 55px;
+  width: 150px;
+
+  background: #fa4141;
+
+  border-radius: 30px;
+
+  font-size: 16px;
+  font-weight: bold;
+  color: white;
+
+  text-transform: capitalize;
+
+  border-bottom: var(--ui-shadow-border);
+  box-shadow: var(--ui-shadow);
+}
+
+th {
+  text-align: left;
+  
+}
+
+td { border-left: 1px solid #000;
+  border-right: 1px solid #000;
+}

retryCache.js

@@ -0,0 +1,21 @@
+const fs = require('fs')
+
+function retryCache(cache) {
+    if (cache == null) {
+        retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+        return retryCacheData
+    }
+    retryCacheData = JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    if (retryCacheData.some(e => e.title === cache.title)) {
+        log.info(cache.title + ' is already in the retry cache.')
+    }
+    else {
+        cache.retryCount = 5
+        retryCacheData.push(cache)
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify(retryCacheData));
+        log.info(cache.title + ' written to retry cache.')
+        return retryCacheData
+    }
+}
+
+module.exports = { retryCache }

routes/api.js

@@ -0,0 +1,28 @@
+const fs = require("fs");
+const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils')
+
+
+module.exports = function (app) {
+    app.get("/api/stats", (req, res) => {
+        retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+        for (let index = 0; index < retryCache.length; index++) {
+            const item = retryCache[index];
+            retryCache[index].newtitle = item.title.replace(/ /g, "‡");
+        }
+        showList = JSON.parse(fs.readFileSync('shows.json'));
+        feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
+        rssTime = nextRssRefresh()
+        linkCheck = nextLinkCheck()
+        lastDownloaded = get_last_downloaded()
+
+        res.json({
+            "ShowList": showList.length,
+            "FeedCache": feedCache.length,
+            "RetryCache": retryCache.length,
+            "RSSCheck": rssTime,
+            "LinkChecker": linkCheck,
+            "LastDownloaded": lastDownloaded
+        });
+    });
+
+}

routes/cache.js

@@ -0,0 +1,25 @@
+const fs = require("fs");
+const {removeShowFromCache} = require('../apiFunctions')
+
+module.exports = function (app) {
+    app.get("/feedCache", (req, res) => {
+        feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json'))
+        res.render("feedCache", { title: "Feed Cache", feedCache: feedCache });
+    });
+
+    app.get("/retryCache", (req, res) => {
+        retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+        for (let index = 0; index < retryCache.length; index++) {
+            const item = retryCache[index];
+            retryCache[index].newtitle = item.title.replace(/ /g, "‡");
+        }
+        res.render("retryCache", { title: "Retry Cache", retryCache: retryCache });
+    });
+
+    app.get('/retryCache/remove', (req, res) => {
+        const showName = req.query.name.replaceAll("‡", " ");
+        console.log(req)
+        removeShowFromCache(showName)
+        res.redirect("/retryCache");
+    });
+}

routes/index.js

@@ -0,0 +1,10 @@
+var fs = require('fs');
+
+module.exports = function(app) {
+    fs.readdirSync(__dirname).forEach(function(file) {
+        if (file === "index.js" || file.substr(file.lastIndexOf('.') + 1) !== 'js')
+            return;
+        var name = file.substr(0, file.indexOf('.'));
+        require('./' + name)(app);
+    });
+}

routes/logFIle.js

@@ -0,0 +1,15 @@
+const lineReader = require("line-reader");
+const Promise = require("bluebird");
+
+module.exports = function (app) {
+    app.get("/logFile", (req, res) => {
+        logFile = []
+        const eachLine = Promise.promisify(lineReader.eachLine);
+        eachLine('jdrssdownloader.log', function (line) {
+            logFile.push(line)
+        }).then(() => {
+            logFile = logFile.slice((logFile.length - 30), logFile.length)
+            res.render("logFile", { title: "App Logs", logFile: logFile.reverse() });
+        });
+    });
+}

routes/root.js

@@ -0,0 +1,14 @@
+const fs = require("fs");
+const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils')
+
+module.exports = function (app) {
+
+    app.get("/", (req, res) => {
+        showListLength = JSON.parse(fs.readFileSync('shows.json')).length
+        feedCacheLength = JSON.parse(fs.readFileSync('./cache/feedCache.json')).length
+        retryCacheLength = JSON.parse(fs.readFileSync('./cache/retryCache.json')).length
+        rssTime = nextRssRefresh()
+        linkCheck = nextLinkCheck()
+        res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: rssTime, linkCheck: linkCheck, lastDownloaded: get_last_downloaded() });
+    });
+}

routes/shows.js

@@ -0,0 +1,49 @@
+const fs = require("fs");
+const { addNewShow, removeShow, editShow } = require('.././apiFunctions')
+const { check, validationResult } = require('express-validator');
+
+
+module.exports = function (app) {
+    app.get("/shows", (req, res) => {
+        showList = JSON.parse(fs.readFileSync('shows.json'))
+        res.render("shows", { title: "Show List", showList: showList });
+    });
+
+    app.get("/shows/add", (req, res) => {
+        res.render("addshow", { title: "Add Show" });
+    });
+
+    app.get("/shows/remove", (req, res) => {
+        showList = JSON.parse(fs.readFileSync('shows.json'))
+        res.render("removeshow", { title: "Remove Show", showList: showList });
+    });
+
+    app.get("/shows/edit", (req, res) => {
+        showList = JSON.parse(fs.readFileSync('shows.json'))
+        res.render("editShow", { title: "Edit Show", showList: showList });
+    });
+
+    app.post('/addNewShow', [
+        check('showName')
+            .isLength({ min: 1 })
+    ], (req, res) => {
+        if (validationResult(req).isEmpty()) {
+            addNewShow(req.body)
+            res.redirect("/shows");
+        } else {
+            log.error('You cannot add a show without a name.')
+            res.redirect("/shows");
+        }
+    }
+    );
+
+    app.post('/removeShow', (req, res) => {
+        removeShow(req.body)
+        res.redirect("/shows");
+    });
+
+    app.post('/editShow', (req, res) => {
+        editShow(req.body)
+        res.redirect("/shows");
+    });
+}

telegramCommunication.js

@@ -0,0 +1,20 @@
+const fs = require("fs");
+const TelegramBot = require('node-telegram-bot-api');
+const token = JSON.parse(fs.readFileSync('config.json')).TelegramBotID;
+const chatId = JSON.parse(fs.readFileSync('config.json')).TelegramChatID;
+
+const bot = new TelegramBot(token, { polling: false });
+
+const telegrambot = (message) => {
+    try {
+        bot.sendMessage(chatId, message, {
+            parse_mode: 'html'
+        });
+    } catch (err) {
+        console.log('Something went wrong when trying to send a Telegram notification', err);
+    }
+}
+
+module.exports = {
+    telegrambot
+}

utils.js

@@ -0,0 +1,61 @@
+const fs = require('fs');
+config = JSON.parse(fs.readFileSync('config.json'))
+var moment = require('moment');
+
+function returnUpdatedDate(date, offset) {
+    var newDate = moment(date);
+    newDate.add(offset, 'm');
+    return new moment(newDate).format("ddd, LTS")
+}
+
+function nextRssRefresh() {
+    return returnUpdatedDate(global.rssRefreshTime, config.RSSFeedRefreshMins)
+}
+
+function nextLinkCheck() {
+    return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins)
+}
+
+function get_last_downloaded() {
+    history = JSON.parse(fs.readFileSync('./cache/downloadHistory.json'))
+    last = history.slice(-1)[0]
+    return last
+}
+
+function create_empty_downloadHistory() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/downloadHistory.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/downloadHistory.json'));
+    }
+}
+
+function create_empty_retry_cache() {
+    try {
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    } catch (error) {
+        fs.writeFileSync('./cache/retryCache.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./cache/retryCache.json'));
+    }
+}
+
+function create_empty_shows_file() {
+    try {
+        return JSON.parse(fs.readFileSync('./shows.json'));
+    } catch (error) {
+        fs.writeFileSync('./shows.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./shows.json'));
+    }
+}
+
+function create_empty_cache_files() {
+    create_empty_downloadHistory()
+    create_empty_retry_cache()
+    create_empty_shows_file() 
+}
+
+module.exports = {
+    nextRssRefresh, nextLinkCheck, get_last_downloaded, create_empty_cache_files
+}
+

views/addshow.pug

@@ -0,0 +1,18 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Add Show
+    div.Message
+      form(action="/addNewShow" method="POST")
+        p Show Name: 
+          input(type="text" name="showName" placeholder="Enter the show to track ")
+        p Quality: 
+          select(name="quality")
+            option(value='720') #{'720p'}
+            option(value='1080') #{'1080p'}
+            option(value='2160') #{'2160p'}
+        input(type="submit", value="Add Show")
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home

views/editShow.pug

@@ -0,0 +1,20 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Edit Show
+    div.Message
+      form(action="/editShow" method="POST")
+        p Show Name: 
+          select(name="showName")
+            each show in showList
+                option(value=show.Name) #{show.Name}
+        p Quality: 
+          select(name="quality")
+            option(value='720') #{'720p'}
+            option(value='1080') #{'1080p'}
+            option(value='2160') #{'2160p'}
+        input(type="submit", value="Edit Show")
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home

views/feedCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Feed Cache
+    if (feedCache.length==0)
+      div.Message
+        h2 No shows in Feed Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              //- th Remove
+          tbody
+            each val, key in feedCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                //- td 
+                //-   a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home

views/index.pug

@@ -0,0 +1,33 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner JDRssDownloader #{version}
+    body
+      div.Message
+        h2 Number of Tracked Shows
+        h3 #{showListLength}
+        h2 Last Downloaded
+        h3 #{lastDownloaded}             
+      div.Message
+        h2 
+          a(href='/feedCache') RSS Feed Cache Size 
+        h3 #{feedCacheLength}
+        h2
+          a(href='/retryCache') Retry Cache Size
+        h3 #{retryCacheLength}
+        h2 Next RSS Refresh
+        h3 #{rssTime}
+        h2 Next Link Check
+        h3 #{linkCheck}   
+    div.NavButtons
+      a(href="/shows")
+          div.NavButton Show List
+      a(href="/shows/add")
+          div.NavButton Add New Show
+      a(href="/shows/edit")
+          div.NavButton Edit Show          
+      a(href="/shows/remove")
+          div.NavButton Remove Show
+      a(href="/logFile")
+          div.NavButton Logs

views/layout.pug

@@ -0,0 +1,13 @@
+block variables
+doctype html
+html
+  head
+    meta(charset="utf-8")
+    link(rel="shortcut icon", href="/favicon.ico")
+    meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=yes")
+    meta(name="theme-color", content="#000000")
+    title #{title} | JDRssDownloader
+    link(rel="stylesheet" href="/style.css")
+  body
+    div#root
+      block layout-content

views/logFile.pug

@@ -0,0 +1,11 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Log File
+    div.Message
+      each val in logFile
+        li= val
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home

views/removeshow.pug

@@ -0,0 +1,15 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Remove Show
+    div.Message
+      form(action="/removeShow" method="POST")
+        p Show Name: 
+          select(name="showName")
+            each show in showList
+                option(value=show.Name) #{show.Name}
+        input(type="submit", value="Remove Show")
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home

views/retryCache.pug

@@ -0,0 +1,26 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Retry Cache
+    if (retryCache.length==0)
+      div.Message
+        h2 No shows in Retry Cache
+    else
+      div.Message
+        table
+          thead
+            tr
+              th Show Name
+              th Remove
+          tbody
+            each val, key in retryCache
+              tr
+                td
+                  a(href=val.link
+                    target="_blank") #{val.title}
+                td 
+                  a(href='/retryCache/remove?name=' + val.newtitle) Remove
+    div.NavButtons
+      a(href="/")
+        div.NavButton Home

views/shows.pug

@@ -0,0 +1,19 @@
+extends layout
+
+block layout-content
+  div.View
+    h1.Banner Show List
+    div.Message
+      table
+        thead
+          tr
+           th Show Name
+           th Quality
+        tbody
+          each val, key in showList
+            tr
+              td= val.Name
+              td= val.Quality
+    div.NavButtons
+      a(href="/")
+          div.NavButton Home