diff --git a/FeedFilter.js b/FeedFilter.js index ce6bf61..3bb61d8 100644 --- a/FeedFilter.js +++ b/FeedFilter.js @@ -3,68 +3,66 @@ const { linkAdder } = require('./JDLinkAdder'); const { getLinksFromURL } = require('./LinkGrabber') const { checkFileName } = require('./checkFileName') const { checkDownloadHistory } = require('./checkDownloadHistory') -const { telegrambot } = require('./telegramCommunication') async function filterFeed() { let myshowlist = JSON.parse(fs.readFileSync('config.json')).Shows let hevcSwitch = JSON.parse(fs.readFileSync('config.json')).OnlyHEVC let feed = JSON.parse(fs.readFileSync('./feedCache.json')); - let retry_show_cache = [] - let urls_to_check = [] + let retryShowCache = [] + let urlsToCheck = [] for (let show of myshowlist) { try { // Find show on feed - let list_filtered_for_show = feed.filter(item => item.title.includes(show.Name)) - if (list_filtered_for_show.length > 0) { - for (let match of list_filtered_for_show) { + let listFilteredForShow = feed.filter(item => item.title.includes(show.Name)) + if (listFilteredForShow.length > 0) { + for (let match of listFilteredForShow) { // If show is found get url then return all links on that page - let full_link_list_from_page = await getLinksFromURL(match.link) + let fullLinkListFromPage = await getLinksFromURL(match.link) if (hevcSwitch) { // Only get urls with HEVC in name - urls_to_check = full_link_list_from_page.filter(item => item.includes('HEVC')) - if (urls_to_check.length == 0) { + urlsToCheck = fullLinkListFromPage.filter(item => item.includes('HEVC')) + if (urlsToCheck.length == 0) { // If no urls with HEVC check for H265 - urls_to_check = full_link_list_from_page.filter(item => item.includes('H265')) + urlsToCheck = fullLinkListFromPage.filter(item => item.includes('H265')) } } else { - urls_to_check = full_link_list_from_page + urlsToCheck = fullLinkListFromPage } // Only keep urls that match show quality - let urls_with_quality_in_url = urls_to_check.filter(item => item.includes(show.Quality)) + let urlsWithQualityInUrl = urlsToCheck.filter(item => item.includes(show.Quality)) // Remove any url trying to direct to a torrent site search - let urls_without_torrent_in_url = urls_with_quality_in_url.filter(item => !item.includes('torrent')) + let urlsWithoutTorrentInUrl = urlsWithQualityInUrl.filter(item => !item.includes('torrent')) // Remove any url that doesn't include MeGusta if (hevcSwitch) { - pre_nitroFlare = urls_without_torrent_in_url.filter(item => item.includes('MeGusta')) + preNitroflare = urlsWithoutTorrentInUrl.filter(item => item.includes('MeGusta')) } else { - pre_nitroFlare = urls_without_torrent_in_url + preNitroflare = urlsWithoutTorrentInUrl } // NitroFlare doesn't group with the rest of the links in JD, remove them. - let remove_nitroflare = pre_nitroFlare.filter(item => !item.includes('nitro')) + let removeNitroflare = preNitroflare.filter(item => !item.includes('nitro')) // Do some stuff - urlObj = checkFileName(remove_nitroflare) - let download_list = urlObj.urlList + urlObj = checkFileName(removeNitroflare) + let downloadList = urlObj.urlList // Send Links to JDdownloader - if (download_list.length !== 0) { + if (downloadList.length !== 0) { if (checkDownloadHistory(urlObj)) { log.info(urlObj.fileName + ' already downloaded, skipped.') break } else { - log.info(download_list.length + ' links for ' + urlObj.fileName + ' have been sent to JDdownloader.') - if (TelegramBotConfig) { - telegrambot(download_list.length + ' links for ' + urlObj.fileName + ' have been sent to JDdownloader.') - } - linkAdder(download_list) + log.tele(downloadList.length + ' links for ' + urlObj.fileName + ' have been sent to JDdownloader.') + linkAdder(downloadList) + global.linkCheckTime = new Date(); } } else { // No HEVC links found - log.info(download_list.length + ' links for ' + show.Name + ' have been found, will recheck next time.') - for (let feed_item of list_filtered_for_show) { - retry_show_cache.push(feed_item) + log.info(downloadList.length + ' links for ' + show.Name + ' have been found, will recheck next time.') + for (let feedItem of listFilteredForShow) { + retryShowCache.push(feedItem) } + global.linkCheckTime = new Date(); } } } else { @@ -73,10 +71,12 @@ async function filterFeed() { } } catch (error) { log.error('Something went wrong ' + error) + global.linkCheckTime = new Date(); } } log.info('Wiping feed cache') - fs.writeFileSync('./feedCache.json', JSON.stringify(retry_show_cache)); + fs.writeFileSync('./feedCache.json', JSON.stringify(retryShowCache)); + global.linkCheckTime = new Date(); } module.exports = { diff --git a/FeedUpdater.js b/FeedUpdater.js index 086124b..6905fb2 100644 --- a/FeedUpdater.js +++ b/FeedUpdater.js @@ -20,8 +20,7 @@ async function feedUpdater() { // Save the file log.info(updatedArray.length + ' items in file cache') fs.writeFileSync('./feedCache.json', JSON.stringify(updatedArray)); - - + global.rssRefreshTime = new Date(); } module.exports = { diff --git a/JDRssDownloader.js b/JDRssDownloader.js index 5f180e9..fbf05ad 100644 --- a/JDRssDownloader.js +++ b/JDRssDownloader.js @@ -1,37 +1,57 @@ const fs = require("fs"); +config = JSON.parse(fs.readFileSync('config.json')) + const { feedUpdater } = require('./FeedUpdater') const { filterFeed } = require('./FeedFilter') const { telegrambot } = require('./telegramCommunication') -const version = require('./package.json').version; +const express = require('express'); +const bodyParser = require('body-parser'); +const cors = require('cors'); +const path = require("path"); +const basicAuth = require('express-basic-auth') +const app = express(); +app.set("views", path.join(__dirname, "views")); +app.set("view engine", "pug"); +app.use(express.static(path.join(__dirname, "public"))); +app.use(cors()); +app.use(bodyParser.urlencoded({ extended: false })); +app.use(bodyParser.json()); +require('./routes')(app); +app.use(basicAuth({ + users: { 'admin': config.AdminPassword }, + challenge: true, +})) -global.TelegramBotConfig = JSON.parse(fs.readFileSync('config.json')).TelegramBot +global.rssRefreshTime = new Date(); +global.linkCheckTime = new Date(); +global.version = require('./package.json').version; global.log = require('simple-node-logger').createSimpleLogger({ logFilePath: 'jdrssdownloader.log', timestampFormat: 'YYYY-MM-DD HH:mm:ss.SSS' }); -async function main() { - log.info('Running JDRssDownloader version ' + version) +log.tele = function () { + var args = Array.prototype.slice.call(arguments), + entry = log.log('info', args); + process.nextTick(function () { + if (config.TelegramBot) { + telegrambot(entry.msg[0]) + } + }); +}; - if (TelegramBotConfig) { - telegrambot('Running JDRssDownloader version ' + version) - } +async function main() { + log.tele('Running JDRssDownloader version ' + global.version) try { - RSSFeedRefreshMins = JSON.parse(fs.readFileSync('config.json')).RSSFeedRefreshMins - JDPostLinksMins = JSON.parse(fs.readFileSync('config.json')).JDPostLinksMins + RSSFeedRefreshMins = config.RSSFeedRefreshMins + JDPostLinksMins = config.JDPostLinksMins } catch (error) { log.error('config.json file is missing.') } - log.info('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') - if (TelegramBotConfig) { - telegrambot('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') - } - log.info('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') - if (TelegramBotConfig) { - telegrambot('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') - } - + log.tele('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') + log.tele('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') + app.listen(config.WebUIPort, () => log.info(`WebUi is listening on ${config.WebUIPort}!`)) setInterval(await feedUpdater, RSSFeedRefreshMins * 60000); setInterval(await filterFeed, JDPostLinksMins * 60000); } diff --git a/README.md b/README.md index c9dd888..a4b94ad 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,16 @@ There is a `config-sample.json` file that needs to be renamed to `config.json`, - JDUserName - Your MyJDownloader Username - JDPassword - Your MyJDownloader Password +- AdminPassword - Password to be set for the WebUI +- WebUIPort - Port for the WebUI to run on - RSSFeed - The url to the rss feed you want to watch (Only tested with - rlsbb) - RSSFeedRefreshMins - How often to poll your rss feed down to local file cache - JDPostLinksMins - How often to check your file cache for your shows and send found links to JDownloader - Autostart - Tells JDownloader to add and start the downloads straight away (true/false) - OnlyHEVC - If false, this will download any files that it finds on the post that matches the quality (true/false) +- TelegramBot - Set to true if you wish to have updates sent via telegramBot +- TelegramBotID - Set this to the id you recieve from TheBotFather +- TelegramChatID - Chat or Group ID for the bot to send messages to - Shows - This needs to be a comma separated list of json objects of the show and quality you want to check for. An example shown below @@ -32,11 +37,16 @@ An example shown below { "JDUserName": "User", "JDPassword": "Pass", + "AdminPassword":"", + "WebUIPort": 3100, "RSSFeed": "https://mypage.com/feed/", "RSSFeedRefreshMins": 10, "JDPostLinksMins": 180, "Autostart": false, "OnlyHEVC": true, + "TelegramBot": true, + "TelegramBotID":"", + "TelegramChatID":123456789, "Shows": [ { "Name": "Obi-Wan Kenobi", diff --git a/apiFunctions.js b/apiFunctions.js new file mode 100644 index 0000000..81e7ac3 --- /dev/null +++ b/apiFunctions.js @@ -0,0 +1,47 @@ +const fs = require('fs'); +const { config } = require('process'); + +async function addNewShow(showData) { + let config = JSON.parse(fs.readFileSync('config.json')) + let exist = false + for (let show of config.Shows) { + if (show.Name == showData.showName) { + exist = true + } + } + if (exist) { + log.error(showData.showName + ' Already exists in list and not added') + } else { + config.Shows.push({ + "Name": showData.showName, + "Quality": showData.quality + }) + try { + fs.writeFileSync('config.json', JSON.stringify(config)); + log.info(showData.showName + ' Added to the list, checking for ' + showData.quality + 'p') + } catch (err) { + console.error(err); + } + } +} + +async function removeShow(showData) { + let config = JSON.parse(fs.readFileSync('config.json')) + + myArray = config.Shows.filter(function (obj) { + return obj.Name !== showData.showName; + }); + + config.Shows = myArray + try { + fs.writeFileSync('config.json', JSON.stringify(config)); + log.info(showData.showName + ' Removed from tracking list.') + } catch (err) { + console.error(err); + } +} + +module.exports = { + addNewShow, removeShow +} + diff --git a/config-sample.json b/config-sample.json index aeef84b..8295556 100644 --- a/config-sample.json +++ b/config-sample.json @@ -1,11 +1,14 @@ { "JDUserName": "", "JDPassword": "", + "AdminPassword":"", + "WebUIPort": 3100, "RSSFeed": "", "RSSFeedRefreshMins": 10, "JDPostLinksMins": 180, "Autostart": false, "OnlyHEVC": true, + "TelegramBot": true, "TelegramBotID":"", "TelegramChatID":123456789, "Shows": [ diff --git a/package.json b/package.json index 3e1985c..4b72bf3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jdrssdownloader", - "version": "1.0.2", + "version": "1.1.0", "description": "", "main": "JDRssDownloader.js", "bin": "JDRssDownloader.js", @@ -12,9 +12,15 @@ "dependencies": { "axios": "^0.27.2", "cheerio": "^1.0.0-rc.11", + "cors": "^2.8.5", + "express": "^4.18.2", + "express-basic-auth": "^1.2.1", "jdownloader-client": "^1.0.0", + "line-reader": "^0.4.0", "lodash": "^4.17.21", + "moment": "^2.29.4", "node-telegram-bot-api": "^0.59.0", + "pug": "^3.0.2", "rss-parser": "^3.12.0", "simple-node-logger": "^21.8.12" }, diff --git a/public/style.css b/public/style.css new file mode 100644 index 0000000..dd4012f --- /dev/null +++ b/public/style.css @@ -0,0 +1,152 @@ +@import url("https://fonts.googleapis.com/css?family=Raleway:800|Merriweather+Sans|Share+Tech+Mono"); + +:root { + --ui-shadow: 0 2px 4px -1px rgba(0, 0, 0, 0.06), + 0 4px 5px 0 rgba(0, 0, 0, 0.06), 0 1px 10px 0 rgba(0, 0, 0, 0.08); + fill: rgba(0, 0, 0, 0.54); + --ui-shadow-border: 1px solid rgba(0, 0, 0, 0.14); +} + +* { + box-sizing: border-box; +} + +html, +body, +#root { + height: 100%; + width: 100%; +} + +body { + margin: 0; + padding: 0; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen", + "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", + sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + -webkit-tap-highlight-color: rgba(255, 255, 255, 0); + background-color: #0e0e0e; +} + +h1, +h2, +h3 { + font-family: "Raleway", sans-serif; + text-transform: uppercase; + + padding: 0; + margin: 0; + + color: #2a3747; +} + +h1 { + font-size: 40px; +} + +a { + color: inherit; + text-decoration: none; + cursor: pointer; + user-select: none; +} + +#root { + display: flex; + flex-direction: column; +} + +.View { + flex: 1; + display: flex; + flex-direction: column; + justify-content: space-between; + align-items: center; + + height: 100%; + width: 100%; + + padding: 20px; + + background-size: cover; + + font-family: "Merriweather Sans", sans-serif; +} + +.Banner { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + width: 100%; + + border-radius: 5px; + + overflow: hidden; + background: white; + padding: 15px; + + font-family: "Share Tech Mono", monospace; + + border-bottom: var(--ui-shadow-border); + box-shadow: var(--ui-shadow); +} + +.Message { + background: white; + padding: 30px; + border-bottom: var(--ui-shadow-border); + box-shadow: var(--ui-shadow); +} + +.Message > .Title { + padding-bottom: 20px; +} + +.Message > .Details { + display: flex; + flex-direction: column; + line-height: 1.5em; +} + +.NavButtons { + display: flex; + width: 100%; + justify-content: space-around; + align-items: center; + padding: 0 20px; +} + +.NavButton { + display: flex; + justify-content: center; + align-items: center; + + height: 55px; + width: 150px; + + background: #fa4141; + + border-radius: 30px; + + font-size: 16px; + font-weight: bold; + color: white; + + text-transform: capitalize; + + border-bottom: var(--ui-shadow-border); + box-shadow: var(--ui-shadow); +} + +th { + text-align: left; + +} + +td { border-left: 1px solid #000; + border-right: 1px solid #000; +} \ No newline at end of file diff --git a/routes/index.js b/routes/index.js new file mode 100644 index 0000000..cece3c0 --- /dev/null +++ b/routes/index.js @@ -0,0 +1,10 @@ +var fs = require('fs'); + +module.exports = function(app) { + fs.readdirSync(__dirname).forEach(function(file) { + if (file === "index.js" || file.substr(file.lastIndexOf('.') + 1) !== 'js') + return; + var name = file.substr(0, file.indexOf('.')); + require('./' + name)(app); + }); +} \ No newline at end of file diff --git a/routes/logFIle.js b/routes/logFIle.js new file mode 100644 index 0000000..5e0d62f --- /dev/null +++ b/routes/logFIle.js @@ -0,0 +1,15 @@ +const lineReader = require("line-reader"); +const Promise = require("bluebird"); + +module.exports = function (app) { + app.get("/logFile", (req, res) => { + logFile = [] + const eachLine = Promise.promisify(lineReader.eachLine); + eachLine('jdrssdownloader.log', function (line) { + logFile.push(line) + }).then(() => { + logFile = logFile.slice((logFile.length - 30), logFile.length) + res.render("logFile", { title: "App Logs", logFile: logFile.reverse() }); + }); + }); +} \ No newline at end of file diff --git a/routes/root.js b/routes/root.js new file mode 100644 index 0000000..ec52316 --- /dev/null +++ b/routes/root.js @@ -0,0 +1,11 @@ +const fs = require("fs"); +const { nextLinkCheck, nextRssRefresh } = require('.././utils') + +module.exports = function (app) { + + app.get("/", (req, res) => { + showListLength = JSON.parse(fs.readFileSync('config.json')).Shows.length + a = + res.render("index", { title: "Home", showListLength: showListLength, version: global.version, rssTime: nextRssRefresh(), linkCheck: nextLinkCheck() }); + }); +} \ No newline at end of file diff --git a/routes/shows.js b/routes/shows.js new file mode 100644 index 0000000..96369c6 --- /dev/null +++ b/routes/shows.js @@ -0,0 +1,28 @@ +const fs = require("fs"); +const { addNewShow, removeShow } = require('.././apiFunctions') + +module.exports = function (app) { + app.get("/shows", (req, res) => { + showList = JSON.parse(fs.readFileSync('config.json')).Shows + res.render("shows", { title: "Show List", showList: showList }); + }); + + app.get("/shows/add", (req, res) => { + res.render("addshow", { title: "Add Show" }); + }); + + app.get("/shows/remove", (req, res) => { + showList = JSON.parse(fs.readFileSync('config.json')).Shows + res.render("removeshow", { title: "Remove Show", showList: showList }); + }); + + app.post('/addNewShow', (req, res) => { + addNewShow(req.body) + res.redirect("/shows"); + }); + + app.post('/removeShow', (req, res) => { + removeShow(req.body) + res.redirect("/shows"); + }); +} diff --git a/utils.js b/utils.js new file mode 100644 index 0000000..0ceaf32 --- /dev/null +++ b/utils.js @@ -0,0 +1,22 @@ +const fs = require('fs'); +config = JSON.parse(fs.readFileSync('config.json')) +var moment = require('moment'); + +function returnUpdatedDate(date, offset) { + var newDate = moment(date); + newDate.add(offset, 'm'); + return new moment(newDate).format("ddd, LTS") +} + +function nextRssRefresh() { + return returnUpdatedDate(global.rssRefreshTime, config.RSSFeedRefreshMins) +} + +function nextLinkCheck() { + return returnUpdatedDate(global.linkCheckTime, config.JDPostLinksMins) +} + +module.exports = { + nextRssRefresh, nextLinkCheck +} + diff --git a/views/addshow.pug b/views/addshow.pug new file mode 100644 index 0000000..3bf8c4c --- /dev/null +++ b/views/addshow.pug @@ -0,0 +1,17 @@ +extends layout + +block layout-content + div.View + h1.Banner Add Show + div.Message + form(action="/addNewShow" method="POST") + p Show Name: + input(type="text" name="showName" placeholder="Enter the show to track ") + p Quality: + select(name="quality") + option(value='720') #{'720p'} + option(value='1080') #{'1080p'} + input(type="submit", value="Add Show") + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/index.pug b/views/index.pug new file mode 100644 index 0000000..b94462d --- /dev/null +++ b/views/index.pug @@ -0,0 +1,22 @@ +extends layout + +block layout-content + div.View + h1.Banner JDRssDownloader #{version} + body + div.Message + h3 Number of Tracked Shows + h1 #{showListLength} + h3 Next RSS Refresh + h1 #{rssTime} + h3 Next Link Check + h1 #{linkCheck} + div.NavButtons + a(href="/shows") + div.NavButton Show List + a(href="/shows/add") + div.NavButton Add New Show + a(href="/shows/remove") + div.NavButton Remove Show + a(href="/logFile") + div.NavButton Logs \ No newline at end of file diff --git a/views/layout.pug b/views/layout.pug new file mode 100644 index 0000000..508cfc0 --- /dev/null +++ b/views/layout.pug @@ -0,0 +1,13 @@ +block variables +doctype html +html + head + meta(charset="utf-8") + link(rel="shortcut icon", href="/favicon.ico") + meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no") + meta(name="theme-color", content="#000000") + title #{title} | JDRssDownloader + link(rel="stylesheet" href="/style.css") + body + div#root + block layout-content \ No newline at end of file diff --git a/views/logFile.pug b/views/logFile.pug new file mode 100644 index 0000000..a238775 --- /dev/null +++ b/views/logFile.pug @@ -0,0 +1,11 @@ +extends layout + +block layout-content + div.View + h1.Banner Log File + div.Message + each val in logFile + li= val + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/removeshow.pug b/views/removeshow.pug new file mode 100644 index 0000000..644cafe --- /dev/null +++ b/views/removeshow.pug @@ -0,0 +1,15 @@ +extends layout + +block layout-content + div.View + h1.Banner Remove Show + div.Message + form(action="/removeShow" method="POST") + p Show Name: + select(name="showName") + each show in showList + option(value=show.Name) #{show.Name} + input(type="submit", value="Remove Show") + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file diff --git a/views/shows.pug b/views/shows.pug new file mode 100644 index 0000000..a4cb3e7 --- /dev/null +++ b/views/shows.pug @@ -0,0 +1,19 @@ +extends layout + +block layout-content + div.View + h1.Banner Show List + div.Message + table + thead + tr + th Show Name + th Quality + tbody + each val, key in showList + tr + td= val.Name + td= val.Quality + div.NavButtons + a(href="/") + div.NavButton Home \ No newline at end of file