diff --git a/.gitignore b/.gitignore index 5ead65e..bfef211 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ config.json package-lock.json jdrssdownloader.log .vscode/launch.json +feedCache.json +dist/jdrssdownloader-linux +dist/jdrssdownloader-win.exe +dist/jdrssdownloader-macos diff --git a/FeedFilter.js b/FeedFilter.js index f9d7140..6a4b1c5 100644 --- a/FeedFilter.js +++ b/FeedFilter.js @@ -1,11 +1,10 @@ const fs = require('fs') const { linkAdder } = require('./JDLinkAdder'); const { getLinksFromURL } = require('./LinkGrabber') -const log = require('simple-node-logger').createSimpleLogger('jdrssdownloader.log'); -async function filterFeed(fileName) { +async function filterFeed() { let myshowlist = JSON.parse(fs.readFileSync('config.json')).Shows - let feed = JSON.parse(fs.readFileSync(fileName)); + let feed = JSON.parse(fs.readFileSync('./feedCache.json')); myshowlist.forEach(async show => { try { @@ -44,7 +43,10 @@ async function filterFeed(fileName) { } catch (error) { log.error('Something went wrong ' + error) } + }) + // log.info('Wiping feed cache') + // fs.writeFileSync(global.fileName, JSON.stringify('[]')); } module.exports = { diff --git a/FeedUpdater.js b/FeedUpdater.js new file mode 100644 index 0000000..086124b --- /dev/null +++ b/FeedUpdater.js @@ -0,0 +1,29 @@ +const fs = require("fs"); +const Parser = require("rss-parser"); +const lodash = require('lodash'); + +async function feedUpdater() { + // Make a new RSS Parser + const parser = new Parser(); + + // Get all the items in the RSS feed + const feed = await parser.parseURL(JSON.parse(fs.readFileSync('config.json')).RSSFeed); + + let items = []; + + if (fs.existsSync('./feedCache.json')) { + items = JSON.parse(fs.readFileSync('./feedCache.json')) + } + // Compare existing cache and new items and merge differences + let updatedArray = lodash.unionBy(feed.items, items, 'title'); + + // Save the file + log.info(updatedArray.length + ' items in file cache') + fs.writeFileSync('./feedCache.json', JSON.stringify(updatedArray)); + + +} + +module.exports = { + feedUpdater +} \ No newline at end of file diff --git a/JDLinkAdder.js b/JDLinkAdder.js index 0ba9d91..5421987 100644 --- a/JDLinkAdder.js +++ b/JDLinkAdder.js @@ -3,7 +3,6 @@ const fs = require("fs"); async function linkAdder(links) { - const client = new JDownloaderClient(JSON.parse(fs.readFileSync('config.json')).JDUserName, JSON.parse(fs.readFileSync('config.json')).JDPassword) await client.connect() const devices = await client.listDevices() diff --git a/JDRssDownloader.js b/JDRssDownloader.js new file mode 100644 index 0000000..7aec98f --- /dev/null +++ b/JDRssDownloader.js @@ -0,0 +1,17 @@ +const fs = require("fs"); +const { feedUpdater } = require('./FeedUpdater') +const { filterFeed } = require('./FeedFilter') +global.log = require('simple-node-logger').createSimpleLogger({ + logFilePath: 'jdrssdownloader.log', + timestampFormat: 'YYYY-MM-DD HH:mm:ss.SSS' +}); + +let RSSFeedRefreshMins = JSON.parse(fs.readFileSync('config.json')).RSSFeedRefreshMins +let JDPostLinksMins = JSON.parse(fs.readFileSync('config.json')).JDPostLinksMins + +log.info('Refreshing RSS Items every ' + RSSFeedRefreshMins + ' Minutes') +log.info('Checking for links and sending to JDdownloader every ' + JDPostLinksMins + ' Minutes') + + +setInterval(feedUpdater, RSSFeedRefreshMins * 60000); +setInterval(filterFeed, JDPostLinksMins * 60000); diff --git a/README.md b/README.md index 81db5c9..bc0b120 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,28 @@ # JDRssDownloader -JDownloder 2 is a great tool, but since V1 has been missing a way to automatically download from RSS feeds, and filter downloads to only download HEVC files. +JDownloader 2 is a great tool, but since V1 has been missing a way to automatically download from RSS feeds, and filter downloads to only download what you want, in my case 720p HEVC files, MeGusta rips by preference. I have put together this simple project to allow me to do that, people may find useful. -You will need NodeJS installed, then you can checkout this repo. +- Automatically check an RSS feed and send to JDownloader +- Uses MyJDownloader API to allow running on separate system +- Local file cache of RSS feed +- Specify time to check RSS feed +- Specify time to check file cache to send links to JDownloader +- Ability to add multiple shows to check for +- Ability to check for different qualities per show you are looking for + -## Configuration +# Configuration There is a ``config-sample.json`` file that needs to be renamed to ``config.json``, after this you can update it with your required settings. - JDUserName - Your MyJDownloader Username -- JDPassword - Your MyJdownloader Password +- JDPassword - Your MyJDownloader Password - RSSFeed - The url to the rss feed you want to watch (Only tested with - https://rlsbb.cc/feed/) - +- RSSFeedRefreshMins - How often to poll your rss feed down to local file cache +- JDPostLinksMins": How often to check your file cache for your shows and send found links to JDownloader - Autostart - Tells JDownloader to add and start the downloads straight away (true/false) -- Shows - This needs to be a comma seperated list of json objects of the show and quality you want to check for. +- Shows - This needs to be a comma separated list of json objects of the show and quality you want to check for. An example shown below @@ -23,6 +31,8 @@ An example shown below "JDUserName": "User", "JDPassword": "Pass", "RSSFeed": "https://rlsbb.cc/feed/", + "RSSFeedRefreshMins": 10, + "JDPostLinksMins": 180, "Autostart": false, "Shows": [ { @@ -36,14 +46,29 @@ An example shown below ] } ``` +# Running + +## Release Version +Either download the version on the releases, as well as the `config-sample.json` and run execute, this is the simplest way, but may not be the latest code, and will not run in the background + +## Source Version +You will need NodeJS installed, then you can checkout this repo. -## Execution For basic usage you can just navigate into the folder and run - - ``npm i`` to install the requirements. -- ``node main.js`` This will execute the process and add the links if they are found. +- ``node JDRssDownloader.js`` This will execute the process and add the links if they are found. -For advanced usage, you can set it to run every hour or something using a cron job, thats what I have setup. +My suggestion would be to use pm2 so it can run "in the background" -## Issues +# Issues -Not alot of testing has gone into this, and I threw it together today in about 2 hours, so there are bound to be issues, please open them and let me know if you find any. +Not alot of testing has gone into this, and I threw it together in a few hours, and only for my use case, so there are bound to be issues, please open them and let me know if you find any. + + +# Future +I have some ideas to make this a bit smarter, at the moment it doesn't clean the cache at all, and will keep sending the same links to JDownloader once they are in the cache, I am working on cleaning them out, but for now the best thing to do it to set JDownloader to automatically mark already downloaded links as finished, then it doesn't bother redownloading all the time. + +Also want to add the ability to look at multiple RSS feeds, this seems quite easy, and I will do in the next couple of weeks. + +# Thanks +Thank for all the people who made any of the modules that I used to create this. \ No newline at end of file diff --git a/config-sample.json b/config-sample.json index 54825f6..574f704 100644 --- a/config-sample.json +++ b/config-sample.json @@ -2,6 +2,8 @@ "JDUserName": "", "JDPassword": "", "RSSFeed": "", + "RSSFeedRefreshMins": 10, + "JDPostLinksMins": 180, "Autostart": false, "Shows": [ { diff --git a/main.js b/main.js deleted file mode 100644 index 9a283be..0000000 --- a/main.js +++ /dev/null @@ -1,34 +0,0 @@ -// Import dependencies -const fs = require("fs"); -const Parser = require("rss-parser"); -const { filterFeed } = require("./FeedFilter"); -const lodash = require('lodash'); -const log = require('simple-node-logger').createSimpleLogger('jdrssdownloader.log'); - -(async function main() { - - - // Make a new RSS Parser - const parser = new Parser(); - - // Get all the items in the RSS feed - const feed = await parser.parseURL(JSON.parse(fs.readFileSync('config.json')).RSSFeed); - - let items = []; - - // Clean up the string and replace reserved characters - const fileName = `${feed.title.replace(/\s+/g, "-").replace(/[/\\?%*:|"<>]/g, '').toLowerCase()}.json`; - - if (fs.existsSync(fileName)) { - items = require(`./${fileName}`); - } - // Compare existing cache and new items and merge differences - let updatedArray = lodash.unionBy(feed.items, items, 'title'); - - // Save the file - log.info(updatedArray.length + ' items in file cache') - fs.writeFileSync(fileName, JSON.stringify(updatedArray)); - - // run next part - filterFeed(fileName) -})(); diff --git a/package.json b/package.json index 3c37de5..eb8b31e 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,8 @@ "name": "jdrssdownloader", "version": "1.0.0", "description": "", - "main": "main.js", + "main": "JDRssDownloader.js", + "bin": "JDRssDownloader.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, @@ -11,10 +12,12 @@ "dependencies": { "axios": "^0.27.2", "cheerio": "^1.0.0-rc.11", - "extract-urls": "^1.3.2", "jdownloader-client": "^1.0.0", "lodash": "^4.17.21", "rss-parser": "^3.12.0", "simple-node-logger": "^21.8.12" + }, + "pkg": { + "outputPath": "dist" } }