diff --git a/.gitignore b/.gitignore index dbf8bde..a24b781 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ downloadHistory.json shows.json shows.json cache/retryCache.json +shows.json diff --git a/LinkGrabber.js b/LinkGrabber.js index 1ad5267..f21cfb2 100644 --- a/LinkGrabber.js +++ b/LinkGrabber.js @@ -1,13 +1,32 @@ const axios = require('axios'); var cheerio = require('cheerio'); +async function flareSolverr(url) { + var data = JSON.stringify({ + "cmd": "request.get", + "url": url, + "maxTimeout": 120000 + }); + var config = { + method: 'post', + url: 'http://127.0.01:8191/v1', + headers: { + 'Content-Type': 'application/json' + }, + data: data + }; + res = await axios(config) + return res.data.solution.response +} + async function getLinksFromURL(url) { try { let links = []; - let httpResponse = await axios.get(url); + let scrape = await flareSolverr(url) + // let httpResponse = await axios.get(url); - let $ = cheerio.load(httpResponse.data); + let $ = cheerio.load(scrape); let linkObjects = $('a'); // get all hyperlinks linkObjects.each((index, element) => { diff --git a/README.md b/README.md index a4b94ad..23e1132 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,168 @@ + # JDRssDownloader + + JDownloader 2 is a great tool, but since V1 has been missing a way to automatically download from RSS feeds, and filter downloads to only download what you want, in my case 720p HEVC files, MeGusta rips by preference. + + I have put together this simple project to allow me to do that, people may find useful. + + - Automatically check an RSS feed and send to JDownloader + - Uses MyJDownloader API to allow running on separate system + - Local file cache of RSS feed + - Specify time to check RSS feed + - Specify time to check file cache to send links to JDownloader + - Ability to add multiple shows to check for + - Ability to check for different qualities per show you are looking for + - Ability to turn OFF only HEVC search + + # Configuration + + There is a `config-sample.json` file that needs to be renamed to `config.json`, after this you can update it with your required settings. + + - JDUserName - Your MyJDownloader Username + - JDPassword - Your MyJDownloader Password + - AdminPassword - Password to be set for the WebUI + - WebUIPort - Port for the WebUI to run on + - RSSFeed - The url to the rss feed you want to watch (Only tested with - rlsbb) + - RSSFeedRefreshMins - How often to poll your rss feed down to local file cache + - JDPostLinksMins - How often to check your file cache for your shows and send found links to JDownloader + - Autostart - Tells JDownloader to add and start the downloads straight away (true/false) + - OnlyHEVC - If false, this will download any files that it finds on the post that matches the quality (true/false) + - TelegramBot - Set to true if you wish to have updates sent via telegramBot + - TelegramBotID - Set this to the id you recieve from TheBotFather + - TelegramChatID - Chat or Group ID for the bot to send messages to -- Shows - This needs to be a comma separated list of json objects of the show and quality you want to check for. + + An example shown below + + ``` + { - "JDUserName": "User", - "JDPassword": "Pass", - "AdminPassword":"", - "WebUIPort": 3100, - "RSSFeed": "https://mypage.com/feed/", - "RSSFeedRefreshMins": 10, - "JDPostLinksMins": 180, - "Autostart": false, - "OnlyHEVC": true, - "TelegramBot": true, - "TelegramBotID":"", - "TelegramChatID":123456789, - "Shows": [ - { - "Name": "Obi-Wan Kenobi", - "Quality": "1080" - }, - { - "Name": "Taskmaster", - "Quality": "720" - } - ] + +"JDUserName": "User", + +"JDPassword": "Pass", + +"AdminPassword":"", + +"WebUIPort": 3100, + +"RSSFeed": "https://mypage.com/feed/", + +"RSSFeedRefreshMins": 10, + +"JDPostLinksMins": 180, + +"Autostart": false, + +"OnlyHEVC": true, + +"TelegramBot": true, + +"TelegramBotID":"", + +"TelegramChatID":123456789, + } + ``` + + # Running + + +## FlareSolverr + +Due to issues with a number of sites I use, I have had to rework the client to now use [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) this allows the tool to bypass/workout Cloudflare site issues, the implication of this is that we need to run a docker image in the background that will actually open the page up in a Chrome window, then return the HTML and get the links as usual, this does slow down the process, as it takes upto 2 mins for it to return HTML from the page, but it works.. + +So you need to have docker installed and to run this command + + docker-compose up -d + +This will run the container in the background, then you can run as usual, and it will proxy all requests via FlareSolverr, please open an issue if you find problems. + ## Release Version + + Either download the version on the releases, as well as the `config-sample.json` and run execute, this is the simplest way, but may not be the latest code, and will not run in the background + + ## Source Version + + You will need NodeJS installed, then you can checkout this repo. + + For basic usage you can just navigate into the folder and run - -- `npm i` to install the requirements. -- `node JDRssDownloader.js` This will execute the process and add the links if they are found. + -My suggestion would be to use pm2 so it can run "in the background" +- `npm i` to install the requirements. + +- `node JDRssDownloader.js` This will execute the process and add the links if they are found. + + + +My suggestion would be to use [pm2](https://pm2.keymetrics.io/docs/usage/quick-start/) so it can run "in the background" + + # Issues + + Not alot of testing has gone into this, and I threw it together in a few hours, and only for my use case, so there are bound to be issues, please open them and let me know if you find any. + + # Future + + I have some ideas to make this a bit smarter and I want to add the ability to look at multiple RSS feeds, this seems quite easy, and I will do in the next couple of weeks. + + # Thanks -Thank for all the people who made any of the modules that I used to create this. + + +Thank for all the people who made any of the modules that I used to create this. \ No newline at end of file diff --git a/config-sample.json b/config-sample.json index 8295556..db8bfdb 100644 --- a/config-sample.json +++ b/config-sample.json @@ -10,11 +10,5 @@ "OnlyHEVC": true, "TelegramBot": true, "TelegramBotID":"", - "TelegramChatID":123456789, - "Shows": [ - { - "Name": "", - "Quality": "720" - } - ] + "TelegramChatID":123456789 } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2c71fd7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +--- +version: "2.1" +services: + flaresolverr: + # DockerHub mirror flaresolverr/flaresolverr:latest + image: ghcr.io/flaresolverr/flaresolverr:latest + container_name: flaresolverr + environment: + - LOG_LEVEL=${LOG_LEVEL:-info} + - LOG_HTML=${LOG_HTML:-false} + - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none} + - TZ=Europe/London + ports: + - "127.0.0.1:${PORT:-8191}:8191" + restart: unless-stopped \ No newline at end of file diff --git a/package.json b/package.json index c2358c0..5ba0958 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jdrssdownloader", - "version": "1.1.2", + "version": "1.2.0", "description": "", "main": "JDRssDownloader.js", "bin": "JDRssDownloader.js", @@ -11,6 +11,8 @@ "license": "ISC", "dependencies": { "axios": "^0.27.2", + "bluebird": "^3.7.2", + "body-parser": "^1.20.2", "cheerio": "^1.0.0-rc.11", "cors": "^2.8.5", "express": "^4.18.2", diff --git a/routes/api.js b/routes/api.js new file mode 100644 index 0000000..2066d32 --- /dev/null +++ b/routes/api.js @@ -0,0 +1,28 @@ +const fs = require("fs"); +const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils') + + +module.exports = function (app) { + app.get("/api/stats", (req, res) => { + retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json')) + for (let index = 0; index < retryCache.length; index++) { + const item = retryCache[index]; + retryCache[index].newtitle = item.title.replace(/ /g, "‡"); + } + showList = JSON.parse(fs.readFileSync('shows.json')); + feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json')); + rssTime = nextRssRefresh() + linkCheck = nextLinkCheck() + lastDownloaded = get_last_downloaded() + + res.json({ + "ShowList": showList.length, + "FeedCache": feedCache.length, + "RetryCache": retryCache.length, + "RSSCheck": rssTime, + "LinkChecker": linkCheck, + "LastDownloaded": lastDownloaded + }); + }); + +} diff --git a/shows.json b/shows.json deleted file mode 100644 index 0637a08..0000000 --- a/shows.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/utils.js b/utils.js index a1731e1..fe19883 100644 --- a/utils.js +++ b/utils.js @@ -40,9 +40,19 @@ function create_empty_retry_cache() { } } +function create_empty_shows_file() { + try { + return JSON.parse(fs.readFileSync('./shows.json')); + } catch (error) { + fs.writeFileSync('./shows.json', JSON.stringify([])); + return JSON.parse(fs.readFileSync('./shows.json')); + } +} + function create_empty_cache_files() { create_empty_downloadHistory() create_empty_retry_cache() + create_empty_shows_file() } module.exports = {