Merge pull request #10 from karl0ss/split_shows

Split shows

.gitignore

@@ -12,3 +12,4 @@ downloadHistory.json
 shows.json
 shows.json
 cache/retryCache.json
+shows.json

LinkGrabber.js

@@ -1,13 +1,32 @@
 const axios = require('axios');
 var cheerio = require('cheerio');
 
+async function flareSolverr(url) {
+    var data = JSON.stringify({
+        "cmd": "request.get",
+        "url": url,
+        "maxTimeout": 120000
+    });
+    var config = {
+        method: 'post',
+        url: 'http://127.0.01:8191/v1',
+        headers: {
+            'Content-Type': 'application/json'
+        },
+        data: data
+    };
+    res = await axios(config)
+    return res.data.solution.response
+}
+
 async function getLinksFromURL(url) {
 
     try {
         let links = [];
-        let httpResponse = await axios.get(url);
+        let scrape = await flareSolverr(url)
+        // let httpResponse = await axios.get(url);
 
-        let $ = cheerio.load(httpResponse.data);
+        let $ = cheerio.load(scrape);
         let linkObjects = $('a'); // get all hyperlinks
 
         linkObjects.each((index, element) => {

README.md

@@ -1,90 +1,168 @@
+
 # JDRssDownloader
 
+  
+
 JDownloader 2 is a great tool, but since V1 has been missing a way to automatically download from RSS feeds, and filter downloads to only download what you want, in my case 720p HEVC files, MeGusta rips by preference.
 
+  
+
 I have put together this simple project to allow me to do that, people may find useful.
 
+  
+
 - Automatically check an RSS feed and send to JDownloader
+
 - Uses MyJDownloader API to allow running on separate system
+
 - Local file cache of RSS feed
+
 - Specify time to check RSS feed
+
 - Specify time to check file cache to send links to JDownloader
+
 - Ability to add multiple shows to check for
+
 - Ability to check for different qualities per show you are looking for
+
 - Ability to turn OFF only HEVC search
 
+  
+
 # Configuration
 
+  
+
 There is a `config-sample.json` file that needs to be renamed to `config.json`, after this you can update it with your required settings.
 
+  
+
 - JDUserName - Your MyJDownloader Username
+
 - JDPassword - Your MyJDownloader Password
+
 - AdminPassword - Password to be set for the WebUI
+
 - WebUIPort - Port for the WebUI to run on
+
 - RSSFeed - The url to the rss feed you want to watch (Only tested with - rlsbb)
+
 - RSSFeedRefreshMins - How often to poll your rss feed down to local file cache
+
 - JDPostLinksMins - How often to check your file cache for your shows and send found links to JDownloader
+
 - Autostart - Tells JDownloader to add and start the downloads straight away (true/false)
+
 - OnlyHEVC - If false, this will download any files that it finds on the post that matches the quality (true/false)
+
 - TelegramBot - Set to true if you wish to have updates sent via telegramBot
+
 - TelegramBotID - Set this to the id you recieve from TheBotFather
+
 - TelegramChatID - Chat or Group ID for the bot to send messages to
-- Shows - This needs to be a comma separated list of json objects of the show and quality you want to check for.
+
+  
 
 An example shown below
 
+  
+
 ```
+
 {
-    "JDUserName": "User",
-    "JDPassword": "Pass",
-    "AdminPassword":"",
-    "WebUIPort": 3100,
-    "RSSFeed": "https://mypage.com/feed/",
-    "RSSFeedRefreshMins": 10,
-    "JDPostLinksMins": 180,
-    "Autostart": false,
-    "OnlyHEVC": true,
-    "TelegramBot": true,
-    "TelegramBotID":"",
-    "TelegramChatID":123456789,
-    "Shows": [
-        {
-            "Name": "Obi-Wan Kenobi",
-            "Quality": "1080"
-        },
-        {
-            "Name": "Taskmaster",
-            "Quality": "720"
-        }
-    ]
+
+"JDUserName": "User",
+
+"JDPassword": "Pass",
+
+"AdminPassword":"",
+
+"WebUIPort": 3100,
+
+"RSSFeed": "https://mypage.com/feed/",
+
+"RSSFeedRefreshMins": 10,
+
+"JDPostLinksMins": 180,
+
+"Autostart": false,
+
+"OnlyHEVC": true,
+
+"TelegramBot": true,
+
+"TelegramBotID":"",
+
+"TelegramChatID":123456789,
+
 }
+
 ```
 
+  
+
 # Running
 
+  
+
+## FlareSolverr
+
+Due to issues with a number of sites I use, I have had to rework the client to now use [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) this allows the tool to bypass/workout Cloudflare site issues, the implication of this is that we need to run a docker image in the background that will actually open the page up in a Chrome window, then return the HTML and get the links as usual, this does slow down the process, as it takes upto 2 mins for it to return HTML from the page, but it works..
+
+So you need to have docker installed and to run this command 
+
+    docker-compose up -d
+
+This will run the container in the background, then you can run as usual, and it will proxy all requests via FlareSolverr, please open an issue if you find problems.  
+
 ## Release Version
 
+  
+
 Either download the version on the releases, as well as the `config-sample.json` and run execute, this is the simplest way, but may not be the latest code, and will not run in the background
 
+  
+
 ## Source Version
 
+  
+
 You will need NodeJS installed, then you can checkout this repo.
 
+  
+
 For basic usage you can just navigate into the folder and run -
 
-- `npm i` to install the requirements.
-- `node JDRssDownloader.js` This will execute the process and add the links if they are found.
+  
 
-My suggestion would be to use pm2 so it can run "in the background"
+-  `npm i` to install the requirements.
+
+-  `node JDRssDownloader.js` This will execute the process and add the links if they are found.
+
+  
+
+My suggestion would be to use [pm2](https://pm2.keymetrics.io/docs/usage/quick-start/) so it can run "in the background"
+
+  
 
 # Issues
 
+  
+
 Not alot of testing has gone into this, and I threw it together in a few hours, and only for my use case, so there are bound to be issues, please open them and let me know if you find any.
 
+  
+
 # Future
 
+  
+
 I have some ideas to make this a bit smarter and I want to add the ability to look at multiple RSS feeds, this seems quite easy, and I will do in the next couple of weeks.
 
+  
+
 # Thanks
 
-Thank for all the people who made any of the modules that I used to create this.
+  
+
+Thank for all the people who made any of the modules that I used to create this.

config-sample.json

@@ -10,11 +10,5 @@
     "OnlyHEVC": true,
     "TelegramBot": true,
     "TelegramBotID":"",
-    "TelegramChatID":123456789,
-    "Shows": [
-        {
-            "Name": "",
-            "Quality": "720"
-        }
-    ]
+    "TelegramChatID":123456789
 }

docker-compose.yml

@@ -0,0 +1,15 @@
+---
+version: "2.1"
+services:
+  flaresolverr:
+    # DockerHub mirror flaresolverr/flaresolverr:latest
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - LOG_HTML=${LOG_HTML:-false}
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
+    ports:
+      - "127.0.0.1:${PORT:-8191}:8191"
+    restart: unless-stopped

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jdrssdownloader",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "description": "",
   "main": "JDRssDownloader.js",
   "bin": "JDRssDownloader.js",
@@ -11,6 +11,8 @@
   "license": "ISC",
   "dependencies": {
     "axios": "^0.27.2",
+    "bluebird": "^3.7.2",
+    "body-parser": "^1.20.2",
     "cheerio": "^1.0.0-rc.11",
     "cors": "^2.8.5",
     "express": "^4.18.2",

routes/api.js

@@ -0,0 +1,28 @@
+const fs = require("fs");
+const { nextLinkCheck, nextRssRefresh, get_last_downloaded } = require('.././utils')
+
+
+module.exports = function (app) {
+    app.get("/api/stats", (req, res) => {
+        retryCache = JSON.parse(fs.readFileSync('./cache/retryCache.json'))
+        for (let index = 0; index < retryCache.length; index++) {
+            const item = retryCache[index];
+            retryCache[index].newtitle = item.title.replace(/ /g, "‡");
+        }
+        showList = JSON.parse(fs.readFileSync('shows.json'));
+        feedCache = JSON.parse(fs.readFileSync('./cache/feedCache.json'));
+        rssTime = nextRssRefresh()
+        linkCheck = nextLinkCheck()
+        lastDownloaded = get_last_downloaded()
+
+        res.json({
+            "ShowList": showList.length,
+            "FeedCache": feedCache.length,
+            "RetryCache": retryCache.length,
+            "RSSCheck": rssTime,
+            "LinkChecker": linkCheck,
+            "LastDownloaded": lastDownloaded
+        });
+    });
+
+}

shows.json

@@ -1 +0,0 @@
-[]

utils.js

@@ -40,9 +40,19 @@ function create_empty_retry_cache() {
     }
 }
 
+function create_empty_shows_file() {
+    try {
+        return JSON.parse(fs.readFileSync('./shows.json'));
+    } catch (error) {
+        fs.writeFileSync('./shows.json', JSON.stringify([]));
+        return JSON.parse(fs.readFileSync('./shows.json'));
+    }
+}
+
 function create_empty_cache_files() {
     create_empty_downloadHistory()
     create_empty_retry_cache()
+    create_empty_shows_file() 
 }
 
 module.exports = {