From 46001b041c36dd7660b4374fab161ad51b49fe4e Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Mon, 20 Jun 2022 12:49:36 +0100 Subject: [PATCH 01/12] initial commit of clovisNyu cli --- GoGoDownloaderCLI.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 GoGoDownloaderCLI.py diff --git a/GoGoDownloaderCLI.py b/GoGoDownloaderCLI.py new file mode 100644 index 0000000..2c0917c --- /dev/null +++ b/GoGoDownloaderCLI.py @@ -0,0 +1,73 @@ +from backend import * +import argparse +import os +import requests +from bs4 import BeautifulSoup + + +def get_archives(archive_file): + + if os.path.isfile(archive_file): + with open(archive_file, "r") as f: + return f.read().split() + + else: + return [] + + +def find_episodes(url): + resp = requests.get(url) + soup = BeautifulSoup(resp.text, "html.parser") + all_episodes = soup.find("ul", {"id": "episode_page"}) + all_episodes = int(all_episodes.get_text().split("-")[-1].strip()) + return all_episodes + + +def main(): + placeholder = "DUMMY" + config = config_check() + parser = argparse.ArgumentParser() + parser.add_argument("url", metavar="url", type=str) + parser.add_argument("--archive", metavar="Archive File", type=str) + parser.add_argument( + "--quality", + metavar="download quality", + choices={"360", "480", "720", "1080"}, + nargs="?", + const="1080", + default="1080", + type=str, + ) + args = parser.parse_args() + + name = args.url.split("/")[-1] + title = name.replace("-", " ").title().strip() + all_episodes = find_episodes(args.url) + + archives = get_archives(args.archive) + downloader = gogoanime( + config, name, args.quality, placeholder, all_episodes, 1, all_episodes, title + ) + + episode_links = [link for link in downloader.get_links() if link not in archives] + dl_links = {} + for ep in episode_links: + ep_num = ep.split("-")[-1] + dl_links[downloader.get_download_link(ep)] = (title, ep_num) + results = downloader.file_downloader(dl_links) + failed = [] + with open(args.archive, "a+") as f: + for ep in episode_links: + ep_num = ep.split("-")[-1] + if os.path.join(title, f"{title} Episode {ep_num}.mp4") in results: + f.write(f"{ep}\n") + else: + failed.append(ep) + + with open("logs.txt", "w+") as f: + for failure in failed: + f.write(f"{failed} failed to download\n") + + +if __name__ == "__main__": + main() From 0dffecc6ffcfb5157f9db43f9a91cb6a1ea87250 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Mon, 20 Jun 2022 12:50:07 +0100 Subject: [PATCH 02/12] ignore the launch.json --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bfb8e83..166d823 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ build/ dist/ main.spec +.vscode/launch.json From a8d54be2d32df8928eae671f204c9a832718ffc2 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Mon, 20 Jun 2022 12:50:22 +0100 Subject: [PATCH 03/12] ignore app.log --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 166d823..2b0100f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build/ dist/ main.spec .vscode/launch.json +app.log From a09ed8c8026faeb42b94c21b661e07f4ea80be7f Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:08:36 +0100 Subject: [PATCH 04/12] ignore bookmarkList --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2b0100f..76bcb62 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ main.spec .vscode/launch.json app.log +bookmarkList.json From 64acd0abb27a2e65181004b38e1f1581421273f3 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:08:52 +0100 Subject: [PATCH 05/12] get_show_from_bookmark --- backend.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/backend.py b/backend.py index 0a42c22..8fe500d 100644 --- a/backend.py +++ b/backend.py @@ -213,6 +213,40 @@ class gogoanime: files = dl.download() return files + def get_show_from_bookmark(self): + bookmarkList = [] + a = dict(auth=gogoanime.get_gogoanime_auth_cookie(self)) + resp = requests.get( + "https://gogoanime.gg/user/bookmark", + cookies=a, + ) + soup = BeautifulSoup(resp.text, "html.parser") + table = soup.find("div", attrs={"class": "article_bookmark"}) + splitTableLines = table.text.split("Remove") + for rows in splitTableLines: + fullRow = " ".join(rows.split()) + if "Anime name" in fullRow: + fullRow = fullRow.replace("Anime name Latest", "") + splitRow = fullRow.split("Latest") + elif fullRow == "Status": + break + else: + fullRow = fullRow.replace("Status ", "") + splitRow = fullRow.split("Latest") + animeName = splitRow[0].strip() + animeDownloadName = animeName.replace(":", "").replace(" ", "-").lower() + episodeNum = splitRow[-1].split()[-1] + bookmarkList.append( + { + "showName": animeName, + "latestEpisode": int(episodeNum), + "downloadURL": f"https://gogoanime.{self.config['CurrentGoGoAnimeDomain']}/{animeDownloadName}-episode-{str(episodeNum)}", + } + ) + with open("bookmarkList.json", "w") as f: + json.dump(bookmarkList, f) + return bookmarkList + @dataclass(init=True) class CustomMessage(Exception): From 9ea3c6a6ce22075ed74df5283e033d762688fd4f Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:09:10 +0100 Subject: [PATCH 06/12] create new CLI to process users bookmarks --- GoGoDownloaderCLI.py | 90 ++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 61 deletions(-) diff --git a/GoGoDownloaderCLI.py b/GoGoDownloaderCLI.py index 2c0917c..202d8fd 100644 --- a/GoGoDownloaderCLI.py +++ b/GoGoDownloaderCLI.py @@ -1,72 +1,40 @@ from backend import * -import argparse -import os -import requests -from bs4 import BeautifulSoup - - -def get_archives(archive_file): - - if os.path.isfile(archive_file): - with open(archive_file, "r") as f: - return f.read().split() - - else: - return [] - - -def find_episodes(url): - resp = requests.get(url) - soup = BeautifulSoup(resp.text, "html.parser") - all_episodes = soup.find("ul", {"id": "episode_page"}) - all_episodes = int(all_episodes.get_text().split("-")[-1].strip()) - return all_episodes def main(): - placeholder = "DUMMY" config = config_check() - parser = argparse.ArgumentParser() - parser.add_argument("url", metavar="url", type=str) - parser.add_argument("--archive", metavar="Archive File", type=str) - parser.add_argument( - "--quality", - metavar="download quality", - choices={"360", "480", "720", "1080"}, - nargs="?", - const="1080", - default="1080", - type=str, - ) - args = parser.parse_args() - - name = args.url.split("/")[-1] - title = name.replace("-", " ").title().strip() - all_episodes = find_episodes(args.url) - - archives = get_archives(args.archive) downloader = gogoanime( - config, name, args.quality, placeholder, all_episodes, 1, all_episodes, title + config, + 1, + config["CLIQuality"], + "a", + 1, + 1, + 1, + config["CLIDownloadLocation"], ) - - episode_links = [link for link in downloader.get_links() if link not in archives] + list = downloader.get_show_from_bookmark() dl_links = {} - for ep in episode_links: - ep_num = ep.split("-")[-1] - dl_links[downloader.get_download_link(ep)] = (title, ep_num) - results = downloader.file_downloader(dl_links) - failed = [] - with open(args.archive, "a+") as f: - for ep in episode_links: - ep_num = ep.split("-")[-1] - if os.path.join(title, f"{title} Episode {ep_num}.mp4") in results: - f.write(f"{ep}\n") - else: - failed.append(ep) - - with open("logs.txt", "w+") as f: - for failure in failed: - f.write(f"{failed} failed to download\n") + for ep in list: + print( + "{OK}Scraping DL for " + ep["showName"] + " Ep " + str(ep["latestEpisode"]) + ) + dl_links[downloader.get_download_link(ep["downloadURL"])] = ( + ep["showName"], + ep["latestEpisode"], + ) + result = downloader.file_downloader(dl_links) + if len(result.errors) > 0: + while len(result.errors) > 0: + print(f"{ERR}{len(result.errors)} links failed retrying.") + print(f"{OK}Re-Scraping Links") + dl_links.clear() + for ep in list: + dl_links[downloader.get_download_link(ep["downloadURL"])] = ( + ep["showName"], + ep["latestEpisode"], + ) + result = downloader.file_downloader(dl_links, overwrite_downloads=0) if __name__ == "__main__": From 7c4a56967886826638257471617c0c1b3e6d4bcf Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:09:19 +0100 Subject: [PATCH 07/12] update config.json --- config.json.default | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.json.default b/config.json.default index 210af6f..d4a7487 100644 --- a/config.json.default +++ b/config.json.default @@ -3,5 +3,7 @@ "GoGoAnime_Password":"", "MaxConcurrentDownloads": 4, "CurrentGoGoAnimeDomain": "gg", - "OverwriteDownloads": 0 + "OverwriteDownloads": 0, + "CLIQuality":"720", + "CLIDownloadLocation": "CLIOutput" } From ee82a26e76355afde74bcbc40d3b825fa935a206 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:22:59 +0100 Subject: [PATCH 08/12] handle special characters in show name --- backend.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend.py b/backend.py index 8fe500d..b852d71 100644 --- a/backend.py +++ b/backend.py @@ -233,8 +233,9 @@ class gogoanime: else: fullRow = fullRow.replace("Status ", "") splitRow = fullRow.split("Latest") - animeName = splitRow[0].strip() - animeDownloadName = animeName.replace(":", "").replace(" ", "-").lower() + animeName = splitRow[0].strip().encode("ascii", "ignore").decode() + animeName = re.sub("[^A-Za-z0-9 ]+", "", animeName) + animeDownloadName = animeName.replace(" ", "-").lower() episodeNum = splitRow[-1].split()[-1] bookmarkList.append( { From b002368705abaa86ea5fe60212de9e76ef2f1dd5 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:32:23 +0100 Subject: [PATCH 09/12] ignore downloadHistory.json --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 76bcb62..78c9960 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ main.spec .vscode/launch.json app.log bookmarkList.json +downloadHistory.json From fc19a690bd2445d8f27941dab103d9cde506dc6f Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 21 Jun 2022 12:32:46 +0100 Subject: [PATCH 10/12] loadDownloadHistory --- GoGoDownloaderCLI.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/GoGoDownloaderCLI.py b/GoGoDownloaderCLI.py index 202d8fd..64fe3c1 100644 --- a/GoGoDownloaderCLI.py +++ b/GoGoDownloaderCLI.py @@ -1,7 +1,24 @@ from backend import * +import json +import io +import os + + +def loadDownloadHistory(): + if os.path.isfile("./downloadHistory.json") and os.access( + "./downloadHistory.json", os.R_OK + ): + return json.load(open("./downloadHistory.json")) + else: + print("Either file is missing or is not readable, creating file...") + with io.open(os.path.join("./", "downloadHistory.json"), "w") as db_file: + db_file.write(json.dumps([])) + return json.load(open("./downloadHistory.json")) def main(): + dh = loadDownloadHistory() + config = config_check() downloader = gogoanime( config, From 8210e0b07b9c5e8bc9a5d6c359c7f80947419032 Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 28 Jun 2022 08:48:05 +0100 Subject: [PATCH 11/12] working CLI tool --- GoGoDownloaderCLI.py | 92 ++++++++++++++++++++++++++++++++++++++------ README.md | 14 ++++--- backend.py | 3 +- config.json.default | 3 +- 4 files changed, 94 insertions(+), 18 deletions(-) diff --git a/GoGoDownloaderCLI.py b/GoGoDownloaderCLI.py index 64fe3c1..c5050c7 100644 --- a/GoGoDownloaderCLI.py +++ b/GoGoDownloaderCLI.py @@ -1,24 +1,81 @@ -from backend import * import json import io import os +import re +from backend import * + + +def renameFile(filename: str): + """_summary_ + + Args: + filename (str): _description_ + + Returns: + _type_: _description_ + """ + newFileName = "".join(re.split("\(|\)|\[|\]", filename)[::2]) + try: + os.rename(filename, newFileName) + return True + except OSError as err: + return err def loadDownloadHistory(): + """Loads the downloadHistory.json, creates it if it doesn't exist + + Returns: + object: download history list + """ if os.path.isfile("./downloadHistory.json") and os.access( "./downloadHistory.json", os.R_OK ): return json.load(open("./downloadHistory.json")) else: - print("Either file is missing or is not readable, creating file...") with io.open(os.path.join("./", "downloadHistory.json"), "w") as db_file: db_file.write(json.dumps([])) return json.load(open("./downloadHistory.json")) +def writeShowToDownloadHistory(showName: str, downloadHistory: list): + """Writes the showName and latestEpisode to the downloadHistory.json file + + Args: + showName (str): _description_ + downloadHistory (list): _description_ + + Returns: + _type_: _description_ + """ + downloadHistory.append(showName) + with io.open(os.path.join("./", "downloadHistory.json"), "w") as db_file: + db_file.write(json.dumps(downloadHistory)) + return json.load(open("./downloadHistory.json")) + + +def readDownloadHistory(fileNameObject: object, downloadHistory: list): + """Reads the downloadHistory.json and checks if the fileName is present + + Args: + fileNameObject (str): _description_ + downloadHistory (list): _description_ + + Returns: + _type_: _description_ + """ + dhFileName = ( + fileNameObject["showName"] + " - " + str(fileNameObject["latestEpisode"]) + ) + if dhFileName not in downloadHistory: + writeShowToDownloadHistory(dhFileName, downloadHistory) + return False + else: + return True + + def main(): dh = loadDownloadHistory() - config = config_check() downloader = gogoanime( config, @@ -33,18 +90,28 @@ def main(): list = downloader.get_show_from_bookmark() dl_links = {} for ep in list: - print( - "{OK}Scraping DL for " + ep["showName"] + " Ep " + str(ep["latestEpisode"]) - ) - dl_links[downloader.get_download_link(ep["downloadURL"])] = ( - ep["showName"], - ep["latestEpisode"], - ) + if readDownloadHistory(ep, dh): + showName = ep["showName"] + " - " + str(ep["latestEpisode"]) + print(f"{IN}{showName} already downloaded") + else: + print( + f"{IN}Scraping DL for " + + ep["showName"] + + " Ep " + + str(ep["latestEpisode"]) + ) + dl_links[downloader.get_download_link(ep["downloadURL"])] = ( + ep["showName"], + ep["latestEpisode"], + ) result = downloader.file_downloader(dl_links) + if config["CleanUpFileName"]: + for file in result.data: + renameFile(file) if len(result.errors) > 0: while len(result.errors) > 0: print(f"{ERR}{len(result.errors)} links failed retrying.") - print(f"{OK}Re-Scraping Links") + print(f"{IN}Re-Scraping Links") dl_links.clear() for ep in list: dl_links[downloader.get_download_link(ep["downloadURL"])] = ( @@ -52,6 +119,9 @@ def main(): ep["latestEpisode"], ) result = downloader.file_downloader(dl_links, overwrite_downloads=0) + if config["CleanUpFileName"]: + for file in result.data: + renameFile(file) if __name__ == "__main__": diff --git a/README.md b/README.md index 62c2176..990f7a6 100644 --- a/README.md +++ b/README.md @@ -57,16 +57,20 @@ If you want to run from source, or are using Linux/Mac you can run directly from ## Usage -The anime name is separated by "-". You can either type it manually, or go to [gogoanime.film](https://gogoanime.film/) and search for the anime you want to download and copy the name from the URL. +The anime name is separated by "-". You can either type it manually, or go to [gogoanime.gg](https://gogoanime.gg/) and search for the anime you want to download and copy the name from the URL. ### Examples ##### One word title -- https://gogoanime.film/category/bakemonogatari >> bakemonogatari -- https://gogoanime.film/category/steinsgate >> steinsgate +- https://gogoanime.gg/category/bakemonogatari >> bakemonogatari +- https://gogoanime.gg/category/steinsgate >> steinsgate ##### Multiple word title -- https://gogoanime.film/category/shadows-house >> shadows-house -- https://gogoanime.film/category/kono-subarashii-sekai-ni-shukufuku-wo- >> kono-subarashii-sekai-ni-shukufuku-wo- +- https://gogoanime.gg/category/shadows-house >> shadows-house +- https://gogoanime.gg/category/kono-subarashii-sekai-ni-shukufuku-wo- >> kono-subarashii-sekai-ni-shukufuku-wo- + + +# GoGoDownloader CLI +I have now also created the GoGoDownloader CLI, this tool can be used to run on a scheduled basis, it will login and get the latest episodes from your GoGoAnime bookmarks, and download the latest episode if it has not been downloaded yet. \ No newline at end of file diff --git a/backend.py b/backend.py index b852d71..9b79c69 100644 --- a/backend.py +++ b/backend.py @@ -214,10 +214,11 @@ class gogoanime: return files def get_show_from_bookmark(self): + print(f"{IN}Loading shows from bookmarks") bookmarkList = [] a = dict(auth=gogoanime.get_gogoanime_auth_cookie(self)) resp = requests.get( - "https://gogoanime.gg/user/bookmark", + f"https://gogoanime.{self.config['CurrentGoGoAnimeDomain']}/user/bookmark", cookies=a, ) soup = BeautifulSoup(resp.text, "html.parser") diff --git a/config.json.default b/config.json.default index d4a7487..5bce992 100644 --- a/config.json.default +++ b/config.json.default @@ -5,5 +5,6 @@ "CurrentGoGoAnimeDomain": "gg", "OverwriteDownloads": 0, "CLIQuality":"720", - "CLIDownloadLocation": "CLIOutput" + "CLIDownloadLocation": "CLIOutput", + "CleanUpFileName": false } From 2d1eda3b9291c439eb26f8ae8da996fec737aa6c Mon Sep 17 00:00:00 2001 From: "karl.hudgell" Date: Tue, 28 Jun 2022 08:48:37 +0100 Subject: [PATCH 12/12] upversion --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 50e47c8..a4f52a5 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.1.1 \ No newline at end of file +3.2.0 \ No newline at end of file