diff --git a/README.md b/README.md index 4d9fde3..7123e9f 100644 --- a/README.md +++ b/README.md @@ -38,17 +38,20 @@ This program has only been tested on Windows 10. Hence, if you encounter any iss - What the flag does: - Regardless of this flag, the program will overwrite any incomplete downloaded file by verifying against the Content-Length header response. + - GDrive downloads are not affected by this flag. - If the size of locally downloaded file does not match with the header response, the program will re-download the file. - Otherwise, if the Content-Length header is not present in the response, the program will only skip the file if the file already exists. - - However, if there's a need to overwrite the skipped files, you can use this flag to do so. - - You should only use this for websites like Pixiv Fanbox which does not return a Content-Length header in their response. + - However, if there's a need to overwrite the skipped files, you can use this flag to do so like for Pixiv Fanbox which does not return a Content-Length header in their response header. + - That said, the program will try to do a clean up of the incomplete downloads by deleting them as of version 1.1.1 and above when you forcefully terminate the program by pressing Ctrl+C so you should not need to use this flag anymore. - Causes: - This is caused by the antivirus program on your PC, flagging the program as a malware and stopping it from executing normally. - This is due to how the programs works, it will try to check the image file size by doing a HEAD request first before doing a GET request for downloading. - In the event that the Content-Length is not in the response header, it will re-download the file if the overwrite flag is set to true. - Usually, this is not a problem for Fantia and Pixiv, but for Pixiv Fanbox, it will not return the Content-Length in the response header, which is why the program will try to re-download all the files and overwrite any existing files. Hence, the antivirus program will flag the program as a ransomware and stop it from executing normally. - Solutions: - - Please exclude `cultured-downloader-cli.exe` or the compiled version of the program from your antivirus software as it can be flagged as a ransomware as previously explained above in the Causes. + - Avoid using the `--overwrite=true` flag and let the program do its clean up when you forcefully terminate the program by pressing Ctrl+C. + - Files that failed to be deleted will be logged so you can manually delete them. + - If you still need to use this flag, please exclude `cultured-downloader-cli.exe` or the compiled version of the program from your antivirus software as it can be flagged as a ransomware as previously explained above in the Causes. - `go run .` will also NOT work as it will still be blocked by the antivirus program. Hence, you will need to build the program first and then run it. - By running `go build . -o cultured-downloader-cli.exe` in the src directory of the project, it will build the program and create an executable file. @@ -64,7 +67,7 @@ This program has only been tested on Windows 10. Hence, if you encounter any iss The example below assumes you are using [Go](https://go.dev/dl/) to run the program. -Otherwise, instead of `go run cultured_downloader.go`, you can run the executable file by typing `./cultured_downloader.exe` for Windows. +Otherwise, instead of `go run . cultured_downloader.go`, you can run the executable file by typing `./cultured_downloader.exe` for Windows. There is also compiled binaries for Linux and macOS in the [releases](https://github.com/KJHJason/Cultured-Downloader-CLI/releases) page. @@ -72,35 +75,47 @@ Note: - For flags that require a value, you can either use the `--flag_name value` format or the `--flag_name=value` format. - For flags that allows multiple values, you can either use the `--flag_name value1,value2` format or the `--flag_name=value1,value2` format. - Quotations like `--flag_name="value1,value2"` are not required but it is recommended to use them. +- For `--text_file=""`, the contents of the text file should be separated by a new line as seen below, +``` + https://fantia.jp/posts/123123 + https://fantia.jp/fanclubs/1234 + https://fantia.jp/fanclubs/1234; 1-12 +``` +- You can add the `; ` after the URL as well! + - Only for: + - Fantia Fanclub URLs + - Pixiv Fanbox Creator URLs + - Pixiv Illustrator URLs + - Pixiv Tag URLs Help: ``` -go run cultured_downloader.go -h +go run . cultured_downloader.go -h ``` Downloading from multiple Fantia Fanclub IDs: ``` -go run cultured_downloader.go fantia --cookie_file="C:\Users\KJHJason\Desktop\fantia.jp_cookies.txt" --fanclub_id 123456,789123 --page_num 1,1-10 --dl_thumbnails=false +go run . cultured_downloader.go fantia --cookie_file="C:\Users\KJHJason\Desktop\fantia.jp_cookies.txt" --fanclub_id 123456,789123 --page_num 1,1-10 --dl_thumbnails=false ``` Downloading from a Pixiv Fanbox Post ID: ``` -go run cultured_downloader.go pixiv_fanbox --session="" --post_id 123456,789123 --gdrive_api_key="" +go run . cultured_downloader.go pixiv_fanbox --session="" --post_id 123456,789123 --gdrive_api_key="" ``` Downloading from a Pixiv Artwork ID (that is a Ugoira): ``` -go run cultured_downloader.go pixiv --session "" --artwork_id 12345678 --ugoira_output_format ".gif" --delete_ugoira_zip=false +go run . cultured_downloader.go pixiv --session "" --artwork_id 12345678 --ugoira_output_format ".gif" --delete_ugoira_zip=false ``` Downloading from multiple Pixiv Artwork IDs: ``` -go run cultured_downloader.go pixiv --refresh_token="" --artwork_id 12345678,87654321 +go run . cultured_downloader.go pixiv --refresh_token="" --artwork_id 12345678,87654321 ``` Downloading from Pixiv using a tag name: ``` -go run cultured_downloader.go pixiv --refresh_token="" --tag_name "tag1,tag2,tag3" --tag_page_num 1,4,2 --rating_mode safe --search_mode s_tag +go run . cultured_downloader.go pixiv --refresh_token="" --tag_name "tag1,tag2,tag3" --tag_page_num 1,4,2 --rating_mode safe --search_mode s_tag ``` ## Base Flags @@ -139,7 +154,7 @@ Usage: cultured-downloader-cli fantia [flags] Flags: - --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. + -c, --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. You can generate a cookie file by using the "Get cookies.txt LOCALLY" extension for your browser. Chrome Extension URL: https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc --dl_attachments Whether to download the attachments of a Fantia post. (default true) @@ -157,7 +172,8 @@ Flags: --post_id strings Fantia post ID(s) to download. For multiple IDs, separate them with a comma. Example: "12345,67891" (without the quotes) - --session string Your _session_id cookie value to use for the requests to Fantia. + -s, --session string Your _session_id cookie value to use for the requests to Fantia. + --text_file string Path to a text file containing Fanclub and/or post URL(s) to download from Fantia. ``` ## Pixiv Fanbox Flags @@ -169,7 +185,7 @@ Usage: cultured-downloader-cli pixiv_fanbox [flags] Flags: - --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. + -c, --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. You can generate a cookie file by using the "Get cookies.txt LOCALLY" extension for your browser. Chrome Extension URL: https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc --creator_id strings Pixiv Fanbox Creator ID(s) to download from. @@ -190,7 +206,8 @@ Flags: --post_id strings Pixiv Fanbox post ID(s) to download. For multiple IDs, separate them with a comma. Example: "12345,67891" (without the quotes) - --session string Your FANBOXSESSID cookie value to use for the requests to Pixiv Fanbox. + -s, --session string Your FANBOXSESSID cookie value to use for the requests to Pixiv Fanbox. + --text_file string Path to a text file containing creator and/or post URL(s) to download from Pixiv Fanbox. ``` @@ -212,9 +229,9 @@ Flags: - all: Include both illustrations, ugoira, and manga artworks Notes: - If you're using the "-pixiv_refresh_token" flag and are downloading by tag names, only "all" is supported. (default "all") - --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. + -c, --cookie_file string Pass in a file path to your saved Netscape/Mozilla generated cookie file to use when downloading. You can generate a cookie file by using the "Get cookies.txt LOCALLY" extension for your browser. - Chrome Extension URL: https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc + Chrome Extension URL: https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc --delete_ugoira_zip Whether to delete the downloaded ugoira zip file after conversion. (default true) --ffmpeg_path string Configure the path to the FFmpeg executable. Download Link: https://ffmpeg.org/download.html (default "ffmpeg") @@ -243,7 +260,7 @@ Flags: - s_tag: Match any post with SIMILAR tag name - s_tag_full: Match any post with the SAME tag name - s_tc: Match any post related by its title or caption (default "s_tag_full") - --session string Your PHPSESSID cookie value to use for the requests to Pixiv. + -s, --session string Your PHPSESSID cookie value to use for the requests to Pixiv. --sort_order string Download Order Options: date, popular, popular_male, popular_female Additionally, you can add the "_d" suffix for a descending order. Example: "popular_d" @@ -257,6 +274,7 @@ Flags: --tag_page_num strings Min and max page numbers to search for corresponding to the order of the supplied tag name(s). Format: "num", "minNum-maxNum", or "" to download all pages Leave blank to search all pages for each tag name. + --text_file string Path to a text file containing artwork, illustrator, and tag name URL(s) to download from Pixiv. --ugoira_output_format string Output format for the ugoira conversion using FFmpeg. Accepted Extensions: .gif, .apng, .webp, .webm, .mp4 (default ".gif") diff --git a/src/cmds/fantia.go b/src/cmds/fantia.go index cdaeead..a0e63bb 100644 --- a/src/cmds/fantia.go +++ b/src/cmds/fantia.go @@ -77,6 +77,7 @@ var ( ) } + utils.PrintWarningMsg() fantia.FantiaDownloadProcess( &fantiaConfig, &fantiaDl, diff --git a/src/cmds/pixiv.go b/src/cmds/pixiv.go index 5fa5a14..50a0b82 100644 --- a/src/cmds/pixiv.go +++ b/src/cmds/pixiv.go @@ -111,6 +111,7 @@ var ( } pixivDlOptions.ValidateArgs() + utils.PrintWarningMsg() pixiv.PixivDownloadProcess( &pixivConfig, &pixivDl, diff --git a/src/cmds/pixiv_fanbox.go b/src/cmds/pixiv_fanbox.go index b6bc828..9db6644 100644 --- a/src/cmds/pixiv_fanbox.go +++ b/src/cmds/pixiv_fanbox.go @@ -79,6 +79,7 @@ var ( } pixivFanboxDlOptions.ValidateArgs() + utils.PrintWarningMsg() pixivfanbox.PixivFanboxDownloadProcess( &pixivFanboxConfig, &pixivFanboxDl, diff --git a/src/utils/useful.go b/src/utils/useful.go index aa51b87..5855d77 100644 --- a/src/utils/useful.go +++ b/src/utils/useful.go @@ -22,12 +22,24 @@ import ( // Prints out a warning message to the user to not stop the program while it is downloading func PrintWarningMsg() { color.Yellow("CAUTION:") - color.Yellow("Please do NOT stop the program while it is downloading.") - color.Yellow("Doing so may result in incomplete downloads and corrupted files.") + color.Yellow("Please do NOT terminate the program while it is downloading unless you really have to!") + color.Yellow("Doing so MAY result in incomplete downloads and corrupted files.") fmt.Println() } -// parse the Netscape cookie file generated by extensions like Get cookies.txt +// For the exported cookies in JSON instead of Netscape format +type ExportedCookies []struct { + Domain string `json:"domain"` + Expire float64 `json:"expirationDate"` + HttpOnly bool `json:"httpOnly"` + Name string `json:"name"` + Path string `json:"path"` + Secure bool `json:"secure"` + Value string `json:"value"` + Session bool `json:"session"` +} + +// parse the Netscape cookie file generated by extensions like Get cookies.txt LOCALLY func ParseNetscapeCookieFile(filePath, sessionId, website string) ([]*http.Cookie, error) { if filePath != "" && sessionId != "" { return nil, fmt.Errorf( @@ -60,66 +72,107 @@ func ParseNetscapeCookieFile(filePath, sessionId, website string) ([]*http.Cooki } var cookies []*http.Cookie - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + if ext := filepath.Ext(filePath); ext == ".txt" { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) - // skip empty lines and comments - if line == "" || strings.HasPrefix(line, "#") { - continue - } + // skip empty lines and comments + if line == "" || strings.HasPrefix(line, "#") { + continue + } - // split the line - cookieInfos := strings.Split(line, "\t") - if len(cookieInfos) < 7 { - // too few values will be ignored - continue - } + // split the line + cookieInfos := strings.Split(line, "\t") + if len(cookieInfos) < 7 { + // too few values will be ignored + continue + } - cookieName := cookieInfos[5] - if cookieName != sessionCookieName { - // not the session cookie - continue + cookieName := cookieInfos[5] + if cookieName != sessionCookieName { + // not the session cookie + continue + } + + // parse the values + cookie := http.Cookie{ + Name: cookieName, + Value: cookieInfos[6], + Domain: cookieInfos[0], + Path: cookieInfos[2], + Secure: cookieInfos[3] == "TRUE", + HttpOnly: true, + SameSite: sessionCookieSameSite, + } + + expiresUnixStr := cookieInfos[4] + if expiresUnixStr != "" { + expiresUnixInt, err := strconv.Atoi(expiresUnixStr) + if err != nil { + // should never happen but just in case + errMsg := fmt.Sprintf( + "error %d: parsing cookie expiration time, \"%s\", more info => %v", + UNEXPECTED_ERROR, + expiresUnixStr, + err, + ) + color.Red(errMsg) + continue + } + if expiresUnixInt > 0 { + cookie.Expires = time.Unix(int64(expiresUnixInt), 0) + } + } + cookies = append(cookies, &cookie) } - // parse the values - cookie := http.Cookie{ - Name: cookieName, - Value: cookieInfos[6], - Domain: cookieInfos[0], - Path: cookieInfos[2], - Secure: cookieInfos[3] == "TRUE", - HttpOnly: true, - SameSite: sessionCookieSameSite, + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf( + "error %d: reading cookie file at %s, more info => %v", + OS_ERROR, + filePath, + err, + ) + } + } else if ext == ".json" { + var exportedCookies ExportedCookies + if err := json.NewDecoder(f).Decode(&exportedCookies); err != nil { + return nil, fmt.Errorf( + "error %d: failed to decode cookie JSON file at %s, more info => %v", + JSON_ERROR, + filePath, + err, + ) } - expiresUnixStr := cookieInfos[4] - if expiresUnixStr != "" { - expiresUnixInt, err := strconv.Atoi(expiresUnixStr) - if err != nil { - // should never happen but just in case - errMsg := fmt.Sprintf( - "error %d: parsing cookie expiration time, \"%s\", more info => %v", - UNEXPECTED_ERROR, - expiresUnixStr, - err, - ) - color.Red(errMsg) + for _, cookie := range exportedCookies { + if cookie.Name != sessionCookieName { + // not the session cookie continue } - if expiresUnixInt > 0 { - cookie.Expires = time.Unix(int64(expiresUnixInt), 0) + + parsedCookie := &http.Cookie{ + Name: cookie.Name, + Value: cookie.Value, + Domain: cookie.Domain, + Path: cookie.Path, + Secure: cookie.Secure, + HttpOnly: cookie.HttpOnly, + SameSite: sessionCookieSameSite, + } + if !cookie.Session { + parsedCookie.Expires = time.Unix(int64(cookie.Expire), 0) } - } - cookies = append(cookies, &cookie) - } - if err := scanner.Err(); err != nil { + cookies = append(cookies, parsedCookie) + } + } else { return nil, fmt.Errorf( - "error %d: reading cookie file at %s, more info => %v", - OS_ERROR, + "error %d: invalid cookie file extension, \"%s\", at %s...\nOnly .txt and .json files are supported", + INPUT_ERROR, + ext, filePath, - err, ) }