From cc113fb78a2b7ebe624216a61bf7b6afe65282d1 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Sat, 19 Nov 2022 23:04:18 +0800 Subject: [PATCH 1/8] Update readme.md --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 38c1e6fc..acfc661d 100644 --- a/readme.md +++ b/readme.md @@ -216,6 +216,7 @@ Q3: Error at process_image(): (, id, dump html, and log file (check on the application folder). Q4: URLError: + - Update version to > pixivutil20221029. - This is because the Pixiv downloader cannot resolve the address to download the images, please try to restart the network connection or do ipconfig /flushdns to refresh the dns cache (windows). From b0d7bf2908900a15988fba9399e909a9ba372d22 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Sun, 20 Nov 2022 20:37:31 +0800 Subject: [PATCH 2/8] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index acfc661d..a4155dba 100644 --- a/readme.md +++ b/readme.md @@ -624,7 +624,7 @@ Please refer run with `--help` for latest information. Codec to be used for encoding webm, default is using `libwebp`. - webpparam - Parameter to be used to encode webm. + Parameter to be used to encode webp. default is `lossless 0 -q:v 90 -loop 0 -vsync 2 -r 999` ## [Ugoira] From cc08b6ad9ebb1956e04d3c93bd80f06737cd34fc Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Mon, 21 Nov 2022 06:04:22 +0800 Subject: [PATCH 3/8] Update readme.md --- readme.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index a4155dba..35fa301d 100644 --- a/readme.md +++ b/readme.md @@ -501,14 +501,14 @@ Please refer run with `--help` for latest information. Number of tags to be used for %tags% meta in filename. Use -1 to use all tags. -- writeimageinfo - - Set to `True` to export the image information to text file. - The filename is following `filename(Manga)Infoformat` + .txt. - writeImageJSON Set to `True` to export the image information to JSON. The filename is following `filename(Manga)Infoformat` + .json. +- writeimageinfo + + Set to `True` to export the image information to text file. + The filename is following `filename(Manga)Infoformat` + .txt. - writeRawJSON Set to `True` to export the image JSON untouched. @@ -516,7 +516,7 @@ Please refer run with `--help` for latest information. - RawJSONFilter Enter the JSON keys which you want to filter out. Keys are seperated by a comma. -- writeSeriesJSON +- includeSeriesJSON Set to `True` to export the series information to JSON. The filename is following `filenameSeriesJSON` + .json. @@ -528,7 +528,7 @@ Please refer run with `--help` for latest information. Set to `True` to export the image information to a .XMP sidecar file, one per image in the album. The data contained within the file is the same but some software requires matching file names to detect the metadata. If set to `True`, then `writeImageXMP` is ignored. - verifyimage - Do image and zip checking after download. Set the value to `True` to enable. + Check if downloaded files is valid image or zip. Set the value to `True` to enable. - writeUrlInDescription Write all url found in the image description to a text file. Set to `True` to enable. The list will be saved to to the application folder as url_list_.txt @@ -702,6 +702,9 @@ Please refer run with `--help` for latest information. For sanitizing filenames with custom rules. Supports regular expressions. For detailed syntax, please refer to 'Bad chars' section. +- customCleanUpRe + + TODO. # Filename Format Syntax Available for filenameFormat, filenameMangaFormat, avatarNameFormat, filenameInfoFormat, From e5d503ec631139364808990db6c523e899179bff Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Mon, 21 Nov 2022 06:19:55 +0800 Subject: [PATCH 4/8] Update PixivConfig.py will https://github.com/Nandaka/PixivUtil2/blob/f2e9ab19fe471c93c0a1697d89f1f9f93f3c39cd/PixivBrowserFactory.py#L60 be affected by the socks5 `getaddrd failed`? --- PixivConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PixivConfig.py b/PixivConfig.py index f3bf5031..a40b462d 100644 --- a/PixivConfig.py +++ b/PixivConfig.py @@ -92,8 +92,8 @@ class PixivConfig(): ConfigItem("Settings", "downloadAvatar", False), ConfigItem("Settings", "useSuppressTags", False), ConfigItem("Settings", "tagsLimit", -1), - ConfigItem("Settings", "writeImageInfo", False), ConfigItem("Settings", "writeImageJSON", False), + ConfigItem("Settings", "writeImageInfo", False), ConfigItem("Settings", "writeRawJSON", False), ConfigItem("Settings", "RawJSONFilter", "id,title,description,alt,userIllusts,storableTags,zoneConfig,extraData,comicPromotion,fanboxPromotion"), @@ -223,7 +223,7 @@ def proxy(self): value = getattr(self, "proxyAddress", None) if not value: return None - match = re.match(r"^(?:(https?|socks[45])://)?([\w.-]+)(:\d+)?$", value) + match = re.match(r"^(?:(https?|socks[45]h?)://)?([\w.-]+)(:\d+)?$", value) if not match: return None scheme, netloc, port = match.groups() From d696c90cd09ff4e884297a47dc1c452e31af20f6 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Wed, 23 Nov 2022 08:43:17 +0800 Subject: [PATCH 5/8] Update readme.md --- readme.md | 67 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/readme.md b/readme.md index 35fa301d..7a13f1a0 100644 --- a/readme.md +++ b/readme.md @@ -503,11 +503,11 @@ Please refer run with `--help` for latest information. Use -1 to use all tags. - writeImageJSON - Set to `True` to export the image information to JSON. + Set to `True` to export the entire original information for the image from the source to JSON. The filename is following `filename(Manga)Infoformat` + .json. - writeimageinfo - Set to `True` to export the image information to text file. + Set to `True` to export the image information to text file. This is a subset info extracted from the `writeImageJSON`. The filename is following `filename(Manga)Infoformat` + .txt. - writeRawJSON @@ -518,7 +518,7 @@ Please refer run with `--help` for latest information. Enter the JSON keys which you want to filter out. Keys are seperated by a comma. - includeSeriesJSON - Set to `True` to export the series information to JSON. + Set to `True` to export the series information to JSON. Non-series artwork doesn't have this info. The filename is following `filenameSeriesJSON` + .json. - writeImageXMP @@ -528,10 +528,10 @@ Please refer run with `--help` for latest information. Set to `True` to export the image information to a .XMP sidecar file, one per image in the album. The data contained within the file is the same but some software requires matching file names to detect the metadata. If set to `True`, then `writeImageXMP` is ignored. - verifyimage - Check if downloaded files is valid image or zip. Set the value to `True` to enable. + Check if downloaded files are valid image or zip. Set the value to `True` to enable. - writeUrlInDescription - Write all url found in the image description to a text file. Set to `True` to enable. The list will be saved to to the application folder as url_list_.txt + Write all url found in the image description to a text file at the root directory. Set to `True` to enable. The list will be saved to to the application folder as url_list_.txt - stripHTMLTagsFromCaption Remove all HTML tags and their contents from the image caption/description when writing metadata to files. The contents of any links will be lost, so consider enabling writeUrlInDescription to retain them. @@ -543,7 +543,7 @@ Please refer run with `--help` for latest information. Use different database. - setLastModified - Set last modified timestamp based on pixiv upload timestamp. + Set last modified timestamp based on pixiv upload timestamp to the file. - useLocalTimezone Use local timezone when setting last modified timestamp/works date. @@ -558,9 +558,16 @@ Please refer run with `--help` for latest information. - maxFileSize Skip if file size is more than minFileSize, set `0` to disable. +- checkLastModified + If the last-modified timestamp of the local files is the same with the uploaded date of the artwork, it'll log "match" and skip to process the current image_id. + Require `setlastmodified = True` in config.ini to work properly +- alwaysCheckFileSize + + Actually, it'll always check the file size. But if `this` is false, if the `overwrite` is also false and this file is recorded in db, it'll skip to process the current image_id. + This will override the image_id checking from db (always fetch the image page to check the remote size). - overwrite - Overwrite old files, set `False` to disable. + If is true, when found file size different, it'll just delete the file (unless the backupOldFile is true), then start to re-download the image. - backupOldFile Set to True to backup old file if the file size is different. @@ -568,10 +575,6 @@ Please refer run with `--help` for latest information. - daylastupdated Only process member_id which were processed at least x days since the last check. -- alwaysCheckFileSize - - Check the file size, if different then it will be downloaded again, set `False` to disable. - This will override the image_id checking from db (always fetch the image page to check the remote size). - checkUpdatedLimit Jump to the next member id if already see n-number of previously downloaded images. @@ -601,10 +604,6 @@ Please refer run with `--help` for latest information. - downloadResized Download the medium size, rather than the original size. -- checkLastModified - - Compare local file's last-modified timestamp with works date. - Require `setlastmodified = True` in config.ini to work properly - skipUnknownSize Skip downloading if the remote size is not known when `alwaysCheckFileSize` is set to True. @@ -612,52 +611,54 @@ Please refer run with `--help` for latest information. ## [FFmpeg] - ffmpeg - Path to ffmpeg executable. + ffmpeg executable path. - ffmpegcodec - Codec to be used for encoding webm, default is using `libvpx-vp9`. + Codec to be used for encoding, default is using `libvpx-vp9`. +- ffmpegExt + + The file extension (container format) to use for encoding. default: `webm`. - ffmpegparam - Parameter to be used to encode webm. default is `-lossless 1 -vsync 2 -r 999 -pix_fmt yuv420p` + Parameter to be used to encode webm, default: `-lossless 0 -crf 15 -b 0 -vsync 0`. - webpcodec Codec to be used for encoding webm, default is using `libwebp`. - webpparam - Parameter to be used to encode webp. - default is `lossless 0 -q:v 90 -loop 0 -vsync 2 -r 999` + Parameter to be used to encode webm, default: `-lossless 0 -compression_level 5 -quality 100 -loop 0 -vsync 0`. ## [Ugoira] - writeugoirainfo - If set to `True`, it will dump the .js to external file. + If set to `True`, it will write the info of ugoira frames to a `filename(Manga)Infoformat`+.zip.js file. `writeImageJSON` contains this info as well. - createugoira If set to `True`, it will create .ugoira file. This is Pixiv own format for animated images. You can use Honeyview to see the animation. -- deleteZipFile +- createwebm - If set to `True`, it will delete the zip files from ugoira. - Only active if `createUgoira = True`. + Set to True to create webm file (video format). The default encoding settings is lossy encoding but high quality with smallest file size. + Required `createUgoira = True` and ffmpeg executeable. +- createwebp + + Set to True to create webp file (image format). The default encoding settings is lossy encoding but high quality with smaller file size. + Required `createUgoira = True` and ffmpeg executeable. - creategif - Set to True to convert ugoira file to gif. + Set to True to convert ugoira file to gif. The default encoding settings is lossy encoding but moderate quality with smaller file size. Required `createUgoira = True` and ffmpeg executeable. - createapng - Set to True to convert ugoira file to animated png. + Set to True to convert ugoira file to animated png. The default encoding settings is lossless encoding but very large file size. Required `createUgoira = True` and ffmpeg executeable. - deleteugoira Set to True to delete original ugoira after conversion. -- createwebm - - Set to True to create webm file (video format). - Required `createUgoira = True` and ffmpeg executeable. -- createwebp +- deleteZipFile - Set to True to create webp file (image format). - Required `createUgoira = True` and ffmpeg executeable. + If set to `True`, it will delete the zip files from ugoira. + Only active if `createUgoira = True`. ## [Filename] - filenameformat From 2f12865e8e17148167247a07053d41e4dcf0cb6d Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Wed, 23 Nov 2022 08:47:47 +0800 Subject: [PATCH 6/8] Update readme.md --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 7a13f1a0..b465dc80 100644 --- a/readme.md +++ b/readme.md @@ -559,6 +559,7 @@ Please refer run with `--help` for latest information. Skip if file size is more than minFileSize, set `0` to disable. - checkLastModified + If the last-modified timestamp of the local files is the same with the uploaded date of the artwork, it'll log "match" and skip to process the current image_id. Require `setlastmodified = True` in config.ini to work properly - alwaysCheckFileSize From 05720c59419b103d8b91862d4d77ab2b2ca76c88 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Wed, 23 Nov 2022 08:54:23 +0800 Subject: [PATCH 7/8] Update PixivConfig.py --- PixivConfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/PixivConfig.py b/PixivConfig.py index a40b462d..58625021 100644 --- a/PixivConfig.py +++ b/PixivConfig.py @@ -187,19 +187,20 @@ class PixivConfig(): ConfigItem("Ugoira", "writeUgoiraInfo", False), ConfigItem("Ugoira", "createUgoira", False), - ConfigItem("Ugoira", "deleteZipFile", False), + ConfigItem("Ugoira", "createWebm", False), + ConfigItem("Ugoira", "createWebp", False), ConfigItem("Ugoira", "createGif", False), ConfigItem("Ugoira", "createApng", False), ConfigItem("Ugoira", "deleteUgoira", False), - ConfigItem("Ugoira", "createWebm", False), - ConfigItem("Ugoira", "createWebp", False), + ConfigItem("Ugoira", "deleteZipFile", False), ConfigItem("DownloadControl", "minFileSize", 0), ConfigItem("DownloadControl", "maxFileSize", 0), + ConfigItem("DownloadControl", "checkLastModified", True), + ConfigItem("DownloadControl", "alwaysCheckFileSize", False), ConfigItem("DownloadControl", "overwrite", False), ConfigItem("DownloadControl", "backupOldFile", False), ConfigItem("DownloadControl", "dayLastUpdated", 7), - ConfigItem("DownloadControl", "alwaysCheckFileSize", False), ConfigItem("DownloadControl", "checkUpdatedLimit", 0), ConfigItem("DownloadControl", "useBlacklistTags", False), ConfigItem("DownloadControl", "useBlacklistTitles", False), @@ -208,7 +209,6 @@ class PixivConfig(): ConfigItem("DownloadControl", "enableInfiniteLoop", False), ConfigItem("DownloadControl", "useBlacklistMembers", False), ConfigItem("DownloadControl", "downloadResized", False), - ConfigItem("DownloadControl", "checkLastModified", True), ConfigItem("DownloadControl", "skipUnknownSize", False), ConfigItem("DownloadControl", "enablePostProcessing", False), ConfigItem("DownloadControl", "postProcessingCmd", ""), From 8f14cad94af969971c63284d2f53f406c6474d9e Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Wed, 23 Nov 2022 23:16:43 +0800 Subject: [PATCH 8/8] Update readme.md --- readme.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readme.md b/readme.md index b465dc80..75b82b81 100644 --- a/readme.md +++ b/readme.md @@ -342,6 +342,9 @@ Please refer run with `--help` for latest information. - r18mode Only list images tagged R18, for member, member's bookmark, and search by tag. Set to `True` to enable. +- r18Type + + 0: All; 1: R18; 2: R18G - dateformat Pixiv DateTime format, leave blank to use default format (YYYY-MM-DD).