From 6ea685d8253f1fd621a6734d504f67515c46e2b7 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 21:05:12 +0100 Subject: [PATCH 01/10] readme wording --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 8a3d055..174ddf8 100644 --- a/readme.md +++ b/readme.md @@ -33,11 +33,11 @@ If you omit the result limit, **ALL** results will be downloaded! That could be as all approx 12000 projects will be queued for download. ### Automate it -### Invoke a scan +### Invoke downloads / a scan ```bash grab-all.py ``` -This will call `grab.py` on all artists and search terms listed in `to-grab.yaml`. +This will call `grab-artists.py` and `grab-search.py` on all artists and search terms listed in `to-grab.yaml`. Files will be saved to `./downloads/{artist_name}/*.{ext}` and `/downloads/search_{search_terms}/*{artist_id}_*.{ext}`. Logs will be saved to `./logs/{artist_name/search_terms}.txt`. From 7f077f5970a16c2fe2b55da7d90e898a923fbdb5 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 21:55:52 +0100 Subject: [PATCH 02/10] typo readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 174ddf8..b1c45b8 100644 --- a/readme.md +++ b/readme.md @@ -22,7 +22,7 @@ Here we have three scripts: ```bash grab-artist.py 'mixppl' ``` -This will grab one individual profile, in this case the user 'mixppl'. You must use the username in profiles url! Not the full name! +This will grab one individual profile, in this case the user 'mixppl'. You must use the username in the profiles url! Not the full name! ### Grab search results ```bash From a300a7ff53ee1c91b170d03736838ce0b6a8b19e Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 22:10:11 +0100 Subject: [PATCH 03/10] readme wording --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index b1c45b8..e4317c3 100644 --- a/readme.md +++ b/readme.md @@ -65,7 +65,7 @@ searches: - terms: robby rotton ``` -The last search term, 'robby rotton' shows that you can also omit `max`. If you do not want to fetch artists, or searches, at all, just delete that yaml-array entirely. +The last search term, 'robby rotton' is to show that you can also omit `max`. If you do not want to fetch artists, or searches, at all, just delete that yaml-array entirely. ## A word on power usage From f017661491371b59da4ba4e1f357b90b8ffde645 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 22:55:27 +0100 Subject: [PATCH 04/10] readme --- readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index e4317c3..7e4136b 100644 --- a/readme.md +++ b/readme.md @@ -37,11 +37,11 @@ as all approx 12000 projects will be queued for download. ```bash grab-all.py ``` -This will call `grab-artists.py` and `grab-search.py` on all artists and search terms listed in `to-grab.yaml`. +This will call `grab-artists.py` and `grab-search.py` on all artists and search terms listed in [`to-grab.yaml`](https://github.com/Leonetienne/Artstation-grabber/blob/master/to-grab.yaml). -Files will be saved to `./downloads/{artist_name}/*.{ext}` and `/downloads/search_{search_terms}/*{artist_id}_*.{ext}`. -Logs will be saved to `./logs/{artist_name/search_terms}.txt`. -Download indices (to skip already downloaded projects) are kept in `./already_saved/{artist_name/search_terms}.txt`. +Files will be saved to `./downloads/{artist_name|search_terms}/`. +Logs will be saved to `./logs/{artist_name|search_terms}.txt`. +Download indices (to skip already downloaded projects) are kept in `./already_saved/{artist_name|search_terms}.txt`. > :warning: Projects already downloaded from an artists-page will be downloaded **again** if they appear in a search term, and vica versa. Artist- and search queries do NOT share download indices! From cbb8106ab2d4fcc6f0a7441fec0457c6f5a8b7eb Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 22:55:43 +0100 Subject: [PATCH 05/10] readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7e4136b..8b4fa33 100644 --- a/readme.md +++ b/readme.md @@ -39,8 +39,8 @@ grab-all.py ``` This will call `grab-artists.py` and `grab-search.py` on all artists and search terms listed in [`to-grab.yaml`](https://github.com/Leonetienne/Artstation-grabber/blob/master/to-grab.yaml). -Files will be saved to `./downloads/{artist_name|search_terms}/`. -Logs will be saved to `./logs/{artist_name|search_terms}.txt`. +Files will be saved to `./downloads/{artist_name|search_terms}/`. +Logs will be saved to `./logs/{artist_name|search_terms}.txt`. Download indices (to skip already downloaded projects) are kept in `./already_saved/{artist_name|search_terms}.txt`. > :warning: Projects already downloaded from an artists-page will be downloaded **again** if they appear in a search term, and vica versa. Artist- and search queries do NOT share download indices! From 2d4634717cec1fc7896b906c405b6c5042d3e144 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 22:56:20 +0100 Subject: [PATCH 06/10] readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 8b4fa33..321f37b 100644 --- a/readme.md +++ b/readme.md @@ -41,7 +41,7 @@ This will call `grab-artists.py` and `grab-search.py` on all artists and search Files will be saved to `./downloads/{artist_name|search_terms}/`. Logs will be saved to `./logs/{artist_name|search_terms}.txt`. -Download indices (to skip already downloaded projects) are kept in `./already_saved/{artist_name|search_terms}.txt`. +Download indices are kept in `./already_saved/{artist_name|search_terms}.txt`. These are needed to skip already downloaded projects. > :warning: Projects already downloaded from an artists-page will be downloaded **again** if they appear in a search term, and vica versa. Artist- and search queries do NOT share download indices! From a2f03bce2c0352cc3f026b1e62ad2679f7cebd4a Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 22:58:21 +0100 Subject: [PATCH 07/10] wording readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 321f37b..ecb9e2d 100644 --- a/readme.md +++ b/readme.md @@ -43,7 +43,7 @@ Files will be saved to `./downloads/{artist_name|search_terms}/`. Logs will be saved to `./logs/{artist_name|search_terms}.txt`. Download indices are kept in `./already_saved/{artist_name|search_terms}.txt`. These are needed to skip already downloaded projects. -> :warning: Projects already downloaded from an artists-page will be downloaded **again** if they appear in a search term, and vica versa. Artist- and search queries do NOT share download indices! +> :warning: Download indices are NOT shared between profiles / search queries! If an image occurs in two different search terms, it will be downloaded twice! ### Configure what to download Simply adjust [`to-grab.yaml`](https://github.com/Leonetienne/Artstation-grabber/blob/master/to-grab.yaml) to your needs. Here is an example: From 09e2a884b03cae15a2207e30460571d54ec9a4fa Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Sun, 30 Jan 2022 23:01:02 +0100 Subject: [PATCH 08/10] readme wording --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index ecb9e2d..0412052 100644 --- a/readme.md +++ b/readme.md @@ -63,9 +63,9 @@ searches: max: 3 - - terms: robby rotton + terms: Robby Rotton ``` -The last search term, 'robby rotton' is to show that you can also omit `max`. If you do not want to fetch artists, or searches, at all, just delete that yaml-array entirely. +The last search term, 'Robby Rotton', is to show that you can also omit `max`. If you do not want to fetch artists, or searches, at all, just delete that yaml-array entirely. ## A word on power usage From 18c686490c5f16117216af421cf5efddb6670ae8 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Mon, 31 Jan 2022 20:04:58 +0100 Subject: [PATCH 09/10] fixed issue that caused to download {LIMIT} images in a search, instead of checking {LIMIT} images --- grab-search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grab-search.py b/grab-search.py index 1663a96..094fd9b 100644 --- a/grab-search.py +++ b/grab-search.py @@ -104,7 +104,8 @@ try: # After downloading all assets, mark the project as downloaded. markPostAsSaved(project_hash_id, search_terms_filename) - projectCounter = projectCounter + 1 + + projectCounter = projectCounter + 1 # Project is already downloaded else: From 61d0f28c994c16ccee86a2795c072f67551b7f44 Mon Sep 17 00:00:00 2001 From: Leonetienne Date: Mon, 31 Jan 2022 20:06:36 +0100 Subject: [PATCH 10/10] fix python syntax --- grab-search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grab-search.py b/grab-search.py index 094fd9b..e892adf 100644 --- a/grab-search.py +++ b/grab-search.py @@ -105,11 +105,11 @@ try: # After downloading all assets, mark the project as downloaded. markPostAsSaved(project_hash_id, search_terms_filename) - projectCounter = projectCounter + 1 - # Project is already downloaded else: logMsg(f"Skipping project '{project_name}' [{project_hash_id}] of artist '{project_artist_name_fullname}' (user-id=[{project_artist_name}]) because it is already downloaded.", "okndl", search_terms_filename) + + projectCounter = projectCounter + 1 logMsg(f"Finished all search result pages of '{search_terms}'... Total pages scanned: {pageCounter}", "okndl", search_terms_filename)