leonetienne/Artstation-grabber
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added socket timeout of two minutes

Browse Source
This commit is contained in:
Leonetienne 2022-01-30 07:11:15 +01:00
parent 7c9caf99c2
commit a6cbc0f4ae
1 changed files with 61 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
grab.py
View File

@ -8,6 +8,7 @@ import re
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime
import time import time
import socket
# SYNPOSIS: # SYNPOSIS:
# To download posts from an artist: # To download posts from an artist:
@ -169,6 +170,8 @@ image_request_headers = [
('accept-language', 'de-DE,de;q=0.9') ('accept-language', 'de-DE,de;q=0.9')
] ]
# 2 minute timeout in case something gets stuck.
socket.setdefaulttimeout(120)
artist_name = str.lower(sys.argv[1]) artist_name = str.lower(sys.argv[1])
@ -186,6 +189,7 @@ Path("./logs/").mkdir(parents=True, exist_ok=True)
# Request project info for artist # Request project info for artist
lastPageReached = False lastPageReached = False
pageCounter = 1 pageCounter = 1
try:
while not lastPageReached: while not lastPageReached:
logMsg(f"Fetching page {pageCounter} of {artist_name}...", "okndl") logMsg(f"Fetching page {pageCounter} of {artist_name}...", "okndl")
projects_data = requests.get(f"https://www.artstation.com/users/{artist_name}/projects.json?page={pageCounter}", headers=project_fetch_headers) projects_data = requests.get(f"https://www.artstation.com/users/{artist_name}/projects.json?page={pageCounter}", headers=project_fetch_headers)
@ -243,3 +247,8 @@ while not lastPageReached:
logMsg(f"Skipping project '{project_name}' [{project_hash_id}] because it is already downloaded.", "okndl") logMsg(f"Skipping project '{project_name}' [{project_hash_id}] because it is already downloaded.", "okndl")
logMsg(f"Finished all pages of {artist_name}... Total pages of this artist scanned: {pageCounter}", "okndl") logMsg(f"Finished all pages of {artist_name}... Total pages of this artist scanned: {pageCounter}", "okndl")
except socket.timeout:
logMsg("Socket timeout of two minutes reached! We'll get 'em next time, boys!", "err")
except:
logMsg("Failed for some reason!", "err")