Added socket timeout of two minutes
This commit is contained in:
parent
7c9caf99c2
commit
a6cbc0f4ae
13
grab.py
13
grab.py
@ -8,6 +8,7 @@ import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import time
|
||||
import socket
|
||||
|
||||
# SYNPOSIS:
|
||||
# To download posts from an artist:
|
||||
@ -169,6 +170,8 @@ image_request_headers = [
|
||||
('accept-language', 'de-DE,de;q=0.9')
|
||||
]
|
||||
|
||||
# 2 minute timeout in case something gets stuck.
|
||||
socket.setdefaulttimeout(120)
|
||||
|
||||
artist_name = str.lower(sys.argv[1])
|
||||
|
||||
@ -186,7 +189,8 @@ Path("./logs/").mkdir(parents=True, exist_ok=True)
|
||||
# Request project info for artist
|
||||
lastPageReached = False
|
||||
pageCounter = 1
|
||||
while not lastPageReached:
|
||||
try:
|
||||
while not lastPageReached:
|
||||
logMsg(f"Fetching page {pageCounter} of {artist_name}...", "okndl")
|
||||
projects_data = requests.get(f"https://www.artstation.com/users/{artist_name}/projects.json?page={pageCounter}", headers=project_fetch_headers)
|
||||
projects = projects_data.json()["data"]
|
||||
@ -242,4 +246,9 @@ while not lastPageReached:
|
||||
else:
|
||||
logMsg(f"Skipping project '{project_name}' [{project_hash_id}] because it is already downloaded.", "okndl")
|
||||
|
||||
logMsg(f"Finished all pages of {artist_name}... Total pages of this artist scanned: {pageCounter}", "okndl")
|
||||
logMsg(f"Finished all pages of {artist_name}... Total pages of this artist scanned: {pageCounter}", "okndl")
|
||||
|
||||
except socket.timeout:
|
||||
logMsg("Socket timeout of two minutes reached! We'll get 'em next time, boys!", "err")
|
||||
except:
|
||||
logMsg("Failed for some reason!", "err")
|
||||
|
Loading…
x
Reference in New Issue
Block a user