239 lines
7.9 KiB
Python
239 lines
7.9 KiB
Python
"""
|
|
Downloads flower vase images (jpg/jpeg/png) into ./TestImg_Vase
|
|
|
|
Uses the Openverse API (https://openverse.org) with OAuth2 client-credentials
|
|
authentication for a higher rate limit than anonymous requests.
|
|
|
|
Setup:
|
|
1. Register an app (free, instant):
|
|
POST https://api.openverse.org/v1/auth_tokens/register/
|
|
body: {"name": "...", "description": "...", "email": "..."}
|
|
-> returns client_id and client_secret
|
|
|
|
2. Set them as environment variables before running:
|
|
export OPENVERSE_CLIENT_ID="your_client_id_here"
|
|
export OPENVERSE_CLIENT_SECRET="your_client_secret_here"
|
|
|
|
3. pip install requests
|
|
4. python scrape_vase_images.py
|
|
|
|
|
|
{"client_id":"xymJfVAfXWq6detaZfQ5UxyIwQHQNQZoCPEOrTfl","client_secret":"Fz5IBZy2NO7okzd84GSUxxMSGdKrBRXMCvaSW3RcJddNfF6QjaUaAkxLszvuDHX2LrJYI0RItt8AcIGFfDpxtMgFVogcmvXWhIn7Mk7bQZxyKo2RE5oEXS2Cz0HoqOpB","name":"vase-scraper","msg":"Check your email for a verification link."}%
|
|
"""
|
|
|
|
import os
|
|
import time
|
|
import requests
|
|
from urllib.parse import urlparse
|
|
|
|
# ---------------- CONFIG ----------------
|
|
# Multiple related queries — Openverse's index for one exact phrase like
|
|
# "flower vase" often has only ~200-300 unique results, so we rotate
|
|
# through several related terms to reach larger totals.
|
|
SEARCH_QUERIES = [
|
|
"flower vase",
|
|
"flower vases",
|
|
"vase of flowers",
|
|
"ceramic vase",
|
|
"glass vase",
|
|
"porcelain vase",
|
|
"vintage vase",
|
|
"vase flowers arrangement",
|
|
"decorative vase",
|
|
"clay vase",
|
|
"vase still life",
|
|
"flower pot vase",
|
|
]
|
|
TARGET_COUNT = 1000 # how many images to download
|
|
SAVE_DIR = "TestImg_Vase"
|
|
SLEEP_MS = 500 # delay between requests, in milliseconds
|
|
PAGE_SIZE = 20 # openverse max per page
|
|
ALLOWED_EXT = (".jpg", ".jpeg", ".png")
|
|
REQUEST_TIMEOUT = 15
|
|
MAX_RETRIES = 5 # retries on 429 / transient errors
|
|
|
|
API_BASE = "https://api.openverse.org/v1"
|
|
SEARCH_URL = f"{API_BASE}/images/"
|
|
TOKEN_URL = f"{API_BASE}/auth_tokens/token/"
|
|
|
|
# --- credentials: placeholders, fill via environment variables ---
|
|
CLIENT_ID = os.environ.get("OPENVERSE_CLIENT_ID", "xymJfVAfXWq6detaZfQ5UxyIwQHQNQZoCPEOrTfl")
|
|
CLIENT_SECRET = os.environ.get("OPENVERSE_CLIENT_SECRET", "Fz5IBZy2NO7okzd84GSUxxMSGdKrBRXMCvaSW3RcJddNfF6QjaUaAkxLszvuDHX2LrJYI0RItt8AcIGFfDpxtMgFVogcmvXWhIn7Mk7bQZxyKo2RE5oEXS2Cz0HoqOpB")
|
|
|
|
HEADERS_BASE = {"User-Agent": "Mozilla/5.0 (educational image dataset builder)"}
|
|
# -----------------------------------------
|
|
|
|
|
|
class TokenManager:
|
|
"""Fetches and auto-refreshes an OAuth2 access token."""
|
|
|
|
def __init__(self, client_id, client_secret):
|
|
self.client_id = client_id
|
|
self.client_secret = client_secret
|
|
self.access_token = None
|
|
self.expires_at = 0 # unix timestamp
|
|
|
|
def get_token(self):
|
|
if self.access_token and time.time() < self.expires_at - 30:
|
|
return self.access_token
|
|
|
|
if self.client_id.startswith("PLACEHOLDER") or self.client_secret.startswith("PLACEHOLDER"):
|
|
print("WARNING: OPENVERSE_CLIENT_ID / OPENVERSE_CLIENT_SECRET not set. "
|
|
"Falling back to anonymous requests (lower rate limit).")
|
|
return None
|
|
|
|
resp = requests.post(
|
|
TOKEN_URL,
|
|
data={
|
|
"grant_type": "client_credentials",
|
|
"client_id": self.client_id,
|
|
"client_secret": self.client_secret,
|
|
},
|
|
headers=HEADERS_BASE,
|
|
timeout=REQUEST_TIMEOUT,
|
|
)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
self.access_token = payload["access_token"]
|
|
self.expires_at = time.time() + payload.get("expires_in", 3600)
|
|
print("Obtained new Openverse access token.")
|
|
return self.access_token
|
|
|
|
def auth_headers(self):
|
|
token = self.get_token()
|
|
headers = dict(HEADERS_BASE)
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
return headers
|
|
|
|
|
|
token_mgr = TokenManager(CLIENT_ID, CLIENT_SECRET)
|
|
|
|
|
|
def sleep(ms=None):
|
|
time.sleep((ms if ms is not None else SLEEP_MS) / 1000.0)
|
|
|
|
|
|
def get_extension(url, content_type):
|
|
path = urlparse(url).path.lower()
|
|
for ext in ALLOWED_EXT:
|
|
if path.endswith(ext):
|
|
return ".jpg" if ext == ".jpeg" else ext
|
|
|
|
if content_type:
|
|
content_type = content_type.lower()
|
|
if "jpeg" in content_type or "jpg" in content_type:
|
|
return ".jpg"
|
|
if "png" in content_type:
|
|
return ".png"
|
|
|
|
return None
|
|
|
|
|
|
def request_with_backoff(method, url, **kwargs):
|
|
"""Wraps requests.get/post with exponential backoff on 429s."""
|
|
delay = 1.0
|
|
resp = None
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
resp = method(url, timeout=REQUEST_TIMEOUT, **kwargs)
|
|
if resp.status_code == 429:
|
|
retry_after = resp.headers.get("Retry-After")
|
|
wait = float(retry_after) if retry_after else delay
|
|
print(f" 429 rate limited. Backing off {wait:.1f}s (attempt {attempt}/{MAX_RETRIES})...")
|
|
time.sleep(wait)
|
|
delay *= 2
|
|
continue
|
|
return resp
|
|
return resp
|
|
|
|
|
|
def fetch_page(query, page):
|
|
params = {"q": query, "page": page, "page_size": PAGE_SIZE, "mature": "false"}
|
|
resp = request_with_backoff(
|
|
requests.get, SEARCH_URL, params=params, headers=token_mgr.auth_headers()
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
|
|
|
|
def download_image(url, dest_path):
|
|
resp = request_with_backoff(
|
|
requests.get, url, headers=token_mgr.auth_headers(), stream=True
|
|
)
|
|
resp.raise_for_status()
|
|
content_type = resp.headers.get("Content-Type", "")
|
|
ext = get_extension(url, content_type)
|
|
if ext is None:
|
|
return None
|
|
|
|
final_path = dest_path + ext
|
|
with open(final_path, "wb") as f:
|
|
for chunk in resp.iter_content(8192):
|
|
f.write(chunk)
|
|
return final_path
|
|
|
|
|
|
def main():
|
|
os.makedirs(SAVE_DIR, exist_ok=True)
|
|
|
|
downloaded = 0
|
|
skipped = 0
|
|
seen_urls = set()
|
|
|
|
print(f"Target: {TARGET_COUNT} images -> ./{SAVE_DIR}/")
|
|
|
|
for query in SEARCH_QUERIES:
|
|
if downloaded >= TARGET_COUNT:
|
|
break
|
|
|
|
print(f"\n--- Searching: '{query}' ---")
|
|
page = 1
|
|
|
|
while downloaded < TARGET_COUNT:
|
|
try:
|
|
data = fetch_page(query, page)
|
|
except requests.RequestException as e:
|
|
print(f"[{query} | page {page}] search failed: {e}. Skipping to next query...")
|
|
break
|
|
|
|
results = data.get("results", [])
|
|
if not results:
|
|
print(f"No more results for '{query}'. Moving to next query.")
|
|
break
|
|
|
|
for item in results:
|
|
if downloaded >= TARGET_COUNT:
|
|
break
|
|
|
|
img_url = item.get("url")
|
|
if not img_url or img_url in seen_urls:
|
|
continue
|
|
seen_urls.add(img_url)
|
|
|
|
dest_stub = os.path.join(SAVE_DIR, f"vase_{downloaded + 1:04d}")
|
|
|
|
try:
|
|
saved_path = download_image(img_url, dest_stub)
|
|
if saved_path:
|
|
downloaded += 1
|
|
print(f"[{downloaded}/{TARGET_COUNT}] saved {saved_path}")
|
|
else:
|
|
skipped += 1
|
|
except requests.RequestException as e:
|
|
skipped += 1
|
|
print(f" skip (download error): {e}")
|
|
|
|
sleep() # rate limit between every request
|
|
|
|
page += 1
|
|
sleep() # rate limit between search/page requests
|
|
|
|
print(f"\nDone. Downloaded {downloaded} images, skipped {skipped}.")
|
|
if downloaded < TARGET_COUNT:
|
|
print(f"Note: only {downloaded} unique images were available across all "
|
|
f"{len(SEARCH_QUERIES)} search terms. Add more terms to SEARCH_QUERIES "
|
|
f"to try for more.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |