""" Downloads flower vase images (jpg/jpeg/png) into ./TestImg_Vase Uses the Openverse API (https://openverse.org) with OAuth2 client-credentials authentication for a higher rate limit than anonymous requests. Setup: 1. Register an app (free, instant): POST https://api.openverse.org/v1/auth_tokens/register/ body: {"name": "...", "description": "...", "email": "..."} -> returns client_id and client_secret 2. Set them as environment variables before running: export OPENVERSE_CLIENT_ID="your_client_id_here" export OPENVERSE_CLIENT_SECRET="your_client_secret_here" 3. pip install requests 4. python scrape_vase_images.py {"client_id":"xymJfVAfXWq6detaZfQ5UxyIwQHQNQZoCPEOrTfl","client_secret":"Fz5IBZy2NO7okzd84GSUxxMSGdKrBRXMCvaSW3RcJddNfF6QjaUaAkxLszvuDHX2LrJYI0RItt8AcIGFfDpxtMgFVogcmvXWhIn7Mk7bQZxyKo2RE5oEXS2Cz0HoqOpB","name":"vase-scraper","msg":"Check your email for a verification link."}% """ import os import time import requests from urllib.parse import urlparse # ---------------- CONFIG ---------------- # Multiple related queries — Openverse's index for one exact phrase like # "flower vase" often has only ~200-300 unique results, so we rotate # through several related terms to reach larger totals. SEARCH_QUERIES = [ "flower vase", "flower vases", "vase of flowers", "ceramic vase", "glass vase", "porcelain vase", "vintage vase", "vase flowers arrangement", "decorative vase", "clay vase", "vase still life", "flower pot vase", ] TARGET_COUNT = 1000 # how many images to download SAVE_DIR = "TestImg_Vase" SLEEP_MS = 500 # delay between requests, in milliseconds PAGE_SIZE = 20 # openverse max per page ALLOWED_EXT = (".jpg", ".jpeg", ".png") REQUEST_TIMEOUT = 15 MAX_RETRIES = 5 # retries on 429 / transient errors API_BASE = "https://api.openverse.org/v1" SEARCH_URL = f"{API_BASE}/images/" TOKEN_URL = f"{API_BASE}/auth_tokens/token/" # --- credentials: placeholders, fill via environment variables --- CLIENT_ID = os.environ.get("OPENVERSE_CLIENT_ID", "xymJfVAfXWq6detaZfQ5UxyIwQHQNQZoCPEOrTfl") CLIENT_SECRET = os.environ.get("OPENVERSE_CLIENT_SECRET", "Fz5IBZy2NO7okzd84GSUxxMSGdKrBRXMCvaSW3RcJddNfF6QjaUaAkxLszvuDHX2LrJYI0RItt8AcIGFfDpxtMgFVogcmvXWhIn7Mk7bQZxyKo2RE5oEXS2Cz0HoqOpB") HEADERS_BASE = {"User-Agent": "Mozilla/5.0 (educational image dataset builder)"} # ----------------------------------------- class TokenManager: """Fetches and auto-refreshes an OAuth2 access token.""" def __init__(self, client_id, client_secret): self.client_id = client_id self.client_secret = client_secret self.access_token = None self.expires_at = 0 # unix timestamp def get_token(self): if self.access_token and time.time() < self.expires_at - 30: return self.access_token if self.client_id.startswith("PLACEHOLDER") or self.client_secret.startswith("PLACEHOLDER"): print("WARNING: OPENVERSE_CLIENT_ID / OPENVERSE_CLIENT_SECRET not set. " "Falling back to anonymous requests (lower rate limit).") return None resp = requests.post( TOKEN_URL, data={ "grant_type": "client_credentials", "client_id": self.client_id, "client_secret": self.client_secret, }, headers=HEADERS_BASE, timeout=REQUEST_TIMEOUT, ) resp.raise_for_status() payload = resp.json() self.access_token = payload["access_token"] self.expires_at = time.time() + payload.get("expires_in", 3600) print("Obtained new Openverse access token.") return self.access_token def auth_headers(self): token = self.get_token() headers = dict(HEADERS_BASE) if token: headers["Authorization"] = f"Bearer {token}" return headers token_mgr = TokenManager(CLIENT_ID, CLIENT_SECRET) def sleep(ms=None): time.sleep((ms if ms is not None else SLEEP_MS) / 1000.0) def get_extension(url, content_type): path = urlparse(url).path.lower() for ext in ALLOWED_EXT: if path.endswith(ext): return ".jpg" if ext == ".jpeg" else ext if content_type: content_type = content_type.lower() if "jpeg" in content_type or "jpg" in content_type: return ".jpg" if "png" in content_type: return ".png" return None def request_with_backoff(method, url, **kwargs): """Wraps requests.get/post with exponential backoff on 429s.""" delay = 1.0 resp = None for attempt in range(1, MAX_RETRIES + 1): resp = method(url, timeout=REQUEST_TIMEOUT, **kwargs) if resp.status_code == 429: retry_after = resp.headers.get("Retry-After") wait = float(retry_after) if retry_after else delay print(f" 429 rate limited. Backing off {wait:.1f}s (attempt {attempt}/{MAX_RETRIES})...") time.sleep(wait) delay *= 2 continue return resp return resp def fetch_page(query, page): params = {"q": query, "page": page, "page_size": PAGE_SIZE, "mature": "false"} resp = request_with_backoff( requests.get, SEARCH_URL, params=params, headers=token_mgr.auth_headers() ) resp.raise_for_status() return resp.json() def download_image(url, dest_path): resp = request_with_backoff( requests.get, url, headers=token_mgr.auth_headers(), stream=True ) resp.raise_for_status() content_type = resp.headers.get("Content-Type", "") ext = get_extension(url, content_type) if ext is None: return None final_path = dest_path + ext with open(final_path, "wb") as f: for chunk in resp.iter_content(8192): f.write(chunk) return final_path def main(): os.makedirs(SAVE_DIR, exist_ok=True) downloaded = 0 skipped = 0 seen_urls = set() print(f"Target: {TARGET_COUNT} images -> ./{SAVE_DIR}/") for query in SEARCH_QUERIES: if downloaded >= TARGET_COUNT: break print(f"\n--- Searching: '{query}' ---") page = 1 while downloaded < TARGET_COUNT: try: data = fetch_page(query, page) except requests.RequestException as e: print(f"[{query} | page {page}] search failed: {e}. Skipping to next query...") break results = data.get("results", []) if not results: print(f"No more results for '{query}'. Moving to next query.") break for item in results: if downloaded >= TARGET_COUNT: break img_url = item.get("url") if not img_url or img_url in seen_urls: continue seen_urls.add(img_url) dest_stub = os.path.join(SAVE_DIR, f"vase_{downloaded + 1:04d}") try: saved_path = download_image(img_url, dest_stub) if saved_path: downloaded += 1 print(f"[{downloaded}/{TARGET_COUNT}] saved {saved_path}") else: skipped += 1 except requests.RequestException as e: skipped += 1 print(f" skip (download error): {e}") sleep() # rate limit between every request page += 1 sleep() # rate limit between search/page requests print(f"\nDone. Downloaded {downloaded} images, skipped {skipped}.") if downloaded < TARGET_COUNT: print(f"Note: only {downloaded} unique images were available across all " f"{len(SEARCH_QUERIES)} search terms. Add more terms to SEARCH_QUERIES " f"to try for more.") if __name__ == "__main__": main()