DataBlue Python SDK — scrape, crawl, search, and map the web.

INSTALL
  pip install datablue

AUTH
  from datablue import DataBlue
  client = DataBlue(api_url="https://api.datablue.dev", api_key="wh_...")
  # Or from env: DATABLUE_API_URL, DATABLUE_API_KEY
  client = DataBlue.from_env()
  # Self-hosted default: api_url="http://localhost:8000"
  # Email/password: client.login("you@example.com", "password")

ASYNC CLIENT
  from datablue import AsyncDataBlue
  async with AsyncDataBlue(api_key="wh_...") as client:
      result = await client.scrape("https://example.com")

METHODS

scrape(url, *, formats=None, only_main_content=True, wait_for=0, timeout=30000,
       include_tags=None, exclude_tags=None, headers=None, cookies=None,
       mobile=False, mobile_device=None, css_selector=None, xpath=None,
       selectors=None, actions=None, extract=None, capture_network=False,
       use_proxy=False, webhook_url=None, webhook_secret=None) -> ScrapeResult
  Scrape a single URL and return structured content.
  formats values: "markdown", "html", "raw_html", "links", "screenshot", "extract"

crawl(url, *, max_pages=100, max_depth=3, concurrency=3, include_paths=None,
      exclude_paths=None, allow_external_links=False, respect_robots_txt=True,
      filter_faceted_urls=True, crawl_strategy=None, scrape_options=None,
      use_proxy=False, webhook_url=None, webhook_secret=None,
      poll_interval=2.0, timeout=300.0, on_progress=None) -> CrawlStatus
  Crawl a website and block until complete.
  crawl_strategy values: "bfs" (breadth-first), "dfs" (depth-first), "bff" (best-first)

start_crawl(url, **crawl_kwargs) -> CrawlJob
  Start a crawl job without blocking. Returns job_id for polling.

get_crawl_status(job_id: str) -> CrawlStatus
  Get current status and results of a crawl job.

cancel_crawl(job_id: str) -> dict
  Cancel an in-progress crawl.

crawl_stream(url, **crawl_kwargs) -> Iterator[CrawlPageData]
  Start a crawl and yield CrawlPageData objects as pages are discovered (NDJSON stream).

crawl_stream_with_callback(url, *, on_document, on_complete=None, on_error=None, **kwargs) -> None
  Start a crawl with callbacks fired for each page. on_document is required.

search(query, *, num_results=5, engine="google", google_api_key=None, google_cx=None,
       brave_api_key=None, formats=None, only_main_content=False, headers=None,
       cookies=None, mobile=False, extract=None, use_proxy=False,
       poll_interval=2.0, timeout=300.0, on_progress=None) -> SearchStatus
  Search the web and scrape result pages; blocks until complete.
  engine values: "google" (default, via SearXNG), "duckduckgo", "brave"

start_search(query, **search_kwargs) -> SearchJob
  Start a search job without blocking.

get_search_status(job_id: str) -> SearchStatus
  Get current status and results of a search job.

map(url, *, search=None, limit=100, include_subdomains=True, use_sitemap=True) -> MapResult
  Discover all URLs on a website using sitemaps and link crawling.

batch_scrape(urls, *, concurrency=5, scrape_options=None) -> list[ScrapeResult]
  Scrape multiple URLs. Sync client returns list; async client also has batch_scrape_iter().

batch_scrape_iter(urls, *, concurrency=5, scrape_options=None) -> AsyncIterator[ScrapeResult]
  (AsyncDataBlue only) Yield ScrapeResult objects as they complete.

RESPONSE FIELDS

ScrapeResult: .success, .data (PageData), .error, .error_code, .job_id
  error_code values: BLOCKED_BY_WAF, CAPTCHA_REQUIRED, TIMEOUT, JS_REQUIRED, NETWORK_ERROR

PageData (result.data):
  .url, .markdown, .fit_markdown, .html, .raw_html, .links, .links_detail,
  .screenshot (base64 PNG), .structured_data, .headings, .images, .extract,
  .citations, .markdown_with_citations, .content_hash, .metadata (PageMetadata)

PageMetadata (result.data.metadata):
  .title, .description, .language, .source_url, .status_code, .word_count,
  .reading_time_seconds, .content_length, .og_image, .canonical_url,
  .favicon, .robots, .response_headers

CrawlStatus (result of crawl()):
  .job_id, .status, .total_pages, .completed_pages, .data (list[CrawlPageData]),
  .total_results, .page, .per_page, .error, .is_complete (bool), .progress (float)

CrawlPageData (item in CrawlStatus.data):
  same as PageData plus: .id, .error, .success

SearchStatus (result of search()):
  .job_id, .status, .query, .total_results, .completed_results,
  .data (list[SearchResultItem]), .error, .is_complete, .progress

SearchResultItem:
  .url, .title, .snippet, .success, .markdown, .html, .links, .extract, .metadata, .error

MapResult: .success, .total, .links (list[LinkResult]), .error, .job_id, .urls (property)
LinkResult: .url, .title, .description, .lastmod, .priority

BROWSER ACTIONS (actions=[])
  {"type": "click", "selector": "#btn"}
  {"type": "wait", "milliseconds": 2000}
  {"type": "scroll", "direction": "down", "amount": 500}
  {"type": "type", "selector": "input", "text": "hello"}
  {"type": "press", "key": "Enter"}
  {"type": "select", "selector": "select#x", "value": "US"}
  {"type": "screenshot"}
  {"type": "hover", "selector": ".menu"}
  {"type": "fill_form", "fields": {"username": "alice"}}
  {"type": "evaluate", "script": "return document.title"}
  {"type": "go_back"}
  {"type": "go_forward"}

ERRORS
  DataBlueError      base; .message, .status_code, .is_retryable, .retry_after, .docs_url
  AuthenticationError  401 — bad or missing API key
  NotFoundError        404 — resource not found
  RateLimitError       429 — .retry_after gives seconds to wait; .is_retryable = True
  ServerError          5xx — auto-retried by SDK; .is_retryable = True
  JobFailedError       job completed with "failed" status; .job_id
  TimeoutError         polling exceeded timeout; .job_id, .elapsed

EXAMPLES

  # Scrape to markdown
  with DataBlue(api_key="wh_...") as client:
      result = client.scrape("https://example.com")
      print(result.data.markdown)

  # Scrape with LLM extraction
  result = client.scrape(
      "https://example.com/product",
      formats=["markdown", "extract"],
      extract={"prompt": "Extract name and price",
               "schema": {"type": "object", "properties": {"name": {"type": "string"}, "price": {"type": "number"}}}},
  )
  print(result.data.extract)

  # Crawl site, stream pages
  for page in client.crawl_stream("https://docs.example.com", max_pages=200):
      print(page.url, len(page.markdown or ""))

  # Search and print results
  status = client.search("python web scraping", num_results=5, formats=["markdown"])
  for item in status.data:
      print(item.title, item.url)

  # Discover URLs
  result = client.map("https://example.com", limit=500)
  print(result.urls)

  # Batch scrape (async)
  async with AsyncDataBlue(api_key="wh_...") as client:
      async for r in client.batch_scrape_iter(urls, concurrency=10):
          print(r.data.url, r.success)
