82 lines
2.5 KiB
Python
82 lines
2.5 KiB
Python
"""Outbound HTTP helper with host-bucket rate limiting and simple retries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from . import host_rate_limit
|
|
from .conf import user_agent
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
|
|
|
|
|
def get(
|
|
url: str,
|
|
*,
|
|
host: str | None = None,
|
|
per_minute: int = 0,
|
|
per_hour: int = 0,
|
|
params: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None,
|
|
timeout: httpx.Timeout = DEFAULT_TIMEOUT,
|
|
max_retries: int = 3,
|
|
) -> httpx.Response:
|
|
"""GET *url*, respecting the per-host budget and retrying on 429/5xx.
|
|
|
|
``host`` defaults to the URL's hostname; pass an explicit value to share a
|
|
bucket across subdomains (e.g. all Celestrak endpoints).
|
|
"""
|
|
bucket_host = (host or host_rate_limit.host_from_url(url)).lower()
|
|
merged_headers = {"User-Agent": user_agent()}
|
|
if headers:
|
|
merged_headers.update(headers)
|
|
|
|
backoff = 1.0
|
|
last_exc: Exception | None = None
|
|
for attempt in range(1, max_retries + 1):
|
|
host_rate_limit.acquire(bucket_host, per_minute, per_hour=per_hour)
|
|
try:
|
|
with httpx.Client(timeout=timeout, follow_redirects=True) as client:
|
|
resp = client.get(url, params=params, headers=merged_headers)
|
|
except httpx.HTTPError as exc:
|
|
last_exc = exc
|
|
logger.warning("http GET %s failed (attempt %d): %s", url, attempt, exc)
|
|
if attempt == max_retries:
|
|
raise
|
|
time.sleep(backoff)
|
|
backoff *= 2
|
|
continue
|
|
|
|
if resp.status_code == 429 or resp.status_code >= 500:
|
|
retry_after = _parse_retry_after(resp.headers.get("Retry-After"))
|
|
sleep_for = retry_after if retry_after is not None else backoff
|
|
logger.info(
|
|
"http GET %s -> %d, retry in %.1fs (attempt %d)",
|
|
url, resp.status_code, sleep_for, attempt,
|
|
)
|
|
if attempt == max_retries:
|
|
resp.raise_for_status()
|
|
time.sleep(sleep_for)
|
|
backoff *= 2
|
|
continue
|
|
|
|
resp.raise_for_status()
|
|
return resp
|
|
|
|
assert last_exc is not None # pragma: no cover
|
|
raise last_exc
|
|
|
|
|
|
def _parse_retry_after(value: str | None) -> float | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return max(0.0, float(value))
|
|
except ValueError:
|
|
return None
|