Initial commit

yksa_kit/http.py

@@ -0,0 +1,82 @@
+"""Outbound HTTP helper with host-bucket rate limiting and simple retries."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+import httpx
+
+from . import host_rate_limit
+from .conf import user_agent
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
+
+
+def get(
+    url: str,
+    *,
+    host: str | None = None,
+    per_minute: int = 0,
+    per_hour: int = 0,
+    params: dict[str, Any] | None = None,
+    headers: dict[str, str] | None = None,
+    timeout: httpx.Timeout = DEFAULT_TIMEOUT,
+    max_retries: int = 3,
+) -> httpx.Response:
+    """GET *url*, respecting the per-host budget and retrying on 429/5xx.
+
+    ``host`` defaults to the URL's hostname; pass an explicit value to share a
+    bucket across subdomains (e.g. all Celestrak endpoints).
+    """
+    bucket_host = (host or host_rate_limit.host_from_url(url)).lower()
+    merged_headers = {"User-Agent": user_agent()}
+    if headers:
+        merged_headers.update(headers)
+
+    backoff = 1.0
+    last_exc: Exception | None = None
+    for attempt in range(1, max_retries + 1):
+        host_rate_limit.acquire(bucket_host, per_minute, per_hour=per_hour)
+        try:
+            with httpx.Client(timeout=timeout, follow_redirects=True) as client:
+                resp = client.get(url, params=params, headers=merged_headers)
+        except httpx.HTTPError as exc:
+            last_exc = exc
+            logger.warning("http GET %s failed (attempt %d): %s", url, attempt, exc)
+            if attempt == max_retries:
+                raise
+            time.sleep(backoff)
+            backoff *= 2
+            continue
+
+        if resp.status_code == 429 or resp.status_code >= 500:
+            retry_after = _parse_retry_after(resp.headers.get("Retry-After"))
+            sleep_for = retry_after if retry_after is not None else backoff
+            logger.info(
+                "http GET %s -> %d, retry in %.1fs (attempt %d)",
+                url, resp.status_code, sleep_for, attempt,
+            )
+            if attempt == max_retries:
+                resp.raise_for_status()
+            time.sleep(sleep_for)
+            backoff *= 2
+            continue
+
+        resp.raise_for_status()
+        return resp
+
+    assert last_exc is not None  # pragma: no cover
+    raise last_exc
+
+
+def _parse_retry_after(value: str | None) -> float | None:
+    if not value:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        return None