Initial commit

tests/settings.py

@@ -0,0 +1,19 @@
+SECRET_KEY = "test-only"
+DEBUG = True
+USE_TZ = True
+PRODUCTION = False
+
+YKSA_SERVICE = "poller"
+
+INSTALLED_APPS = [
+    "django.contrib.contenttypes",
+    "django.contrib.auth",
+    "yksa_kit",
+    "tests.testapp",
+]
+
+DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}}
+
+CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
+
+CELERY_TASK_ALWAYS_EAGER = True

tests/test_runner.py

@@ -0,0 +1,169 @@
+"""What the runner has to guarantee: every exit leaves the source schedulable.
+
+The bugs this suite exists for are all the same shape — a poll that ends without
+re-arming next_poll_at, so the dispatcher either hammers the source every minute
+or drops it forever.
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+
+import pytest
+from django.utils import timezone
+
+from yksa_kit.choices import RunStatus
+from yksa_poller.adapters import AdapterRegistry, BaseAdapter
+from yksa_poller.runner import Poller, cleanup_runs, due_source_ids, run_source
+
+from tests.testapp.models import Kind, Run, Satellite, Source
+
+ADAPTERS = AdapterRegistry()
+
+
+@ADAPTERS.register(Kind.HTTP)
+class TwoRecords(BaseAdapter):
+    def fetch(self):
+        yield {"id": 1}
+        yield {"id": 2}
+
+
+class Exploding(BaseAdapter):
+    def fetch(self):
+        raise RuntimeError("upstream on fire")
+        yield  # pragma: no cover
+
+
+class Unconfigured(BaseAdapter):
+    def fetch(self):
+        raise NotImplementedError("no credentials for this source")
+        yield  # pragma: no cover
+
+
+def poller(adapters=None, **kwargs):
+    return Poller(
+        source_model=Source,
+        run_model=Run,
+        adapters=adapters if adapters is not None else ADAPTERS,
+        persist=kwargs.pop("persist", lambda src, rec: True),
+        **kwargs,
+    )
+
+
+@pytest.fixture
+def source(db):
+    return Source.objects.create(name="Upstream", slug="upstream", kind=Kind.HTTP)
+
+
+@pytest.mark.django_db
+def test_a_successful_poll_counts_and_reschedules(source):
+    before = source.next_poll_at
+    result = run_source(poller(), source.pk)
+
+    assert result["fetched"] == 2
+    assert result["new"] == 2
+    source.refresh_from_db()
+    assert source.last_status == RunStatus.SUCCESS
+    assert source.success_count == 1
+    assert source.next_poll_at > before
+    assert source.runs.get().status == RunStatus.SUCCESS
+
+
+@pytest.mark.django_db
+def test_only_new_records_are_counted_as_new(source):
+    result = run_source(poller(persist=lambda src, rec: rec["id"] == 1), source.pk)
+    assert (result["fetched"], result["new"]) == (2, 1)
+
+
+@pytest.mark.django_db
+def test_a_failing_poll_still_reschedules_and_reraises(source):
+    before = source.next_poll_at
+    with pytest.raises(RuntimeError):
+        run_source(poller(adapters={Kind.HTTP: Exploding}), source.pk)
+
+    source.refresh_from_db()
+    assert source.last_status == RunStatus.FAILED
+    assert source.failure_count == 1
+    assert source.next_poll_at > before
+    assert "upstream on fire" in source.runs.get().error
+
+
+@pytest.mark.django_db
+def test_an_unconfigured_adapter_is_skipped_not_failed(source):
+    """A source nobody finished configuring must not look like an outage."""
+    result = run_source(poller(adapters={Kind.HTTP: Unconfigured}), source.pk)
+
+    assert result["status"] == RunStatus.SKIPPED
+    source.refresh_from_db()
+    assert source.failure_count == 0
+    assert source.runs.get().status == RunStatus.SKIPPED
+
+
+@pytest.mark.django_db
+def test_a_kind_with_no_adapter_is_skipped(source):
+    source.kind = Kind.PUSH
+    source.save()
+    result = run_source(poller(), source.pk)
+    assert result["status"] == RunStatus.SKIPPED
+    assert "no adapter" in source.runs.get().error
+
+
+@pytest.mark.django_db
+def test_a_disabled_source_does_nothing_at_all(source):
+    source.is_enabled = False
+    source.save()
+    assert run_source(poller(), source.pk)["skipped"] is True
+    assert source.runs.count() == 0
+
+
+@pytest.mark.django_db
+def test_track_carries_extra_fields_onto_the_source(source):
+    """tdas uses this for last_rx_at, the lower bound of its next query."""
+    stamp = timezone.now()
+
+    def track(src, record, extra):
+        extra["last_run_at"] = stamp
+
+    run_source(poller(track=track), source.pk)
+    source.refresh_from_db()
+    assert source.last_run_at == stamp
+
+
+@pytest.mark.django_db
+def test_dispatch_picks_only_due_enabled_pollable_sources():
+    past = timezone.now() - timedelta(minutes=1)
+    future = timezone.now() + timedelta(hours=1)
+    due = Source.objects.create(name="due", slug="due", kind=Kind.HTTP, next_poll_at=past)
+    Source.objects.create(name="later", slug="later", kind=Kind.HTTP, next_poll_at=future)
+    Source.objects.create(name="off", slug="off", kind=Kind.HTTP, next_poll_at=past, is_enabled=False)
+    Source.objects.create(name="pushed", slug="pushed", kind=Kind.PUSH, next_poll_at=past)
+
+    ids = due_source_ids(poller(unpollable_kinds=(Kind.PUSH,)))
+    assert ids == [due.pk]
+
+
+@pytest.mark.django_db
+def test_cleanup_deletes_only_old_runs(source):
+    fresh = Run.objects.create(source=source, status=RunStatus.SUCCESS)
+    old = Run.objects.create(source=source, status=RunStatus.SUCCESS)
+    # started_at is auto_now_add, so age has to be forced.
+    Run.objects.filter(pk=old.pk).update(
+        started_at=timezone.now() - timedelta(days=40)
+    )
+
+    assert cleanup_runs(Run, retention_days=30)["deleted"] == 1
+    assert list(Run.objects.values_list("pk", flat=True)) == [fresh.pk]
+
+
+@pytest.mark.django_db
+def test_subscription_ids_skip_satellites_missing_the_identifier(source):
+    """An upstream query built from a blank identifier returns either nothing or
+    everything, and both are wrong."""
+    with_norad = Satellite.objects.create(norad_cat_id="25544", internal_id="iss")
+    Satellite.objects.create(norad_cat_id="", internal_id="unlaunched")
+    source.tracked_satellites.set(Satellite.objects.all())
+
+    assert source.subscription_id_values() == [with_norad.norad_cat_id]
+
+    source.subscription_id_kind = "internal"
+    assert sorted(source.subscription_id_values()) == ["iss", "unlaunched"]

tests/testapp/models.py

@@ -0,0 +1,27 @@
+"""A minimal service, standing in for tdas and odms."""
+
+from django.db import models
+
+from yksa_poller.models import AbstractPollingSource, AbstractSourceRun
+
+
+class Kind(models.TextChoices):
+    HTTP = "http", "HTTP"
+    PUSH = "push", "Push (fed from outside)"
+
+
+class Satellite(models.Model):
+    norad_cat_id = models.CharField(max_length=16, blank=True)
+    internal_id = models.SlugField(max_length=64)
+
+
+class Source(AbstractPollingSource):
+    SUBSCRIPTION_ID_FIELDS = {"norad": "norad_cat_id", "internal": "internal_id"}
+
+    kind = models.CharField(max_length=32, choices=Kind.choices)
+    subscription_id_kind = models.CharField(max_length=16, default="norad")
+    tracked_satellites = models.ManyToManyField(Satellite, blank=True)
+
+
+class Run(AbstractSourceRun):
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs")