fix(input): correct lon mapping
This commit is contained in:
parent
19557f3a62
commit
84d1664b29
20 changed files with 1197 additions and 137 deletions
|
|
@ -161,6 +161,57 @@ func (m *Manager) SelectFor(t time.Time, lat, lng float64) weather.WindField {
|
|||
return nil
|
||||
}
|
||||
|
||||
// FieldFor returns the global dataset whose epoch is epoch, loading it from
|
||||
// storage on demand when it is stored but not currently active.
|
||||
//
|
||||
// Being stored is enough. A caller asking for a specific run — an archival
|
||||
// forecast, a reproducible comparison — must not have to make it the service's
|
||||
// active dataset first, and asking for it must not repoint the service: Load
|
||||
// appends, and Active keeps returning the first global it finds.
|
||||
//
|
||||
// Errors when no stored dataset carries that epoch, so a request naming a
|
||||
// dataset the service cannot serve is refused rather than quietly answered from
|
||||
// a different one.
|
||||
//
|
||||
// ponytail: loaded datasets are never evicted, so a long-lived service asked for
|
||||
// many distinct epochs accumulates one mmap and fd each. Add LRU eviction here
|
||||
// if that becomes real; the files are mmap-backed, so the cost is address space
|
||||
// and descriptors, not resident memory.
|
||||
func (m *Manager) FieldFor(ctx context.Context, epoch time.Time) (weather.WindField, error) {
|
||||
if f := m.activeFieldAt(epoch); f != nil {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
stored, err := m.store.List()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list stored datasets: %w", err)
|
||||
}
|
||||
for _, id := range stored {
|
||||
if !id.Subset.IsGlobal() || !id.Epoch.Equal(epoch) {
|
||||
continue
|
||||
}
|
||||
if err := m.Load(ctx, id); err != nil {
|
||||
return nil, fmt.Errorf("load %s: %w", id.Filename(), err)
|
||||
}
|
||||
if f := m.activeFieldAt(epoch); f != nil {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("dataset %s is not stored", epoch.UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// activeFieldAt returns the loaded global field with exactly this epoch, or nil.
|
||||
func (m *Manager) activeFieldAt(epoch time.Time) weather.WindField {
|
||||
m.activeMu.RLock()
|
||||
defer m.activeMu.RUnlock()
|
||||
for _, d := range m.active {
|
||||
if d.ID.Subset.IsGlobal() && d.ID.Epoch.Equal(epoch) {
|
||||
return d.Field
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadedDatasets returns snapshots of every currently-loaded dataset.
|
||||
func (m *Manager) LoadedDatasets() []LoadedDatasetInfo {
|
||||
m.activeMu.RLock()
|
||||
|
|
@ -309,36 +360,48 @@ func (m *Manager) Load(ctx context.Context, id DatasetID) error {
|
|||
//
|
||||
// Returns the JobID started, or empty string when nothing was scheduled.
|
||||
func (m *Manager) Refresh(ctx context.Context, freshnessTTL time.Duration) (string, error) {
|
||||
if a := m.activeGlobal(); a != nil && time.Since(a.ID.Epoch) < freshnessTTL {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if datasets, err := m.store.List(); err == nil {
|
||||
for _, id := range datasets {
|
||||
if !id.Subset.IsGlobal() {
|
||||
continue
|
||||
}
|
||||
if time.Since(id.Epoch) > freshnessTTL {
|
||||
continue
|
||||
}
|
||||
if a := m.activeGlobal(); a != nil && a.ID.Equals(id) {
|
||||
return "", nil
|
||||
}
|
||||
if err := m.Load(ctx, id); err == nil {
|
||||
return "", nil
|
||||
// Get something usable loaded first, whatever its age.
|
||||
//
|
||||
// freshnessTTL answers one question only: go fetch something newer? It must
|
||||
// never decide whether data already on disk may be read. While one check
|
||||
// served both, a stored run older than the TTL was skipped even when it was
|
||||
// the only dataset present — active stayed empty, every prediction answered
|
||||
// "no dataset loaded" with gigabytes of usable wind on disk, and with no
|
||||
// reachable origin that state was permanent.
|
||||
if m.activeGlobal() == nil {
|
||||
if stored, err := m.store.List(); err == nil {
|
||||
for _, id := range stored { // newest first
|
||||
if !id.Subset.IsGlobal() {
|
||||
continue
|
||||
}
|
||||
if err := m.Load(ctx, id); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
active := m.activeGlobal()
|
||||
if active != nil && time.Since(active.ID.Epoch) < freshnessTTL {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
latest, err := m.src.LatestEpoch(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("latest epoch: %w", err)
|
||||
}
|
||||
id := DatasetID{Epoch: latest}
|
||||
if a := m.activeGlobal(); a != nil && !latest.After(a.ID.Epoch) {
|
||||
if active != nil && !latest.After(active.ID.Epoch) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Another replica sharing this volume may have committed it already.
|
||||
id := DatasetID{Epoch: latest}
|
||||
if m.store.Exists(id) {
|
||||
if err := m.Load(ctx, id); err == nil {
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
jobID := m.Download(id)
|
||||
go m.loadAfterCompletion(jobID, id)
|
||||
return jobID, nil
|
||||
|
|
|
|||
166
internal/datasets/manager_test.go
Normal file
166
internal/datasets/manager_test.go
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
package datasets
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"predictor-refactored/internal/weather"
|
||||
)
|
||||
|
||||
// stubField is a WindField that carries only its epoch.
|
||||
type stubField struct{ epoch time.Time }
|
||||
|
||||
func (f stubField) Wind(_, _, _, _ float64) (weather.Sample, error) {
|
||||
return weather.Sample{}, nil
|
||||
}
|
||||
func (f stubField) Epoch() time.Time { return f.epoch }
|
||||
func (f stubField) Source() string { return "fake" }
|
||||
|
||||
// fakeSource records what it was asked to open and can simulate an unreachable
|
||||
// origin via latestErr.
|
||||
type fakeSource struct {
|
||||
latest time.Time
|
||||
latestErr error
|
||||
opened []DatasetID
|
||||
}
|
||||
|
||||
func (s *fakeSource) ID() string { return "fake" }
|
||||
|
||||
func (s *fakeSource) LatestEpoch(context.Context) (time.Time, error) {
|
||||
if s.latestErr != nil {
|
||||
return time.Time{}, s.latestErr
|
||||
}
|
||||
return s.latest, nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) Download(context.Context, DatasetID, Storage, ProgressSink, Throttle) error {
|
||||
return errors.New("fakeSource does not download")
|
||||
}
|
||||
|
||||
func (s *fakeSource) Open(_ context.Context, id DatasetID, _ Storage) (weather.WindField, error) {
|
||||
s.opened = append(s.opened, id)
|
||||
return stubField{epoch: id.Epoch}, nil
|
||||
}
|
||||
|
||||
func (s *fakeSource) Coverage(id DatasetID) Coverage {
|
||||
return Coverage{
|
||||
Region: Region{MinLat: -90, MaxLat: 90, MinLng: 0, MaxLng: 360},
|
||||
StartTime: id.Epoch,
|
||||
EndTime: id.Epoch.Add(192 * time.Hour),
|
||||
}
|
||||
}
|
||||
|
||||
// fakeStore is an in-memory Storage holding a fixed set of committed datasets,
|
||||
// newest first, as LocalStore.List promises.
|
||||
type fakeStore struct{ ids []DatasetID }
|
||||
|
||||
func (s *fakeStore) SourceID() string { return "fake" }
|
||||
func (s *fakeStore) Path(id DatasetID) string { return "/dev/null/" + id.Filename() }
|
||||
|
||||
func (s *fakeStore) Exists(id DatasetID) bool {
|
||||
for _, have := range s.ids {
|
||||
if have.Equals(id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *fakeStore) List() ([]DatasetID, error) { return s.ids, nil }
|
||||
func (s *fakeStore) Remove(DatasetID) error { return nil }
|
||||
func (s *fakeStore) BeginWrite(DatasetID) (TempHandle, error) { return nil, errors.New("not used") }
|
||||
func (s *fakeStore) Lock(context.Context) (func(), error) { return func() {}, nil }
|
||||
|
||||
// TestRefreshLoadsAStoredDatasetOlderThanTheFreshnessTTL pins the distinction
|
||||
// the freshness TTL is allowed to make.
|
||||
//
|
||||
// A dataset on disk is usable wind data whatever its age. The TTL answers "go
|
||||
// fetch something newer?", not "may I read what is already here?". While both
|
||||
// questions shared one check, an archival run that was the only dataset present
|
||||
// got skipped, active stayed empty, and every prediction answered "no dataset
|
||||
// loaded" with gigabytes of usable data on disk — and, with no reachable origin,
|
||||
// forever.
|
||||
func TestRefreshLoadsAStoredDatasetOlderThanTheFreshnessTTL(t *testing.T) {
|
||||
old := time.Now().UTC().Add(-30 * 24 * time.Hour).Truncate(time.Hour)
|
||||
src := &fakeSource{latestErr: errors.New("origin unreachable")}
|
||||
store := &fakeStore{ids: []DatasetID{{Epoch: old}}}
|
||||
m := New(src, store, nil, nil)
|
||||
|
||||
// The origin probe is expected to fail; loading what is on disk is not.
|
||||
_, _ = m.Refresh(context.Background(), 48*time.Hour)
|
||||
|
||||
if m.Active() == nil {
|
||||
t.Fatal("Active() is nil: the stored dataset was never loaded")
|
||||
}
|
||||
if got := m.Active().Epoch(); !got.Equal(old) {
|
||||
t.Errorf("loaded epoch = %s, want %s", got, old)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefreshPrefersTheNewestStoredDataset guards the ordering the fix relies
|
||||
// on: falling back to disk must not mean falling back to the oldest file there.
|
||||
func TestRefreshPrefersTheNewestStoredDataset(t *testing.T) {
|
||||
newer := time.Now().UTC().Add(-10 * 24 * time.Hour).Truncate(time.Hour)
|
||||
older := newer.Add(-20 * 24 * time.Hour)
|
||||
src := &fakeSource{latestErr: errors.New("origin unreachable")}
|
||||
store := &fakeStore{ids: []DatasetID{{Epoch: newer}, {Epoch: older}}}
|
||||
m := New(src, store, nil, nil)
|
||||
|
||||
_, _ = m.Refresh(context.Background(), 48*time.Hour)
|
||||
|
||||
if m.Active() == nil {
|
||||
t.Fatal("Active() is nil")
|
||||
}
|
||||
if got := m.Active().Epoch(); !got.Equal(newer) {
|
||||
t.Errorf("loaded epoch = %s, want the newest stored %s", got, newer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFieldForLoadsAStoredDatasetThatIsNotActive covers asking for a specific
|
||||
// run by epoch — the archival-forecast case. Being stored is enough; the caller
|
||||
// should not have to make it the active dataset first.
|
||||
func TestFieldForLoadsAStoredDatasetThatIsNotActive(t *testing.T) {
|
||||
active := time.Now().UTC().Truncate(time.Hour)
|
||||
archive := active.Add(-30 * 24 * time.Hour)
|
||||
src := &fakeSource{latest: active}
|
||||
store := &fakeStore{ids: []DatasetID{{Epoch: active}, {Epoch: archive}}}
|
||||
m := New(src, store, nil, nil)
|
||||
|
||||
if err := m.Load(context.Background(), DatasetID{Epoch: active}); err != nil {
|
||||
t.Fatalf("Load active: %v", err)
|
||||
}
|
||||
|
||||
field, err := m.FieldFor(context.Background(), archive)
|
||||
if err != nil {
|
||||
t.Fatalf("FieldFor(%s): %v", archive, err)
|
||||
}
|
||||
if got := field.Epoch(); !got.Equal(archive) {
|
||||
t.Errorf("epoch = %s, want %s", got, archive)
|
||||
}
|
||||
// The active dataset must be unaffected: asking for an archival run for one
|
||||
// prediction should not repoint the service at it.
|
||||
if got := m.Active().Epoch(); !got.Equal(active) {
|
||||
t.Errorf("Active() moved to %s, want %s", got, active)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFieldForRejectsAnEpochThatIsNotStored is the half that matters most: a
|
||||
// requested dataset that cannot be served must say so rather than quietly
|
||||
// substituting a different one, which is what ignoring the parameter did.
|
||||
func TestFieldForRejectsAnEpochThatIsNotStored(t *testing.T) {
|
||||
active := time.Now().UTC().Truncate(time.Hour)
|
||||
src := &fakeSource{latest: active}
|
||||
store := &fakeStore{ids: []DatasetID{{Epoch: active}}}
|
||||
m := New(src, store, nil, nil)
|
||||
if err := m.Load(context.Background(), DatasetID{Epoch: active}); err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
missing := active.Add(-365 * 24 * time.Hour)
|
||||
field, err := m.FieldFor(context.Background(), missing)
|
||||
if err == nil {
|
||||
t.Fatalf("FieldFor(%s) returned field with epoch %s, want an error", missing, field.Epoch())
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue