gsn/predictor
1
Fork 0
Code Issues Pull requests Projects Releases Activity

feat: polish & windviz & deploy

Browse source
This commit is contained in:
Anatoly Antonov 2026-05-30 06:29:39 +09:00
parent 81b8e763bd
commit 465ad00f7b
78 changed files with 20622 additions and 2154 deletions
Download patch file Download diff file Expand all files Collapse all files

48
internal/datasets/manager.go
View file

@ -27,16 +27,16 @@ const (
// JobInfo is the externally-visible snapshot of a download job.
type JobInfo struct {
ID string
Source string
Dataset DatasetID
Status JobStatus
StartedAt time.Time
EndedAt *time.Time
Err string
Total int
Done int
Bytes int64
ID string
Source string
Dataset DatasetID
Status JobStatus
StartedAt time.Time
EndedAt *time.Time
Err string
Total int
Done int
Bytes int64
}
type jobEntry struct {
@ -75,9 +75,9 @@ func (e *jobEntry) snapshot() JobInfo {
type jobProgress struct{ e *jobEntry }
func (p jobProgress) SetTotal(n int) { p.e.total.Store(int64(n)) }
func (p jobProgress) StepComplete() { p.e.done.Add(1) }
func (p jobProgress) Bytes(n int64) { p.e.bytes.Add(n) }
func (p jobProgress) SetTotal(n int) { p.e.total.Store(int64(n)) }
func (p jobProgress) StepComplete() { p.e.done.Add(1) }
func (p jobProgress) Bytes(n int64) { p.e.bytes.Add(n) }
// loadedDataset bundles a loaded WindField with its identity and coverage.
type loadedDataset struct {
@ -387,7 +387,7 @@ func (m *Manager) runDownload(ctx context.Context, e *jobEntry) {
zap.String("job", e.id),
zap.String("dataset", e.dataset.Filename()))
err := m.src.Download(ctx, e.dataset, m.store, jobProgress{e: e}, m.throttle)
err := m.downloadLocked(ctx, e)
now := time.Now().UTC()
e.mu.Lock()
@ -410,6 +410,26 @@ func (m *Manager) runDownload(ctx context.Context, e *jobEntry) {
zap.NamedError("err", err))
}
// downloadLocked runs the source download while holding the storage's
// cross-process lock, so multiple replicas sharing a node-local dataset
// volume coordinate instead of each fetching ~9 GB. After acquiring the lock
// it re-checks existence: if another replica committed the dataset while this
// one waited, it skips the download and lets the caller load the committed file.
func (m *Manager) downloadLocked(ctx context.Context, e *jobEntry) error {
release, err := m.store.Lock(ctx)
if err != nil {
return fmt.Errorf("acquire download lock: %w", err)
}
defer release()
if m.store.Exists(e.dataset) {
m.log.Info("dataset committed by another instance while waiting; skipping download",
zap.String("dataset", e.dataset.Filename()))
return nil
}
return m.src.Download(ctx, e.dataset, m.store, jobProgress{e: e}, m.throttle)
}
func (m *Manager) completeShortCircuit(ctx context.Context, e *jobEntry) {
_ = ctx
defer m.inFlight.Delete(e.dataset.Filename())