engine refactor
This commit is contained in:
parent
9e663db9dc
commit
81b8e763bd
37 changed files with 3532 additions and 1639 deletions
|
|
@ -3,29 +3,33 @@
|
|||
//
|
||||
// Endpoints:
|
||||
//
|
||||
// GET /api/v1/admin/datasets list stored epochs
|
||||
// POST /api/v1/admin/datasets trigger a download
|
||||
// DELETE /api/v1/admin/datasets/{epoch} delete a stored epoch
|
||||
// GET /api/v1/admin/jobs list all jobs
|
||||
// GET /api/v1/admin/jobs/{id} fetch one job
|
||||
// DELETE /api/v1/admin/jobs/{id} cancel a running job
|
||||
// GET /api/v1/admin/datasets list stored datasets
|
||||
// POST /api/v1/admin/datasets trigger a download
|
||||
// DELETE /api/v1/admin/datasets/{name} delete a stored dataset by filename
|
||||
// GET /api/v1/admin/jobs list all jobs
|
||||
// GET /api/v1/admin/jobs/{id} fetch one job
|
||||
// DELETE /api/v1/admin/jobs/{id} cancel a running job
|
||||
// GET /api/v1/admin/status service status summary
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"predictor-refactored/internal/api/httpjson"
|
||||
"predictor-refactored/internal/datasets"
|
||||
)
|
||||
|
||||
// Handler serves all /api/v1/admin/* endpoints.
|
||||
type Handler struct {
|
||||
mgr *datasets.Manager
|
||||
log *zap.Logger
|
||||
mgr *datasets.Manager
|
||||
start time.Time
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
// New wires an admin handler.
|
||||
|
|
@ -33,52 +37,94 @@ func New(mgr *datasets.Manager, log *zap.Logger) *Handler {
|
|||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
return &Handler{mgr: mgr, log: log}
|
||||
return &Handler{mgr: mgr, start: time.Now().UTC(), log: log}
|
||||
}
|
||||
|
||||
// Register installs admin routes on mux. Routes are mounted under
|
||||
// /api/v1/admin/...
|
||||
// Register installs admin routes on mux.
|
||||
func (h *Handler) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/admin/datasets", h.listDatasets)
|
||||
mux.HandleFunc("POST /api/v1/admin/datasets", h.triggerDownload)
|
||||
mux.HandleFunc("DELETE /api/v1/admin/datasets/{epoch}", h.deleteDataset)
|
||||
mux.HandleFunc("DELETE /api/v1/admin/datasets/{name}", h.deleteDataset)
|
||||
mux.HandleFunc("GET /api/v1/admin/jobs", h.listJobs)
|
||||
mux.HandleFunc("GET /api/v1/admin/jobs/{id}", h.getJob)
|
||||
mux.HandleFunc("DELETE /api/v1/admin/jobs/{id}", h.cancelJob)
|
||||
mux.HandleFunc("GET /api/v1/admin/status", h.status)
|
||||
}
|
||||
|
||||
// datasetDTO is the JSON shape of one stored dataset.
|
||||
type datasetDTO struct {
|
||||
Filename string `json:"filename"`
|
||||
Epoch string `json:"epoch"`
|
||||
Subset *subsetDTO `json:"subset,omitempty"`
|
||||
Coverage *coverageDTO `json:"coverage,omitempty"`
|
||||
Loaded bool `json:"loaded"`
|
||||
}
|
||||
|
||||
type subsetDTO struct {
|
||||
Region *datasets.Region `json:"region,omitempty"`
|
||||
HourRange *datasets.HourRange `json:"hour_range,omitempty"`
|
||||
Members []int `json:"members,omitempty"`
|
||||
}
|
||||
|
||||
type coverageDTO struct {
|
||||
Region datasets.Region `json:"region"`
|
||||
StartTime string `json:"start_time"`
|
||||
EndTime string `json:"end_time"`
|
||||
}
|
||||
|
||||
// listDatasets handles GET /api/v1/admin/datasets.
|
||||
func (h *Handler) listDatasets(w http.ResponseWriter, _ *http.Request) {
|
||||
epochs, err := h.mgr.ListEpochs()
|
||||
stored, err := h.mgr.ListEpochs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
active := ""
|
||||
if a := h.mgr.Active(); a != nil {
|
||||
active = a.Epoch().UTC().Format(time.RFC3339)
|
||||
loaded := h.mgr.LoadedDatasets()
|
||||
loadedByName := make(map[string]datasets.LoadedDatasetInfo, len(loaded))
|
||||
for _, ld := range loaded {
|
||||
loadedByName[ld.ID.Filename()] = ld
|
||||
}
|
||||
|
||||
out := struct {
|
||||
Source string `json:"source"`
|
||||
Active string `json:"active,omitempty"`
|
||||
Epochs []string `json:"epochs"`
|
||||
}{
|
||||
Source: h.mgr.Source(),
|
||||
Active: active,
|
||||
}
|
||||
for _, e := range epochs {
|
||||
out.Epochs = append(out.Epochs, e.UTC().Format(time.RFC3339))
|
||||
Source string `json:"source"`
|
||||
Datasets []datasetDTO `json:"datasets"`
|
||||
}{Source: h.mgr.Source(), Datasets: make([]datasetDTO, 0, len(stored))}
|
||||
|
||||
for _, id := range stored {
|
||||
dto := datasetDTO{
|
||||
Filename: id.Filename(),
|
||||
Epoch: id.Epoch.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !id.Subset.IsGlobal() {
|
||||
dto.Subset = &subsetDTO{
|
||||
Region: id.Subset.Region,
|
||||
HourRange: id.Subset.HourRange,
|
||||
Members: id.Subset.Members,
|
||||
}
|
||||
}
|
||||
if ld, ok := loadedByName[id.Filename()]; ok {
|
||||
dto.Loaded = true
|
||||
dto.Coverage = &coverageDTO{
|
||||
Region: ld.Coverage.Region,
|
||||
StartTime: ld.Coverage.StartTime.UTC().Format(time.RFC3339),
|
||||
EndTime: ld.Coverage.EndTime.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
out.Datasets = append(out.Datasets, dto)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// triggerDownload handles POST /api/v1/admin/datasets.
|
||||
//
|
||||
// Body: {"epoch": "2026-03-28T06:00:00Z"} OR {"latest": true}.
|
||||
// Body:
|
||||
// {"latest": true} — refresh the latest global dataset
|
||||
// {"epoch": "2026-03-28T06:00:00Z", "subset": {...}} — explicit dataset
|
||||
func (h *Handler) triggerDownload(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Latest bool `json:"latest,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Latest bool `json:"latest,omitempty"`
|
||||
Subset *datasets.SubsetSpec `json:"subset,omitempty"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid body: "+err.Error())
|
||||
|
|
@ -89,7 +135,6 @@ func (h *Handler) triggerDownload(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
var epoch time.Time
|
||||
if body.Latest {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
|
@ -102,29 +147,40 @@ func (h *Handler) triggerDownload(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
epoch, err = time.Parse(time.RFC3339, body.Epoch)
|
||||
epoch, err := time.Parse(time.RFC3339, body.Epoch)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid epoch: "+err.Error())
|
||||
return
|
||||
}
|
||||
jobID := h.mgr.Download(epoch)
|
||||
id := datasets.DatasetID{Epoch: epoch.UTC()}
|
||||
if body.Subset != nil {
|
||||
id.Subset = *body.Subset
|
||||
}
|
||||
jobID := h.mgr.Download(id)
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{"job_id": jobID})
|
||||
}
|
||||
|
||||
// deleteDataset handles DELETE /api/v1/admin/datasets/{epoch}.
|
||||
// deleteDataset handles DELETE /api/v1/admin/datasets/{name}.
|
||||
//
|
||||
// {name} is the dataset filename (DatasetID.Filename()) as returned by GET.
|
||||
func (h *Handler) deleteDataset(w http.ResponseWriter, r *http.Request) {
|
||||
rawEpoch := r.PathValue("epoch")
|
||||
epoch, err := time.Parse(time.RFC3339, rawEpoch)
|
||||
name := r.PathValue("name")
|
||||
stored, err := h.mgr.ListEpochs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid epoch: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := h.mgr.RemoveEpoch(epoch); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
for _, id := range stored {
|
||||
if id.Filename() == name {
|
||||
if err := h.mgr.Remove(id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "dataset not found")
|
||||
}
|
||||
|
||||
// listJobs handles GET /api/v1/admin/jobs.
|
||||
|
|
@ -158,24 +214,59 @@ func (h *Handler) cancelJob(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// status handles GET /api/v1/admin/status — a consolidated dashboard view.
|
||||
func (h *Handler) status(w http.ResponseWriter, _ *http.Request) {
|
||||
jobs := h.mgr.ListJobs()
|
||||
stored, _ := h.mgr.ListEpochs()
|
||||
loaded := h.mgr.LoadedDatasets()
|
||||
|
||||
counts := map[string]int{}
|
||||
for _, j := range jobs {
|
||||
counts[string(j.Status)]++
|
||||
}
|
||||
var mem runtime.MemStats
|
||||
runtime.ReadMemStats(&mem)
|
||||
|
||||
resp := struct {
|
||||
Source string `json:"source"`
|
||||
Uptime string `json:"uptime"`
|
||||
Goroutines int `json:"goroutines"`
|
||||
MemoryMB uint64 `json:"memory_mb"`
|
||||
JobsByStatus map[string]int `json:"jobs_by_status"`
|
||||
Stored int `json:"stored_datasets"`
|
||||
Loaded int `json:"loaded_datasets"`
|
||||
}{
|
||||
Source: h.mgr.Source(),
|
||||
Uptime: time.Since(h.start).Round(time.Second).String(),
|
||||
Goroutines: runtime.NumGoroutine(),
|
||||
MemoryMB: mem.Alloc / 1024 / 1024,
|
||||
JobsByStatus: counts,
|
||||
Stored: len(stored),
|
||||
Loaded: len(loaded),
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
type jobDTO struct {
|
||||
ID string `json:"id"`
|
||||
Source string `json:"source"`
|
||||
Epoch string `json:"epoch"`
|
||||
Status string `json:"status"`
|
||||
StartedAt string `json:"started_at"`
|
||||
EndedAt string `json:"ended_at,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
Total int `json:"total_units"`
|
||||
Done int `json:"done_units"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
ID string `json:"id"`
|
||||
Source string `json:"source"`
|
||||
Dataset string `json:"dataset"`
|
||||
Epoch string `json:"epoch"`
|
||||
Status string `json:"status"`
|
||||
StartedAt string `json:"started_at"`
|
||||
EndedAt string `json:"ended_at,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
Total int `json:"total_units"`
|
||||
Done int `json:"done_units"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
func toDTO(j datasets.JobInfo) jobDTO {
|
||||
dto := jobDTO{
|
||||
ID: j.ID,
|
||||
Source: j.Source,
|
||||
Epoch: j.Epoch.UTC().Format(time.RFC3339),
|
||||
Dataset: j.Dataset.Filename(),
|
||||
Epoch: j.Dataset.Epoch.UTC().Format(time.RFC3339),
|
||||
Status: string(j.Status),
|
||||
StartedAt: j.StartedAt.UTC().Format(time.RFC3339),
|
||||
Err: j.Err,
|
||||
|
|
@ -189,18 +280,5 @@ func toDTO(j datasets.JobInfo) jobDTO {
|
|||
return dto
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, body any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, description string) {
|
||||
writeJSON(w, status, map[string]any{
|
||||
"error": map[string]string{
|
||||
"type": http.StatusText(status),
|
||||
"description": description,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
var writeJSON = httpjson.Write
|
||||
var writeError = httpjson.Error
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue