276 lines
8.2 KiB
Go
276 lines
8.2 KiB
Go
// Command compare-tawhiri runs identical predictions against a local predictor
|
|
// and a hosted Tawhiri instance and reports how closely they agree.
|
|
//
|
|
// To make the comparison test the engine rather than data drift, it discovers
|
|
// the local predictor's loaded GFS run via /ready and asks Tawhiri to use the
|
|
// same run (the `dataset` parameter), so both integrate identical wind data.
|
|
// It compares the burst apex (terrain-independent) and the landing point
|
|
// (terrain-dependent) separately, since without the ruaumoko elevation dataset
|
|
// the local predictor terminates descent at sea level while Tawhiri uses
|
|
// ground elevation.
|
|
//
|
|
// Usage:
|
|
//
|
|
// compare-tawhiri --server http://localhost:8080 # built-in suite
|
|
// compare-tawhiri --lat 52.2 --lng 0.1 --burst 30000 # single site
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"text/tabwriter"
|
|
"time"
|
|
)
|
|
|
|
func main() {
|
|
var (
|
|
server = flag.String("server", "http://localhost:8080", "local predictor base URL")
|
|
tawhiri = flag.String("tawhiri", "https://api.v2.sondehub.org/tawhiri", "hosted Tawhiri base URL")
|
|
lat = flag.Float64("lat", math.NaN(), "launch latitude (single-site mode)")
|
|
lng = flag.Float64("lng", math.NaN(), "launch longitude (single-site mode)")
|
|
alt = flag.Float64("alt", 0, "launch altitude m")
|
|
ascent = flag.Float64("ascent-rate", 5, "ascent rate m/s")
|
|
burst = flag.Float64("burst", 30000, "burst altitude m")
|
|
descent = flag.Float64("descent-rate", 5, "descent rate m/s")
|
|
launch = flag.String("launch", "", "launch time RFC3339 (default: epoch + 3h)")
|
|
align = flag.Bool("align-dataset", true, "ask Tawhiri to use the local predictor's GFS run")
|
|
)
|
|
flag.Parse()
|
|
|
|
epoch, err := fetchActiveEpoch(*server)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "local /ready:", err)
|
|
os.Exit(1)
|
|
}
|
|
fmt.Printf("local dataset epoch: %s\n", epoch.Format(time.RFC3339))
|
|
|
|
launchTime := epoch.Add(3 * time.Hour)
|
|
if *launch != "" {
|
|
launchTime, err = time.Parse(time.RFC3339, *launch)
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "invalid --launch:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
datasetParam := ""
|
|
if *align {
|
|
datasetParam = epoch.Format(time.RFC3339)
|
|
}
|
|
|
|
sites := suite()
|
|
if !math.IsNaN(*lat) && !math.IsNaN(*lng) {
|
|
sites = []site{{name: "custom", lat: *lat, lng: *lng}}
|
|
}
|
|
|
|
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
|
fmt.Fprintln(tw, "\nsite\tburst Δ\tlanding Δ\tapex alt Δ\tland alt Δ\tasc pts\tdesc pts\tnotes")
|
|
fmt.Fprintln(tw, "----\t-------\t---------\t----------\t----------\t-------\t--------\t-----")
|
|
|
|
var worst float64
|
|
compared := 0
|
|
for _, s := range sites {
|
|
p := params{lat: s.lat, lng: s.lng, alt: *alt, launch: launchTime,
|
|
ascent: *ascent, burst: *burst, descent: *descent}
|
|
|
|
ours, err := predict(*server+"/api/v1/prediction", p, "")
|
|
if err != nil {
|
|
fmt.Fprintf(tw, "%s\tlocal error: %v\n", s.name, err)
|
|
continue
|
|
}
|
|
theirs, err := predict(*tawhiri, p, datasetParam)
|
|
if err != nil {
|
|
fmt.Fprintf(tw, "%s\ttawhiri error: %v\n", s.name, err)
|
|
continue
|
|
}
|
|
compared++
|
|
|
|
burstD := haversine(ours.apexLat, ours.apexLng, theirs.apexLat, theirs.apexLng)
|
|
landD := haversine(ours.landLat, ours.landLng, theirs.landLat, theirs.landLng)
|
|
if landD > worst {
|
|
worst = landD
|
|
}
|
|
note := ""
|
|
if theirs.dataset != "" && ours.dataset != "" && theirs.dataset != ours.dataset {
|
|
note = fmt.Sprintf("dataset mismatch (theirs=%s)", theirs.dataset)
|
|
}
|
|
fmt.Fprintf(tw, "%s\t%.0f m\t%.2f km\t%.0f m\t%.0f m\t%d/%d\t%d/%d\t%s\n",
|
|
s.name, burstD, landD/1000,
|
|
math.Abs(ours.apexAlt-theirs.apexAlt), math.Abs(ours.landAlt-theirs.landAlt),
|
|
ours.ascPts, theirs.ascPts, ours.descPts, theirs.descPts, note)
|
|
}
|
|
tw.Flush()
|
|
|
|
if compared == 0 {
|
|
fmt.Println("\nVERDICT: NO COMPARISONS (every site errored — see rows above)")
|
|
os.Exit(1)
|
|
}
|
|
fmt.Printf("\ncompared %d/%d sites; worst landing distance: %.2f km\n", compared, len(sites), worst/1000)
|
|
switch {
|
|
case worst < 1000:
|
|
fmt.Println("VERDICT: MATCH (all landings < 1 km — engine agrees with Tawhiri)")
|
|
case worst < 50000:
|
|
fmt.Println("VERDICT: CLOSE (< 50 km — consistent with elevation/dataset differences)")
|
|
default:
|
|
fmt.Println("VERDICT: DIVERGENT (> 50 km — investigate)")
|
|
os.Exit(2)
|
|
}
|
|
}
|
|
|
|
type site struct {
|
|
name string
|
|
lat, lng float64
|
|
}
|
|
|
|
// suite is a small set of diverse launch points: UK (lands on land/sea
|
|
// depending on winds), mid-Atlantic and mid-Pacific (ocean landings, so the
|
|
// sea-level-vs-terrain difference vanishes), and southern hemisphere.
|
|
func suite() []site {
|
|
return []site{
|
|
{"cambridge-uk", 52.2135, 0.0964},
|
|
{"mid-atlantic", 35.0, -40.0},
|
|
{"mid-pacific", 0.0, -160.0},
|
|
{"new-zealand", -41.3, 174.8},
|
|
{"colorado-us", 39.0, -105.5},
|
|
}
|
|
}
|
|
|
|
type params struct {
|
|
lat, lng, alt float64
|
|
launch time.Time
|
|
ascent, burst, descent float64
|
|
}
|
|
|
|
type result struct {
|
|
apexLat, apexLng, apexAlt float64
|
|
landLat, landLng, landAlt float64
|
|
ascPts, descPts int
|
|
dataset string
|
|
}
|
|
|
|
func predict(endpoint string, p params, dataset string) (result, error) {
|
|
// Tawhiri requires longitude in [0, 360); normalize so both endpoints get
|
|
// the same request. Returned trajectory longitudes are [-180, 180] on both
|
|
// sides, so the comparison stays consistent.
|
|
lng := p.lng
|
|
if lng < 0 {
|
|
lng += 360
|
|
}
|
|
q := url.Values{}
|
|
q.Set("launch_latitude", fmt.Sprintf("%.4f", p.lat))
|
|
q.Set("launch_longitude", fmt.Sprintf("%.4f", lng))
|
|
q.Set("launch_altitude", fmt.Sprintf("%.0f", p.alt))
|
|
q.Set("launch_datetime", p.launch.Format(time.RFC3339))
|
|
q.Set("ascent_rate", fmt.Sprintf("%.2f", p.ascent))
|
|
q.Set("burst_altitude", fmt.Sprintf("%.0f", p.burst))
|
|
q.Set("descent_rate", fmt.Sprintf("%.2f", p.descent))
|
|
if dataset != "" {
|
|
q.Set("dataset", dataset)
|
|
}
|
|
|
|
full := endpoint + "?" + q.Encode()
|
|
var body []byte
|
|
var status int
|
|
var lastErr error
|
|
for range 3 {
|
|
resp, err := http.Get(full)
|
|
if err != nil {
|
|
lastErr = err
|
|
time.Sleep(time.Second)
|
|
continue
|
|
}
|
|
body, _ = io.ReadAll(resp.Body)
|
|
status = resp.StatusCode
|
|
resp.Body.Close()
|
|
lastErr = nil
|
|
break
|
|
}
|
|
if lastErr != nil {
|
|
return result{}, lastErr
|
|
}
|
|
if status != 200 {
|
|
return result{}, fmt.Errorf("HTTP %d: %s", status, truncate(string(body), 160))
|
|
}
|
|
|
|
var doc struct {
|
|
Prediction []struct {
|
|
Stage string `json:"stage"`
|
|
Trajectory []struct {
|
|
Latitude float64 `json:"latitude"`
|
|
Longitude float64 `json:"longitude"`
|
|
Altitude float64 `json:"altitude"`
|
|
} `json:"trajectory"`
|
|
} `json:"prediction"`
|
|
Request struct {
|
|
Dataset string `json:"dataset"`
|
|
} `json:"request"`
|
|
}
|
|
if err := json.Unmarshal(body, &doc); err != nil {
|
|
return result{}, err
|
|
}
|
|
|
|
var r result
|
|
r.dataset = doc.Request.Dataset
|
|
for _, st := range doc.Prediction {
|
|
if len(st.Trajectory) == 0 {
|
|
continue
|
|
}
|
|
last := st.Trajectory[len(st.Trajectory)-1]
|
|
switch st.Stage {
|
|
case "ascent":
|
|
r.ascPts = len(st.Trajectory)
|
|
r.apexLat, r.apexLng, r.apexAlt = last.Latitude, last.Longitude, last.Altitude
|
|
case "descent":
|
|
r.descPts = len(st.Trajectory)
|
|
r.landLat, r.landLng, r.landAlt = last.Latitude, last.Longitude, last.Altitude
|
|
}
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
type readinessResp struct {
|
|
Status string `json:"status"`
|
|
DatasetTime string `json:"dataset_time"`
|
|
}
|
|
|
|
func fetchActiveEpoch(base string) (time.Time, error) {
|
|
resp, err := http.Get(base + "/ready")
|
|
if err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(resp.Body)
|
|
if resp.StatusCode != 200 {
|
|
return time.Time{}, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
|
}
|
|
var r readinessResp
|
|
if err := json.Unmarshal(body, &r); err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
if r.Status != "ok" {
|
|
return time.Time{}, fmt.Errorf("server status %q (no dataset loaded yet)", r.Status)
|
|
}
|
|
return time.Parse(time.RFC3339, r.DatasetTime)
|
|
}
|
|
|
|
func haversine(lat1, lng1, lat2, lng2 float64) float64 {
|
|
const R = 6371000.0
|
|
phi1 := lat1 * math.Pi / 180
|
|
phi2 := lat2 * math.Pi / 180
|
|
dphi := (lat2 - lat1) * math.Pi / 180
|
|
dlam := (lng2 - lng1) * math.Pi / 180
|
|
a := math.Sin(dphi/2)*math.Sin(dphi/2) + math.Cos(phi1)*math.Cos(phi2)*math.Sin(dlam/2)*math.Sin(dlam/2)
|
|
return R * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n] + "…"
|
|
}
|