updated downloader

This commit is contained in:
straitz 2026-03-22 16:29:53 +09:00
parent ca95e06ab7
commit 8e9f117799
30 changed files with 1209 additions and 698 deletions

114
scripts/compare.go Normal file
View file

@ -0,0 +1,114 @@
package main
import (
"encoding/json"
"fmt"
"math"
"os"
)
type Point struct {
Datetime string `json:"datetime"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
Altitude float64 `json:"altitude"`
}
type Stage struct {
Stage string `json:"stage"`
Trajectory []Point `json:"trajectory"`
}
type Prediction struct {
Prediction []Stage `json:"prediction"`
}
func haversine(lat1, lon1, lat2, lon2 float64) float64 {
R := 6371000.0
phi1, phi2 := lat1*math.Pi/180, lat2*math.Pi/180
dphi := (lat2 - lat1) * math.Pi / 180
dlam := (lon2 - lon1) * math.Pi / 180
a := math.Sin(dphi/2)*math.Sin(dphi/2) + math.Cos(phi1)*math.Cos(phi2)*math.Sin(dlam/2)*math.Sin(dlam/2)
return R * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
}
func load(path string) Prediction {
data, _ := os.ReadFile(path)
var p Prediction
json.Unmarshal(data, &p)
return p
}
func main() {
our := load("c:/tmp/our.json")
taw := load("c:/tmp/tawhiri.json")
// Find burst and landing points
var ourBurst, ourLand, tawBurst, tawLand Point
for _, s := range our.Prediction {
t := s.Trajectory
if s.Stage == "ascent" {
ourBurst = t[len(t)-1]
}
if s.Stage == "descent" {
ourLand = t[len(t)-1]
}
}
for _, s := range taw.Prediction {
t := s.Trajectory
if s.Stage == "ascent" {
tawBurst = t[len(t)-1]
}
if s.Stage == "descent" {
tawLand = t[len(t)-1]
}
}
fmt.Println("=== Burst Point ===")
fmt.Printf(" Our: lat=%.4f, lon=%.4f, alt=%.0f, time=%s\n", ourBurst.Latitude, ourBurst.Longitude, ourBurst.Altitude, ourBurst.Datetime)
fmt.Printf(" Tawhiri: lat=%.4f, lon=%.4f, alt=%.0f, time=%s\n", tawBurst.Latitude, tawBurst.Longitude, tawBurst.Altitude, tawBurst.Datetime)
burstDist := haversine(ourBurst.Latitude, ourBurst.Longitude, tawBurst.Latitude, tawBurst.Longitude)
fmt.Printf(" Distance: %.2f km\n", burstDist/1000)
fmt.Println()
fmt.Println("=== Landing Point ===")
fmt.Printf(" Our: lat=%.4f, lon=%.4f, alt=%.0f, time=%s\n", ourLand.Latitude, ourLand.Longitude, ourLand.Altitude, ourLand.Datetime)
fmt.Printf(" Tawhiri: lat=%.4f, lon=%.4f, alt=%.0f, time=%s\n", tawLand.Latitude, tawLand.Longitude, tawLand.Altitude, tawLand.Datetime)
landDist := haversine(ourLand.Latitude, ourLand.Longitude, tawLand.Latitude, tawLand.Longitude)
fmt.Printf(" Distance: %.2f km\n", landDist/1000)
fmt.Println()
fmt.Println("=== Trajectory Comparison (every 10 min) ===")
ourPts := map[string]Point{}
tawPts := map[string]Point{}
for _, s := range our.Prediction {
for _, p := range s.Trajectory {
ourPts[p.Datetime] = p
}
}
for _, s := range taw.Prediction {
for _, p := range s.Trajectory {
tawPts[p.Datetime] = p
}
}
// Collect common times
var common []string
for _, s := range our.Prediction {
for _, p := range s.Trajectory {
if _, ok := tawPts[p.Datetime]; ok {
common = append(common, p.Datetime)
}
}
}
for i, t := range common {
if i%10 == 0 {
o := ourPts[t]
tw := tawPts[t]
d := haversine(o.Latitude, o.Longitude, tw.Latitude, tw.Longitude)
fmt.Printf(" %s: dist=%.2f km (our: %.3f,%.3f vs taw: %.3f,%.3f)\n",
t, d/1000, o.Latitude, o.Longitude, tw.Latitude, tw.Longitude)
}
}
}

44
scripts/rebuild_cube.go Normal file
View file

@ -0,0 +1,44 @@
package main
import (
"context"
"fmt"
"os"
"time"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
)
func main() {
ctx := context.Background()
cfg := &grib.Config{
Dir: "C:/tmp/grib",
TTL: 48 * time.Hour,
CacheTTL: 1 * time.Hour,
Parallel: 8,
}
svc, err := grib.New(cfg)
if err != nil {
fmt.Printf("Error: %v\n", err)
return
}
// Delete old cube to force rebuild
cubePath := "C:/tmp/grib/20260212_12.cube"
if err := os.Remove(cubePath); err != nil && !os.IsNotExist(err) {
fmt.Printf("Remove cube error: %v\n", err)
} else {
fmt.Println("Old cube removed")
}
// Update will download missing pgrb2b files and rebuild cube
fmt.Println("Starting update (download pgrb2b + rebuild cube)...")
start := time.Now()
if err := svc.Update(ctx); err != nil {
fmt.Printf("Update error: %v\n", err)
return
}
fmt.Printf("Done in %v\n", time.Since(start))
}

36
scripts/test_download.go Normal file
View file

@ -0,0 +1,36 @@
package main
import (
"context"
"fmt"
"time"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
)
func main() {
ctx := context.Background()
// Найти последний доступный прогноз
run, err := grib.GetLatestModelRun(ctx)
if err != nil {
fmt.Printf("Error finding model run: %v\n", err)
return
}
fmt.Printf("Found model run: %v\n", run)
// Создать downloader
dl := grib.NewPartialDownloader("C:/tmp/grib", 8)
// Запустить загрузку
start := time.Now()
fmt.Println("Starting download...")
err = dl.Run(ctx, run)
if err != nil {
fmt.Printf("Download error: %v\n", err)
return
}
fmt.Printf("Download completed in %v\n", time.Since(start))
}

60
scripts/test_gh.go Normal file
View file

@ -0,0 +1,60 @@
package main
import (
"encoding/binary"
"fmt"
"math"
"os"
mmap "github.com/edsrzf/mmap-go"
)
var pressureLevels = []float64{
1000, 975, 950, 925, 900, 875, 850, 825, 800, 775,
750, 725, 700, 675, 650, 625, 600, 575, 550, 525,
500, 475, 450, 425, 400, 375, 350, 325, 300, 275,
250, 225, 200, 175, 150, 125, 100, 70, 50, 30,
20, 10, 7, 5, 3, 2, 1,
}
func main() {
f, _ := os.Open("C:/tmp/grib/20260212_12.cube")
mm, _ := mmap.Map(f, mmap.RDONLY, 0)
defer mm.Unmap()
defer f.Close()
const (
nT = 97
nP = 47
nLat = 721
nLon = 1440
)
bytesPerVar := int64(nT * nP * nLat * nLon * 4)
val := func(varIdx, ti, pi, y, x int) float32 {
idx := (((ti*nP + pi) * nLat) + y) * nLon + x
off := int64(varIdx)*bytesPerVar + int64(idx)*4
bits := binary.LittleEndian.Uint32(mm[off : off+4])
return math.Float32frombits(bits)
}
// Check gh values at lat=52.2N (y=(90-52.2)*4=151.2 → y=151), lon=0.1E (x=0.1*4=0.4 → x=0)
// Time step 9 (9 hours into forecast)
ti := 9
y := 151
x := 0
fmt.Println("GH values at (52.25N, 0E), t=+9h:")
fmt.Printf("%8s %8s %10s\n", "Level", "hPa", "GH(m)")
for pi := 0; pi < nP; pi++ {
gh := val(0, ti, pi, y, x)
fmt.Printf("%8d %8.0f %10.1f\n", pi, pressureLevels[pi], gh)
}
fmt.Println("\nU-wind values at same point:")
fmt.Printf("%8s %8s %10s\n", "Level", "hPa", "U(m/s)")
for pi := 0; pi < nP; pi++ {
u := val(1, ti, pi, y, x)
fmt.Printf("%8d %8.0f %10.2f\n", pi, pressureLevels[pi], u)
}
}

38
scripts/test_grib_read.go Normal file
View file

@ -0,0 +1,38 @@
package main
import (
"fmt"
"os"
"github.com/nilsmagnus/grib/griblib"
)
func main() {
f, err := os.Open("C:/tmp/grib/gfs.t18z.pgrb2.0p25.f000")
if err != nil {
fmt.Printf("Error opening file: %v\n", err)
return
}
defer f.Close()
messages, err := griblib.ReadMessages(f)
if err != nil {
fmt.Printf("Error reading GRIB: %v\n", err)
return
}
fmt.Printf("Found %d messages\n\n", len(messages))
for i, m := range messages {
product := m.Section4.ProductDefinitionTemplate
if product.ParameterCategory != 2 || product.ParameterNumber != 2 {
continue // only u-wind
}
fmt.Printf("UGRD Msg %d: SurfType=%d SurfValue=%d SurfScale=%d DataLen=%d\n",
i,
product.FirstSurface.Type,
product.FirstSurface.Value,
product.FirstSurface.Scale,
len(m.Data()))
}
}

View file

@ -0,0 +1,87 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
)
func main() {
ctx := context.Background()
// Инициализируем GRIB сервис
cfg := &grib.Config{
Dir: "C:/tmp/grib",
TTL: 48 * time.Hour,
CacheTTL: 1 * time.Hour,
Parallel: 8,
}
svc, err := grib.New(cfg)
if err != nil {
fmt.Printf("Error creating service: %v\n", err)
return
}
// Обновляем данные (создаёт куб)
fmt.Println("Updating GRIB data (building cube)...")
start := time.Now()
if err := svc.Update(ctx); err != nil {
fmt.Printf("Update error: %v\n", err)
return
}
fmt.Printf("Cube built in %v\n", time.Since(start))
// Тестируем извлечение ветра
fmt.Println("\nTesting wind extraction...")
lat, lon, alt := 52.2, 0.1, 10000.0
ts := time.Date(2026, 2, 11, 12, 0, 0, 0, time.UTC)
wind, err := svc.Extract(ctx, lat, lon, alt, ts)
if err != nil {
fmt.Printf("Extract error: %v\n", err)
return
}
fmt.Printf("Wind at (%.2f, %.2f, %.0fm) at %v:\n", lat, lon, alt, ts)
fmt.Printf(" U (east): %.2f m/s\n", wind[0])
fmt.Printf(" V (north): %.2f m/s\n", wind[1])
// Сравниваем с Tawhiri
fmt.Println("\nComparing with Tawhiri API...")
tawhiriURL := fmt.Sprintf(
"https://api.v2.sondehub.org/tawhiri?launch_latitude=%.2f&launch_longitude=%.2f&launch_altitude=0&launch_datetime=%s&ascent_rate=5&burst_altitude=30000&descent_rate=5",
lat, lon, ts.Format(time.RFC3339),
)
resp, err := http.Get(tawhiriURL)
if err != nil {
fmt.Printf("Tawhiri request error: %v\n", err)
return
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
var tawhiriResp map[string]interface{}
json.Unmarshal(body, &tawhiriResp)
// Выводим финальную точку приземления
if prediction, ok := tawhiriResp["prediction"].([]interface{}); ok {
for _, stage := range prediction {
stageMap := stage.(map[string]interface{})
if stageMap["stage"] == "descent" {
trajectory := stageMap["trajectory"].([]interface{})
if len(trajectory) > 0 {
last := trajectory[len(trajectory)-1].(map[string]interface{})
fmt.Printf("\nTawhiri landing point:\n")
fmt.Printf(" Lat: %.4f\n", last["latitude"])
fmt.Printf(" Lon: %.4f\n", last["longitude"])
}
}
}
}
}

View file

@ -1,89 +0,0 @@
package main
import (
"context"
"fmt"
"log"
"os"
"time"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
)
func main() {
ctx := context.Background()
// Create S3 downloader
downloader, err := grib.NewS3Downloader(
"/tmp/grib_test",
4, // parallel downloads
"noaa-gfs-bdp-pds",
"us-east-1",
)
if err != nil {
log.Fatalf("Failed to create S3 downloader: %v", err)
}
// Ensure directory exists
if err := os.MkdirAll("/tmp/grib_test", 0o755); err != nil {
log.Fatalf("Failed to create directory: %v", err)
}
// Find nearest run (6-hour intervals: 00, 06, 12, 18 UTC)
now := time.Now().UTC()
hour := now.Hour() - (now.Hour() % 6)
// Use data from 6 hours ago to ensure it's available
run := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC).Add(-6 * time.Hour)
fmt.Printf("Testing S3 download for run: %s\n", run.Format("2006-01-02 15:04 MST"))
// List available files first
runStr := run.Format("20060102")
fmt.Printf("Listing available files for %s/%02d...\n", runStr, run.Hour())
files, err := downloader.ListAvailableFiles(ctx, runStr, run.Hour())
if err != nil {
log.Fatalf("Failed to list files: %v", err)
}
fmt.Printf("Found %d files in S3:\n", len(files))
if len(files) > 0 {
// Show first 5 files
for i, file := range files {
if i >= 5 {
fmt.Printf("... and %d more files\n", len(files)-5)
break
}
fmt.Printf(" - %s\n", file)
}
}
// Try downloading just first 3 forecast hours (f000, f001, f002)
fmt.Println("\nTesting download of first 3 forecast hours...")
testRun := run
// Create a timeout context for the download
downloadCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
if err := downloader.Run(downloadCtx, testRun); err != nil {
log.Fatalf("Failed to download: %v", err)
}
fmt.Println("\nDownload completed successfully!")
// Check downloaded files
entries, err := os.ReadDir("/tmp/grib_test")
if err != nil {
log.Fatalf("Failed to read directory: %v", err)
}
fmt.Printf("\nDownloaded %d files:\n", len(entries))
for i, entry := range entries {
if i >= 10 {
fmt.Printf("... and %d more files\n", len(entries)-10)
break
}
info, _ := entry.Info()
fmt.Printf(" - %s (%.2f MB)\n", entry.Name(), float64(info.Size())/1024/1024)
}
}

View file

@ -1,68 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"log"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func main() {
ctx := context.Background()
// Create AWS config with anonymous credentials
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion("us-east-1"),
config.WithCredentialsProvider(aws.AnonymousCredentials{}),
)
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
client := s3.NewFromConfig(cfg)
// Try to download a single file
bucket := "noaa-gfs-bdp-pds"
key := "gfs.20251020/00/atmos/gfs.t00z.pgrb2.0p50.f000"
fmt.Printf("Downloading: s3://%s/%s\n", bucket, key)
input := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
result, err := client.GetObject(ctx, input)
if err != nil {
log.Fatalf("Failed to get object: %v", err)
}
defer result.Body.Close()
// Create output file
outFile := "/tmp/test_grib.part"
f, err := os.Create(outFile)
if err != nil {
log.Fatalf("Failed to create file: %v", err)
}
defer f.Close()
// Copy data
written, err := io.Copy(f, result.Body)
if err != nil {
log.Fatalf("Failed to copy data: %v (wrote %d bytes)", err, written)
}
fmt.Printf("Successfully downloaded %d bytes\n", written)
// Rename
if err := os.Rename(outFile, "/tmp/test_grib"); err != nil {
log.Fatalf("Failed to rename: %v", err)
}
fmt.Println("Download complete!")
}

55
scripts/test_wind.go Normal file
View file

@ -0,0 +1,55 @@
package main
import (
"context"
"fmt"
"time"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
)
func main() {
ctx := context.Background()
cfg := &grib.Config{
Dir: "C:/tmp/grib",
TTL: 48 * time.Hour,
CacheTTL: 1 * time.Hour,
Parallel: 8,
}
svc, err := grib.New(cfg)
if err != nil {
fmt.Printf("Error: %v\n", err)
return
}
if err := svc.Update(ctx); err != nil {
fmt.Printf("Update error: %v\n", err)
return
}
// Test wind at lat=52.2, lon=0.1 at various altitudes
// Run is 2026-02-12T12:00Z, request time 21:00Z = +9 hours
ts := time.Date(2026, 2, 12, 21, 0, 0, 0, time.UTC)
lat, lon := 52.2, 0.1
fmt.Println("Wind at (52.2°N, 0.1°E) at 2026-02-12T21:00Z:")
fmt.Printf("%8s %8s %8s\n", "Alt(m)", "U(m/s)", "V(m/s)")
for _, alt := range []float64{0, 1000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000} {
w, err := svc.Extract(ctx, lat, lon, alt, ts)
if err != nil {
fmt.Printf("%8.0f Error: %v\n", alt, err)
continue
}
fmt.Printf("%8.0f %8.2f %8.2f\n", alt, w[0], w[1])
}
// Also test at a few nearby points to check spatial consistency
fmt.Println("\nWind at 10km altitude, varying longitude:")
for _, testLon := range []float64{0.0, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 350.0, 359.75} {
w, _ := svc.Extract(ctx, lat, testLon, 10000, ts)
fmt.Printf(" lon=%6.2f: U=%8.2f V=%8.2f\n", testLon, w[0], w[1])
}
}