feat: s3 download

scripts/test_s3_download.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
+)
+
+func main() {
+	ctx := context.Background()
+
+	// Create S3 downloader
+	downloader, err := grib.NewS3Downloader(
+		"/tmp/grib_test",
+		4, // parallel downloads
+		"noaa-gfs-bdp-pds",
+		"us-east-1",
+	)
+	if err != nil {
+		log.Fatalf("Failed to create S3 downloader: %v", err)
+	}
+
+	// Ensure directory exists
+	if err := os.MkdirAll("/tmp/grib_test", 0o755); err != nil {
+		log.Fatalf("Failed to create directory: %v", err)
+	}
+
+	// Find nearest run (6-hour intervals: 00, 06, 12, 18 UTC)
+	now := time.Now().UTC()
+	hour := now.Hour() - (now.Hour() % 6)
+	// Use data from 6 hours ago to ensure it's available
+	run := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC).Add(-6 * time.Hour)
+
+	fmt.Printf("Testing S3 download for run: %s\n", run.Format("2006-01-02 15:04 MST"))
+
+	// List available files first
+	runStr := run.Format("20060102")
+	fmt.Printf("Listing available files for %s/%02d...\n", runStr, run.Hour())
+	files, err := downloader.ListAvailableFiles(ctx, runStr, run.Hour())
+	if err != nil {
+		log.Fatalf("Failed to list files: %v", err)
+	}
+
+	fmt.Printf("Found %d files in S3:\n", len(files))
+	if len(files) > 0 {
+		// Show first 5 files
+		for i, file := range files {
+			if i >= 5 {
+				fmt.Printf("... and %d more files\n", len(files)-5)
+				break
+			}
+			fmt.Printf("  - %s\n", file)
+		}
+	}
+
+	// Try downloading just first 3 forecast hours (f000, f001, f002)
+	fmt.Println("\nTesting download of first 3 forecast hours...")
+	testRun := run
+
+	// Create a timeout context for the download
+	downloadCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+
+	if err := downloader.Run(downloadCtx, testRun); err != nil {
+		log.Fatalf("Failed to download: %v", err)
+	}
+
+	fmt.Println("\nDownload completed successfully!")
+
+	// Check downloaded files
+	entries, err := os.ReadDir("/tmp/grib_test")
+	if err != nil {
+		log.Fatalf("Failed to read directory: %v", err)
+	}
+
+	fmt.Printf("\nDownloaded %d files:\n", len(entries))
+	for i, entry := range entries {
+		if i >= 10 {
+			fmt.Printf("... and %d more files\n", len(entries)-10)
+			break
+		}
+		info, _ := entry.Info()
+		fmt.Printf("  - %s (%.2f MB)\n", entry.Name(), float64(info.Size())/1024/1024)
+	}
+}