step one

internal/datasets/gfs/idx.go

@@ -0,0 +1,125 @@
+package gfs
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// IdxEntry is one parsed line from a NOAA GRIB .idx file.
+//
+// Example line: "15:1207405:d=2024010100:HGT:1000 mb:0 hour fcst:"
+type IdxEntry struct {
+	Index     int
+	Offset    int64
+	Variable  string
+	LevelMB   int   // 0 when the level is not isobaric
+	Hour      int   // forecast hour; 0 for analysis ("anl"); -1 if unparseable
+	EndOffset int64 // computed from the next entry's Offset; -1 for the final entry
+}
+
+// Length returns the byte length of this GRIB message, or -1 if unknown
+// (the final entry in an idx file).
+func (e *IdxEntry) Length() int64 {
+	if e.EndOffset <= 0 {
+		return -1
+	}
+	return e.EndOffset - e.Offset
+}
+
+// ParseIdx parses a .idx file body. Unparseable lines are silently skipped.
+func ParseIdx(body []byte) []IdxEntry {
+	lines := strings.Split(string(body), "\n")
+	var entries []IdxEntry
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Split(line, ":")
+		if len(parts) < 7 {
+			continue
+		}
+		idx, err := strconv.Atoi(parts[0])
+		if err != nil {
+			continue
+		}
+		off, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			continue
+		}
+		entries = append(entries, IdxEntry{
+			Index:     idx,
+			Offset:    off,
+			Variable:  parts[3],
+			LevelMB:   parseLevelMB(parts[4]),
+			Hour:      parseHour(parts[5]),
+			EndOffset: -1,
+		})
+	}
+	for i := 0; i < len(entries)-1; i++ {
+		entries[i].EndOffset = entries[i+1].Offset
+	}
+	return entries
+}
+
+// FilterIdx returns entries matching one of the wanted variables at a known
+// pressure level with a computable byte length.
+func FilterIdx(entries []IdxEntry, wanted map[string]bool) []IdxEntry {
+	var out []IdxEntry
+	for _, e := range entries {
+		if !wanted[e.Variable] || e.LevelMB <= 0 || e.Length() <= 0 {
+			continue
+		}
+		out = append(out, e)
+	}
+	return out
+}
+
+func parseLevelMB(s string) int {
+	s = strings.TrimSpace(s)
+	if !strings.HasSuffix(s, " mb") {
+		return 0
+	}
+	n, err := strconv.Atoi(strings.TrimSuffix(s, " mb"))
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+func parseHour(s string) int {
+	s = strings.TrimSpace(s)
+	if s == "anl" {
+		return 0
+	}
+	n, err := strconv.Atoi(strings.TrimSuffix(s, " hour fcst"))
+	if err != nil {
+		return -1
+	}
+	return n
+}
+
+// ByteRange is one HTTP range download corresponding to one GRIB message.
+type ByteRange struct {
+	Start int64
+	End   int64 // inclusive
+	Entry IdxEntry
+}
+
+// EntriesToRanges converts idx entries to inclusive HTTP byte ranges.
+func EntriesToRanges(entries []IdxEntry) []ByteRange {
+	out := make([]ByteRange, 0, len(entries))
+	for _, e := range entries {
+		if e.Length() <= 0 {
+			continue
+		}
+		out = append(out, ByteRange{Start: e.Offset, End: e.EndOffset - 1, Entry: e})
+	}
+	return out
+}
+
+// FormatRange returns an HTTP Range header value for the byte range.
+func (r ByteRange) FormatRange() string {
+	return fmt.Sprintf("bytes=%d-%d", r.Start, r.End)
+}