feat: s3 download

This commit is contained in:
Anatoly Antonov 2025-10-20 19:10:07 +09:00
parent a850615e1f
commit c4f355a32e
15 changed files with 590 additions and 109 deletions

View file

@ -1,21 +1,18 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"context"
"git.intra.yksa.space/gsn/predictor/internal/jobs/grib/updater"
"git.intra.yksa.space/gsn/predictor/internal/pkg/errcodes"
"git.intra.yksa.space/gsn/predictor/internal/pkg/grib"
"git.intra.yksa.space/gsn/predictor/internal/pkg/log"
"git.intra.yksa.space/gsn/predictor/internal/service"
"git.intra.yksa.space/gsn/predictor/internal/transport/rest"
"git.intra.yksa.space/gsn/predictor/internal/transport/rest/handler"
"git.intra.yksa.space/gsn/predictor/pkg/scheduler"
env "github.com/caarlos0/env/v11"
"go.uber.org/zap"
)
@ -29,11 +26,6 @@ func main() {
defer lg.Sync()
ctx := log.ToCtx(context.Background(), lg)
cfg, err := loadConfig()
if err != nil {
log.Ctx(ctx).Fatal("failed to load configuration", zap.Error(err))
}
schedulerConfig, err := scheduler.NewConfig()
if err != nil {
log.Ctx(ctx).Fatal("failed to load scheduler configuration", zap.Error(err))
@ -44,14 +36,12 @@ func main() {
log.Ctx(ctx).Fatal("failed to load GRIB updater configuration", zap.Error(err))
}
gribService, err := grib.New(grib.ServiceConfig{
Dir: cfg.GribDir,
TTL: cfg.GribTTL,
CacheTTL: cfg.GribCacheTTL,
Parallel: cfg.GribParallel,
Client: cfg.CreateHTTPClient(),
DatasetURL: cfg.GribDatasetURL,
})
gribCfg, err := grib.NewConfig()
if err != nil {
log.Ctx(ctx).Fatal("failed to load GRIB configuration", zap.Error(err))
}
gribService, err := grib.New(gribCfg)
if err != nil {
log.Ctx(ctx).Fatal("failed to initialize GRIB service", zap.Error(err))
}
@ -67,7 +57,7 @@ func main() {
}
}()
svc, err := service.New(cfg, gribService)
svc, err := service.New(gribService)
if err != nil {
log.Ctx(ctx).Fatal("failed to initialize service", zap.Error(err))
}
@ -103,13 +93,7 @@ func main() {
lg.Info("scheduler started")
}
lg.Info("service started successfully",
zap.String("grib_dir", cfg.GribDir),
zap.Duration("grib_ttl", cfg.GribTTL),
zap.Duration("grib_cache_ttl", cfg.GribCacheTTL),
zap.Int("grib_parallel", cfg.GribParallel),
zap.Bool("scheduler_enabled", schedulerConfig.Enabled),
zap.Duration("grib_update_interval", gribUpdaterConfig.Interval))
lg.Info("service started successfully")
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
@ -127,15 +111,3 @@ func main() {
lg.Info("scheduler stopped")
}
}
func loadConfig() (*service.Config, error) {
cfg := &service.Config{}
if err := env.ParseWithOptions(cfg, env.Options{
PrefixTagName: servicePrefix + "_",
}); err != nil {
return nil, errcodes.Wrap(err, "failed to parse configuration")
}
return cfg, nil
}