AINPP products: MVK - /standard/v8/hourly, NOW - /now/latest, NRT - /realtime_ver/v8/latest

download_gsmap_data.py

import ftplib
import os
from datetime import datetime, timedelta
import requests
import pathlib
import tempfile
import gzip
import numpy as np
from dotenv import load_dotenv
load_dotenv()

FTP_HOST   = 'hokusai.eorc.jaxa.jp'
BASE_DIR   = '/now/latest'
user = os.environ['FTP_JAXA_USER']
password = os.environ['FTP_JAXA_PASSWORD']
product = 'gsmap_now'
HORIZON = 6 # hours


# Coordenadas globais
glat_min, glat_max = -60.0, 60.0
glon_min, glon_max = -180.0, 180.0

# south america
rlat_min, rlat_max = -55.0, 13.0
rlon_min, rlon_max = -83.0, -33.0

NROW = 680
NCOL = 500

def read_data(filepath):
    with gzip.open(filepath, mode='rb') as handle:
        data = np.frombuffer(handle.read(), dtype=np.float32).reshape(1200, 3600)
        data = np.roll(data, shift=1800, axis=1)
    return data

def crop_data(data, lat_min, lat_max, lon_min, lon_max):
    lat_resolution = (glat_max - glat_min) / data.shape[0]
    lat_idx_min = int((glat_max - lat_max) / lat_resolution)
    lat_idx_max = int((glat_max - lat_min) / lat_resolution)
    
    lon_resolution = (glon_max - glon_min) / data.shape[1]
    lon_idx_min = int((lon_min - glon_min) / lon_resolution)
    lon_idx_max = int((lon_max - glon_min) / lon_resolution)
    
    cropped_data = data[lat_idx_min:lat_idx_max, lon_idx_min:lon_idx_max]
    return cropped_data


remote_dir = f"{BASE_DIR}"
with ftplib.FTP(FTP_HOST) as ftp:
    ftp.login(user, password)
    ftp.cwd(remote_dir)
    files = []
    ftp.retrlines('LIST', lambda x: files.append(x.split()[-1]))
    files = sorted(set((filter(lambda x: x.startswith(product) and x.endswith('00.dat.gz'), files))))[:HORIZON]
    input_data = []
    datetime_labels = []
    for file in files:
        datetime_label = datetime.strptime(file, f"{product}.%Y%m%d.%H00.dat.gz")
        with tempfile.TemporaryDirectory() as tmp_dir:
            local_path = pathlib.Path(tmp_dir) / file
            with open(local_path, 'wb') as f:
                ftp.retrbinary(f"RETR {file}", f.write)
            print(f"Downloaded: {local_path}")
            data = read_data(local_path)
            data = crop_data(data, rlat_min, rlat_max, rlon_min, rlon_max)
            input_data.append(data)
            datetime_labels.append(datetime_label)
            print(f"Processed: {file} at {datetime_label}")
    input_data = np.array(input_data).astype(np.float32)