Author: Benjamin Marchant
License: CC BY 4.0
from datetime import date
from os import path
from calendar import monthrange
import urllib.request
import urllib.request, json
import pprint
import os
import glob
import netCDF4
import random
import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import numpy.ma as ma
output_dir = '/Volumes/HD2/Datasets/Research/NASA'
To download a file from NASA LAADS DAAC, you need to create a token. Here are the steps to follow:
Visit this website: https://ladsweb.modaps.eosdis.nasa.gov/
Create a free account.
From the drop-down profile menu located in the top right corner, select "Generate Token."
Copy your generated token.
Please replace "the token below" with your own token. As an example, I have put in my own token.
token = 'Ym1hcmNoYW50OlltVnVhbUZ0YVc0dWJXRnlZMmhoYm5SQWJtRnpZUzVuYjNZPToxNjI0NzE3MjEwOjEwZmNhNWU4ODVlNzc3OGUyMzE3NzFkZjNmNjUwMGIzNjVhMDY4ZWY'
opener = urllib.request.build_opener()
opener.addheaders = [('Authorization', 'Bearer {}'.format(token))]
urllib.request.install_opener(opener)
year = 2019
month = 8
day = 1
platform = 'AQUA'
instrument = 'MODIS'
product_category = 61
product = 'MYD14'
d0 = date(year, 1, 1)
d1 = date(year, month, day)
delta = d1 - d0
count_of_day = delta.days + 1
ladsweb_url = 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/{}/{}/{:04d}/{:03d}.json'.format(product_category,product,year,count_of_day)
print(ladsweb_url)
with urllib.request.urlopen(ladsweb_url) as url:
data = json.loads(url.read().decode())
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(data['content'][:10])
os.system( 'mkdir {}/{}'.format(output_dir,platform) )
os.system( 'mkdir {}/{}/{}'.format(output_dir,platform,instrument) )
os.system( 'mkdir {}/{}/{}/{}'.format(output_dir,platform,instrument,product))
os.system( 'mkdir {}/{}/{}/{}/{:04d}'.format(output_dir,platform,instrument,product,year) )
os.system( 'mkdir {}/{}/{}/{}/{:04d}/{:04d}_{:02d}_{:02d}'.format(output_dir,platform,instrument,product,year,year,month,day) )
%%time
log_error = []
for file in data['content']:
file_name = file['name']
#print(file_name)
target_dir = '{}/{}/{}/{}/{:04d}/{:04d}_{:02d}_{:02d}/{}'.format(output_dir,platform,instrument,product,year,year,month,day,file_name)
if not path.exists( target_dir ):
try:
ladsweb_url = 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/{}/{}/{:04d}/{:03d}/{}'.format(product_category,product,year,count_of_day,file_name)
urllib.request.urlretrieve(ladsweb_url,target_dir)
except:
log_error.append(file)
print('error')
file_list = glob.glob('{}/{}/{}/{}/{:04d}/{:04d}_{:02d}_{:02d}/*.hdf'.format(output_dir,platform,instrument,product,year,year,month,day))
len(file_list)
tot_size = 0
for file in file_list:
file_size = os.path.getsize(file) * 0.001 # in KB
tot_size += file_size
print( str( round(file_size,2) ) + ' KB')
tot_size
365 * 4 * 20 / ( 60 * 24 )
%%time
year = 2019
month = 8
platform = 'AQUA'
instrument = 'MODIS'
product_category = 61
product = 'MYD14'
log_error = []
for day in range(1, monthrange(year, month)[1]+1):
d0 = date(year, 1, 1)
d1 = date(year, month, day)
delta = d1 - d0
count_of_day = delta.days + 1
ladsweb_url = 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/{}/{}/{:04d}/{:03d}.json'.format(product_category,product,year,count_of_day)
with urllib.request.urlopen(ladsweb_url) as url:
data = json.loads(url.read().decode())
os.system( 'mkdir {}/{}/{}/{}/{:04d}'.format(output_dir,platform,instrument,product,year) )
os.system( 'mkdir {}/{}/{}/{}/{:04d}/{:04d}_{:02d}_{:02d}'.format(output_dir,platform,instrument,product,year,year,month,day) )
for file in data['content']:
file_name = file['name']
#print(file_name)
target_dir = '{}/{}/{}/{}/{:04d}/{:04d}_{:02d}_{:02d}/{}'.format(output_dir,platform,instrument,product,year,year,month,day,file_name)
if not path.exists( target_dir ):
try:
ladsweb_url = 'https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/{}/{}/{:04d}/{:03d}/{}'.format(product_category,product,year,count_of_day,file_name)
urllib.request.urlretrieve(ladsweb_url,target_dir)
except:
log_error.append(file)