Items

Utilities for fetching item catalogs

catalogs = get_item_catalogs()

CPU times: user 20.4 ms, sys: 6.03 ms, total: 26.4 ms
Wall time: 1.59 s

catalog1805 = catalogs[catalogs.yearmonth == '201805']

href1805 = catalog1805.iloc[0].href; href1805

'https://globalnightlight.s3.amazonaws.com/npp_201805/npp_201805_catalog.json'

baseurl = catalog1805.iloc[0].baseurl; baseurl

'https://globalnightlight.s3.amazonaws.com'

folder1805 = catalog1805.iloc[0].folder; folder1805

'npp_201805'

data1805 = get_data(href1805)

CPU times: user 64.7 ms, sys: 6.24 ms, total: 71 ms
Wall time: 2.44 s

df1805 = make_df(data1805,rel='item')

---

get_item_href

 get_item_href (href, folder, baseurl)

assert get_item_href('./SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops.rade9.co.json',
              '201805',
              'https://globalnightlight.s3.amazonaws.com') == 'https://globalnightlight.s3.amazonaws.com/201805/SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops.rade9.co.json'

---

split_href_type

 split_href_type (href, first=True)

assert split_href_type('./SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops.rade9.co.json') == 'SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops'

assert split_href_type('./SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops.rade9.co.json', first=False) == 'rade9.co.json'

---

transform_items_df

 transform_items_df (items_df, folder, baseurl)

items_df = pd.DataFrame(
    data=dict(rel=['item'],
              type=['application/json'],
              href=['./SVDNB_npp_d20180501_t0004098_e0009502_b33718_c20180501060951522600_noac_ops.rade9.co.json']
             ))

items_df = transform_items_df(items_df,'201805','https://globalnightlight.s3.amazonaws.com')

assert set(['base_url', 'folder', 'href', 'item_href', 'stem', 'suffix']) == set(items_df.columns.values)

---

find_stem_components

 find_stem_components (stem)

assert find_stem_components('SVDNB_npp_d20180901_t0002302_e0008088_b35463_c20180901040811139620_nobc_ops') == ['SVDNB_npp',
 '20180901',
 '0002302',
 '0008088',
 '35463',
 '20180901040811139620',
 'nobc',
 'ops']

---

make_vflag_file

 make_vflag_file (o)

items_df2 = pd.DataFrame(dict(start_date=['20150504'], first_scantime=['1335358'], end_scantime=['1341162'], orbital_nbr=['18219']))

assert (items_df2.apply(make_vflag_file,axis=1) == pd.Series(['npp_d20150504_t1335358_e1341162_b18219.vflag.co.tif'])).all(axis=None)

---

make_vflag_href

 make_vflag_href (o)

items_df3 = pd.DataFrame(dict(base_url=['https://globalnightlight.s3.amazonaws.com'], folder=['npp_201504'], vflag_file=['npp_d20150504_t1335358_e1341162_b18219.vflag.co.tif']))

assert (items_df3.apply(make_vflag_href, axis=1) == pd.Series(['https://globalnightlight.s3.amazonaws.com/npp_201504/npp_d20150504_t1335358_e1341162_b18219.vflag.co.tif'])).all(axis=None)

---

split_stem_components

 split_stem_components (df)

items_df4 = pd.DataFrame(
    dict(stem=['SVDNB_npp_d20180901_t0002302_e0008088_b35463_c20180901040811139620_nobc_ops'],
         base_url=['https://globalnightlight.s3.amazonaws.com'], 
         folder=['npp_201809']
         ))

items_df4 = split_stem_components(items_df4)

assert set(items_df4.columns.values) == set([
 'base_url',
 'create_datetime',
 'data_domain',
 'data_origin',
 'end_scantime',
 'first_scantime',
 'folder',
 'orbital_nbr',
 'product_id',
 'start_date',
 'stem',
 'vflag_file',
 'vflag_href'])

---

get_monthly_items

 get_monthly_items (href, folder, baseurl)

items1805 = get_monthly_items(href1805,folder1805, baseurl)

CPU times: user 177 ms, sys: 7.41 ms, total: 184 ms
Wall time: 2.59 s

assert len(items1805) == 3600

assert set(items1805.columns.values) == set(['href','item_href','suffix', *items_df4.columns.values])