s3-inventory.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3d503b15",
   "metadata": {},
   "source": [
    "# Data Inventory\n",
    "\n",
    "Lists collections stored in AWS S3 under a specified prefix but not catalogued in NASA MAAP STAC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2e321a9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "from pathlib import Path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c94fdbb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -U pystac-client\n",
    "from pystac_client import Client"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e07e4fa6",
   "metadata": {},
   "source": [
    "## Connecting to a MAAP STAC using Pystac Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1e0a6cb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "URL = 'https://stac.maap-project.org/'\n",
    "cat = Client.open(URL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed442394",
   "metadata": {},
   "source": [
    "## Listing all the STAC IDs of all the Collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "776cd79b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ESACCI_Biomass_L4_AGB_V3_100m_2018\n",
      "ESACCI_Biomass_L4_AGB_V3_100m_2017\n",
      "ESACCI_Biomass_L4_AGB_V3_100m_2010\n",
      "Landsat8_SurfaceReflectance\n",
      "Global_PALSAR2_PALSAR_FNF\n",
      "Global_Forest_Change_2000-2017\n",
      "AFRISAR_DLR2\n",
      "AfriSAR_UAVSAR_KZ\n",
      "AfriSAR_UAVSAR_Ungeocoded_Covariance\n",
      "AfriSAR_UAVSAR_Normalization_Area\n",
      "AfriSAR_UAVSAR_Geocoded_SLC\n",
      "AfriSAR_UAVSAR_Geocoded_Covariance\n",
      "GlobCover_09\n",
      "GlobCover_05_06\n",
      "GEDI_CalVal_Field_Data\n",
      "AfriSAR_UAVSAR_Coreg_SLC\n",
      "GEDI_CalVal_Lidar_Data_Compressed\n",
      "GEDI_CalVal_Lidar_Data\n",
      "ALOS_PSR_RTC_HIGH\n",
      "ABoVE_UAVSAR_PALSAR\n",
      "AFRISAR_DLR\n",
      "BIOSAR1\n",
      "NASA_JPL_global_agb_mean_2020\n",
      "icesat2-boreal\n",
      "ESACCI_Biomass_L4_AGB_V4_100m_2020\n",
      "ESACCI_Biomass_L4_AGB_V4_100m_2019\n",
      "ESACCI_Biomass_L4_AGB_V4_100m_2018\n",
      "ESACCI_Biomass_L4_AGB_V4_100m_2017\n",
      "ICESat2_Boreal_AGB_tindex_average\n",
      "Number of collections in STAC :29\n"
     ]
    }
   ],
   "source": [
    "\n",
    "stac_collection = []\n",
    "for collection in cat.get_all_collections():                                  \n",
    "    print(collection.id)\n",
    "    stac_collection.append(collection.id)\n",
    "print(f\"Number of collections in STAC :{len(stac_collection)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "670d3da2",
   "metadata": {},
   "source": [
    "## Listing all the Collections under the prefix 'file-staging/nasa-map/' from S3 Bucket 'nasa-maap-data-store'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "11b913cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['30mATL08_boreal___0', 'ABLVIS1B___001', 'ABLVIS2___001', 'ABLVIS2___1', 'ABoVE_UAVSAR_PALSAR___1', 'AFLVIS1B___001', 'AFLVIS2___001', 'ALOS_PSR_L1.5___1', 'ALOS_PSR_RTC_HIGH___1', 'ATL03___004', 'ATL03___005', 'ATL08_001_user-added___0', 'ATL08_ARD-beta___001', 'ATL08_LAS___004', 'ATL08___004', 'ATL08___005', 'AfriSAR_AGB_Maps_1681___1', 'AfriSAR_KingAir_B200_flight_tracks_Gabon___1', 'AfriSAR_LVIS_Footprint_Cover_1591___1', 'AfriSAR_Mondah_Field_Data_1580___1', 'AfriSAR_UAVSAR_Coreg_SLC___1', 'AfriSAR_UAVSAR_Geocoded_Covariance___1', 'AfriSAR_UAVSAR_Geocoded_SLC___1', 'AfriSAR_UAVSAR_KZ___1', 'AfriSAR_UAVSAR_MLC___2', 'AfriSAR_UAVSAR_Normalization_Area___1', 'AfriSAR_UAVSAR_SLC___2', 'AfriSAR_UAVSAR_Ungeocoded_Covariance___1', 'Afrisar_LVIS_Biomass_VProfiles_1775___1', 'ESACCI_Biomass_L4_AGB_V4_100m_2017', 'ESACCI_Biomass_L4_AGB_V4_100m_2018', 'ESACCI_Biomass_L4_AGB_V4_100m_2019', 'ESACCI_Biomass_L4_AGB_V4_100m_2020', 'EnvStress___1', 'GEDI01_B___001', 'GEDI01_B___002', 'GEDI02_A___001', 'GEDI02_A___002', 'GEDI02_B___002', 'GEDI_CalVal_Field_Data___1', 'GEDI_CalVal_Field_Data___2', 'GEDI_CalVal_Field_USA_Sonoma___1', 'GEDI_CalVal_Lidar_Data_Compressed___2', 'GEDI_CalVal_Lidar_Data___1', 'GEDI_CalVal_Lidar_Data___2', 'GEDI_CalVal_Lidar_USA_Sonoma___1', 'GlobCover_05_06___2.2', 'GlobCover_09___2.3', 'Global_Forest_Change_2000-2017___1.5', 'Global_PALSAR2_PALSAR_FNF___1', 'Global_PALSAR2_PALSAR_Mosiac___1', 'LVISF1B___001', 'LVISF1B___1', 'LVISF2___001', 'LVISF2___1', 'Landsat7_SurfaceReflectance___1', 'Landsat8_SurfaceReflectance___1', 'NISAR_RSLC___001', 'PolInSAR_Canopy_Height_1589___1', 'Polarimetric_CT_1601___1', 'Polarimetric_CT_user-added___0', 'Polarimetric_height_profile_1577___1', 'SENTINEL-1A_DP_GRD_HIGH___1', 'SENTINEL-1B_DP_GRD_HIGH___1', 'SRTMGL1_COD___001', 'SRTMGL1___003', 'USERSHARED_EXAMPLE___001', 'USER_SHARED_EXAMPLE_1205___001', 'USER_SHARED_EXAMPLE___001', 'USER_SHARED_EX_0125___001', 'USER_SHARED_EX_06163___001', 'USER_SHARED_EX_06164___001', 'USER_SHARED_EX_0616___001', 'USER_SHARED_EX_1209___001', 'USER_SHARED_EX_1210___001', 'USER_SHARED_EX_1211___001', 'USER_SHARED_EX_1229___001', 'afriSAR_mondah_field_data___1', 'icesat2-boreal', 'lvis_Level_1b___2.0.1', 'lvis_collection_level_1B___2.0.1', 'lvis_collection_level_2___2.0.1', 'nceo-africa-2017', 'polarimetric_ct___1', 'srtm_L1___003', 'topo_a41_r9', 'uavsar_afrisar_collection_s3___000', 'user_shared_landsat8___001']\n",
      "Number of collections : 88\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "s3 = boto3.client('s3')\n",
    "collections = []\n",
    "bucket_name = 'nasa-maap-data-store'\n",
    "\n",
    "prefix = 'file-staging/nasa-map/'\n",
    "\n",
    "response = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix, Delimiter='/')\n",
    "\n",
    "common_prefixes = response.get('CommonPrefixes')\n",
    "\n",
    "for common_prefix in common_prefixes:\n",
    "    path = Path(common_prefix['Prefix'])\n",
    "    collections.append(path.parts[2])\n",
    "print(collections)\n",
    "print(f\"Number of collections : {len(collections)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88edf8f2",
   "metadata": {},
   "source": [
    "## Finding collections present in S3 but not in STAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b389dd99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['30mATL08_boreal___0',\n",
       " 'ABLVIS1B___001',\n",
       " 'ABLVIS2___001',\n",
       " 'ABLVIS2___1',\n",
       " 'AFLVIS1B___001',\n",
       " 'AFLVIS2___001',\n",
       " 'ALOS_PSR_L1.5___1',\n",
       " 'ATL03___004',\n",
       " 'ATL03___005',\n",
       " 'ATL08_001_user-added___0',\n",
       " 'ATL08_ARD-beta___001',\n",
       " 'ATL08_LAS___004',\n",
       " 'ATL08___004',\n",
       " 'ATL08___005',\n",
       " 'AfriSAR_AGB_Maps_1681___1',\n",
       " 'AfriSAR_KingAir_B200_flight_tracks_Gabon___1',\n",
       " 'AfriSAR_LVIS_Footprint_Cover_1591___1',\n",
       " 'AfriSAR_Mondah_Field_Data_1580___1',\n",
       " 'AfriSAR_UAVSAR_MLC___2',\n",
       " 'AfriSAR_UAVSAR_SLC___2',\n",
       " 'Afrisar_LVIS_Biomass_VProfiles_1775___1',\n",
       " 'EnvStress___1',\n",
       " 'GEDI01_B___001',\n",
       " 'GEDI01_B___002',\n",
       " 'GEDI02_A___001',\n",
       " 'GEDI02_A___002',\n",
       " 'GEDI02_B___002',\n",
       " 'GEDI_CalVal_Field_USA_Sonoma___1',\n",
       " 'GEDI_CalVal_Lidar_USA_Sonoma___1',\n",
       " 'Global_PALSAR2_PALSAR_Mosiac___1',\n",
       " 'LVISF1B___001',\n",
       " 'LVISF1B___1',\n",
       " 'LVISF2___001',\n",
       " 'LVISF2___1',\n",
       " 'Landsat7_SurfaceReflectance___1',\n",
       " 'NISAR_RSLC___001',\n",
       " 'PolInSAR_Canopy_Height_1589___1',\n",
       " 'Polarimetric_CT_1601___1',\n",
       " 'Polarimetric_CT_user-added___0',\n",
       " 'Polarimetric_height_profile_1577___1',\n",
       " 'SENTINEL-1A_DP_GRD_HIGH___1',\n",
       " 'SENTINEL-1B_DP_GRD_HIGH___1',\n",
       " 'SRTMGL1_COD___001',\n",
       " 'SRTMGL1___003',\n",
       " 'USERSHARED_EXAMPLE___001',\n",
       " 'USER_SHARED_EXAMPLE_1205___001',\n",
       " 'USER_SHARED_EXAMPLE___001',\n",
       " 'USER_SHARED_EX_0125___001',\n",
       " 'USER_SHARED_EX_06163___001',\n",
       " 'USER_SHARED_EX_06164___001',\n",
       " 'USER_SHARED_EX_0616___001',\n",
       " 'USER_SHARED_EX_1209___001',\n",
       " 'USER_SHARED_EX_1210___001',\n",
       " 'USER_SHARED_EX_1211___001',\n",
       " 'USER_SHARED_EX_1229___001',\n",
       " 'afriSAR_mondah_field_data___1',\n",
       " 'lvis_Level_1b___2.0.1',\n",
       " 'lvis_collection_level_1B___2.0.1',\n",
       " 'lvis_collection_level_2___2.0.1',\n",
       " 'nceo-africa-2017',\n",
       " 'polarimetric_ct___1',\n",
       " 'srtm_L1___003',\n",
       " 'topo_a41_r9',\n",
       " 'uavsar_afrisar_collection_s3___000',\n",
       " 'user_shared_landsat8___001']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stac_collection = [i.lower() for i in stac_collection]\n",
    "stac_collection_string = ','.join(all_stac_collection)\n",
    "s3_stac_missing = []\n",
    "for collection in collections: \n",
    "    try:\n",
    "        slice_idx = collection.index('__')\n",
    "        collection_name = collection[:slice_idx]\n",
    "    except ValueError:\n",
    "        collection_name = collection\n",
    "    if collection_name.lower() not in stac_collection_string:\n",
    "        s3_stac_missing.append(collection)\n",
    "s3_stac_missing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90afbb63",
   "metadata": {},
   "source": [
    "## Listing collections missing from STAC (present in S3) and their size in GB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "11a9de07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing collection: 30mATL08_boreal___0\n",
      "Collection size: 0.008224004 GB\n",
      "Processing collection: ABLVIS1B___001\n",
      "Collection size: 1038.502559429 GB\n",
      "Processing collection: ABLVIS2___001\n",
      "Collection size: 111.568167566 GB\n",
      "Processing collection: ABLVIS2___1\n",
      "Collection size: 111.562109695 GB\n",
      "Processing collection: AFLVIS1B___001\n",
      "Collection size: 110.156656369 GB\n",
      "Processing collection: AFLVIS2___001\n",
      "Collection size: 16.356217403 GB\n",
      "Processing collection: ALOS_PSR_L1.5___1\n",
      "Collection size: 382.603999518 GB\n",
      "Processing collection: ATL03___004\n",
      "Collection size: 261251.882255514 GB\n",
      "Processing collection: ATL03___005\n",
      "Collection size: 5.567143698 GB\n",
      "Processing collection: ATL08_001_user-added___0\n",
      "Collection size: 0.070835492 GB\n",
      "Processing collection: ATL08_ARD-beta___001\n",
      "Collection size: 3588.897073543 GB\n",
      "Processing collection: ATL08_LAS___004\n",
      "Collection size: 0.000193003 GB\n",
      "Processing collection: ATL08___004\n",
      "Collection size: 14601.77024958 GB\n",
      "Processing collection: ATL08___005\n",
      "Collection size: 16413.889679864 GB\n",
      "Processing collection: AfriSAR_AGB_Maps_1681___1\n",
      "Collection size: 0.000590104 GB\n",
      "Processing collection: AfriSAR_KingAir_B200_flight_tracks_Gabon___1\n",
      "Collection size: 0.007555728 GB\n",
      "Processing collection: AfriSAR_LVIS_Footprint_Cover_1591___1\n",
      "Collection size: 37.840724546 GB\n",
      "Processing collection: AfriSAR_Mondah_Field_Data_1580___1\n",
      "Collection size: 0.001129139 GB\n",
      "Processing collection: AfriSAR_UAVSAR_MLC___2\n",
      "Collection size: 24.46611537 GB\n",
      "Processing collection: AfriSAR_UAVSAR_SLC___2\n",
      "Collection size: 193.408049996 GB\n",
      "Processing collection: Afrisar_LVIS_Biomass_VProfiles_1775___1\n",
      "Collection size: 0.022575442 GB\n",
      "Processing collection: EnvStress___1\n",
      "Collection size: 0.091770254 GB\n",
      "Processing collection: GEDI01_B___001\n",
      "Collection size: 9823.114468932 GB\n",
      "Processing collection: GEDI01_B___002\n",
      "Collection size: 102862.982947515 GB\n",
      "Processing collection: GEDI02_A___001\n",
      "Collection size: 6780.859142293 GB\n",
      "Processing collection: GEDI02_A___002\n",
      "Collection size: 72031.590427204 GB\n",
      "Processing collection: GEDI02_B___002\n",
      "Collection size: 16321.582304808 GB\n",
      "Processing collection: GEDI_CalVal_Field_USA_Sonoma___1\n",
      "Collection size: 8.1665e-05 GB\n",
      "Processing collection: GEDI_CalVal_Lidar_USA_Sonoma___1\n",
      "Collection size: 0.006670521 GB\n",
      "Processing collection: Global_PALSAR2_PALSAR_Mosiac___1\n",
      "Collection size: 10261.220825591 GB\n",
      "Processing collection: LVISF1B___001\n",
      "Collection size: 2430.50204507 GB\n",
      "Processing collection: LVISF1B___1\n",
      "Collection size: 2433.40009817 GB\n",
      "Processing collection: LVISF2___001\n",
      "Collection size: 322.688184792 GB\n",
      "Processing collection: LVISF2___1\n",
      "Collection size: 322.688184792 GB\n",
      "Processing collection: Landsat7_SurfaceReflectance___1\n",
      "Collection size: 861.447167257 GB\n",
      "Processing collection: NISAR_RSLC___001\n",
      "Collection size: 6.529547312 GB\n",
      "Processing collection: PolInSAR_Canopy_Height_1589___1\n",
      "Collection size: 0.242233139 GB\n",
      "Processing collection: Polarimetric_CT_1601___1\n",
      "Collection size: 0.470294785 GB\n",
      "Processing collection: Polarimetric_CT_user-added___0\n",
      "Collection size: 0.014311064 GB\n",
      "Processing collection: Polarimetric_height_profile_1577___1\n",
      "Collection size: 15.134655664 GB\n",
      "Processing collection: SENTINEL-1A_DP_GRD_HIGH___1\n",
      "Collection size: 1.124610778 GB\n",
      "Processing collection: SENTINEL-1B_DP_GRD_HIGH___1\n",
      "Collection size: 14.107654742 GB\n",
      "Processing collection: SRTMGL1_COD___001\n",
      "Collection size: 103.851279983 GB\n",
      "Processing collection: SRTMGL1___003\n",
      "Collection size: 104.786053331 GB\n",
      "Processing collection: USERSHARED_EXAMPLE___001\n",
      "Collection size: 0.022636047 GB\n",
      "Processing collection: USER_SHARED_EXAMPLE_1205___001\n",
      "Collection size: 0.009871429 GB\n",
      "Processing collection: USER_SHARED_EXAMPLE___001\n",
      "Collection size: 0.015459762 GB\n",
      "Processing collection: USER_SHARED_EX_0125___001\n",
      "Collection size: 0.009378136 GB\n",
      "Processing collection: USER_SHARED_EX_06163___001\n",
      "Collection size: 2.165e-06 GB\n",
      "Processing collection: USER_SHARED_EX_06164___001\n",
      "Collection size: 2.165e-06 GB\n",
      "Processing collection: USER_SHARED_EX_0616___001\n",
      "Collection size: 0.005469671 GB\n",
      "Processing collection: USER_SHARED_EX_1209___001\n",
      "Collection size: 0.009871429 GB\n",
      "Processing collection: USER_SHARED_EX_1210___001\n",
      "Collection size: 0.010123199 GB\n",
      "Processing collection: USER_SHARED_EX_1211___001\n",
      "Collection size: 0.009871429 GB\n",
      "Processing collection: USER_SHARED_EX_1229___001\n",
      "Collection size: 0.009871429 GB\n",
      "Processing collection: afriSAR_mondah_field_data___1\n",
      "Collection size: 0.001108972 GB\n",
      "Processing collection: lvis_Level_1b___2.0.1\n",
      "Collection size: 34.253866857 GB\n",
      "Processing collection: lvis_collection_level_1B___2.0.1\n",
      "Collection size: 0.290501203 GB\n",
      "Processing collection: lvis_collection_level_2___2.0.1\n",
      "Collection size: 16.646824236 GB\n",
      "Processing collection: nceo-africa-2017\n",
      "Collection size: 2.334248013 GB\n",
      "Processing collection: polarimetric_ct___1\n",
      "Collection size: 0.47009867 GB\n",
      "Processing collection: srtm_L1___003\n",
      "Collection size: 104.735077121 GB\n",
      "Processing collection: topo_a41_r9\n",
      "Collection size: 0.109985654 GB\n",
      "Processing collection: uavsar_afrisar_collection_s3___000\n",
      "Collection size: 571.400147521 GB\n",
      "Processing collection: user_shared_landsat8___001\n",
      "Collection size: 0.246407284 GB\n"
     ]
    }
   ],
   "source": [
    "size_map = {}\n",
    "s3 = boto3.resource('s3')\n",
    "data_bucket = s3.Bucket(\"nasa-maap-data-store\")\n",
    "for collection in s3_stac_missing:\n",
    "    collection_size = 0\n",
    "    print(f\"Processing collection: {collection}\")\n",
    "    for obj in data_bucket.objects.filter(Prefix=f\"file-staging/nasa-map/{collection}\"):\n",
    "        collection_size+=obj.size\n",
    "    print(f\"Collection size: {collection_size/1000000000} GB\")\n",
    "    size_map[collection] = collection_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dccc9a0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_base",
   "language": "python",
   "name": "conda_base"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}