sentinel_aois_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Outputs from this notebook is saved in `./data/files_list` folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "from itertools import product as itrprod\n",
    "import asyncio\n",
    "import json\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_DIR = \"./data/files_list\"\n",
    "os.makedirs(SAVE_DIR, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "AU_POLYGONS = [\n",
    "    'POLYGON((154.073685 -26.398506,151.56311 -25.801189,151.075897 -27.40662,153.624039 -28.014174,154.073685 -26.398506))',\n",
    "    'POLYGON((146.474747 -16.033461,144.151413 -15.494384,143.742279 -17.110708,146.086823 -17.657339,146.474747 -16.033461))',\n",
    "    'POLYGON((150.214127 -28.315512,152.713028 -27.71974,152.206833 -26.059097,149.74646 -26.644444,150.214127 -28.315512))',\n",
    "    'POLYGON((116.691261 -27.688679,114.187599 -27.092514,113.688507 -28.695972,116.231606 -29.302624,116.691261 -27.688679))',\n",
    "]\n",
    "\n",
    "AN_POLYGONS = [\n",
    "    'POLYGON((173.982986 -78.4249729999985,172.122543 -76.221466,179.0336 -75.837616,-177.961258 -77.972656,173.982986 -78.4249729999985))',\n",
    "    'POLYGON((111.136292 -67.7959589999999,115.933327 -66.5219569999998,113.435944 -65.0743559999997,108.809639 -66.2770999999998,111.136292 -67.7959589999999))',\n",
    "    'POLYGON((172.382263 -78.6327819999986,171.732635 -76.398155,163.953552 -76.383263,163.121597 -78.6150589999986,172.382263 -78.6327819999986))',\n",
    "    'POLYGON((70.583374 -54.4052539999951,74.212395 -53.5243909999941,73.09771 -51.946536999992,69.587746 -52.7966349999932,70.583374 -54.4052539999951))',\n",
    "    'POLYGON((178.576126 -71.618423,173.430893 -70.1199569999999,176.938004 -68.7651059999999,-178.032867 -70.1673429999999,178.576126 -71.618423))',\n",
    "    'POLYGON((79.214729 -70.0986789999999,74.184479 -68.6963419999999,71.050514 -69.9258959999999,76.189308 -71.414604,79.214729 -70.0986789999999))',\n",
    "    'POLYGON((12.562656 -72.541679,17.732925 -70.976974,13.91748 -69.6723859999999,8.82353100000002 -71.134811,12.562656 -72.541679))',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "S1_DATA_DIR = \"/g/data/fj7/Copernicus/Sentinel-1/C-SAR/\"\n",
    "S1_AOI_LIST_AU = [\n",
    "    \"25S150E-30S155E\", \n",
    "    \"15S145E-20S150E\", \n",
    "    \"25S150E-30S155E\",\n",
    "    \"25S110E-30S115E\",\n",
    "] # Australia\n",
    "S1_AOI_LIST_AN = [\n",
    "    \"75S150W-80S145W\",\n",
    "    \"65S110E-70S115E\",\n",
    "    \"75S165E-80S170E\",\n",
    "    \"70S155E-75S160E\",\n",
    "    \"65S070E-70S075E\",\n",
    "] # Antarctica\n",
    "\n",
    "S1_QUERY_YEARS_AU = [\"2023\", \"2024\"]\n",
    "S1_QUERY_YEARS_AN = [\"2023\", \"2024\"]\n",
    "S1_QUERY_PRODS = [\"GRD\", \"SLC\"]\n",
    "MONTH_RANGE = [\"{:02d}\".format(m) for m in range(1, 13)]\n",
    "S1_CASE_LIST_AU = list(itrprod(S1_QUERY_PRODS, S1_QUERY_YEARS_AU, MONTH_RANGE))\n",
    "S1_CASE_LIST_AN = list(itrprod(S1_QUERY_PRODS, S1_QUERY_YEARS_AN, MONTH_RANGE))\n",
    "\n",
    "S1_AU_FILENAME = \"s1_au.csv\"\n",
    "S1_AN_FILENAME = \"s1_an.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "S2_DATA_DIR = \"/g/data/fj7/Copernicus/Sentinel-2/MSI/\"\n",
    "S2_AOI_LIST_AU = [\n",
    "    \"T56JNQ\", \n",
    "    \"T55KCB\", \n",
    "    \"T56JKQ\",\n",
    "    \"T50JKP\",\n",
    "]\n",
    "S2_AOI_LIST_AN = [\n",
    "    \"T09CWQ\",\n",
    "    \"T49DDH\",\n",
    "    \"T58CEU\",\n",
    "    \"T57DVB\",\n",
    "    \"T43DDC\",\n",
    "    \"T33DVA\",\n",
    "]\n",
    "S2_QUERY_YEARS_AU = [\"2023\", \"2024\"]\n",
    "S2_QUERY_YEARS_AN = [\"2023\", \"2024\"] #[str(x) for x in list(range(2020,2025))]\n",
    "S2_QUERY_PRODS = [\"L1C\", \"L2A\"]\n",
    "MONTH_RANGE = [\"{:02d}\".format(m) for m in range(1, 13)]\n",
    "S2_CASE_LIST_AU = list(itrprod(S2_QUERY_PRODS, S2_QUERY_YEARS_AU, MONTH_RANGE))\n",
    "S2_CASE_LIST_AN = list(itrprod(S2_QUERY_PRODS, S2_QUERY_YEARS_AN, MONTH_RANGE))\n",
    "\n",
    "S2_AU_FILENAME = \"s2_au.csv\"\n",
    "S2_AN_FILENAME = \"s2_an.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_s1_filenames(\n",
    "    polygon_list:str, \n",
    "    years:list[str], \n",
    "    output_file_path:str,\n",
    "    products:list[str]=[\"GRD\", \"SLC\"],\n",
    "):\n",
    "    \"\"\"\n",
    "    Get scene names for S1 via direct requet to SARA server\n",
    "    \"\"\"\n",
    "    s1_names = []\n",
    "    if os.path.isfile(output_file_path):\n",
    "        with open(output_file_path, \"r\") as f:\n",
    "            for l in f:\n",
    "                s1_names.append(l.strip())\n",
    "    else:\n",
    "        start = f\"{years[0]}-01-01\"\n",
    "        end = f\"{years[1]}-12-12\"\n",
    "        for poly in polygon_list:\n",
    "            for prod in products:\n",
    "                page = 1\n",
    "                query_resp = [\"start\"]\n",
    "                while query_resp != []:\n",
    "                    query = f\"https://copernicus.nci.org.au/sara.server/1.0/api/collections/S1/search.json?_pretty=1&geometry={poly}&startDate={start}&completionDate={end}&instrument=C-SAR&sensor=IW&maxRecords=500&productType={prod}&page={page}\"\n",
    "                    response = json.loads(requests.get(query).content)\n",
    "                    query_resp = [r[\"properties\"][\"title\"] for r in response[\"features\"]]\n",
    "                    s1_names.extend(query_resp)\n",
    "                    page += 1\n",
    "        with open(output_file_path, \"w\") as f:\n",
    "            for n in s1_names:\n",
    "                f.write(f\"{n}\\n\")\n",
    "    return s1_names\n",
    "\n",
    "def save_file_list(file_list:dict, save_path:str) -> None:\n",
    "    \"\"\"\n",
    "    Saves the retrieved data.\n",
    "    \"\"\"\n",
    "    with open(save_path, \"w\") as f:\n",
    "        for (k, v) in file_list.items():\n",
    "            for filename in v:\n",
    "                f.write(f\"{k},{filename}\\n\")\n",
    "    return None\n",
    "\n",
    "async def find_all_files_for_case(query_case:tuple, sat_data_dir:str) -> bool:\n",
    "    \"\"\"\n",
    "    Finds all files for a selected case of product/year/month\n",
    "    \"\"\"\n",
    "    case_path = os.path.join(sat_data_dir, query_case[0], query_case[1], f\"{query_case[1]}-{query_case[2]}\")\n",
    "    print(f\"Retrieving files for {case_path}\", end=\"\\r\")\n",
    "    return glob.glob(case_path + \"/*/*.zip\")\n",
    "\n",
    "async def find_aoi_files(aoi:str, all_files:list[str]) -> list[str]:\n",
    "    \"\"\"\n",
    "    Filters all files and finds files for the area of interest.\n",
    "    \"\"\"\n",
    "    print(f\"filtering files for {aoi}\", end=\"\\r\")\n",
    "    return list(filter(lambda p: aoi in p, all_files))\n",
    "\n",
    "def flatten(l:list[list]) -> list:\n",
    "    \"\"\"\n",
    "    Flattens the list\n",
    "    \"\"\"\n",
    "    return[x for xs in l for x in xs]\n",
    "\n",
    "# Not sure if async runs well in notebook\n",
    "async def find_files_for_aios_async(\n",
    "        query_cases:list[tuple], \n",
    "        sat_data_dir:str, \n",
    "        aoi_list:list[str], \n",
    "    ) -> dict:\n",
    "    \"\"\"\n",
    "    Asyncronously finds the files for an AOI list given as list of identifiers based on a combination of produt/year/month from NCI Copernicus databse.\n",
    "    Set `is_s1` to True for Sentinel-1.\n",
    "    \"\"\"\n",
    "    all_files_async = [find_all_files_for_case(c, sat_data_dir) for c in query_cases]\n",
    "    all_files = await asyncio.gather(*all_files_async)\n",
    "    all_files = flatten(all_files)\n",
    "    print(\"\")\n",
    "    \n",
    "    aoi_files_async = [find_aoi_files(aoi, all_files) for aoi in aoi_list]\n",
    "    aoi_files = await asyncio.gather(*aoi_files_async)\n",
    "    print(\"\")\n",
    "    return dict(map(lambda k, v: (k, v), aoi_list, aoi_files))\n",
    "\n",
    "# syncronous function for all cases and AOIs at the same time. Could take long\n",
    "def find_files_for_aios(\n",
    "        query_cases:list[tuple], \n",
    "        sat_data_dir:str, \n",
    "        aoi_list:list[str], \n",
    "    ) -> dict:\n",
    "    \"\"\"\n",
    "    Finds the files for an AOI list given as list of identifiers based on a combination of produt/year/month from NCI Copernicus databse.\n",
    "    Set `is_s1` to True for Sentinel-1.\n",
    "    \"\"\"\n",
    "    all_files = []\n",
    "    aoi_files = []\n",
    "    for c in query_cases:\n",
    "        case_path = os.path.join(sat_data_dir, c[0], c[1], f\"{c[1]}-{c[2]}\")\n",
    "        print(\"\\r\", f\"Retrieving files for {case_path}\", end=\"\")\n",
    "        all_files.extend(glob.glob(case_path + \"/*/*.zip\"))\n",
    "\n",
    "    print(\"\")\n",
    "    aoi_files = {}\n",
    "    for aoi in aoi_list:\n",
    "        print(\"\\r\", f\"filtering files for {aoi}\", end=\"\")\n",
    "        aoi_files[aoi] = list(filter(lambda p: aoi in p, all_files))\n",
    "    \n",
    "    print(\"\")\n",
    "    return aoi_files\n",
    "\n",
    "def find_files_for_s1_aois(nci_files_dict:dict, s1_file_names:list[str]) -> dict:\n",
    "    \"\"\"\n",
    "    Finds the files found from SARA server inside the AOI files retrieved from NCI\n",
    "    \"\"\"\n",
    "    files_dict = {}\n",
    "    for k, v in nci_files_dict.items():\n",
    "        nci_files = [os.path.splitext(os.path.basename(f))[0] for f in v]\n",
    "        found_idx = [nci_files.index(f) for f in s1_file_names if f in nci_files]\n",
    "        found = [v[idx] for idx in found_idx]\n",
    "        files_dict[k] = found\n",
    "    return files_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25S150E-30S155E => 1940\n",
      "15S145E-20S150E => 465\n",
      "25S110E-30S115E => 80\n"
     ]
    }
   ],
   "source": [
    "s1_au_names = get_s1_filenames(AU_POLYGONS, S1_QUERY_YEARS_AU, \"data/outputs/temp_s1_au.txt\") # this needs internet connection if the output file does not already exist.\n",
    "s1_aoi_files_au_nci = await find_files_for_aios_async(S1_CASE_LIST_AU, S1_DATA_DIR, S1_AOI_LIST_AU)\n",
    "s1_aoi_files_au = find_files_for_s1_aois(s1_aoi_files_au_nci, s1_au_names)\n",
    "for key in s1_aoi_files_au.keys():\n",
    "    print(f\"{key} => {len(s1_aoi_files_au[key])}\")\n",
    "\n",
    "save_file_list(s1_aoi_files_au, os.path.join(SAVE_DIR, S1_AU_FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving files for /g/data/fj7/Copernicus/Sentinel-1/C-SAR/SLC/2024/2024-12\n",
      "filtering files for 65S070E-70S075E\n",
      "75S150W-80S145W => 430\n",
      "65S110E-70S115E => 761\n",
      "75S165E-80S170E => 300\n",
      "70S155E-75S160E => 302\n",
      "65S070E-70S075E => 660\n"
     ]
    }
   ],
   "source": [
    "s1_an_names = get_s1_filenames(AN_POLYGONS, S1_QUERY_YEARS_AN, \"data/outputs/temp_s1_an.txt\") # this needs internet connection if the output file does not already exist.\n",
    "s1_aoi_files_an_nci = await find_files_for_aios_async(S1_CASE_LIST_AN, S1_DATA_DIR, S1_AOI_LIST_AN)\n",
    "s1_aoi_files_an = find_files_for_s1_aois(s1_aoi_files_an_nci, s1_an_names)\n",
    "for key in s1_aoi_files_an.keys():\n",
    "    print(f\"{key} => {len(s1_aoi_files_an[key])}\")\n",
    "\n",
    "save_file_list(s1_aoi_files_an, os.path.join(SAVE_DIR, S1_AN_FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T56JNQ => 576\n",
      "T55KCB => 580\n",
      "T56JKQ => 583\n",
      "T50JKP => 592\n"
     ]
    }
   ],
   "source": [
    "s2_aoi_files_au = await find_files_for_aios_async(S2_CASE_LIST_AU, S2_DATA_DIR, S2_AOI_LIST_AU)\n",
    "for key in s2_aoi_files_au.keys():\n",
    "    print(f\"{key} => {len(s2_aoi_files_au[key])}\")\n",
    "\n",
    "save_file_list(s2_aoi_files_au, os.path.join(SAVE_DIR, S2_AU_FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving files for /g/data/fj7/Copernicus/Sentinel-2/MSI/L2A/2024/2024-12\n",
      "filtering files for T33DVA\n",
      "T09CWQ => 0\n",
      "T49DDH => 198\n",
      "T58CEU => 502\n",
      "T57DVB => 170\n",
      "T43DDC => 257\n",
      "T33DVA => 0\n"
     ]
    }
   ],
   "source": [
    "s2_aoi_files_an = await find_files_for_aios_async(S2_CASE_LIST_AN, S2_DATA_DIR, S2_AOI_LIST_AN)\n",
    "for key in s2_aoi_files_an.keys():\n",
    "    print(f\"{key} => {len(s2_aoi_files_an[key])}\")\n",
    "\n",
    "save_file_list(s2_aoi_files_an, os.path.join(SAVE_DIR, S2_AN_FILENAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}