updated config for aus

README.md

@@ -33,6 +33,15 @@ sh setup.sh
 ```bash
 source run_process.sh
 ```
+- On an AWS instance with a large number of jobs, run with nohup to ensure process is not killed if an SSH disconnection occurs.
+```bash
+conda activate rtc_opera
+nohup ./run_process.sh
+```
+- in a new terminal, run the following to trace the output
+```bash
+tail -f nohup.out
+```
 
 # Common errors
 - ensure credentials have been set before running the setup script. If this was not the case, update the credentials in the credentials folder and run the folllowing:

configs/OPERA-rtc-20m-aus.yaml

@@ -76,8 +76,8 @@ runconfig:
 
           # Options to run geo2rdr
           geo2rdr:
-              threshold: 1.0e-8
-              numiter: 25
+              threshold: 1.0e-7
+              numiter: 50
 
           # Options to run rdr2geo
           rdr2geo:
@@ -94,10 +94,10 @@ runconfig:
           apply_rtc: True
 
           # Apply bistatic delay correction
-          apply_bistatic_delay_correction: False
+          apply_bistatic_delay_correction: True
 
           # Apply static tropospheric delay correction
-          apply_static_tropospheric_delay_correction: False
+          apply_static_tropospheric_delay_correction: True
 
           # DEM interpolation method
           # Choices - 'sinc', 'bilinear', 'bicubic', 'nearest', 'biquintic'
@@ -143,10 +143,10 @@ runconfig:
               memory_mode: auto
 
               # OPTIONAL - Processing upsampling factor applied to input geogrid
-              geogrid_upsampling: 2
+              geogrid_upsampling: 1
 
               # Save the incidence angle
-              save_incidence_angle: False
+              save_incidence_angle: True
 
               # Save the local-incidence angle
               save_local_inc_angle: True

configs/OPERA-rtc-30m-ant.yaml

@@ -0,0 +1,227 @@
+runconfig:
+  name: rtc_s1_workflow_default
+
+  groups:
+      primary_executable:
+          product_type: RTC_S1
+
+      pge_name_group:
+          pge_name: RTC_S1_PGE
+
+      input_file_group:
+          # Required. List of SAFE files (min=1)
+          safe_file_path: [SAFE_PATH]
+
+          orbit_file_path: [ORBIT_PATH]
+
+          # Optional. Burst ID to process (empty for all bursts)
+          # burst_id: [t069_147173_iw1, t069_147174_iw1]
+          # burst_id: [t071_151225_iw1, t071_151226_iw1]
+
+      dynamic_ancillary_file_group:
+          # Digital elevation model
+          # dem_file: /data/dem/glo_30/S1B_IW_SLC__1SSH_20190223T222639_20190223T222706_015079_01C2E9_1D63_dem.tif
+          dem_file: DEM_PATH
+
+          # Digital elevation model description
+          dem_file_description:
+
+      static_ancillary_file_group:
+
+          # burst database sqlite file
+          burst_database_file:
+
+      product_group:
+          processing_type: 'CUSTOM'
+
+          # Directory where PGE will place results
+          product_path: .
+
+          # Directory where SAS writes temporary data
+          scratch_path: OPERA_SCRATCH_FOLDER
+
+          # If option `save_bursts` is set, output bursts are saved to:
+          #     {output_dir}/{burst_id}/{product_id}{suffix}.{ext}
+          # If option `save_mosaics` is set, output mosaics are saved to:
+          #     {output_dir}/{product_id}{suffix}.{ext}
+          # If the field `product_id`` is left empty, the prefix "rtc_product"
+          # will be used instead.
+          # `suffix` is only used when there are multiple output files.
+          # `ext` is determined by geocoding_options.output_imagery_format.
+          output_dir: OPERA_OUTPUT_FOLDER
+          product_id: OPERA_L2_RTC-SCENE_NAME
+
+          save_bursts: True
+          save_mosaics: True
+          output_imagery_format: COG
+          output_imagery_compression: DEFLATE
+          output_imagery_nbits: 32
+
+          # Optional. Save secondary layers (e.g., inc. angle) within 
+          # the HDF5 file
+          save_secondary_layers_as_hdf5: True
+
+          # Save RTC-S1 metadata in the HDF5 format
+          # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or
+          # 'COG', and enabled by default for `output_imagery_format` equal
+          # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True
+          save_metadata: True
+
+      processing:
+
+          # Check if ancillary input cover entirely output products
+          check_ancillary_inputs_coverage: True
+
+          polarization: POLARIZATION_TYPE
+
+          # Options to run geo2rdr
+          geo2rdr:
+              threshold: 1.0e-8
+              numiter: 25
+
+          # Options to run rdr2geo
+          rdr2geo:
+              threshold: 1.0e-7
+              numiter: 25
+
+          # Apply absolute radiometric correction
+          apply_absolute_radiometric_correction: True
+
+          # Apply thermal noise correction
+          apply_thermal_noise_correction: True
+
+          # OPTIONAL - Apply RTC
+          apply_rtc: True
+
+          # Apply bistatic delay correction
+          apply_bistatic_delay_correction: False
+
+          # Apply static tropospheric delay correction
+          apply_static_tropospheric_delay_correction: False
+
+          # DEM interpolation method
+          # Choices - 'sinc', 'bilinear', 'bicubic', 'nearest', 'biquintic'
+          dem_interpolation_method: bilinear
+
+          # OPTIONAL - to control behavior of RTC module
+          # (only applicable if geocode.apply_rtc is True)
+          rtc:
+              # OPTIONAL - Choices:
+              # "gamma0" (default)
+              # "sigma0"
+              output_type: gamma0
+
+              # OPTIONAL - Choices:
+              # "bilinear_distribution" (default)
+              # "area_projection"
+              algorithm_type: area_projection
+
+              # OPTIONAL - Choices:
+              # "beta0" (default)
+              # "sigma0"
+              input_terrain_radiometry: beta0
+
+              # OPTIONAL - Minimum RTC area factor in dB
+              rtc_min_value_db: -30
+
+              # RTC DEM upsampling
+              dem_upsampling: 2
+
+          # OPTIONAL - Mechanism to specify output posting and DEM
+          geocoding:
+
+              # OPTIONAL - Apply valid-samples sub-swath masking
+              apply_valid_samples_sub_swath_masking: True
+
+              # OPTIONAL - Apply shadow masking
+              apply_shadow_masking: True
+
+              # OPTIONAL -
+              algorithm_type: area_projection
+
+              # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default)
+              memory_mode: auto
+
+              # OPTIONAL - Processing upsampling factor applied to input geogrid
+              geogrid_upsampling: 2
+
+              # Save the incidence angle
+              save_incidence_angle: False
+
+              # Save the local-incidence angle
+              save_local_inc_angle: True
+
+              # Save the projection angle
+              save_projection_angle: False
+
+              # Save the RTC ANF compuated with the projection angle method
+              save_rtc_anf_projection_angle: True
+
+              # Save the range slope angle
+              save_range_slope: False
+
+              # Save the number of looks used to compute RTC-S1
+              save_nlooks: True
+
+              # Save the RTC area normalization factor (ANF) used to generate
+              # the RTC product
+              save_rtc_anf: True
+
+              # Save the RTC area normalization factor (ANF) gamma0 to sigma0
+              save_rtc_anf_gamma0_to_sigma0: True
+
+              # Save interpolated DEM used to compute RTC-S1
+              save_dem: True
+
+              # Save layover shadow mask
+              save_mask: True
+
+              # OPTIONAL - Absolute radiometric correction
+              abs_rad_cal: 1
+
+              # OPTIONAL - Clip values above threshold
+              clip_max:
+
+              # OPTIONAL - Clip values below threshold
+              clip_min:
+
+              # OPTIONAL - Double sampling of the radar-grid
+              # input sampling in the range direction
+              upsample_radargrid: False
+
+              # Fields to populate the products' metadata required by
+              # CEOS Analysis Ready Data specifications
+              estimated_geometric_accuracy_bias_x:
+              estimated_geometric_accuracy_bias_y:
+              estimated_geometric_accuracy_stddev_x:
+              estimated_geometric_accuracy_stddev_y:
+
+              bursts_geogrid:
+                  output_epsg: 3031
+                  x_posting: 30
+                  y_posting: 30
+                  x_snap: 30
+                  y_snap: 30
+                  top_left:
+                      x:
+                      y:
+                  bottom_right:
+                      x:
+                      y:
+
+
+          mosaicking:
+              # OPTIONAL - Choices: "average", "first", "bursts_center" (default)
+              mosaic_mode: first
+              mosaic_geogrid: 
+                  output_epsg: 3031 
+                  x_posting: 30
+                  y_posting: 30
+                  x_snap: 30
+                  y_snap: 30
+                  top_left:
+                      x:
+                      y:
+                  bottom_right:
+                      x:
+                      y:

rtc_otf.py

@@ -438,16 +438,16 @@ def run_process(args):
 
         if otf_cfg['delete_local_files']:
             logging.info(f'PROCESS 4: Clear files locally')
-            # #clear downloads
-            # for file_ in [scene_zip,
-            #             DEM_PATH,
-            #             ORBIT_PATH,
-            #             #opera_config_path,
-            #             ]:
-            #     logging.info(f'Deleteing {file_}')
-            #     os.remove(file_)
-            # logging.info(f'Clearing SAFE directory: {ORIGINAL_SAFE_PATH}')
-            # shutil.rmtree(ORIGINAL_SAFE_PATH)
+            #clear downloads
+            for file_ in [scene_zip,
+                        DEM_PATH,
+                        ORBIT_PATH,
+                        opera_config_path,
+                        ]:
+                logging.info(f'Deleteing {file_}')
+                os.remove(file_)
+            logging.info(f'Clearing SAFE directory: {ORIGINAL_SAFE_PATH}')
+            shutil.rmtree(ORIGINAL_SAFE_PATH)
             if otf_cfg['apply_ETAD']:
                 logging.info(f'Clearing ETAD corrected SAFE directory: {ETAD_SAFE_PATH}')
                 shutil.rmtree(ETAD_SAFE_PATH)

setup.sh

@@ -1,4 +1,4 @@
-git clone --branch v1.0.1 --single-branch  https://github.com/opera-adt/RTC.git RTC_2
+git clone --branch v1.0.1 --single-branch  https://github.com/opera-adt/RTC.git RTC
 
 # build container
 cp -fr build_docker_image_otf.sh RTC