From ee2a86849810daed94ada3c6e4a2ddea2adb3046 Mon Sep 17 00:00:00 2001 From: abradley60 Date: Fri, 6 Sep 2024 05:16:55 +0000 Subject: [PATCH] updated config for aus --- README.md | 9 ++ configs/OPERA-rtc-20m-aus.yaml | 12 +- configs/OPERA-rtc-30m-ant.yaml | 227 +++++++++++++++++++++++++++++++++ rtc_otf.py | 20 +-- setup.sh | 2 +- 5 files changed, 253 insertions(+), 17 deletions(-) create mode 100644 configs/OPERA-rtc-30m-ant.yaml diff --git a/README.md b/README.md index 61bf31f..2bf2f9c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,15 @@ sh setup.sh ```bash source run_process.sh ``` +- On an AWS instance with a large number of jobs, run with nohup to ensure process is not killed if an SSH disconnection occurs. +```bash +conda activate rtc_opera +nohup ./run_process.sh +``` +- in a new terminal, run the following to trace the output +```bash +tail -f nohup.out +``` # Common errors - ensure credentials have been set before running the setup script. If this was not the case, update the credentials in the credentials folder and run the folllowing: diff --git a/configs/OPERA-rtc-20m-aus.yaml b/configs/OPERA-rtc-20m-aus.yaml index 932acdb..3fe4864 100644 --- a/configs/OPERA-rtc-20m-aus.yaml +++ b/configs/OPERA-rtc-20m-aus.yaml @@ -76,8 +76,8 @@ runconfig: # Options to run geo2rdr geo2rdr: - threshold: 1.0e-8 - numiter: 25 + threshold: 1.0e-7 + numiter: 50 # Options to run rdr2geo rdr2geo: @@ -94,10 +94,10 @@ runconfig: apply_rtc: True # Apply bistatic delay correction - apply_bistatic_delay_correction: False + apply_bistatic_delay_correction: True # Apply static tropospheric delay correction - apply_static_tropospheric_delay_correction: False + apply_static_tropospheric_delay_correction: True # DEM interpolation method # Choices - 'sinc', 'bilinear', 'bicubic', 'nearest', 'biquintic' @@ -143,10 +143,10 @@ runconfig: memory_mode: auto # OPTIONAL - Processing upsampling factor applied to input geogrid - geogrid_upsampling: 2 + geogrid_upsampling: 1 # Save the incidence angle - save_incidence_angle: False + save_incidence_angle: True # Save the local-incidence angle save_local_inc_angle: True diff --git a/configs/OPERA-rtc-30m-ant.yaml b/configs/OPERA-rtc-30m-ant.yaml new file mode 100644 index 0000000..63c02df --- /dev/null +++ b/configs/OPERA-rtc-30m-ant.yaml @@ -0,0 +1,227 @@ +runconfig: + name: rtc_s1_workflow_default + + groups: + primary_executable: + product_type: RTC_S1 + + pge_name_group: + pge_name: RTC_S1_PGE + + input_file_group: + # Required. List of SAFE files (min=1) + safe_file_path: [SAFE_PATH] + + orbit_file_path: [ORBIT_PATH] + + # Optional. Burst ID to process (empty for all bursts) + # burst_id: [t069_147173_iw1, t069_147174_iw1] + # burst_id: [t071_151225_iw1, t071_151226_iw1] + + dynamic_ancillary_file_group: + # Digital elevation model + # dem_file: /data/dem/glo_30/S1B_IW_SLC__1SSH_20190223T222639_20190223T222706_015079_01C2E9_1D63_dem.tif + dem_file: DEM_PATH + + # Digital elevation model description + dem_file_description: + + static_ancillary_file_group: + + # burst database sqlite file + burst_database_file: + + product_group: + processing_type: 'CUSTOM' + + # Directory where PGE will place results + product_path: . + + # Directory where SAS writes temporary data + scratch_path: OPERA_SCRATCH_FOLDER + + # If option `save_bursts` is set, output bursts are saved to: + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} + # If option `save_mosaics` is set, output mosaics are saved to: + # {output_dir}/{product_id}{suffix}.{ext} + # If the field `product_id`` is left empty, the prefix "rtc_product" + # will be used instead. + # `suffix` is only used when there are multiple output files. + # `ext` is determined by geocoding_options.output_imagery_format. + output_dir: OPERA_OUTPUT_FOLDER + product_id: OPERA_L2_RTC-SCENE_NAME + + save_bursts: True + save_mosaics: True + output_imagery_format: COG + output_imagery_compression: DEFLATE + output_imagery_nbits: 32 + + # Optional. Save secondary layers (e.g., inc. angle) within + # the HDF5 file + save_secondary_layers_as_hdf5: True + + # Save RTC-S1 metadata in the HDF5 format + # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or + # 'COG', and enabled by default for `output_imagery_format` equal + # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True + save_metadata: True + + processing: + + # Check if ancillary input cover entirely output products + check_ancillary_inputs_coverage: True + + polarization: POLARIZATION_TYPE + + # Options to run geo2rdr + geo2rdr: + threshold: 1.0e-8 + numiter: 25 + + # Options to run rdr2geo + rdr2geo: + threshold: 1.0e-7 + numiter: 25 + + # Apply absolute radiometric correction + apply_absolute_radiometric_correction: True + + # Apply thermal noise correction + apply_thermal_noise_correction: True + + # OPTIONAL - Apply RTC + apply_rtc: True + + # Apply bistatic delay correction + apply_bistatic_delay_correction: False + + # Apply static tropospheric delay correction + apply_static_tropospheric_delay_correction: False + + # DEM interpolation method + # Choices - 'sinc', 'bilinear', 'bicubic', 'nearest', 'biquintic' + dem_interpolation_method: bilinear + + # OPTIONAL - to control behavior of RTC module + # (only applicable if geocode.apply_rtc is True) + rtc: + # OPTIONAL - Choices: + # "gamma0" (default) + # "sigma0" + output_type: gamma0 + + # OPTIONAL - Choices: + # "bilinear_distribution" (default) + # "area_projection" + algorithm_type: area_projection + + # OPTIONAL - Choices: + # "beta0" (default) + # "sigma0" + input_terrain_radiometry: beta0 + + # OPTIONAL - Minimum RTC area factor in dB + rtc_min_value_db: -30 + + # RTC DEM upsampling + dem_upsampling: 2 + + # OPTIONAL - Mechanism to specify output posting and DEM + geocoding: + + # OPTIONAL - Apply valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: True + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: True + + # OPTIONAL - + algorithm_type: area_projection + + # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) + memory_mode: auto + + # OPTIONAL - Processing upsampling factor applied to input geogrid + geogrid_upsampling: 2 + + # Save the incidence angle + save_incidence_angle: False + + # Save the local-incidence angle + save_local_inc_angle: True + + # Save the projection angle + save_projection_angle: False + + # Save the RTC ANF compuated with the projection angle method + save_rtc_anf_projection_angle: True + + # Save the range slope angle + save_range_slope: False + + # Save the number of looks used to compute RTC-S1 + save_nlooks: True + + # Save the RTC area normalization factor (ANF) used to generate + # the RTC product + save_rtc_anf: True + + # Save the RTC area normalization factor (ANF) gamma0 to sigma0 + save_rtc_anf_gamma0_to_sigma0: True + + # Save interpolated DEM used to compute RTC-S1 + save_dem: True + + # Save layover shadow mask + save_mask: True + + # OPTIONAL - Absolute radiometric correction + abs_rad_cal: 1 + + # OPTIONAL - Clip values above threshold + clip_max: + + # OPTIONAL - Clip values below threshold + clip_min: + + # OPTIONAL - Double sampling of the radar-grid + # input sampling in the range direction + upsample_radargrid: False + + # Fields to populate the products' metadata required by + # CEOS Analysis Ready Data specifications + estimated_geometric_accuracy_bias_x: + estimated_geometric_accuracy_bias_y: + estimated_geometric_accuracy_stddev_x: + estimated_geometric_accuracy_stddev_y: + + bursts_geogrid: + output_epsg: 3031 + x_posting: 30 + y_posting: 30 + x_snap: 30 + y_snap: 30 + top_left: + x: + y: + bottom_right: + x: + y: + + + mosaicking: + # OPTIONAL - Choices: "average", "first", "bursts_center" (default) + mosaic_mode: first + mosaic_geogrid: + output_epsg: 3031 + x_posting: 30 + y_posting: 30 + x_snap: 30 + y_snap: 30 + top_left: + x: + y: + bottom_right: + x: + y: diff --git a/rtc_otf.py b/rtc_otf.py index dd47650..b12909a 100644 --- a/rtc_otf.py +++ b/rtc_otf.py @@ -438,16 +438,16 @@ def run_process(args): if otf_cfg['delete_local_files']: logging.info(f'PROCESS 4: Clear files locally') - # #clear downloads - # for file_ in [scene_zip, - # DEM_PATH, - # ORBIT_PATH, - # #opera_config_path, - # ]: - # logging.info(f'Deleteing {file_}') - # os.remove(file_) - # logging.info(f'Clearing SAFE directory: {ORIGINAL_SAFE_PATH}') - # shutil.rmtree(ORIGINAL_SAFE_PATH) + #clear downloads + for file_ in [scene_zip, + DEM_PATH, + ORBIT_PATH, + opera_config_path, + ]: + logging.info(f'Deleteing {file_}') + os.remove(file_) + logging.info(f'Clearing SAFE directory: {ORIGINAL_SAFE_PATH}') + shutil.rmtree(ORIGINAL_SAFE_PATH) if otf_cfg['apply_ETAD']: logging.info(f'Clearing ETAD corrected SAFE directory: {ETAD_SAFE_PATH}') shutil.rmtree(ETAD_SAFE_PATH) diff --git a/setup.sh b/setup.sh index a0af5f9..476efa4 100644 --- a/setup.sh +++ b/setup.sh @@ -1,4 +1,4 @@ -git clone --branch v1.0.1 --single-branch https://github.com/opera-adt/RTC.git RTC_2 +git clone --branch v1.0.1 --single-branch https://github.com/opera-adt/RTC.git RTC # build container cp -fr build_docker_image_otf.sh RTC