Merge branch 'dev' into pam/1385_moment_restfreq

CHANGELOG.md

@@ -7,15 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
-* Support PV image generation along a polyline region ([#1341](https://github.com/CARTAvis/carta-backend/issues/1341)).
-* Support setting rest frequency for moment image generation ([#1385](https://github.com/CARTAvis/carta-backend/issues/1385)).
+* Add support for PV image generation along a polyline region ([#1341](https://github.com/CARTAvis/carta-backend/issues/1341)).
+* Add support for loading remote FITS files from the hips2fits server ([#1379](https://github.com/CARTAvis/carta-backend/issues/1379)).
+* Add support for setting rest frequency for moment image generation ([#1385](https://github.com/CARTAvis/carta-backend/issues/1385)).
 
 ### Fixed
-* Fixed crash when loading non-image HDU by URL ([#1365](https://github.com/CARTAvis/carta-backend/issues/1365)).
+* Fix crash when loading non-image HDU by URL ([#1365](https://github.com/CARTAvis/carta-backend/issues/1365)).
 * Fix crash when parsing FITS header long value ([#1366](https://github.com/CARTAvis/carta-backend/issues/1366)).
 * Fix incorrect parsing of SPECSYS value for ATCA FITS header ([#1375](https://github.com/CARTAvis/carta-backend/issues/1375)).
 * Fix hdf5 image distortion after animation stops ([#1368](https://github.com/CARTAvis/carta-backend/issues/1368)).
 * Fix matched polygon region approximation crash ([#1383](https://github.com/CARTAvis/carta-backend/issues/1383)).
+* Fix save image/export regions bug which could cause directory overwrite or deletion ([#1377](https://github.com/CARTAvis/carta-backend/issues/1377)).
 
 ### Changed
 * Move the loader cache to separate files ([#1021](https://github.com/CARTAvis/carta-backend/issues/1021)).

CMakeLists.txt

@@ -265,7 +265,9 @@ set(SOURCE_FILES
         src/Util/Stokes.cc
         src/Util/String.cc
         src/Util/Token.cc
-        src/ImageFitter/ImageFitter.cc)
+        src/ImageFitter/ImageFitter.cc
+        src/Util/RemoteFiles.cc
+        src/Util/RemoteFiles.h)
 
 add_definitions(-DHAVE_HDF5 -DPUGIXML_COMPACT)
 add_executable(carta_backend ${SOURCE_FILES})

src/Frame/Frame.cc

@@ -1916,23 +1916,52 @@ void Frame::SaveFile(const std::string& root_folder, const CARTA::SaveFile& save
     fs::path output_filename(save_file_msg.output_file_name());
     fs::path directory(save_file_msg.output_file_directory());
     CARTA::FileType output_file_type(save_file_msg.output_file_type());
+    bool overwrite(save_file_msg.overwrite());
 
     // Set response message
     int file_id(save_file_msg.file_id());
     save_file_ack.set_file_id(file_id);
-    bool success(false);
-    casacore::String message;
+    std::string message;
+
+    if (output_filename.empty()) {
+        message = "Cannot save image with no filename.";
+        save_file_ack.set_success(false);
+        save_file_ack.set_message(message);
+        return;
+    }
 
     // Get the full resolved name of the output image
     fs::path temp_path = fs::path(root_folder) / directory;
     fs::path abs_path = fs::absolute(temp_path);
     output_filename = abs_path / output_filename;
 
-    if (output_filename.string() == in_file) {
-        message = "The source file can not be overwritten!";
-        save_file_ack.set_success(success);
-        save_file_ack.set_message(message);
-        return;
+    if (fs::exists(output_filename)) {
+        if (output_filename.string() == in_file) {
+            message = "Cannot overwrite the source image.";
+        } else if (fs::is_other(output_filename)) {
+            // Not a file, directory, or symlink
+            message = "Cannot overwrite existing path: not a file or directory.";
+        } else if (CasacoreImageType(output_filename.string()) == casacore::ImageOpener::UNKNOWN) {
+            // Not an image
+            if (fs::is_directory(output_filename)) {
+                // Never overwrite directory
+                message = "Cannot overwrite existing directory.";
+            } else if (!overwrite) {
+                // Only overwrite file or symlink with confirmation
+                message = "Cannot overwrite existing file or symlink.";
+                save_file_ack.set_overwrite_confirmation_required(true);
+            }
+        } else if (!overwrite) {
+            // Only overwrite image with confirmation
+            message = "Cannot overwrite existing image.";
+            save_file_ack.set_overwrite_confirmation_required(true);
+        }
+
+        if (!message.empty()) {
+            save_file_ack.set_success(false);
+            save_file_ack.set_message(message);
+            return;
+        }
     }
 
     double rest_freq(save_file_msg.rest_freq());
@@ -1991,7 +2020,7 @@ void Frame::SaveFile(const std::string& root_folder, const CARTA::SaveFile& save
             } else {
                 image = casacore::SubImage<float>(sub_image, casacore::AxesSpecifier(false), true).cloneII();
             }
-        } catch (casacore::AipsError error) {
+        } catch (const casacore::AipsError& error) {
             message = error.getMesg();
             save_file_ack.set_success(false);
             save_file_ack.set_message(message);
@@ -2004,16 +2033,15 @@ void Frame::SaveFile(const std::string& root_folder, const CARTA::SaveFile& save
     if (change_rest_freq) {
         casacore::CoordinateSystem coord_sys = image->coordinates();
         casacore::String error_msg("");
-        bool success = coord_sys.setRestFrequency(error_msg, casacore::Quantity(rest_freq, casacore::Unit("Hz")));
-        if (success) {
-            success = image->setCoordinateInfo(coord_sys);
-        }
-        if (!success) {
-            spdlog::warn("Failed to set new rest freq; use header rest freq instead: {}", error_msg);
+        if (coord_sys.setRestFrequency(error_msg, casacore::Quantity(rest_freq, casacore::Unit("Hz")))) {
+            if (!image->setCoordinateInfo(coord_sys)) {
+                spdlog::warn("Failed to set new rest freq; using header rest freq instead: {}", error_msg);
+            }
         }
     }
 
     // Export image data to file
+    bool success(false);
     try {
         std::unique_lock<std::mutex> ulock(_image_mutex); // Lock the image while saving the file
         {
@@ -2030,15 +2058,16 @@ void Frame::SaveFile(const std::string& root_folder, const CARTA::SaveFile& save
             }
         }
         ulock.unlock(); // Unlock the image
-    } catch (casacore::AipsError error) {
+
+        if (success) {
+            spdlog::info("Exported a {} file \'{}\'.", FileTypeString[output_file_type], output_filename.string());
+        }
+    } catch (const casacore::AipsError& error) {
         message += error.getMesg();
         save_file_ack.set_success(false);
         save_file_ack.set_message(message);
         return;
     }
-    if (success) {
-        spdlog::info("Exported a {} file \'{}\'.", FileTypeString[output_file_type], output_filename.string());
-    }
 
     // Remove the root folder from the ack message
     if (!root_folder.empty()) {
@@ -2057,7 +2086,7 @@ void Frame::SaveFile(const std::string& root_folder, const CARTA::SaveFile& save
 // Input output_filename as file path
 // Input message as a return message, which may contain error message
 // Return a bool if this functionality success
-bool Frame::ExportCASAImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, casacore::String& message) {
+bool Frame::ExportCASAImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, std::string& message) {
     bool success(false);
 
     // Remove the old image file if it has a same file name
@@ -2104,7 +2133,7 @@ bool Frame::ExportCASAImage(casacore::ImageInterface<casacore::Float>& image, fs
 // Input output_filename as file path
 // Input message as a return message, which may contain error message
 // Return a bool if this functionality success
-bool Frame::ExportFITSImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, casacore::String& message) {
+bool Frame::ExportFITSImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, std::string& message) {
     bool success = false;
     bool prefer_velocity;
     bool optical_velocity;

src/Frame/Frame.h

@@ -252,8 +252,8 @@ class Frame {
     bool HasSpectralConfig(const SpectralConfig& config);
 
     // Export image
-    bool ExportCASAImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, casacore::String& message);
-    bool ExportFITSImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, casacore::String& message);
+    bool ExportCASAImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, std::string& message);
+    bool ExportFITSImage(casacore::ImageInterface<casacore::Float>& image, fs::path output_filename, std::string& message);
     void ValidateChannelStokes(std::vector<int>& channels, std::vector<int>& stokes, const CARTA::SaveFile& save_file_msg);
     casacore::Slicer GetExportImageSlicer(const CARTA::SaveFile& save_file_msg, casacore::IPosition image_shape);
     casacore::Slicer GetExportRegionSlicer(const CARTA::SaveFile& save_file_msg, casacore::IPosition image_shape,

src/ImageData/CartaFitsImage.cc

@@ -27,7 +27,7 @@
 
 using namespace carta;
 
-CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu)
+CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu, bool is_http)
     : casacore::ImageInterface<float>(),
       _filename(filename),
       _hdu(hdu),
@@ -37,10 +37,13 @@ CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu)
       _equiv_bitpix(-32), // assume float
       _has_blanks(false),
       _pixel_mask(nullptr),
+      _is_http(is_http),
       _is_copy(false) {
-    casacore::File ccfile(filename);
-    if (!ccfile.exists() || !ccfile.isReadable()) {
-        throw(casacore::AipsError("FITS file is not readable or does not exist."));
+    if (!is_http) {
+        casacore::File ccfile(filename);
+        if (!ccfile.exists() || !ccfile.isReadable()) {
+            throw(casacore::AipsError("FITS file is not readable or does not exist."));
+        }
     }
 
     SetUpImage();
@@ -237,7 +240,11 @@ fitsfile* CartaFitsImage::OpenFile() {
         fits_open_file(&fptr, _filename.c_str(), iomode, &status);
 
         if (status) {
-            throw(casacore::AipsError("Error opening FITS file."));
+            fits_report_error(stdout, status);
+            char err_text[30];
+            fits_get_errstatus(status, err_text);
+            std::string error(err_text);
+            throw(casacore::AipsError("Error opening FITS file: " + error));
         }
 
         // Advance to requested hdu
@@ -446,7 +453,9 @@ void CartaFitsImage::GetFitsHeaderString(int& nheaders, std::string& hdrstr) {
     fits_free_memory(*header, &free_status);
 
     // Done with file setup
-    CloseFile();
+    if (!_is_http) {
+        CloseFile();
+    }
 }
 
 void CartaFitsImage::SetFitsHeaderStrings(int nheaders, const std::string& header) {

src/ImageData/CartaFitsImage.h

@@ -28,7 +28,7 @@ static std::unordered_map<int, casacore::DataType> bitpix_types(
 class CartaFitsImage : public casacore::ImageInterface<float> {
 public:
     // Construct an image from a pre-existing file.
-    CartaFitsImage(const std::string& filename, unsigned int hdu = 0);
+    CartaFitsImage(const std::string& filename, unsigned int hdu = 0, bool is_http = false);
     // Copy constructor
     CartaFitsImage(const CartaFitsImage& other);
     ~CartaFitsImage() override;
@@ -119,6 +119,8 @@ class CartaFitsImage : public casacore::ImageInterface<float> {
 
     // Whether is a copy of the other CartaFitsImage
     bool _is_copy;
+    // Whether the file is a remote file
+    bool _is_http;
 };
 
 } // namespace carta

src/ImageData/FileLoader.cc

@@ -32,6 +32,8 @@ FileLoader* FileLoader::GetLoader(const std::string& filename, const std::string
         return new ExprLoader(filename, directory);
     } else if (IsCompressedFits(filename)) {
         return new FitsLoader(filename, true);
+    } else if (IsRemoteHttpFile(filename)) {
+        return new FitsLoader(filename, false, true);
     }
 
     switch (CasacoreImageType(filename)) {

src/ImageData/FitsLoader.cc

@@ -19,7 +19,8 @@
 
 namespace carta {
 
-FitsLoader::FitsLoader(const std::string& filename, bool is_gz) : FileLoader(filename, "", is_gz) {}
+FitsLoader::FitsLoader(const std::string& filename, bool is_gz, bool is_http)
+    : FileLoader(filename, "", is_gz, is_http), _is_http(is_http) {}
 
 FitsLoader::~FitsLoader() {
     // Remove decompressed fits.gz file
@@ -34,7 +35,7 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
     // Open image file as a casacore::ImageInterface<float>
 
     // Convert string to FITS hdu number
-    casacore::uInt hdu_num(FileInfo::GetFitsHdu(hdu));
+    casacore::uInt hdu_num(_is_http ? 0 : FileInfo::GetFitsHdu(hdu));
 
     if (!_image || (hdu_num != _hdu_num)) {
         bool gz_mem_ok(true);
@@ -62,17 +63,23 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
             }
         }
 
-        std::string error;
-        auto num_headers = GetNumImageHeaders(_filename, hdu_num, error);
-        if (num_headers == 0) {
-            throw(casacore::AipsError(error));
-        }
-
         // Default is casacore::FITSImage; if fails, try CartaFitsImage
         bool use_casacore_fits(true);
-        if (num_headers > 2000) {
-            // casacore::FITSImage parses HISTORY
+
+        if (_is_http) {
             use_casacore_fits = false;
+        } else {
+            std::string error;
+            auto num_headers = GetNumImageHeaders(_filename, hdu_num, error);
+
+            if (num_headers == 0) {
+                throw(casacore::AipsError(error));
+            }
+
+            if (num_headers > 2000) {
+                // casacore::FITSImage parses HISTORY
+                use_casacore_fits = false;
+            }
         }
 
         try {
@@ -86,6 +93,8 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
                     // use casacore for unzipped FITS file
                     _image.reset(new casacore::FITSImage(_unzip_file, 0, hdu_num));
                 }
+            } else if (_is_http) {
+                _image.reset(new CartaFitsImage(_filename, hdu_num, true));
             } else if (use_casacore_fits) {
                 if (Is64BitBeamsTable(_filename)) {
                     use_casacore_fits = false;
@@ -106,7 +115,9 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
                     spdlog::error(err.getMesg());
                 }
             } else {
-                spdlog::error(err.getMesg());
+                auto error = err.getMesg();
+                spdlog::error(error);
+                throw(casacore::AipsError(error));
             }
         }
 
@@ -179,7 +190,8 @@ int FitsLoader::GetNumImageHeaders(const std::string& filename, int hdu, std::st
 void FitsLoader::ResetImageBeam(unsigned int hdu_num) {
     // Remove restoring beam not if not in header entries.
     // If supporting AIPS beam, use last beam from history instead.
-    if (!_image) {
+    // Remote files can't have restoring beams
+    if (!_image || _is_http) {
         return;
     }
 

src/ImageData/FitsLoader.h

@@ -13,12 +13,13 @@ namespace carta {
 
 class FitsLoader : public FileLoader {
 public:
-    FitsLoader(const std::string& filename, bool is_gz = false);
+    FitsLoader(const std::string& filename, bool is_gz = false, bool is_http = false);
     ~FitsLoader();
 
 private:
     std::string _unzip_file;
     casacore::uInt _hdu_num;
+    bool _is_http;
 
     void AllocateImage(const std::string& hdu) override;
     int GetNumImageHeaders(const std::string& filename, int hdu, std::string& error);