Angus/hips remote image load

CHANGELOG.md

@@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
-* Support PV image generation along a polyline region ([#1341](https://github.com/CARTAvis/carta-backend/issues/1341)).
+* Added support for PV image generation along a polyline region ([#1341](https://github.com/CARTAvis/carta-backend/issues/1341)).
+* Added support for loading remote FITS files from the hips2fits server ([#1379](https://github.com/CARTAvis/carta-backend/issues/1379)).
 
 ### Fixed
 * Fixed crash when loading non-image HDU by URL ([#1365](https://github.com/CARTAvis/carta-backend/issues/1365)).

CMakeLists.txt

@@ -265,7 +265,9 @@ set(SOURCE_FILES
         src/Util/Stokes.cc
         src/Util/String.cc
         src/Util/Token.cc
-        src/ImageFitter/ImageFitter.cc)
+        src/ImageFitter/ImageFitter.cc
+        src/Util/RemoteFiles.cc
+        src/Util/RemoteFiles.h)
 
 add_definitions(-DHAVE_HDF5 -DPUGIXML_COMPACT)
 add_executable(carta_backend ${SOURCE_FILES})

src/ImageData/CartaFitsImage.cc

@@ -27,7 +27,7 @@
 
 using namespace carta;
 
-CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu)
+CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu, bool is_http)
     : casacore::ImageInterface<float>(),
       _filename(filename),
       _hdu(hdu),
@@ -37,10 +37,13 @@ CartaFitsImage::CartaFitsImage(const std::string& filename, unsigned int hdu)
       _equiv_bitpix(-32), // assume float
       _has_blanks(false),
       _pixel_mask(nullptr),
+      _is_http(is_http),
       _is_copy(false) {
-    casacore::File ccfile(filename);
-    if (!ccfile.exists() || !ccfile.isReadable()) {
-        throw(casacore::AipsError("FITS file is not readable or does not exist."));
+    if (!is_http) {
+        casacore::File ccfile(filename);
+        if (!ccfile.exists() || !ccfile.isReadable()) {
+            throw(casacore::AipsError("FITS file is not readable or does not exist."));
+        }
     }
 
     SetUpImage();
@@ -237,7 +240,10 @@ fitsfile* CartaFitsImage::OpenFile() {
         fits_open_file(&fptr, _filename.c_str(), iomode, &status);
 
         if (status) {
-            throw(casacore::AipsError("Error opening FITS file."));
+            char err_text[30];
+            fits_get_errstatus(status, err_text);
+            std::string error(err_text);
+            throw(casacore::AipsError("Error opening FITS file: " + error));
         }
 
         // Advance to requested hdu
@@ -446,7 +452,9 @@ void CartaFitsImage::GetFitsHeaderString(int& nheaders, std::string& hdrstr) {
     fits_free_memory(*header, &free_status);
 
     // Done with file setup
-    CloseFile();
+    if (!_is_http) {
+        CloseFile();
+    }
 }
 
 void CartaFitsImage::SetFitsHeaderStrings(int nheaders, const std::string& header) {

src/ImageData/CartaFitsImage.h

@@ -28,7 +28,7 @@ static std::unordered_map<int, casacore::DataType> bitpix_types(
 class CartaFitsImage : public casacore::ImageInterface<float> {
 public:
     // Construct an image from a pre-existing file.
-    CartaFitsImage(const std::string& filename, unsigned int hdu = 0);
+    CartaFitsImage(const std::string& filename, unsigned int hdu = 0, bool is_http = false);
     // Copy constructor
     CartaFitsImage(const CartaFitsImage& other);
     ~CartaFitsImage() override;
@@ -119,6 +119,8 @@ class CartaFitsImage : public casacore::ImageInterface<float> {
 
     // Whether is a copy of the other CartaFitsImage
     bool _is_copy;
+    // Whether the file is a remote file
+    bool _is_http;
 };
 
 } // namespace carta

src/ImageData/FileLoader.cc

@@ -32,6 +32,8 @@ FileLoader* FileLoader::GetLoader(const std::string& filename, const std::string
         return new ExprLoader(filename, directory);
     } else if (IsCompressedFits(filename)) {
         return new FitsLoader(filename, true);
+    } else if (IsRemoteHttpFile(filename)) {
+        return new FitsLoader(filename, false, true);
     }
 
     switch (CasacoreImageType(filename)) {

src/ImageData/FitsLoader.cc

@@ -19,7 +19,8 @@
 
 namespace carta {
 
-FitsLoader::FitsLoader(const std::string& filename, bool is_gz) : FileLoader(filename, "", is_gz) {}
+FitsLoader::FitsLoader(const std::string& filename, bool is_gz, bool is_http)
+    : FileLoader(filename, "", is_gz, is_http), _is_http(is_http) {}
 
 FitsLoader::~FitsLoader() {
     // Remove decompressed fits.gz file
@@ -34,7 +35,7 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
     // Open image file as a casacore::ImageInterface<float>
 
     // Convert string to FITS hdu number
-    casacore::uInt hdu_num(FileInfo::GetFitsHdu(hdu));
+    casacore::uInt hdu_num(_is_http ? 0 : FileInfo::GetFitsHdu(hdu));
 
     if (!_image || (hdu_num != _hdu_num)) {
         bool gz_mem_ok(true);
@@ -62,17 +63,23 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
             }
         }
 
-        std::string error;
-        auto num_headers = GetNumImageHeaders(_filename, hdu_num, error);
-        if (num_headers == 0) {
-            throw(casacore::AipsError(error));
-        }
-
         // Default is casacore::FITSImage; if fails, try CartaFitsImage
         bool use_casacore_fits(true);
-        if (num_headers > 2000) {
-            // casacore::FITSImage parses HISTORY
+
+        if (_is_http) {
             use_casacore_fits = false;
+        } else {
+            std::string error;
+            auto num_headers = GetNumImageHeaders(_filename, hdu_num, error);
+
+            if (num_headers == 0) {
+                throw(casacore::AipsError(error));
+            }
+
+            if (num_headers > 2000) {
+                // casacore::FITSImage parses HISTORY
+                use_casacore_fits = false;
+            }
         }
 
         try {
@@ -86,6 +93,8 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
                     // use casacore for unzipped FITS file
                     _image.reset(new casacore::FITSImage(_unzip_file, 0, hdu_num));
                 }
+            } else if (_is_http) {
+                _image.reset(new CartaFitsImage(_filename, hdu_num, true));
             } else if (use_casacore_fits) {
                 if (Is64BitBeamsTable(_filename)) {
                     use_casacore_fits = false;
@@ -106,7 +115,9 @@ void FitsLoader::AllocateImage(const std::string& hdu) {
                     spdlog::error(err.getMesg());
                 }
             } else {
-                spdlog::error(err.getMesg());
+                auto error = err.getMesg();
+                spdlog::error(error);
+                throw(casacore::AipsError(error));
             }
         }
 
@@ -179,7 +190,8 @@ int FitsLoader::GetNumImageHeaders(const std::string& filename, int hdu, std::st
 void FitsLoader::ResetImageBeam(unsigned int hdu_num) {
     // Remove restoring beam not if not in header entries.
     // If supporting AIPS beam, use last beam from history instead.
-    if (!_image) {
+    // Remote files can't have restoring beams
+    if (!_image || _is_http) {
         return;
     }
 

src/ImageData/FitsLoader.h

@@ -13,12 +13,13 @@ namespace carta {
 
 class FitsLoader : public FileLoader {
 public:
-    FitsLoader(const std::string& filename, bool is_gz = false);
+    FitsLoader(const std::string& filename, bool is_gz = false, bool is_http = false);
     ~FitsLoader();
 
 private:
     std::string _unzip_file;
     casacore::uInt _hdu_num;
+    bool _is_http;
 
     void AllocateImage(const std::string& hdu) override;
     int GetNumImageHeaders(const std::string& filename, int hdu, std::string& error);

src/Session/Session.cc

@@ -24,6 +24,7 @@
 #include "FileList/FileInfoLoader.h"
 #include "FileList/FitsHduList.h"
 #include "ImageData/CompressedFits.h"
+#include "ImageData/FitsLoader.h"
 #include "ImageGenerators/ImageGenerator.h"
 #include "Logger/Logger.h"
 #include "OnMessageTask.h"
@@ -32,6 +33,7 @@
 #include "Util/App.h"
 #include "Util/File.h"
 #include "Util/Message.h"
+#include "Util/RemoteFiles.h"
 
 #ifdef _ARM_ARCH_
 #include <sse2neon/sse2neon.h>
@@ -53,6 +55,7 @@ Session::Session(uWS::WebSocket<false, true, PerSocketData>* ws, uWS::Loop* loop
       _loop(loop),
       _id(id),
       _address(address),
+      _remote_file_index(-1),
       _table_controller(std::make_unique<TableController>()),
       _region_handler(nullptr),
       _file_list_handler(file_list_handler),
@@ -2404,3 +2407,44 @@ void Session::CloseCachedImage(const std::string& directory, const std::string&
         }
     }
 }
+void Session::OnRemoteFileRequest(const CARTA::RemoteFileRequest& message, uint32_t request_id) {
+    auto file_id(message.file_id());
+
+    CARTA::RemoteFileResponse response;
+    std::string url, err_message;
+    bool success = GenerateUrlFromRequest(message, url, err_message);
+    if (success) {
+        spdlog::info("Fetching remote file from url {}", url);
+        auto loader = _loaders.Get(url, "");
+
+        CARTA::OpenFileAck ack;
+        try {
+            loader->OpenFile("0");
+
+            std::string remote_file_name;
+            auto index = ++_remote_file_index;
+            if (index > 0) {
+                remote_file_name = fmt::format("remote_file{}.fits", index);
+            } else {
+                remote_file_name = "remote_file.fits";
+            }
+            spdlog::info("Opening remote file: {}", remote_file_name);
+
+            auto image = loader->GetImage();
+
+            success = OnOpenFile(file_id, remote_file_name, image, response.mutable_open_file_ack());
+            if (success) {
+                response.set_message("File opened successfully");
+            }
+        } catch (const casacore::AipsError& err) {
+            err_message = err.getMesg();
+            response.set_message(err_message);
+            success = false;
+        }
+    } else {
+        response.set_message(err_message);
+    }
+    response.set_success(success);
+
+    SendEvent(CARTA::REMOTE_FILE_RESPONSE, request_id, response);
+}

src/Session/Session.h

@@ -95,6 +95,7 @@ class Session {
     void OnSetVectorOverlayParameters(const CARTA::SetVectorOverlayParameters& message);
     void OnStopPvPreview(const CARTA::StopPvPreview& stop_pv_preview);
     void OnClosePvPreview(const CARTA::ClosePvPreview& close_pv_preview);
+    void OnRemoteFileRequest(const CARTA::RemoteFileRequest& message, uint32_t request_id);
 
     void AddToSetChannelQueue(CARTA::SetImageChannels message, uint32_t request_id) {
         std::pair<CARTA::SetImageChannels, uint32_t> rp;
@@ -287,6 +288,9 @@ class Session {
     int _last_file_id;
     std::mutex _frame_mutex;
 
+    // Suffix for opening multiple remote files
+    int _remote_file_index;
+
     const std::unique_ptr<TableController> _table_controller;
 
     // Handler for region creation, import/export, requirements, and data

src/Session/SessionManager.cc

@@ -511,6 +511,14 @@ void SessionManager::OnMessage(WSType* ws, std::string_view sv_message, uWS::OpC
                     }
                     break;
                 }
+                case CARTA::EventType::REMOTE_FILE_REQUEST: {
+                    CARTA::RemoteFileRequest message;
+                    if (message.ParseFromArray(event_buf, event_length)) {
+                        session->OnRemoteFileRequest(message, head.request_id);
+                        message_parsed = true;
+                    }
+                    break;
+                }
                 default: {
                     spdlog::warn("Bad event type {}!", event_type);
                     break;

src/Util/File.cc

@@ -34,6 +34,10 @@ bool IsCompressedFits(const std::string& filename) {
     return false;
 }
 
+bool IsRemoteHttpFile(const std::string& filename) {
+    return filename.find("http") == 0;
+}
+
 bool IsGzMagicNumber(uint32_t magic_number) {
     std::string hex_string = fmt::format("{:#x}", magic_number);
     return (hex_string.length() > 4) && (hex_string.substr(hex_string.length() - 4) == "8b1f");

src/Util/File.h

@@ -31,6 +31,7 @@ CARTA::CatalogFileType GuessTableType(const std::string& path_string, bool check
 uint32_t GetMagicNumber(const std::string& filename);
 bool IsCompressedFits(const std::string& filename);
 bool IsGzMagicNumber(uint32_t magic_number);
+bool IsRemoteHttpFile(const std::string& filename);
 
 // directory functions
 int GetNumItems(const std::string& path);

src/Util/Message.cc

@@ -692,6 +692,23 @@ CARTA::ListProgress Message::ListProgress(
     message.set_percentage(percentage);
     return message;
 }
+CARTA::RemoteFileRequest Message::RemoteFileRequest(int32_t file_id, const string& hips, const string& wcs, int32_t width, int32_t height,
+    const string& projection, float fov, float ra, float dec, const string& coordsys, float rotation_angle, const string& object) {
+    CARTA::RemoteFileRequest message;
+    message.set_file_id(file_id);
+    message.set_hips(hips);
+    message.set_wcs(wcs);
+    message.set_width(width);
+    message.set_height(height);
+    message.set_projection(projection);
+    message.set_fov(fov);
+    message.set_ra(ra);
+    message.set_dec(dec);
+    message.set_coordsys(coordsys);
+    message.set_rotation_angle(rotation_angle);
+    message.set_object(object);
+    return message;
+}
 
 void FillHistogram(CARTA::Histogram* histogram, int32_t num_bins, double bin_width, double first_bin_center,
     const std::vector<int32_t>& bins, double mean, double std_dev) {

src/Util/Message.h

@@ -26,6 +26,7 @@
 #include <carta-protobuf/region_requirements.pb.h>
 #include <carta-protobuf/region_stats.pb.h>
 #include <carta-protobuf/register_viewer.pb.h>
+#include <carta-protobuf/remote_file_request.pb.h>
 #include <carta-protobuf/resume_session.pb.h>
 #include <carta-protobuf/save_file.pb.h>
 #include <carta-protobuf/scripting.pb.h>
@@ -133,6 +134,9 @@ class Message {
     static CARTA::PvRequest PvRequest(
         int32_t file_id, int32_t region_id, int32_t width, int z_min = -1, int32_t z_max = -1, bool reverse = false, bool keep = false);
     static CARTA::PvProgress PvProgress(int32_t file_id, float progress, int32_t preview_id = 0);
+    static CARTA::RemoteFileRequest RemoteFileRequest(int32_t file_id, const std::string& hips, const std::string& wcs, int32_t width,
+        int32_t height, const std::string& projection, float fov, float ra, float dec, const std::string& coordsys, float rotation_angle,
+        const std::string& object);
     static CARTA::FittingProgress FittingProgress(int32_t file_id, float progress);
     static CARTA::RegionHistogramData RegionHistogramData(
         int32_t file_id, int32_t region_id, int32_t channel, int32_t stokes, float progress, const carta::HistogramConfig& hist_config);