Add prints to debug random issue

BlueBrain · Jul 9, 2024 · 4395943 · 4395943
1 parent 5602aad
commit 4395943
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 4 deletions.
diff --git a/src/data/node.cpp b/src/data/node.cpp
@@ -1,6 +1,8 @@
 #include <algorithm>
 #include <cassert>
 #include <stdexcept>
+#include <iostream>
+#include "../library/sonatareport.h"
 
 #include "node.h"
 
@@ -53,8 +55,45 @@ void Node::update_elements(std::vector<uint32_t> element_ids, std::vector<double
 }
 
 
-void Node::fill_data(std::vector<float>::iterator it) {
+void Node::fill_data(std::vector<float>& report, size_t position, double step) {
     assert(elements_.empty() || element_handles_.empty());
+
+    {
+        size_t num_elements_to_copy = 0;
+
+        if (!elements_.empty()) {
+            num_elements_to_copy = elements_.size();
+        } else if (!element_handles_.empty()) {
+            num_elements_to_copy = element_handles_.size();
+        }
+
+        size_t total_size_report = report.size();
+        size_t start_position = position;
+
+        if (start_position + num_elements_to_copy > total_size_report) {
+
+            size_t overflow = (start_position + num_elements_to_copy) - total_size_report;
+
+            std::cerr << "ERROR: Overflow detected in Node::fill_data()\n";
+            std::cerr << "Rank: " << SonataReport::rank_ << "\n";
+            std::cerr << "Report Allocated Size: " << total_size_report << "\n";
+            std::cerr << "Elements to Write: " << num_elements_to_copy << "\n";
+            std::cerr << "Overflow: " << overflow << "\n";
+
+            abort();
+        }
+    }
+
+    std::set<uint64_t> specific_node_ids = {2482561, 2537489, 2584861, 2644780, 2705687, 2761049, 2807052, 2854979, 2920895, 2974962, 3028685, 3074604, 3098646, 3116703};
+    if (step == 1360.0) {
+        if (specific_node_ids.find(node_id_) != specific_node_ids.end()) {
+            std::cout << "Rank: " << SonataReport::rank_ <<", Step: " << step << ", Node ID: " << node_id_ << ", Report size: " << report.size() << ", Position: " << position << std::endl;
+            for (const auto& elem : elements_) {
+                std::cout << " ** Pointer value: " << *elem << std::endl;
+            }
+        }
+    }
+    std::vector<float>::iterator it = report.begin() + position;
     if (!elements_.empty()) {
         std::transform(elements_.begin(), elements_.end(), it, [](auto elem) -> float {
             return *elem;

diff --git a/src/data/node.h b/src/data/node.h
@@ -13,7 +13,7 @@ class Node
   public:
     explicit Node(uint64_t node_id);
     virtual ~Node() = default;
-    void fill_data(std::vector<float>::iterator it);
+    void fill_data(std::vector<float>& report, size_t position, double step);
     void refresh_pointers(std::function<double*(double*)> refresh_function);
     virtual void add_element(double* element_value, uint32_t element_id);
     virtual void add_element(std::function<double()> element_value, uint32_t element_id);

diff --git a/src/data/sonata_data.cpp b/src/data/sonata_data.cpp
@@ -143,7 +143,7 @@ void SonataData::record_data(double step, const std::vector<uint64_t>& node_ids)
         uint64_t current_node_id = kv.second->get_node_id();
         // Check if node is set to be recorded (found in nodeids)
         if (std::find(node_ids.begin(), node_ids.end(), current_node_id) != node_ids.end()) {
-            kv.second->fill_data(report_buffer_.begin() + local_position);
+            kv.second->fill_data(report_buffer_, local_position, step);
             nodes_recorded_.insert(current_node_id);
         }
         local_position += kv.second->get_num_elements();
@@ -176,7 +176,7 @@ void SonataData::record_data(double step) {
             local_position);
     }
     for (auto& kv : *nodes_) {
-        kv.second->fill_data(report_buffer_.begin() + local_position);
+        kv.second->fill_data(report_buffer_, local_position, step);
         local_position += kv.second->get_num_elements();
     }
     current_step_++;
@@ -346,6 +346,12 @@ void SonataData::write_data(const std::vector<float>& buffered_data, uint32_t st
                       report_name_,
                       population_name_);
     }
+    if (SonataReport::rank_ == 62) {
+        logger->debug("First 14 elements of buffered data:");
+        for (size_t i = 0; i < std::min(buffered_data.size(), static_cast<size_t>(14)); ++i) {
+            logger->debug("Element {}: {}", i, buffered_data[i]);
+        }
+    }
     hdf5_writer_->write_2D(buffered_data, steps_to_write, total_elements_);
     remaining_steps_ -= steps_to_write;
     if (SonataReport::rank_ == 0) {