Skip to content

Commit

Permalink
added correction for corrupt files in offload
Browse files Browse the repository at this point in the history
  • Loading branch information
gferraro committed Aug 6, 2024
1 parent add2125 commit 96feb9f
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 87 deletions.
206 changes: 120 additions & 86 deletions src/core1_sub_tasks.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,111 +166,133 @@ pub fn offload_flash_storage_and_events(
// do some offloading.
let mut file_count = 0;
flash_storage.begin_offload();
let mut file_start = true;
let mut part_count = 0;
let mut success: bool = true;
let mut counter = timer.get_counter();
let mut start_block = 0;
let mut start_page = 0;
// TODO: Could speed this up slightly using cache_random_read interleaving on flash storage.
// Probably doesn't matter though.
while let Some(((part, crc, block_index, page_index), is_last, spi)) =
flash_storage.get_file_part()
{
if watchdog.is_some() {
watchdog.as_mut().unwrap().feed();
}
pi_spi.enable(spi, resets);
let transfer_type = if file_start && !is_last {
start_block = block_index;
start_page = 0;
ExtTransferMessage::BeginFileTransfer
} else if !file_start && !is_last {
ExtTransferMessage::ResumeFileTransfer
} else if is_last {
ExtTransferMessage::EndFileTransfer
} else if file_start && is_last {
ExtTransferMessage::BeginAndEndFileTransfer
} else {
crate::unreachable!("Invalid file transfer state");
};

let crc_check = Crc::<u16>::new(&CRC_16_XMODEM);
let current_crc = crc_check.checksum(&part);
if current_crc != crc {
warn!(
"Data corrupted at part #{} ({}:{}) in transfer to or from flash memory",
part_count, block_index, page_index
);
}

let mut attempts = 0;
'transfer_part: loop {
loop {
let mut file_start = true;
let mut part_count = 0;
let mut counter = timer.get_counter();
let mut start_block = 0;
let mut start_page = 0;
let mut file_end = false;
// TODO: Could speed this up slightly using cache_random_read interleaving on flash storage.
// Probably doesn't matter though.
while let Some(((part, crc, block_index, page_index), is_last, spi)) =
flash_storage.get_file_part()
{
file_end = is_last;
if watchdog.is_some() {
watchdog.as_mut().unwrap().feed();
}
let did_transfer =
pi_spi.send_message(transfer_type, &part, current_crc, dma, timer, resets);
counter = timer.get_counter();
if !did_transfer {
attempts += 1;
if attempts % 10 == 0 {
info!("Failed {}", attempts);
pi_spi.enable(spi, resets);
let transfer_type = if file_start && !is_last {
start_block = block_index;
start_page = 0;
ExtTransferMessage::BeginFileTransfer
} else if !file_start && !is_last {
ExtTransferMessage::ResumeFileTransfer
} else if is_last {
ExtTransferMessage::EndFileTransfer
} else if file_start && is_last {
ExtTransferMessage::BeginAndEndFileTransfer
} else {
crate::unreachable!("Invalid file transfer state");
};

let crc_check = Crc::<u16>::new(&CRC_16_XMODEM);
let current_crc = crc_check.checksum(&part);
if current_crc != crc {
warn!(
"Data corrupted at part #{} ({}:{}) in transfer to or from flash memory",
part_count, block_index, page_index
);
}

let mut attempts = 0;
'transfer_part: loop {
if watchdog.is_some() {
watchdog.as_mut().unwrap().feed();
}
if attempts > 100 {
success = false;
let did_transfer =
pi_spi.send_message(transfer_type, &part, current_crc, dma, timer, resets);
counter = timer.get_counter();
if !did_transfer {
attempts += 1;
if attempts % 10 == 0 {
info!("Failed {}", attempts);
}
if attempts > 100 {
success = false;
break 'transfer_part;
}
//takes tc2-agent about this long to poll again will fail a lot otherwise
let time_since = (timer.get_counter() - counter).to_micros();
if time_since < TIME_BETWEEN_TRANSFER {
delay.delay_us((TIME_BETWEEN_TRANSFER - time_since) as u32);
}
} else {
break 'transfer_part;
}
//takes tc2-agent about this long to poll again will fail a lot otherwise
let time_since = (timer.get_counter() - counter).to_micros();
if time_since < TIME_BETWEEN_TRANSFER {
delay.delay_us((TIME_BETWEEN_TRANSFER - time_since) as u32);
}
} else {
break 'transfer_part;
}
}

// Give spi peripheral back to flash storage.
if let Some(spi) = pi_spi.disable() {
flash_storage.take_spi(spi, resets, clock_freq.Hz());
// Give spi peripheral back to flash storage.
if let Some(spi) = pi_spi.disable() {
flash_storage.take_spi(spi, resets, clock_freq.Hz());
}
if !success {
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::FileOffloadFailed,
time.get_timestamp_micros(&timer),
),
flash_storage,
);
break;
} else if is_last {
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::OffloadedRecording(
(((start_block as u64) << 32) | start_page as u64) as u64,
),
time.get_timestamp_micros(&timer),
),
flash_storage,
);
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::OffloadedRecording(
(((block_index as u64) << 32) | page_index as u64) as u64,
),
time.get_timestamp_micros(&timer),
),
flash_storage,
);
}
part_count += 1;
if is_last {
file_count += 1;
info!("Offloaded {} file(s)", file_count);
file_start = true;
} else {
file_start = false;
}
}
if !success {
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::FileOffloadFailed,
time.get_timestamp_micros(&timer),
),
flash_storage,
);
break;
} else if is_last {
if !file_end && part_count > 0 {
//possible corrupt file
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::OffloadedRecording(
(((start_block as u64) << 32) | start_page as u64) as u64,
),
LoggerEventKind::CorruptFile,
time.get_timestamp_micros(&timer),
),
flash_storage,
);
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::OffloadedRecording(
(((block_index as u64) << 32) | page_index as u64) as u64,
),
time.get_timestamp_micros(&timer),
),
flash_storage,
info!(
"Corrupt file trying next block {}",
flash_storage.current_block_index + 1
);
}
part_count += 1;
if is_last {
file_count += 1;
info!("Offloaded {} file(s)", file_count);
file_start = true;
flash_storage.set_current_position(flash_storage.current_block_index + 1, 0);
} else {
file_start = false;
break;
}
}
if success {
Expand Down Expand Up @@ -340,9 +362,11 @@ pub fn offload_file(
let mut success: bool = true;
let mut counter;
let mut last_block: isize = 0;
let mut file_end = false;
while let Some(((part, crc, block_index, page_index), is_last, spi)) =
flash_storage.get_file_part()
{
file_end = is_last;
if watchdog.is_some() {
watchdog.as_mut().unwrap().feed();
}
Expand Down Expand Up @@ -418,6 +442,16 @@ pub fn offload_file(
}
file_start = false;
}
if !file_end && part_count > 0 {
//possible corrupt file
event_logger.log_event(
LoggerEvent::new(
LoggerEventKind::CorruptFile,
time.get_timestamp_micros(&timer),
),
flash_storage,
);
}

if success {
info!("Completed file offload, transferred {} files", file_count);
Expand Down
4 changes: 3 additions & 1 deletion src/event_logger.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ pub enum LoggerEventKind {
RecordingNotFinished,
FileOffloadFailed,
LogOffloadFailed,

OffloadedLogs,
CorruptFile,
}

impl Into<u16> for LoggerEventKind {
Expand Down Expand Up @@ -74,6 +74,7 @@ impl Into<u16> for LoggerEventKind {
FileOffloadFailed => 27,
OffloadedLogs => 28,
LogOffloadFailed => 29,
CorruptFile => 30,
}
}
}
Expand Down Expand Up @@ -113,6 +114,7 @@ impl TryFrom<u16> for LoggerEventKind {
27 => Ok(FileOffloadFailed),
28 => Ok(OffloadedLogs),
29 => Ok(LogOffloadFailed),
30 => Ok(CorruptFile),
_ => Err(()),
}
}
Expand Down

0 comments on commit 96feb9f

Please sign in to comment.