feat: add readahead routines (#18)

* feat(kernel/markov): add markov and markov state

* feat(kernel/memstat): add memory stats gathering

* refactor(kernel/utils): add bytes to kb conversion function

* feat(kernel/state): implement readahead

* refactor(kernel/memstat): clean up the code

* refactor(kernel/state): use Default instead of manually filling up fields

* perf(kernel): use rayon to speed up routines

* feat(kernel/map): add set_block method

* feat(kernel/utils): add readahead helper function

* refactor(kernel/state): use HashSet instead of BTreeSet

* refactor(kernel/memstat): rearrange method chain calls for better clarity

* refactor(kernel/state): move logging within conditional check

crates/kernel/Cargo.toml

@@ -4,11 +4,16 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bitflags = "2.6.0"
 config = { version = "0.1.0", path = "../config" }
 educe = { version = "0.6.0", default-features = false, features = ["Debug", "Eq", "Hash", "Ord", "PartialEq", "PartialOrd"] }
+humansize = "2.1.3"
+itertools = "0.13.0"
 libc = "0.2.159"
+nix = { version = "0.29.0", features = ["fs"] }
 parking_lot = "0.12.3"
 procfs = "0.16.0"
+rayon = "1.10.0"
 sysinfo = "0.31.4"
 thiserror.workspace = true
 tokio = { version = "1.40.0", features = ["sync", "time"] }
@@ -17,6 +22,11 @@ tracing.workspace = true
 [dev-dependencies]
 pretty_assertions = "1.4.1"
 proptest.workspace = true
+tempfile = "3.13.0"
+
+[features]
+default = []
+fiemap = []
 
 [lints]
 workspace = true

crates/kernel/src/error.rs

@@ -5,4 +5,13 @@ pub enum Error {
 
     #[error("Failed to read procfs info: {0}")]
     ProcfsReadFailed(#[from] procfs::ProcError),
+
+    #[error("Procfs field does not exist: {0}")]
+    ProcfsFieldDoesNotExist(String),
+
+    #[error("Failed to read file: {0}")]
+    FileReadFailed(#[from] std::io::Error),
+
+    #[error("Failed to readahead: {0}")]
+    ReadaheadFailed(#[from] nix::Error),
 }

crates/kernel/src/exe/inner.rs

@@ -2,6 +2,7 @@
 
 use crate::ExeMap;
 use educe::Educe;
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use std::{collections::HashSet, path::PathBuf};
 
 #[derive(Default, Clone, Educe)]
@@ -12,7 +13,7 @@ pub struct ExeInner {
     #[educe(Debug(ignore))]
     pub exemaps: HashSet<ExeMap>,
 
-    pub size: usize,
+    pub size: u64,
 
     pub seq: u64,
 
@@ -48,11 +49,11 @@ impl ExeInner {
 
     pub fn with_exemaps(&mut self, exemaps: HashSet<ExeMap>) -> &mut Self {
         self.exemaps = exemaps;
-        let size: usize = self
+        let size = self
             .exemaps
-            .iter()
+            .par_iter()
             .map(|map| map.map.length())
-            .fold(0usize, |acc, x| acc.wrapping_add(x));
+            .reduce(|| 0, |acc, x| acc.wrapping_add(x));
         self.size = self.size.wrapping_add(size);
         self
     }
@@ -67,10 +68,12 @@ impl ExeInner {
 
     pub fn bid_in_maps(&self, last_running_timestamp: u64) {
         if self.is_running(last_running_timestamp) {
-            self.exemaps.iter().for_each(|v| v.map.increase_lnprob(1.));
+            self.exemaps
+                .par_iter()
+                .for_each(|v| v.map.increase_lnprob(1.));
         } else {
             self.exemaps
-                .iter()
+                .par_iter()
                 .for_each(|v| v.map.set_lnprob(self.lnprob));
         }
     }

crates/kernel/src/exe/mod.rs

@@ -41,7 +41,7 @@ impl Exe {
         self.0.lock().lnprob = 0.0;
     }
 
-    pub fn size(&self) -> usize {
+    pub fn size(&self) -> u64 {
         self.0.lock().size
     }
 
@@ -81,8 +81,8 @@ mod tests {
     prop_compose! {
         fn arbitrary_map()(
             path in ".*",
-            offset in 0..usize::MAX,
-            length in 0..usize::MAX,
+            offset in 0..u64::MAX,
+            length in 0..u64::MAX,
         ) -> Map {
             Map::new(path, offset, length)
         }
@@ -98,10 +98,10 @@ mod tests {
     proptest! {
         #[test]
         fn exe_sums_map_sizes(exemaps in hash_set(arbitrary_exemap(), 0..2000)) {
-            let map_sizes: usize = exemaps
+            let map_sizes: u64 = exemaps
                 .iter()
                 .map(|m| m.map.length())
-                .fold(0usize, |acc, x| acc.wrapping_add(x));
+                .fold(0, |acc, x| acc.wrapping_add(x));
             let exe = Exe::new("foo").with_exemaps(exemaps);
             let exe_size = exe.size();
 

crates/kernel/src/lib.rs

@@ -2,11 +2,15 @@ mod error;
 mod exe;
 mod exemap;
 mod map;
+mod markov;
+mod memstat;
 mod state;
 pub mod utils;
 
 pub use error::Error;
 pub use exe::Exe;
 pub use exemap::ExeMap;
 pub use map::{Map, RuntimeStats};
+pub use markov::{Markov, MarkovState};
+pub use memstat::MemStat;
 pub use state::State;

crates/kernel/src/map/inner.rs

@@ -1,6 +1,7 @@
+use crate::Error;
 use educe::Educe;
 use parking_lot::Mutex;
-use std::path::PathBuf;
+use std::{os::linux::fs::MetadataExt, path::PathBuf};
 
 /// Runtime statistics related to the map.
 #[derive(Debug, Default, Clone, Copy)]
@@ -12,7 +13,7 @@ pub struct RuntimeStats {
     pub seq: u64,
 
     /// On-disk location of the start of the map.
-    pub block: u64,
+    pub block: Option<u64>,
     // private: u64,
 }
 
@@ -23,23 +24,37 @@ pub struct MapInner {
     pub path: PathBuf,
 
     /// Offset of the mapped section in bytes.
-    pub offset: usize,
+    pub offset: u64,
 
     /// Length of the mapped section in bytes.
-    pub length: usize,
+    pub length: u64,
 
     /// Runtime statistics related to the map.
     #[educe(Eq(ignore), Ord(ignore), Hash(ignore))]
     pub runtime: Mutex<RuntimeStats>,
 }
 
 impl MapInner {
-    pub fn new(path: impl Into<PathBuf>, offset: usize, length: usize) -> Self {
+    pub fn new(path: impl Into<PathBuf>, offset: u64, length: u64) -> Self {
         Self {
             path: path.into(),
             length,
             offset,
             ..Default::default()
         }
     }
+
+    /// For now the `use_inode` parameter does nothing.
+    pub fn set_block(&self) -> Result<(), Error> {
+        // in case we can get block, set 0 to not retry
+        self.runtime.lock().block = Some(0);
+        let meta = self.path.metadata()?;
+
+        #[cfg(feature = "fiemap")]
+        {
+            // TODO: if (!use_inode) { ... }
+        }
+        self.runtime.lock().block = Some(meta.st_ino());
+        Ok(())
+    }
 }

crates/kernel/src/map/mod.rs

@@ -1,16 +1,20 @@
 mod inner;
 
+use crate::Error;
 use inner::MapInner;
 pub use inner::RuntimeStats;
-use std::{path::PathBuf, sync::Arc};
+use std::{
+    path::{Path, PathBuf},
+    sync::Arc,
+};
 
 #[derive(Debug, Default, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
 pub struct Map {
     inner: Arc<MapInner>,
 }
 
 impl Map {
-    pub fn new(path: impl Into<PathBuf>, offset: usize, length: usize) -> Self {
+    pub fn new(path: impl Into<PathBuf>, offset: u64, length: u64) -> Self {
         Self {
             inner: Arc::new(MapInner::new(path, offset, length)),
         }
@@ -20,18 +24,26 @@ impl Map {
         self.inner.runtime.lock().lnprob
     }
 
+    pub fn path(&self) -> &Path {
+        &self.inner.path
+    }
+
     pub fn seq(&self) -> u64 {
         self.inner.runtime.lock().seq
     }
 
-    pub fn block(&self) -> u64 {
+    pub fn block(&self) -> Option<u64> {
         self.inner.runtime.lock().block
     }
 
-    pub fn length(&self) -> usize {
+    pub fn length(&self) -> u64 {
         self.inner.length
     }
 
+    pub fn offset(&self) -> u64 {
+        self.inner.offset
+    }
+
     pub fn set_seq(&self, seq: u64) {
         self.inner.runtime.lock().seq = seq;
     }
@@ -47,6 +59,10 @@ impl Map {
     pub fn set_lnprob(&self, lnprob: f32) {
         self.inner.runtime.lock().lnprob = lnprob;
     }
+
+    pub fn set_block(&self) -> Result<(), Error> {
+        self.inner.set_block()
+    }
 }
 
 #[cfg(test)]
@@ -58,8 +74,8 @@ mod tests {
     prop_compose! {
         fn arbitrary_map()(
             path in ".*",
-            offset in 0..usize::MAX,
-            length in 0..usize::MAX,
+            offset in 0..u64::MAX,
+            length in 0..u64::MAX,
             lnprob: f32,
             seq in 0..u64::MAX,
         ) -> Map {

crates/kernel/src/markov/markov_state.rs

@@ -0,0 +1,32 @@
+use bitflags::bitflags;
+
+bitflags! {
+    #[repr(transparent)]
+    #[derive(Debug, Default, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)]
+    pub struct MarkovState: u8 {
+        // avoiding zero-bit flag since it is always contained, but is never
+        // intersected
+        const NeitherRunning = 0b001;
+        const ExeARunning = 0b010;
+        const ExeBRunning = 0b100;
+        const BothRunning = 0b110;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn test_markov_state_flags() {
+        assert_eq!(
+            MarkovState::BothRunning,
+            MarkovState::ExeARunning | MarkovState::ExeBRunning
+        );
+        assert_eq!(
+            MarkovState::BothRunning | MarkovState::ExeARunning,
+            MarkovState::BothRunning,
+        );
+    }
+}