Skip to content
This repository has been archived by the owner on Nov 7, 2024. It is now read-only.

Commit

Permalink
chunking: Add prior build's packing structure as argument to
Browse files Browse the repository at this point in the history
basic_packing in preparation of its consumption
  • Loading branch information
RishabhSaini committed Apr 17, 2023
1 parent 6c3460a commit 21f074b
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 9 deletions.
24 changes: 17 additions & 7 deletions lib/src/chunking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -244,10 +244,11 @@ impl Chunking {
repo: &ostree::Repo,
rev: &str,
meta: ObjectMetaSized,
max_layers: Option<NonZeroU32>,
max_layers: &Option<NonZeroU32>,
prior_build_metadata: &Option<Vec<Vec<String>>>,
) -> Result<Self> {
let mut r = Self::new(repo, rev)?;
r.process_mapping(meta, max_layers)?;
r.process_mapping(meta, max_layers, prior_build_metadata)?;
Ok(r)
}

Expand All @@ -261,7 +262,8 @@ impl Chunking {
pub fn process_mapping(
&mut self,
meta: ObjectMetaSized,
max_layers: Option<NonZeroU32>,
max_layers: &Option<NonZeroU32>,
prior_build_metadata: &Option<Vec<Vec<String>>>,
) -> Result<()> {
self.max = max_layers
.unwrap_or(NonZeroU32::new(MAX_CHUNKS).unwrap())
Expand Down Expand Up @@ -292,7 +294,11 @@ impl Chunking {
.unwrap();

// TODO: Compute bin packing in a better way
let packing = basic_packing(sizes, NonZeroU32::new(self.max).unwrap());
let packing = basic_packing(
sizes,
NonZeroU32::new(self.max).unwrap(),
prior_build_metadata,
);

for bin in packing.into_iter() {
let first = bin[0];
Expand Down Expand Up @@ -395,7 +401,11 @@ fn sort_packing(packing: &mut [ChunkedComponents]) {
/// - Take the "tail" (all components past maximum), and group by source package
/// - If we have fewer components than bins, we're done
/// - Take the whole tail and group them toether (this is the overly simplistic part)
fn basic_packing(components: &[ObjectSourceMetaSized], bins: NonZeroU32) -> Vec<ChunkedComponents> {
fn basic_packing<'a>(
components: &'a [ObjectSourceMetaSized],
bins: NonZeroU32,
prior_build_metadata: &'a Option<Vec<Vec<String>>>,
) -> Vec<ChunkedComponents<'a>> {
// let total_size: u64 = components.iter().map(|v| v.size).sum();
// let avg_size: u64 = total_size / components.len() as u64;
let mut r = Vec::new();
Expand Down Expand Up @@ -451,7 +461,7 @@ mod test {
fn test_packing_basics() -> Result<()> {
// null cases
for v in [1u32, 7].map(|v| NonZeroU32::new(v).unwrap()) {
assert_eq!(basic_packing(&[], v).len(), 0);
assert_eq!(basic_packing(&[], v, &None).len(), 0);
}
Ok(())
}
Expand All @@ -462,7 +472,7 @@ mod test {
serde_json::from_reader(flate2::read::GzDecoder::new(FCOS_CONTENTMETA))?;
let total_size = contentmeta.iter().map(|v| v.size).sum::<u64>();

let packing = basic_packing(&contentmeta, NonZeroU32::new(MAX_CHUNKS).unwrap());
let packing = basic_packing(&contentmeta, NonZeroU32::new(MAX_CHUNKS).unwrap(), &None);
assert!(!contentmeta.is_empty());
// We should fit into the assigned chunk size
assert_eq!(packing.len() as u32, MAX_CHUNKS);
Expand Down
12 changes: 11 additions & 1 deletion lib/src/container/encapsulate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,15 @@ fn build_oci(
let mut manifest = ocidir::new_empty_manifest().build().unwrap();

let chunking = contentmeta
.map(|meta| crate::chunking::Chunking::from_mapping(repo, commit, meta, opts.max_layers))
.map(|meta| {
crate::chunking::Chunking::from_mapping(
repo,
commit,
meta,
&opts.max_layers,
&opts.prior_build_metadata,
)
})
.transpose()?;
// If no chunking was provided, create a logical single chunk.
let chunking = chunking
Expand Down Expand Up @@ -381,6 +389,8 @@ pub struct ExportOpts {
// TODO semver-break: remove this
/// Use only the standard OCI version label
pub no_legacy_version_label: bool,
/// Prevent major change in packaging structure by taking previous builds in order of priority
pub prior_build_metadata: Option<Vec<Vec<String>>>,
}

impl ExportOpts {
Expand Down
3 changes: 2 additions & 1 deletion lib/src/container/ocidir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,8 @@ mod tests {
let mut config = oci_image::ImageConfigurationBuilder::default()
.build()
.unwrap();
w.push_layer(&mut manifest, &mut config, root_layer, "root");
let annotations: Option<HashMap<String, String>> = None;
w.push_layer(&mut manifest, &mut config, root_layer, "root", annotations);
let config = w.write_config(config)?;
manifest.set_config(config);
w.replace_with_single_manifest(manifest.clone(), oci_image::Platform::default())?;
Expand Down

0 comments on commit 21f074b

Please sign in to comment.