From bc79114a3ad57029185e4c1a8fe650f4c1bc7b47 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Wed, 24 Jan 2024 19:36:04 +0800 Subject: [PATCH 1/4] CA-372059: refactor the type of host in `squeeze.ml` use `Set` instead of `List` because `set_difference` implemented using `List` is quadratic cost Signed-off-by: Luca Zhang --- ocaml/squeezed/lib/squeeze.ml | 212 ++++++++-------- ocaml/squeezed/src/squeeze_xen.ml | 358 ++++++++++++++-------------- ocaml/squeezed/test/squeeze_test.ml | 10 +- 3 files changed, 288 insertions(+), 292 deletions(-) diff --git a/ocaml/squeezed/lib/squeeze.ml b/ocaml/squeezed/lib/squeeze.ml index e41575c1d97..33f24fd7e10 100644 --- a/ocaml/squeezed/lib/squeeze.ml +++ b/ocaml/squeezed/lib/squeeze.ml @@ -84,28 +84,20 @@ let domain_to_string_pairs (x : domain) = ; ("inaccuracy_kib", i64 x.inaccuracy_kib) ] -module IntMap = Map.Make (struct - type t = int +module DomainSet = Set.Make (struct + type t = domain - let compare = compare + let compare {domid= domid_a; _} {domid= domid_b; _} = + Int.compare domid_a domid_b end) (** Per-Host data *) type host = { - domains: domain list (** VMs running on this host *) - ; domid_to_domain: domain IntMap.t (** total free memory on this host *) - ; free_mem_kib: int64 + domains: DomainSet.t (** VMs running on this host *) + ; free_mem_kib: int64 (** total free memory on this host *) } -let make_host ~domains ~free_mem_kib = - { - domains - ; domid_to_domain= - List.fold_left - (fun map domain -> IntMap.add domain.domid domain map) - IntMap.empty domains - ; free_mem_kib - } +let make_host ~domains ~free_mem_kib = {domains; free_mem_kib} let string_pairs_to_string (x : (string * string) list) = String.concat "; " (List.map (fun (k, v) -> k ^ "=" ^ v) x) @@ -113,7 +105,12 @@ let string_pairs_to_string (x : (string * string) list) = let domain_to_string d = string_pairs_to_string (domain_to_string_pairs d) let host_to_string_pairs (x : host) = - let domains = String.concat "; " (List.map domain_to_string x.domains) in + let domains = + DomainSet.fold + (fun domain acc -> domain_to_string domain :: acc) + x.domains [] + |> String.concat "; " + in [ ("domains", "[" ^ domains ^ "]") ; ("free_mem_kib", Int64.to_string x.free_mem_kib) @@ -137,10 +134,6 @@ let ( +* ) = Int64.add let ( ** ) = Int64.mul -let set_difference a b = List.filter (fun x -> not (List.mem x b)) a - -let sum = List.fold_left ( +* ) 0L - (** The value returned from the 'free_memory' function *) type result = | Success (** enough memory is now available *) @@ -210,7 +203,7 @@ module Stuckness_monitor = struct (** Update our internal state given a snapshot of the outside world *) let update (x : t) (host : host) (now : float) = - List.iter + DomainSet.iter (fun (domain : domain) -> let direction = direction_of_actual domain.inaccuracy_kib domain.memory_actual_kib @@ -255,11 +248,16 @@ module Stuckness_monitor = struct host.domains ; (* Clear out dead domains just in case someone keeps *) (* one of these things around for a long time. *) - let live_domids = List.map (fun domain -> domain.domid) host.domains in + let module IntSet = Set.Make (Int) in + let live_domids = + DomainSet.fold + (fun d acc -> IntSet.add d.domid acc) + host.domains IntSet.empty + in let to_delete = Hashtbl.fold (fun domid _ acc -> - if List.mem domid live_domids then acc else domid :: acc + if IntSet.mem domid live_domids then acc else domid :: acc ) x.per_domain [] in @@ -335,7 +333,9 @@ module Proportional = struct Int64.of_float (gamma *. Int64.to_float (range domain)) in (* gamma = the proportion where 0 <= gamma <= 1 *) - let total_range = sum (List.map range domains) in + let total_range = + DomainSet.fold (fun domain acc -> range domain +* acc) domains Int64.zero + in let gamma' = Int64.to_float surplus_memory_kib /. Int64.to_float total_range in @@ -352,9 +352,11 @@ module Proportional = struct Int64.of_float (Int64.to_float total_range *. (gamma' -. gamma)) ) ) ; - List.map - (fun domain -> (domain, domain.dynamic_min_kib +* allocate gamma domain)) - domains + DomainSet.fold + (fun domain acc -> + (domain, domain.dynamic_min_kib +* allocate gamma domain) :: acc + ) + domains [] (* Given a set of domains and a host free memory target, return balloon target adjustments *) @@ -362,7 +364,9 @@ module Proportional = struct host_target_kib = (* If all domains balloon down to dynamic_min: *) let maximum_free_mem_kib = - host.free_mem_kib +* sum (List.map (min_freeable ?fistpoints) host.domains) + DomainSet.fold + (fun domain acc -> acc +* min_freeable ?fistpoints domain) + host.domains host.free_mem_kib in let surplus_memory_kib = max 0L (maximum_free_mem_kib -* host_target_kib) in allocate_memory_in_proportion verbose surplus_memory_kib host.domains @@ -374,10 +378,11 @@ module Squeezer = struct (** State maintained between invocations of the algorithm function *) type t = { stuckness: Stuckness_monitor.t - ; non_active_domids: int list (* domids are unique and constant *) + ; non_active_domains: DomainSet.t (* domids are unique and constant *) } - let make () = {stuckness= Stuckness_monitor.make (); non_active_domids= []} + let make () = + {stuckness= Stuckness_monitor.make (); non_active_domains= DomainSet.empty} (** Takes a view of the host state and amount of free memory desired and returns a list of ballooning actions which may help achieve the goal. *) @@ -386,31 +391,29 @@ module Squeezer = struct (* 1. Compute which domains are still considered active *) Stuckness_monitor.update x.stuckness host now ; let active_domains = - List.filter + DomainSet.filter (fun domain -> domain.can_balloon && Stuckness_monitor.domid_is_active x.stuckness domain.domid now ) host.domains in - let non_active_domids = - List.map (fun d -> d.domid) (set_difference host.domains active_domains) - in - let declared_inactive_domids = - set_difference non_active_domids x.non_active_domids + let non_active_domains = DomainSet.diff host.domains active_domains in + let declared_inactive_domains = + DomainSet.diff non_active_domains x.non_active_domains in - let declared_active_domids = - set_difference x.non_active_domids non_active_domids + let declared_active_domains = + DomainSet.diff x.non_active_domains non_active_domains in if verbose then ( - List.iter - (fun d -> debug "domid %d has been declared inactive" d) - declared_inactive_domids ; - List.iter - (fun d -> debug "domid %d has been declared active" d) - declared_active_domids + DomainSet.iter + (fun d -> debug "domid %d has been declared inactive" d.domid) + declared_inactive_domains ; + DomainSet.iter + (fun d -> debug "domid %d has been declared active" d.domid) + declared_active_domains ) ; - let x = {x with non_active_domids} in + let x = {x with non_active_domains} in (* 2. Compute how we would adjust the domain memory targets *) let targets = Policy.compute_target_adjustments ~fistpoints verbose @@ -418,8 +421,9 @@ module Squeezer = struct host_target_kib in let maximum_possible_free_memory_kib = - host.free_mem_kib - +* sum (List.map (min_freeable ~fistpoints) active_domains) + DomainSet.fold + (fun domain acc -> min_freeable ~fistpoints domain +* acc) + active_domains host.free_mem_kib in if verbose then debug @@ -446,7 +450,7 @@ module Squeezer = struct freeing in (* Have all the non-stuck domains reached their current targets? *) - let all p xs = List.fold_left ( && ) true (List.map p xs) in + let all p xs = DomainSet.for_all p xs in let hit_target domain = has_hit_target domain.inaccuracy_kib domain.memory_actual_kib domain.target_kib @@ -463,8 +467,9 @@ module Squeezer = struct let success = success_condition host.free_mem_kib in let target_too_big = maximum_possible_free_memory_kib < host_target_kib in let no_target_changes = - List.filter (fun (domain, target) -> domain.target_kib <> target) targets - = [] + List.for_all + (fun ({target_kib; _}, target) -> target_kib = target) + targets in if verbose then debug @@ -485,20 +490,10 @@ module Squeezer = struct (if allocation_phase then "allocation phase" else "freeing phase") ; (* If we have to blame a domain for being stuck, we don't blame it if it can't balloon. *) - let non_active_domains = - List.concat - (List.map - (fun d -> - try [IntMap.find d host.domid_to_domain] with Not_found -> [] - ) - non_active_domids - ) - in - let non_active_can_balloon_domains = - List.filter (fun d -> d.can_balloon) non_active_domains - in let non_active_can_balloon_domids = - List.map (fun d -> d.domid) non_active_can_balloon_domains + DomainSet.fold + (fun d acc -> if d.can_balloon then d.domid :: acc else acc) + non_active_domains [] in (* 1. In all cases we wait for all targets to be reached and to be stable. 2. If targets are reached and stable we evaluate our success condition @@ -515,7 +510,7 @@ module Squeezer = struct else Failed non_active_can_balloon_domids in - (x, declared_active_domids, declared_inactive_domids, action) + (x, declared_active_domains, declared_inactive_domains, action) end module Gnuplot = struct @@ -535,7 +530,7 @@ module Gnuplot = struct let write_row oc host cols time = Printf.fprintf oc "%.2f %Ld " time host.free_mem_kib ; - List.iter + DomainSet.iter (fun domain -> if List.mem Dynamic_min cols then Printf.fprintf oc " %Ld " domain.dynamic_min_kib ; @@ -556,33 +551,33 @@ module Gnuplot = struct Printf.fprintf oc "plot \"%s.dat\" using 1:2 title \"free host memory\" with lines " stem ; let col = ref 3 in - List.iter - (fun domain -> + DomainSet.iter + (fun {domid; _} -> if List.mem Dynamic_min cols then ( Printf.fprintf oc ", \"%s.dat\" using 1:%d title \"domid %d dynamic_min\" with \ points " - stem !col domain.domid ; + stem !col domid ; incr col ) ; if List.mem Dynamic_max cols then ( Printf.fprintf oc ", \"%s.dat\" using 1:%d title \"domid %d dynamic_max\" with \ points " - stem !col domain.domid ; + stem !col domid ; incr col ) ; if List.mem Memory_actual cols then ( Printf.fprintf oc ", \"%s.dat\" using 1:%d title \"domid %d memory_actual\" with \ lines " - stem !col domain.domid ; + stem !col domid ; incr col ) ; if List.mem Target cols then ( Printf.fprintf oc ", \"%s.dat\" using 1:%d title \"domid %d target\" with lines " stem - !col domain.domid ; + !col domid ; incr col ) ) @@ -617,13 +612,13 @@ let change_host_free_memory ?fistpoints io required_mem_kib success_condition = free_mem = %Ld KiB" required_mem_kib io.target_host_free_mem_kib host.free_mem_kib ; let active_domains = - List.filter (fun domain -> domain.can_balloon) host.domains + DomainSet.filter (fun domain -> domain.can_balloon) host.domains in let hit_target domain = has_hit_target domain.inaccuracy_kib domain.memory_actual_kib domain.target_kib in - let all_targets_reached = List.for_all hit_target active_domains in + let all_targets_reached = DomainSet.for_all hit_target active_domains in if io.verbose then debug "change_host_free_memory all VM target meet %B" all_targets_reached ; let finished = @@ -637,7 +632,7 @@ let change_host_free_memory ?fistpoints io required_mem_kib success_condition = while not !finished do let t = io.gettimeofday () in let host_debug_string, host = io.make_host () in - let acc', _declared_active_domids, declared_inactive_domids, result = + let acc', _declared_active_domains, declared_inactive_domains, result = Squeezer.one_iteration ?fistpoints io.verbose success_condition !acc host required_mem_kib t in @@ -671,48 +666,47 @@ let change_host_free_memory ?fistpoints io required_mem_kib success_condition = [] in let new_target_direction domain = - let new_target = - if List.mem_assoc domain.domid new_targets then - Some (List.assoc domain.domid new_targets) - else - None - in - match new_target with + match List.assoc_opt domain.domid new_targets with | None -> string_of_direction None | Some t -> string_of_direction (direction_of_int64 domain.target_kib t) in let debug_string = - String.concat "; " - (host_debug_string - :: List.map - (fun domain -> - short_string_of_domain domain ^ new_target_direction domain - ) - host.domains - ) + let domain_infos = + DomainSet.fold + (fun domain acc -> + let info = + short_string_of_domain domain ^ new_target_direction domain + in + info :: acc + ) + host.domains [] + in + String.concat "; " (host_debug_string :: domain_infos) in debug "%s" debug_string ; (* For each domid, decide what maxmem should be *) - let maxmems = - IntMap.mapi - (fun domid domain -> - if List.mem domid declared_inactive_domids then + DomainSet.iter + (fun domain -> + let mem = + if DomainSet.mem domain declared_inactive_domains then (* CA-41832: clip the target of an 'inactive' domain to within the dynamic min-max range. The danger here is that a domain might be using less than dynamic min now, but might suddenly wake up and allocate memory belonging to someone else later. *) let ideal_kib = min domain.target_kib domain.memory_actual_kib in min domain.dynamic_max_kib (max domain.dynamic_min_kib ideal_kib) - else if List.mem_assoc domid new_targets then - List.assoc domid new_targets else - domain.target_kib - ) - host.domid_to_domain - in - IntMap.iter io.domain_setmaxmem maxmems ; + match List.assoc_opt domain.domid new_targets with + | None -> + domain.target_kib + | Some t -> + t + in + io.domain_setmaxmem domain.domid mem + ) + host.domains ; match result with | Success -> if io.verbose then @@ -771,16 +765,13 @@ let free_memory_range ?fistpoints io min_kib max_kib = } in let host = snd (io.make_host ()) in - let host' = {host with domains= domain :: host.domains} in + let host' = {host with domains= DomainSet.add domain host.domains} in let adjustments = Policy.compute_target_adjustments io.verbose host' io.target_host_free_mem_kib in let target = - if List.mem_assoc domain adjustments then - List.assoc domain adjustments - else - min_kib + match List.assoc_opt domain adjustments with None -> min_kib | Some a -> a in debug "free_memory_range ideal target = %Ld" target ; change_host_free_memory ?fistpoints io (target +* io.target_host_free_mem_kib) @@ -807,18 +798,21 @@ let balance_memory ?fistpoints io = (** Return true if the host memory is currently unbalanced and needs rebalancing *) let is_host_memory_unbalanced ?fistpoints io = let debug_string, host = io.make_host () in - let domains = List.map (fun d -> (d.domid, d)) host.domains in let _, _, _, result = Squeezer.one_iteration ?fistpoints false (is_balanced io) (Squeezer.make ()) host io.target_host_free_mem_kib (io.gettimeofday ()) in let is_new_target a = - let existing = (List.assoc a.action_domid domains).target_kib in - a.new_target_kib <> existing + let existing_target_kib = + host.domains + |> DomainSet.find_first_opt (fun d -> a.action_domid = d.domid) + |> Option.fold ~none:Int64.minus_one ~some:(fun d -> d.target_kib) + in + a.new_target_kib <> existing_target_kib in match result with | AdjustTargets ts -> - if List.fold_left ( || ) false (List.map is_new_target ts) then ( + if List.exists is_new_target ts then ( debug "Memory is not currently balanced" ; debug "%s" debug_string ; true diff --git a/ocaml/squeezed/src/squeeze_xen.ml b/ocaml/squeezed/src/squeeze_xen.ml index 667108cdf5c..f4ba7e5accd 100644 --- a/ocaml/squeezed/src/squeeze_xen.ml +++ b/ocaml/squeezed/src/squeeze_xen.ml @@ -23,6 +23,9 @@ open Squeezed_xenstore let with_lock = Xapi_stdext_threads.Threadext.Mutex.execute +module IntSet = Set.Make (Int) +module DomainSet = Squeeze.DomainSet + module D = Debug.Make (struct let name = "squeeze_xen" end) open D @@ -57,8 +60,6 @@ let ( -* ) = Int64.sub let mib = 1024L -let set_difference a b = List.filter (fun x -> not (List.mem x b)) a - (** Same as xen commandline *) let low_mem_emergency_pool = 1L ** mib @@ -147,11 +148,15 @@ module Domain = struct let current_domains = Xenctrl.domain_getinfolist xc 0 in with_lock m (fun () -> let alive_domids = - List.map (fun d -> d.Xenctrl.domid) current_domains + List.fold_left + (fun acc d -> IntSet.add d.Xenctrl.domid acc) + IntSet.empty current_domains in - let known_domids = Hashtbl.fold (fun k _ acc -> k :: acc) cache [] in - let gone_domids = set_difference known_domids alive_domids in - List.iter + let known_domids = + Hashtbl.fold (fun k _ acc -> IntSet.add k acc) cache IntSet.empty + in + let gone_domids = IntSet.diff known_domids alive_domids in + IntSet.iter (fun d -> debug "Remove domid %d in cache" d ; Hashtbl.remove cache d @@ -186,11 +191,6 @@ module Domain = struct ) interesting_paths in - let module IntSet = Set.Make (struct - type t = int - - let compare = compare - end) in let watching_domids = ref IntSet.empty in let watch_token domid = Printf.sprintf "squeezed:domain-%d" domid in let look_for_different_domains () = @@ -527,20 +527,27 @@ let when_domain_was_last_cooperative : (int, float) Hashtbl.t = let update_cooperative_table host = let now = Unix.gettimeofday () in - let alive_domids = List.map (fun d -> d.Squeeze.domid) host.Squeeze.domains in + let domains = host.Squeeze.domains in + let alive_domids = + DomainSet.fold + (fun d acc -> IntSet.add d.Squeeze.domid acc) + domains IntSet.empty + in let known_domids = - Hashtbl.fold (fun k _ acc -> k :: acc) when_domain_was_last_cooperative [] + Hashtbl.fold + (fun k _ acc -> IntSet.add k acc) + when_domain_was_last_cooperative IntSet.empty in - let gone_domids = set_difference known_domids alive_domids in - List.iter (Hashtbl.remove when_domain_was_last_cooperative) gone_domids ; - let arrived_domids = set_difference alive_domids known_domids in + let gone_domids = IntSet.diff known_domids alive_domids in + IntSet.iter (Hashtbl.remove when_domain_was_last_cooperative) gone_domids ; + let arrived_domids = IntSet.diff alive_domids known_domids in (* Assume domains are initially co-operative *) - List.iter + IntSet.iter (fun x -> Hashtbl.replace when_domain_was_last_cooperative x now) arrived_domids ; (* Main business: mark any domain which is on or above target OR which cannot balloon as co-operative *) - List.iter + DomainSet.iter (fun d -> if (not d.Squeeze.can_balloon) @@ -549,7 +556,7 @@ let update_cooperative_table host = then Hashtbl.replace when_domain_was_last_cooperative d.Squeeze.domid now ) - host.Squeeze.domains + domains (** Update all the flags in xenstore *) let update_cooperative_flags cnx = @@ -604,157 +611,151 @@ let make_host ~verbose ~xc = in let cnx = xc in let domains = - List.concat - (List.map - (fun di -> - try - let domain_type = Domain.get_domain_type cnx di.Xenctrl.domid in - let memory_actual_kib = - Xenctrl.pages_to_kib - (Int64.of_nativeint di.Xenctrl.total_memory_pages) - in - let memory_shadow_kib = - match domain_type with - | `hvm | `pvh | `pv_in_pvh -> ( - try - Memory.kib_of_mib - (Int64.of_int - (Xenctrl.shadow_allocation_get xc di.Xenctrl.domid) - ) - with _ -> 0L - ) - | `pv -> - 0L - in - (* dom0 is special for some reason *) - let memory_max_kib = - if di.Xenctrl.domid = 0 then - 0L - else - Domain.maxmem_of_dominfo di domain_type - in - let can_balloon = - Domain.get_feature_balloon cnx di.Xenctrl.domid - in - let has_guest_agent = - Domain.get_guest_agent cnx di.Xenctrl.domid - in - let has_booted = can_balloon || has_guest_agent in - (* Once the domain tells us it has booted, we assume it's not - currently ballooning and record the offset between memory_actual - and target. We assume this is constant over the lifetime of the - domain. *) - let offset_kib : int64 = - if not has_booted then - 0L - else - try Domain.get_memory_offset cnx di.Xenctrl.domid - with Xs_protocol.Enoent _ -> - (* Our memory_actual_kib value was sampled before reading - xenstore which means there is a slight race. The race is - probably only noticable in the hypercall simulator. - However we can fix it by resampling memory_actual *after* - noticing the feature-balloon flag. *) - let target_kib = Domain.get_target cnx di.Xenctrl.domid in - let memory_actual_kib' = - Xenctrl.pages_to_kib - (Int64.of_nativeint - (Xenctrl.domain_getinfo xc di.Xenctrl.domid) - .Xenctrl.total_memory_pages - ) - in - let offset_kib = memory_actual_kib' -* target_kib in - debug "domid %d just %s; calibrating memory-offset = %Ld KiB" - di.Xenctrl.domid - ( match (can_balloon, has_guest_agent) with - | true, false -> - "advertised a balloon driver" - | true, true -> - "started a guest agent and advertised a balloon driver" - | false, true -> - "started a guest agent (but has no balloon driver)" - | false, false -> - "N/A" (*impossible: see if has_booted above *) - ) - offset_kib ; - Domain.set_memory_offset_noexn cnx di.Xenctrl.domid - offset_kib ; - offset_kib - in - let memory_actual_kib = memory_actual_kib -* offset_kib in - let domain = - { - Squeeze.domid= di.Xenctrl.domid - ; can_balloon - ; dynamic_min_kib= 0L - ; dynamic_max_kib= 0L - ; target_kib= 0L - ; memory_actual_kib= 0L - ; memory_max_kib - ; inaccuracy_kib= 4L - } - in - - (* If the domain has never run (detected by being paused, not - shutdown and clocked up no CPU time) then we'll need to consider - the domain's "initial-reservation". Note that the other fields - won't necessarily have been created yet. *) - - (* If the domain has yet to boot properly then we assume it is - using at least its "initial-reservation". *) - if not has_booted then ( - let initial_reservation_kib = - Domain.get_initial_reservation cnx di.Xenctrl.domid - in - let unaccounted_kib = - max 0L - (initial_reservation_kib - -* memory_actual_kib - -* memory_shadow_kib - ) - in - reserved_kib := Int64.add !reserved_kib unaccounted_kib ; - [ - { - domain with - Squeeze.dynamic_min_kib= memory_max_kib - ; dynamic_max_kib= memory_max_kib - ; target_kib= memory_max_kib - ; memory_actual_kib= memory_max_kib - } - ] - ) else - let target_kib = Domain.get_target cnx di.Xenctrl.domid in - (* min and max are written separately; if we notice they *) - (* are missing set them both to the target for now. *) - let min_kib, max_kib = - try - ( Domain.get_dynamic_min cnx di.Xenctrl.domid - , Domain.get_dynamic_max cnx di.Xenctrl.domid - ) - with _ -> (target_kib, target_kib) - in - [ - { - domain with - Squeeze.dynamic_min_kib= min_kib - ; dynamic_max_kib= max_kib - ; target_kib - ; memory_actual_kib - } - ] - with - | Xs_protocol.Enoent _ -> - (* useful debug message is printed by the Domain.read* functions *) - [] - | e -> - if verbose then - debug "Skipping domid %d: %s" di.Xenctrl.domid - (Printexc.to_string e) ; - [] - ) - domain_infolist + List.fold_left + (fun acc di -> + try + let domain_type = Domain.get_domain_type cnx di.Xenctrl.domid in + let memory_actual_kib = + Xenctrl.pages_to_kib + (Int64.of_nativeint di.Xenctrl.total_memory_pages) + in + let memory_shadow_kib = + match domain_type with + | `hvm | `pvh | `pv_in_pvh -> ( + try + Memory.kib_of_mib + (Int64.of_int + (Xenctrl.shadow_allocation_get xc di.Xenctrl.domid) + ) + with _ -> 0L + ) + | `pv -> + 0L + in + (* dom0 is special for some reason *) + let memory_max_kib = + if di.Xenctrl.domid = 0 then + 0L + else + Domain.maxmem_of_dominfo di domain_type + in + let can_balloon = Domain.get_feature_balloon cnx di.Xenctrl.domid in + let has_guest_agent = Domain.get_guest_agent cnx di.Xenctrl.domid in + let has_booted = can_balloon || has_guest_agent in + (* Once the domain tells us it has booted, we assume it's not + currently ballooning and record the offset between memory_actual + and target. We assume this is constant over the lifetime of the + domain. *) + let offset_kib : int64 = + if not has_booted then + 0L + else + try Domain.get_memory_offset cnx di.Xenctrl.domid + with Xs_protocol.Enoent _ -> + (* Our memory_actual_kib value was sampled before reading + xenstore which means there is a slight race. The race is + probably only noticable in the hypercall simulator. + However we can fix it by resampling memory_actual *after* + noticing the feature-balloon flag. *) + let target_kib = Domain.get_target cnx di.Xenctrl.domid in + let memory_actual_kib' = + Xenctrl.pages_to_kib + (Int64.of_nativeint + (Xenctrl.domain_getinfo xc di.Xenctrl.domid) + .Xenctrl.total_memory_pages + ) + in + let offset_kib = memory_actual_kib' -* target_kib in + debug "domid %d just %s; calibrating memory-offset = %Ld KiB" + di.Xenctrl.domid + ( match (can_balloon, has_guest_agent) with + | true, false -> + "advertised a balloon driver" + | true, true -> + "started a guest agent and advertised a balloon driver" + | false, true -> + "started a guest agent (but has no balloon driver)" + | false, false -> + "N/A" (*impossible: see if has_booted above *) + ) + offset_kib ; + Domain.set_memory_offset_noexn cnx di.Xenctrl.domid offset_kib ; + offset_kib + in + let memory_actual_kib = memory_actual_kib -* offset_kib in + let domain = + { + Squeeze.domid= di.Xenctrl.domid + ; can_balloon + ; dynamic_min_kib= 0L + ; dynamic_max_kib= 0L + ; target_kib= 0L + ; memory_actual_kib= 0L + ; memory_max_kib + ; inaccuracy_kib= 4L + } + in + + (* If the domain has never run (detected by being paused, not + shutdown and clocked up no CPU time) then we'll need to consider + the domain's "initial-reservation". Note that the other fields + won't necessarily have been created yet. *) + + (* If the domain has yet to boot properly then we assume it is + using at least its "initial-reservation". *) + if not has_booted then ( + let initial_reservation_kib = + Domain.get_initial_reservation cnx di.Xenctrl.domid + in + let unaccounted_kib = + max 0L + (initial_reservation_kib + -* memory_actual_kib + -* memory_shadow_kib + ) + in + reserved_kib := Int64.add !reserved_kib unaccounted_kib ; + + DomainSet.add + { + domain with + Squeeze.dynamic_min_kib= memory_max_kib + ; dynamic_max_kib= memory_max_kib + ; target_kib= memory_max_kib + ; memory_actual_kib= memory_max_kib + } + acc + ) else + let target_kib = Domain.get_target cnx di.Xenctrl.domid in + (* min and max are written separately; if we notice they *) + (* are missing set them both to the target for now. *) + let min_kib, max_kib = + try + ( Domain.get_dynamic_min cnx di.Xenctrl.domid + , Domain.get_dynamic_max cnx di.Xenctrl.domid + ) + with _ -> (target_kib, target_kib) + in + DomainSet.add + { + domain with + Squeeze.dynamic_min_kib= min_kib + ; dynamic_max_kib= max_kib + ; target_kib + ; memory_actual_kib + } + acc + with + | Xs_protocol.Enoent _ -> + (* useful debug message is printed by the Domain.read* functions *) + acc + | e -> + if verbose then + debug "Skipping domid %d: %s" di.Xenctrl.domid + (Printexc.to_string e) ; + acc ) + DomainSet.empty domain_infolist in (* For the host free memory we sum the free pages and the pages needing scrubbing: we don't want to adjust targets simply because the scrubber is @@ -786,17 +787,14 @@ let make_host ~verbose ~xc = (* It's always safe to _decrease_ a domain's maxmem towards target. This catches the case where a toolstack creates a domain with maxmem = static_max and target < static_max (eg CA-36316) *) - let updates = - Squeeze.IntMap.fold - (fun domid domain updates -> - if domain.Squeeze.target_kib < Domain.get_maxmem xc domid then - Squeeze.IntMap.add domid domain.Squeeze.target_kib updates - else - updates - ) - host.Squeeze.domid_to_domain Squeeze.IntMap.empty - in - Squeeze.IntMap.iter (Domain.set_maxmem_noexn xc) updates ; + DomainSet.iter + (fun domain -> + let domid = domain.Squeeze.domid + and target_kib = domain.Squeeze.target_kib in + if target_kib < Domain.get_maxmem xc domid then + Domain.set_maxmem_noexn xc domid target_kib + ) + domains ; ( Printf.sprintf "F%Ld S%Ld R%Ld T%Ld" free_pages_kib scrub_pages_kib !reserved_kib total_pages_kib , host diff --git a/ocaml/squeezed/test/squeeze_test.ml b/ocaml/squeezed/test/squeeze_test.ml index c1af6b9bddb..774caa2f088 100644 --- a/ocaml/squeezed/test/squeeze_test.ml +++ b/ocaml/squeezed/test/squeeze_test.ml @@ -384,7 +384,7 @@ let verify_memory_is_guaranteed_free host kib = considered ok *) let extreme domain = domain.target_kib +* domain.inaccuracy_kib in let increase domain = extreme domain -* domain.memory_actual_kib in - let total = List.fold_left ( +* ) 0L (List.map increase host.domains) in + let total = DomainSet.fold (fun d acc -> increase d +* acc) host.domains 0L in if host.free_mem_kib -* total < kib then failwith (Printf.sprintf @@ -441,8 +441,12 @@ let simulate scenario = let i = ref 0 in let gettimeofday () = float_of_int !i /. 10. in let make_host () = - Squeeze.make_host ~free_mem_kib:!host_free_mem_kib - ~domains:(List.map (fun d -> d#get_domain) !all_domains) + let domains = + List.fold_left + (fun acc d -> DomainSet.add d#get_domain acc) + DomainSet.empty !all_domains + in + Squeeze.make_host ~free_mem_kib:!host_free_mem_kib ~domains in let wait _ = incr i ; From d49f33bda88dd41d2c4fdd8b08e43d456889def2 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Thu, 25 Jan 2024 11:18:34 +0800 Subject: [PATCH 2/4] CA-372059: delete the unused code Signed-off-by: Luca Zhang --- ocaml/squeezed/lib/squeeze.ml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ocaml/squeezed/lib/squeeze.ml b/ocaml/squeezed/lib/squeeze.ml index 33f24fd7e10..aa29982a6b8 100644 --- a/ocaml/squeezed/lib/squeeze.ml +++ b/ocaml/squeezed/lib/squeeze.ml @@ -99,23 +99,6 @@ type host = { let make_host ~domains ~free_mem_kib = {domains; free_mem_kib} -let string_pairs_to_string (x : (string * string) list) = - String.concat "; " (List.map (fun (k, v) -> k ^ "=" ^ v) x) - -let domain_to_string d = string_pairs_to_string (domain_to_string_pairs d) - -let host_to_string_pairs (x : host) = - let domains = - DomainSet.fold - (fun domain acc -> domain_to_string domain :: acc) - x.domains [] - |> String.concat "; " - in - [ - ("domains", "[" ^ domains ^ "]") - ; ("free_mem_kib", Int64.to_string x.free_mem_kib) - ] - (** The ballooning algorithm returns a list of actions to perform *) type action = { action_domid: int (** domid of domain to operate on *) From 5f6c195ca079da45de7ace66f43b4178d8ded4fb Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Fri, 26 Jan 2024 10:52:16 +0800 Subject: [PATCH 3/4] CA-372059: use `Opt.value` instead of `match` and use `find_opt` to find a domain Signed-off-by: Luca Zhang --- ocaml/squeezed/lib/squeeze.ml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ocaml/squeezed/lib/squeeze.ml b/ocaml/squeezed/lib/squeeze.ml index aa29982a6b8..4816b6fc794 100644 --- a/ocaml/squeezed/lib/squeeze.ml +++ b/ocaml/squeezed/lib/squeeze.ml @@ -681,11 +681,8 @@ let change_host_free_memory ?fistpoints io required_mem_kib success_condition = let ideal_kib = min domain.target_kib domain.memory_actual_kib in min domain.dynamic_max_kib (max domain.dynamic_min_kib ideal_kib) else - match List.assoc_opt domain.domid new_targets with - | None -> - domain.target_kib - | Some t -> - t + List.assoc_opt domain.domid new_targets + |> Option.value ~default:domain.target_kib in io.domain_setmaxmem domain.domid mem ) @@ -754,7 +751,7 @@ let free_memory_range ?fistpoints io min_kib max_kib = io.target_host_free_mem_kib in let target = - match List.assoc_opt domain adjustments with None -> min_kib | Some a -> a + List.assoc_opt domain adjustments |> Option.value ~default:min_kib in debug "free_memory_range ideal target = %Ld" target ; change_host_free_memory ?fistpoints io (target +* io.target_host_free_mem_kib) @@ -788,10 +785,10 @@ let is_host_memory_unbalanced ?fistpoints io = let is_new_target a = let existing_target_kib = host.domains - |> DomainSet.find_first_opt (fun d -> a.action_domid = d.domid) - |> Option.fold ~none:Int64.minus_one ~some:(fun d -> d.target_kib) + |> DomainSet.find_opt (domain_make a.action_domid false 0L 0L 0L 0L 0L 0L) + |> Option.map (fun d -> d.target_kib) in - a.new_target_kib <> existing_target_kib + Some a.new_target_kib <> existing_target_kib in match result with | AdjustTargets ts -> From 229d48619d3196af676de929863beb6820b387e2 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Fri, 26 Jan 2024 10:54:27 +0800 Subject: [PATCH 4/4] CA-372059: add an interface for squeeze.ml Signed-off-by: Luca Zhang --- ocaml/squeezed/lib/squeeze.ml | 35 ------------- ocaml/squeezed/lib/squeeze.mli | 92 ++++++++++++++++++++++++++++++++++ quality-gate.sh | 2 +- 3 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 ocaml/squeezed/lib/squeeze.mli diff --git a/ocaml/squeezed/lib/squeeze.ml b/ocaml/squeezed/lib/squeeze.ml index 4816b6fc794..30b203c7e8c 100644 --- a/ocaml/squeezed/lib/squeeze.ml +++ b/ocaml/squeezed/lib/squeeze.ml @@ -23,8 +23,6 @@ (* Make debug printing work both when linked into a daemon and from the commandline *) -let start = Unix.gettimeofday () - module D = Debug.Make (struct let name = "squeeze" end) open D @@ -71,19 +69,6 @@ let domain_make domid can_balloon dynamic_min_kib target_kib dynamic_max_kib ; inaccuracy_kib } -let domain_to_string_pairs (x : domain) = - let i64 = Int64.to_string and i = string_of_int in - [ - ("domid", i x.domid) - ; ("can_balloon", string_of_bool x.can_balloon) - ; ("dynamic_min_kib", i64 x.dynamic_min_kib) - ; ("target_kib", i64 x.target_kib) - ; ("dynamic_max_kib", i64 x.dynamic_max_kib) - ; ("memory_actual_kib", i64 x.memory_actual_kib) - ; ("memory_max_kib", i64 x.memory_max_kib) - ; ("inaccuracy_kib", i64 x.inaccuracy_kib) - ] - module DomainSet = Set.Make (struct type t = domain @@ -105,12 +90,6 @@ type action = { ; new_target_kib: int64 (** new balloon target to set *) } -let action_to_string_pairs (x : action) = - [ - ("domid", string_of_int x.action_domid) - ; ("new_target_kib", Int64.to_string x.new_target_kib) - ] - let ( -* ) = Int64.sub let ( +* ) = Int64.add @@ -274,14 +253,6 @@ let min_freeable ?(fistpoints = []) domain = -* (2L ** domain.inaccuracy_kib) ) -(** The minimum amount we will allocate by setting target = dynamic_max *) -let min_allocatable domain = - max 0L - (domain.dynamic_max_kib - -* domain.memory_actual_kib - -* (2L ** domain.inaccuracy_kib) - ) - (** The range between dynamic_min and dynamic_max i.e. the total amount we may vary the balloon target NOT the total amount the memory_actual may vary. *) let range domain = max 0L (domain.dynamic_max_kib -* domain.dynamic_min_kib) @@ -723,12 +694,6 @@ let change_host_free_memory ?fistpoints io required_mem_kib success_condition = io.wait 1. done -let free_memory fistpoints io required_mem_kib = - change_host_free_memory ?fistpoints io - (required_mem_kib +* io.target_host_free_mem_kib) (fun x -> - x >= required_mem_kib +* io.target_host_free_mem_kib - ) - let free_memory_range ?fistpoints io min_kib max_kib = (* First compute the 'ideal' amount of free memory based on the proportional allocation policy *) diff --git a/ocaml/squeezed/lib/squeeze.mli b/ocaml/squeezed/lib/squeeze.mli new file mode 100644 index 00000000000..b915e56f268 --- /dev/null +++ b/ocaml/squeezed/lib/squeeze.mli @@ -0,0 +1,92 @@ +val manage_domain_zero : bool ref + +val domain_zero_dynamic_min : int64 ref + +val domain_zero_dynamic_max : int64 option ref + +val boot_time_host_free_memory_constant_count_min : int ref + +val boot_time_host_free_memory_check_interval : float ref + +type domain = { + domid: int + ; can_balloon: bool + ; dynamic_min_kib: int64 + ; target_kib: int64 + ; dynamic_max_kib: int64 + ; memory_actual_kib: int64 + ; memory_max_kib: int64 + ; inaccuracy_kib: int64 +} + +val domain_make : + int -> bool -> int64 -> int64 -> int64 -> int64 -> int64 -> int64 -> domain + +module DomainSet : Set.S with type elt = domain + +type host = {domains: DomainSet.t; free_mem_kib: int64} + +val make_host : domains:DomainSet.t -> free_mem_kib:int64 -> host + +type action = {action_domid: int; new_target_kib: int64} + +val ( -* ) : int64 -> int64 -> int64 + +val ( +* ) : int64 -> int64 -> int64 + +val ( ** ) : int64 -> int64 -> int64 + +type result = Success | Failed of int list | AdjustTargets of action list + +val has_hit_target : int64 -> int64 -> int64 -> bool + +module Stuckness_monitor : sig + type per_domain_state = { + mutable last_actual_kib: int64 + ; mutable last_makingprogress_time: float + ; mutable stuck: bool + } + + type t = {per_domain: (int, per_domain_state) Hashtbl.t} +end + +type fistpoint = DisableTwoPhaseTargetSets | DisableInaccuracyCompensation + +val min_freeable : ?fistpoints:fistpoint list -> domain -> int64 + +module Gnuplot : sig + type colspec = Dynamic_min | Memory_actual | Dynamic_max | Target + + val write_header : out_channel -> colspec list -> unit + + val write_row : out_channel -> host -> colspec list -> float -> unit + + val write_gp : string -> host -> colspec list -> unit +end + +type io = { + verbose: bool + ; make_host: unit -> string * host + ; domain_setmaxmem: int -> int64 -> unit + ; execute_action: action -> unit + ; wait: float -> unit + ; gettimeofday: unit -> float + ; target_host_free_mem_kib: int64 + ; free_memory_tolerance_kib: int64 +} + +exception Cannot_free_this_much_memory of int64 * int64 + +exception Domains_refused_to_cooperate of int list + +val change_host_free_memory : + ?fistpoints:fistpoint list -> io -> int64 -> (int64 -> bool) -> unit + +val free_memory_range : + ?fistpoints:fistpoint list -> io -> int64 -> int64 -> int64 + +val is_balanced : ?fistpoints:'a -> io -> int64 -> bool + +val balance_memory : ?fistpoints:fistpoint list -> io -> unit + +val is_host_memory_unbalanced : ?fistpoints:fistpoint list -> io -> bool diff --git a/quality-gate.sh b/quality-gate.sh index 15133234a82..d0a1a7ee296 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -25,7 +25,7 @@ verify-cert () { } mli-files () { - N=519 + N=518 # do not count ml files from the tests in ocaml/{tests/perftest/quicktest} MLIS=$(git ls-files -- '**/*.mli' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) MLS=$(git ls-files -- '**/*.ml' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;)