Clear XDE underlay when destroying virtual hardware

OPTE now prevents itself from being unloaded if its underlay state is
set. Currently, underlay setup is performed only once, and it seems to be
the case that XDE can be unloaded in some scenarios (e.g., `a4x2`
setup).

However, a consequence is that removing the driver requires an extra
operation to explicitly clear the underlay state. This PR adds this
operation to the `cargo xtask virtual-hardware destroy` command.

This is currently blocked on opte#485 being approved/merged.

Closes #5314.

Cargo.toml

@@ -305,14 +305,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

dev-tools/xtask/src/virtual_hardware.rs

@@ -100,6 +100,7 @@ const IPADM: &'static str = "/usr/sbin/ipadm";
 const MODINFO: &'static str = "/usr/sbin/modinfo";
 const MODUNLOAD: &'static str = "/usr/sbin/modunload";
 const NETSTAT: &'static str = "/usr/bin/netstat";
+const OPTEADM: &'static str = "/opt/oxide/opte/bin/opteadm";
 const PFEXEC: &'static str = "/usr/bin/pfexec";
 const PING: &'static str = "/usr/sbin/ping";
 const SWAP: &'static str = "/usr/sbin/swap";
@@ -240,8 +241,17 @@ fn unload_xde_driver() -> Result<()> {
         println!("xde driver already unloaded");
         return Ok(());
     };
-    println!("unloading xde driver");
+    println!("unloading xde driver:\na) clearing underlay...");
+    let mut cmd = Command::new(PFEXEC);
+    cmd.args([OPTEADM, "clear-xde-underlay"]);
+    if let Err(e) = execute(cmd) {
+        // This is explicitly non-fatal: the underlay is only set when
+        // sled-agent is running. We still need to be able to tear
+        // down the driver if we immediately call create->destroy.
+        println!("\tFailed or already unset: {e}");
+    }
 
+    println!("b) unloading module...");
     let mut cmd = Command::new(PFEXEC);
     cmd.arg(MODUNLOAD);
     cmd.arg("-i");