jmpesp
commented
10 months ago Adding a few probes:
@@ -280,13 +283,14 @@ impl<T> Tracking<T> {
/// Record keeping for [`NoneOutstanding`] futures emitted by [`Tracking`]
struct TrackingWait {
+ device_id: u64,
empty: bool,
gen: usize,
wake: Vec<Waker>,
}
impl TrackingWait {
- fn new() -> Self {
- Self { empty: true, gen: 1, wake: Vec::new() }
+ fn new(device_id: u64) -> Self {
+ Self { device_id, empty: true, gen: 1, wake: Vec::new() }
}
fn set_empty(&mut self) {
self.empty = true;
@@ -320,9 +324,14 @@ impl Future for NoneOutstanding {
cx: &mut Context<'_>,
) -> Poll<Self::Output> {
let mut wguard = self.wait.lock().unwrap();
+ probes::none_outstanding_poll!(|| (wguard.device_id));
+
if wguard.empty {
+ probes::none_outstanding_ready!(|| (wguard.device_id));
Poll::Ready(())
} else {
+ probes::none_outstanding_must_wait!(|| (wguard.device_id));
+
// Add us to the waker list if the TrackingWait generation is newer
// than this future. That list is cleaned up by draining its
// entries when waking them (forcing interested futures to re-addShows:
device id 1 none outstanding poll
device id 1 none outstanding ready!
device id 2 none outstanding poll
device id 2 none outstanding must wait
device id 3 none outstanding poll
device id 3 none outstanding ready!The instance spec shows the following for storage devices:
storage_devices: {
"doom-disk": NvmeDisk(
NvmeDisk {
backend_name: "doom-disk",
pci_path: PciPath {
bus: 0,
device: 17,
function: 0,
},
},
),
"doom-install": NvmeDisk(
NvmeDisk {
backend_name: "doom-install",
pci_path: PciPath {
bus: 0,
device: 18,
function: 0,
},
},
),
"cloud-init": VirtioDisk(
VirtioDisk {
backend_name: "cloud-init",
pci_path: PciPath {
bus: 0,
device: 24,
function: 0,
},
},
),
},Device id 2 probably (?) corresponds to doom-install[^1], the second entry in this list, which is the nvme device for which the pause future isn't returning.
[^1]: I'm naming instances after id software games these days
pfmooney
luqmana
mkeeter
If Crucible takes too long to process some IO, Linux resets the NVMe controller, and eventually disables the controller and kicks out the disk:
I then tried to
sudo powerofffrom within the instance, and it's stuck waiting for thepci-nvme-0.18.0pause future to complete:This instance is stuck in the
Stoppingstate