switchyard_protocol/
infra.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Model-pool observations supplied by the serving integration.
5
6use crate::ModelId;
7use std::{collections::BTreeMap, time::Instant};
8
9/// Observations keyed by model, without worker identities.
10pub type InfraSignals = BTreeMap<ModelId, ModelInfraSignals>;
11
12/// Pool observations and local request accounting.
13#[derive(Clone, Debug, Default)]
14pub struct ModelInfraSignals {
15    /// None means unknown, never idle.
16    pub load: Option<PoolLoad>,
17    /// Requests still in flight through this host.
18    pub local_inflight: u64,
19}
20
21/// Aggregated scheduler counts. The host is responsible for pool coverage.
22#[derive(Clone, Debug)]
23pub struct PoolLoad {
24    /// Oldest underlying observation, expressed in the host's monotonic clock.
25    pub sampled_at: Instant,
26    /// Lowest queue depth among the pool's engines.
27    pub min_waiting: u64,
28    /// Highest queue depth among the pool's engines.
29    pub max_waiting: u64,
30    /// Total running requests across the pool.
31    pub running: u64,
32}
switchyard_protocol/infra.rs

switchyard_protocol/
infra.rs
