switchyard_protocol/
infra.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Serving observations supplied by the host for infrastructure-aware routing.
5
6use crate::ModelId;
7use std::{collections::BTreeMap, time::Instant};
8
9/// Model observations. Missing models or workers mean unknown, never idle.
10pub type InfraSignals = BTreeMap<ModelId, ModelInfraSignals>;
11
12/// Backend worker gauges and requests still in flight through this host.
13#[derive(Clone, Debug, Default)]
14pub struct ModelInfraSignals {
15    /// One entry per independently scheduled worker, keyed by configured worker ID.
16    pub workers: BTreeMap<String, WorkerLoad>,
17    /// All requests sent to this model that have not completed at this host.
18    pub local_inflight: u64,
19    /// Subset of local requests moved from the efficient tier.
20    pub local_spillovers: u64,
21}
22
23/// Running and waiting gauges from one worker scrape.
24#[derive(Clone, Debug)]
25pub struct WorkerLoad {
26    /// Local monotonic time at the start of collection.
27    pub sampled_at: Instant,
28    /// Backend requests currently executing.
29    pub running: u64,
30    /// Backend requests waiting for execution capacity.
31    pub waiting: u64,
32}
switchyard_protocol/infra.rs

switchyard_protocol/
infra.rs
