Coverage Report

Created: 2026-07-09 17:15

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/runner/work/lyquor/lyquor/platform/node/src/availability.rs
Line
Count
Source
1
//! Deployment image-availability certification worker.
2
//!
3
//! Deployments registered while an availability committee is active enter the
4
//! bartender registry as `Pending` and are not hosted anywhere. This worker
5
//! watches bartender's `AvailabilityPending` log topic (and periodically
6
//! sweeps the registry so restarts and missed events self-heal), pulls each
7
//! pending deployment's image through the node's [`ImageResolver`] — which
8
//! content-verifies the bytes against the digest — and, on success, records
9
//! the verified digest in bartender's node-local instance state and proposes
10
//! an availability certificate. Once a threshold of committee nodes validates
11
//! the claim, the certified `attest_available` call flips the deployment
12
//! `Live` and hosting proceeds through the normal registration path.
13
14
use std::sync::Arc;
15
use std::time::Duration;
16
17
use lyquor_api::call::CallParams;
18
use lyquor_hosting::{ImageResolver, Lyquid};
19
use lyquor_primitives::{Address, AvailabilityPendingEvent, B256, LyquidID, LyteLog, encode_by_fields};
20
use lyquor_vm::scheduler::{RunOptions, RunSource};
21
use tokio::sync::mpsc;
22
use tokio_util::sync::CancellationToken;
23
use tokio_util::task::TaskTracker;
24
25
/// How often the worker re-checks the registry for deployments that are still
26
/// pending (bounded re-probe of unretrievable images).
27
const SWEEP_INTERVAL: Duration = Duration::from_secs(15);
28
29
/// Background worker that certifies image availability for pending deployments.
30
pub struct AvailabilityWatcher {
31
    shutdown: CancellationToken,
32
    task_tracker: TaskTracker,
33
}
34
35
impl AvailabilityWatcher {
36
    /// Start the availability worker.
37
0
    pub fn start(bartender: Lyquid, image_resolver: Arc<ImageResolver>, log_stream: mpsc::Receiver<LyteLog>) -> Self {
38
0
        let shutdown = CancellationToken::new();
39
0
        let task_tracker = TaskTracker::new();
40
41
0
        let token = shutdown.clone();
42
0
        task_tracker.spawn(async move {
43
0
            tokio::select! {
44
0
                () = token.cancelled() => {}
45
0
                () = watch(bartender, image_resolver, log_stream) => {}
46
            }
47
0
        });
48
49
0
        Self { shutdown, task_tracker }
50
0
    }
51
52
    /// Stop the worker and wait for it to finish.
53
0
    pub async fn shutdown(self) {
54
0
        self.shutdown.cancel();
55
0
        self.task_tracker.close();
56
0
        self.task_tracker.wait().await;
57
0
    }
58
}
59
60
0
async fn watch(bartender: Lyquid, image_resolver: Arc<ImageResolver>, mut log_stream: mpsc::Receiver<LyteLog>) {
61
0
    let mut sweep = tokio::time::interval(SWEEP_INTERVAL);
62
0
    sweep.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
63
    loop {
64
0
        tokio::select! {
65
0
            _ = sweep.tick() => {
66
0
                let pending = match call_bartender::<Vec<AvailabilityPendingEvent>>(
67
0
                    &bartender,
68
0
                    "get_pending_deployments",
69
0
                    encode_by_fields!().into(),
70
                )
71
0
                .await
72
                {
73
0
                    Ok(pending) => pending,
74
0
                    Err(e) => {
75
0
                        tracing::debug!("Availability sweep failed to list pending deployments: {e}");
76
0
                        continue;
77
                    }
78
                };
79
0
                for event in pending {
80
0
                    try_certify(&bartender, &image_resolver, &event).await;
81
                }
82
            }
83
0
            log = log_stream.recv() => {
84
0
                let Some(log) = log else { break };
85
0
                let Some(event) = lyquor_primitives::decode_object::<AvailabilityPendingEvent>(&log.data) else {
86
0
                    continue;
87
                };
88
0
                try_certify(&bartender, &image_resolver, &event).await;
89
            }
90
        }
91
    }
92
0
}
93
94
/// Pull and verify one pending deployment's image, then flag it and propose
95
/// the availability certificate. Failures are left for the next sweep.
96
0
async fn try_certify(bartender: &Lyquid, image_resolver: &ImageResolver, event: &AvailabilityPendingEvent) {
97
    let AvailabilityPendingEvent {
98
0
        id,
99
0
        nth,
100
0
        image_digest,
101
0
        repo_hint,
102
0
    } = event;
103
0
    if let Some(hint) = repo_hint.as_deref() {
104
0
        image_resolver.remember_repository_hint(*image_digest, hint);
105
0
    }
106
    // `ensure_local` stores the pack content-addressed by digest, so success
107
    // means this node holds bytes matching the registered digest.
108
0
    if let Err(e) = image_resolver.ensure_local(*image_digest).await {
109
0
        tracing::debug!("Availability: image {image_digest} for {id}[{nth}] not retrievable yet: {e}");
110
0
        return;
111
0
    }
112
0
    match call_bartender::<bool>(
113
0
        bartender,
114
0
        "note_image_verified",
115
0
        encode_by_fields!(image_digest: B256 = *image_digest).into(),
116
    )
117
0
    .await
118
    {
119
0
        Ok(_) => {}
120
0
        Err(e) => {
121
0
            tracing::warn!("Availability: failed to record verified image {image_digest} for {id}[{nth}]: {e}");
122
0
            return;
123
        }
124
    }
125
    // Every committee node proposes independently; duplicate certificates are
126
    // deduplicated by the certified call's nonce replay protection, and
127
    // `certify_availability` no-ops once the deployment is no longer Pending.
128
0
    match call_bartender::<bool>(
129
0
        bartender,
130
0
        "certify_availability",
131
0
        encode_by_fields!(id: LyquidID = *id, nth: u32 = *nth).into(),
132
    )
133
0
    .await
134
    {
135
0
        Ok(true) => tracing::info!("Availability: certified image {image_digest} for {id}[{nth}]"),
136
        Ok(false) => {
137
0
            tracing::debug!("Availability: certificate for {id}[{nth}] not formed (no quorum yet or already settled)");
138
        }
139
0
        Err(e) => tracing::warn!("Availability: certification call for {id}[{nth}] failed: {e}"),
140
    }
141
0
}
142
143
0
async fn call_bartender<T: for<'a> serde::Deserialize<'a> + Send + 'static>(
144
0
    bartender: &Lyquid, method: &str, input: lyquor_primitives::Bytes,
145
0
) -> Result<T, lyquor_vm::instance::Error> {
146
0
    let call = {
147
0
        let bar_ln = bartender.latest_number().await;
148
0
        bartender
149
0
            .call_instance_func_decoded(
150
0
                bar_ln,
151
0
                CallParams::builder()
152
0
                    .caller(Address::ZERO)
153
0
                    .method(method.into())
154
0
                    .input(input)
155
0
                    .build(),
156
0
                RunOptions::new(RunSource::InstanceCall),
157
0
            )
158
0
            .await
159
    };
160
0
    call.await
161
0
}