/home/runner/work/lyquor/lyquor/platform/node/src/availability.rs
Line | Count | Source |
1 | | //! Deployment image-availability certification worker. |
2 | | //! |
3 | | //! Deployments registered while an availability committee is active enter the |
4 | | //! bartender registry as `Pending` and are not hosted anywhere. This worker |
5 | | //! watches bartender's `AvailabilityPending` log topic (and periodically |
6 | | //! sweeps the registry so restarts and missed events self-heal), pulls each |
7 | | //! pending deployment's image through the node's [`ImageResolver`] — which |
8 | | //! content-verifies the bytes against the digest — and, on success, records |
9 | | //! the verified digest in bartender's node-local instance state and proposes |
10 | | //! an availability certificate. Once a threshold of committee nodes validates |
11 | | //! the claim, the certified `attest_available` call flips the deployment |
12 | | //! `Live` and hosting proceeds through the normal registration path. |
13 | | |
14 | | use std::sync::Arc; |
15 | | use std::time::Duration; |
16 | | |
17 | | use lyquor_api::call::CallParams; |
18 | | use lyquor_hosting::{ImageResolver, Lyquid}; |
19 | | use lyquor_primitives::{Address, AvailabilityPendingEvent, B256, LyquidID, LyteLog, encode_by_fields}; |
20 | | use lyquor_vm::scheduler::{RunOptions, RunSource}; |
21 | | use tokio::sync::mpsc; |
22 | | use tokio_util::sync::CancellationToken; |
23 | | use tokio_util::task::TaskTracker; |
24 | | |
25 | | /// How often the worker re-checks the registry for deployments that are still |
26 | | /// pending (bounded re-probe of unretrievable images). |
27 | | const SWEEP_INTERVAL: Duration = Duration::from_secs(15); |
28 | | |
29 | | /// Background worker that certifies image availability for pending deployments. |
30 | | pub struct AvailabilityWatcher { |
31 | | shutdown: CancellationToken, |
32 | | task_tracker: TaskTracker, |
33 | | } |
34 | | |
35 | | impl AvailabilityWatcher { |
36 | | /// Start the availability worker. |
37 | 0 | pub fn start(bartender: Lyquid, image_resolver: Arc<ImageResolver>, log_stream: mpsc::Receiver<LyteLog>) -> Self { |
38 | 0 | let shutdown = CancellationToken::new(); |
39 | 0 | let task_tracker = TaskTracker::new(); |
40 | | |
41 | 0 | let token = shutdown.clone(); |
42 | 0 | task_tracker.spawn(async move { |
43 | 0 | tokio::select! { |
44 | 0 | () = token.cancelled() => {} |
45 | 0 | () = watch(bartender, image_resolver, log_stream) => {} |
46 | | } |
47 | 0 | }); |
48 | | |
49 | 0 | Self { shutdown, task_tracker } |
50 | 0 | } |
51 | | |
52 | | /// Stop the worker and wait for it to finish. |
53 | 0 | pub async fn shutdown(self) { |
54 | 0 | self.shutdown.cancel(); |
55 | 0 | self.task_tracker.close(); |
56 | 0 | self.task_tracker.wait().await; |
57 | 0 | } |
58 | | } |
59 | | |
60 | 0 | async fn watch(bartender: Lyquid, image_resolver: Arc<ImageResolver>, mut log_stream: mpsc::Receiver<LyteLog>) { |
61 | 0 | let mut sweep = tokio::time::interval(SWEEP_INTERVAL); |
62 | 0 | sweep.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); |
63 | | loop { |
64 | 0 | tokio::select! { |
65 | 0 | _ = sweep.tick() => { |
66 | 0 | let pending = match call_bartender::<Vec<AvailabilityPendingEvent>>( |
67 | 0 | &bartender, |
68 | 0 | "get_pending_deployments", |
69 | 0 | encode_by_fields!().into(), |
70 | | ) |
71 | 0 | .await |
72 | | { |
73 | 0 | Ok(pending) => pending, |
74 | 0 | Err(e) => { |
75 | 0 | tracing::debug!("Availability sweep failed to list pending deployments: {e}"); |
76 | 0 | continue; |
77 | | } |
78 | | }; |
79 | 0 | for event in pending { |
80 | 0 | try_certify(&bartender, &image_resolver, &event).await; |
81 | | } |
82 | | } |
83 | 0 | log = log_stream.recv() => { |
84 | 0 | let Some(log) = log else { break }; |
85 | 0 | let Some(event) = lyquor_primitives::decode_object::<AvailabilityPendingEvent>(&log.data) else { |
86 | 0 | continue; |
87 | | }; |
88 | 0 | try_certify(&bartender, &image_resolver, &event).await; |
89 | | } |
90 | | } |
91 | | } |
92 | 0 | } |
93 | | |
94 | | /// Pull and verify one pending deployment's image, then flag it and propose |
95 | | /// the availability certificate. Failures are left for the next sweep. |
96 | 0 | async fn try_certify(bartender: &Lyquid, image_resolver: &ImageResolver, event: &AvailabilityPendingEvent) { |
97 | | let AvailabilityPendingEvent { |
98 | 0 | id, |
99 | 0 | nth, |
100 | 0 | image_digest, |
101 | 0 | repo_hint, |
102 | 0 | } = event; |
103 | 0 | if let Some(hint) = repo_hint.as_deref() { |
104 | 0 | image_resolver.remember_repository_hint(*image_digest, hint); |
105 | 0 | } |
106 | | // `ensure_local` stores the pack content-addressed by digest, so success |
107 | | // means this node holds bytes matching the registered digest. |
108 | 0 | if let Err(e) = image_resolver.ensure_local(*image_digest).await { |
109 | 0 | tracing::debug!("Availability: image {image_digest} for {id}[{nth}] not retrievable yet: {e}"); |
110 | 0 | return; |
111 | 0 | } |
112 | 0 | match call_bartender::<bool>( |
113 | 0 | bartender, |
114 | 0 | "note_image_verified", |
115 | 0 | encode_by_fields!(image_digest: B256 = *image_digest).into(), |
116 | | ) |
117 | 0 | .await |
118 | | { |
119 | 0 | Ok(_) => {} |
120 | 0 | Err(e) => { |
121 | 0 | tracing::warn!("Availability: failed to record verified image {image_digest} for {id}[{nth}]: {e}"); |
122 | 0 | return; |
123 | | } |
124 | | } |
125 | | // Every committee node proposes independently; duplicate certificates are |
126 | | // deduplicated by the certified call's nonce replay protection, and |
127 | | // `certify_availability` no-ops once the deployment is no longer Pending. |
128 | 0 | match call_bartender::<bool>( |
129 | 0 | bartender, |
130 | 0 | "certify_availability", |
131 | 0 | encode_by_fields!(id: LyquidID = *id, nth: u32 = *nth).into(), |
132 | | ) |
133 | 0 | .await |
134 | | { |
135 | 0 | Ok(true) => tracing::info!("Availability: certified image {image_digest} for {id}[{nth}]"), |
136 | | Ok(false) => { |
137 | 0 | tracing::debug!("Availability: certificate for {id}[{nth}] not formed (no quorum yet or already settled)"); |
138 | | } |
139 | 0 | Err(e) => tracing::warn!("Availability: certification call for {id}[{nth}] failed: {e}"), |
140 | | } |
141 | 0 | } |
142 | | |
143 | 0 | async fn call_bartender<T: for<'a> serde::Deserialize<'a> + Send + 'static>( |
144 | 0 | bartender: &Lyquid, method: &str, input: lyquor_primitives::Bytes, |
145 | 0 | ) -> Result<T, lyquor_vm::instance::Error> { |
146 | 0 | let call = { |
147 | 0 | let bar_ln = bartender.latest_number().await; |
148 | 0 | bartender |
149 | 0 | .call_instance_func_decoded( |
150 | 0 | bar_ln, |
151 | 0 | CallParams::builder() |
152 | 0 | .caller(Address::ZERO) |
153 | 0 | .method(method.into()) |
154 | 0 | .input(input) |
155 | 0 | .build(), |
156 | 0 | RunOptions::new(RunSource::InstanceCall), |
157 | 0 | ) |
158 | 0 | .await |
159 | | }; |
160 | 0 | call.await |
161 | 0 | } |