/home/runner/work/lyquor/lyquor/platform/node/src/availability.rs
Line | Count | Source |
1 | | //! Deployment image-availability certification worker. |
2 | | //! |
3 | | //! Registrations that enter `Pending` remain there until their content digest |
4 | | //! is admitted or a deployment deadline receives a certified negative verdict. |
5 | | //! This worker watches bartender's `AvailabilityPending` log topic (and periodically sweeps the |
6 | | //! registry so restarts and missed events self-heal), pulls each referenced |
7 | | //! image through the node's [`ImageResolver`], and proposes a digest-scoped |
8 | | //! availability certificate. One certificate admits every deployment that |
9 | | //! references the same immutable content. |
10 | | |
11 | | use std::sync::Arc; |
12 | | use std::time::Duration; |
13 | | |
14 | | use lyquor_api::call::CallParams; |
15 | | use lyquor_hosting::{ImageResolver, Lyquid}; |
16 | | use lyquor_primitives::{Address, AvailabilityPendingEvent, B256, LyteLog, encode_by_fields}; |
17 | | use lyquor_vm::scheduler::{RunOptions, RunSource}; |
18 | | use tokio::sync::mpsc; |
19 | | use tokio_util::sync::CancellationToken; |
20 | | |
21 | | /// How often the worker re-checks the registry for deployments that are still |
22 | | /// pending (bounded re-probe of unretrievable images). |
23 | | const SWEEP_INTERVAL: Duration = Duration::from_secs(15); |
24 | | |
25 | | /// Run the availability watcher until its task group is cancelled. |
26 | 0 | pub async fn run_availability_watcher( |
27 | 0 | bartender: Lyquid, image_resolver: Arc<ImageResolver>, log_stream: mpsc::Receiver<LyteLog>, |
28 | 0 | shutdown: CancellationToken, |
29 | 0 | ) { |
30 | 0 | tokio::select! { |
31 | | biased; |
32 | 0 | () = shutdown.cancelled() => {} |
33 | 0 | () = watch(bartender, image_resolver, log_stream) => {} |
34 | | } |
35 | 0 | } |
36 | | |
37 | 0 | async fn watch(bartender: Lyquid, image_resolver: Arc<ImageResolver>, mut log_stream: mpsc::Receiver<LyteLog>) { |
38 | 0 | let mut sweep = tokio::time::interval(SWEEP_INTERVAL); |
39 | 0 | sweep.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); |
40 | | loop { |
41 | 0 | tokio::select! { |
42 | 0 | _ = sweep.tick() => { |
43 | 0 | let pending = match call_bartender::<Vec<AvailabilityPendingEvent>>( |
44 | 0 | &bartender, |
45 | 0 | "get_pending_deployments", |
46 | 0 | encode_by_fields!().into(), |
47 | | ) |
48 | 0 | .await |
49 | | { |
50 | 0 | Ok(pending) => pending, |
51 | 0 | Err(e) => { |
52 | 0 | tracing::debug!("Availability sweep failed to list pending deployments: {e}"); |
53 | 0 | continue; |
54 | | } |
55 | | }; |
56 | 0 | for event in pending { |
57 | 0 | try_certify(&bartender, &image_resolver, &event).await; |
58 | | } |
59 | | } |
60 | 0 | log = log_stream.recv() => { |
61 | 0 | let Some(log) = log else { break }; |
62 | 0 | let Some(event) = lyquor_primitives::decode_object::<AvailabilityPendingEvent>(&log.data) else { |
63 | 0 | continue; |
64 | | }; |
65 | 0 | try_certify(&bartender, &image_resolver, &event).await; |
66 | | } |
67 | | } |
68 | | } |
69 | 0 | } |
70 | | |
71 | | /// Pull and verify one pending deployment's image, record the local result, |
72 | | /// then propose the matching positive or negative certificate. |
73 | 0 | async fn try_certify(bartender: &Lyquid, image_resolver: &ImageResolver, event: &AvailabilityPendingEvent) { |
74 | | let AvailabilityPendingEvent { |
75 | 0 | id, |
76 | 0 | nth, |
77 | 0 | image_digest, |
78 | 0 | repo_hint, |
79 | 0 | } = event; |
80 | 0 | if let Some(hint) = repo_hint.as_deref() { |
81 | 0 | image_resolver.remember_repository_hint(*image_digest, hint); |
82 | 0 | } |
83 | | // `ensure_local` stores the pack content-addressed by digest, so success |
84 | | // means this node holds bytes matching the registered digest. |
85 | 0 | let available = match image_resolver.ensure_local(*image_digest).await { |
86 | 0 | Ok(()) => true, |
87 | 0 | Err(e) => { |
88 | 0 | tracing::debug!("Availability: image {image_digest} for {id}[{nth}] not retrievable yet: {e}"); |
89 | 0 | false |
90 | | } |
91 | | }; |
92 | 0 | match call_bartender::<bool>( |
93 | 0 | bartender, |
94 | 0 | "note_image_probe", |
95 | 0 | encode_by_fields!(image_digest: B256 = *image_digest, available: bool = available).into(), |
96 | | ) |
97 | 0 | .await |
98 | | { |
99 | 0 | Ok(_) => {} |
100 | 0 | Err(e) => { |
101 | 0 | tracing::warn!("Availability: failed to record image probe for {image_digest} at {id}[{nth}]: {e}"); |
102 | 0 | return; |
103 | | } |
104 | | } |
105 | 0 | let method = if available { |
106 | 0 | "certify_availability" |
107 | | } else { |
108 | 0 | "certify_unavailability" |
109 | | }; |
110 | | // Every committee node proposes independently. Duplicate positive |
111 | | // proposals become no-ops after admission; negative proposals remain |
112 | | // deployment-scoped and are checked again at their sequenced position. |
113 | 0 | match call_bartender::<bool>( |
114 | 0 | bartender, |
115 | 0 | method, |
116 | 0 | encode_by_fields!(image_digest: B256 = *image_digest).into(), |
117 | | ) |
118 | 0 | .await |
119 | | { |
120 | 0 | Ok(true) if available => tracing::info!("Availability: certified image {image_digest} for {id}[{nth}]"), |
121 | 0 | Ok(true) => tracing::info!("Availability: certified unavailable image {image_digest} for {id}[{nth}]"), |
122 | 0 | Ok(false) if available => { |
123 | 0 | tracing::debug!("Availability: positive certificate for {image_digest} not formed (no quorum yet)"); |
124 | | } |
125 | 0 | Ok(false) => tracing::debug!( |
126 | | "Availability: negative certificate for {image_digest} not submitted or not formed (deadline or quorum pending)" |
127 | | ), |
128 | 0 | Err(e) => { |
129 | 0 | tracing::warn!("Availability: {method} call for {image_digest} from {id}[{nth}] failed: {e}"); |
130 | | } |
131 | | } |
132 | 0 | } |
133 | | |
134 | 0 | async fn call_bartender<T: for<'a> serde::Deserialize<'a> + Send + 'static>( |
135 | 0 | bartender: &Lyquid, method: &str, input: lyquor_primitives::Bytes, |
136 | 0 | ) -> Result<T, lyquor_vm::instance::Error> { |
137 | 0 | let call = { |
138 | 0 | let bar_ln = bartender.latest_number().await; |
139 | 0 | bartender |
140 | 0 | .call_instance_func_decoded( |
141 | 0 | bar_ln, |
142 | 0 | CallParams::builder() |
143 | 0 | .caller(Address::ZERO) |
144 | 0 | .method(method.into()) |
145 | 0 | .input(input) |
146 | 0 | .build(), |
147 | 0 | RunOptions::new(RunSource::InstanceCall), |
148 | 0 | ) |
149 | 0 | .await |
150 | | }; |
151 | 0 | call.await |
152 | 0 | } |