Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 52 additions & 8 deletions crates/cli/src/commands/admin/heal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ use serde::Serialize;
use super::get_admin_client;
use crate::exit_code::ExitCode;
use crate::output::Formatter;
use rc_core::admin::{AdminApi, HealScanMode, HealStartRequest, HealStatus, HealTaskRequest};
use rc_core::admin::{
AdminApi, HealRuntimeState, HealScanMode, HealStartRequest, HealStatus, HealTaskRequest,
};

const HEAL_STOP_SUCCESS_MESSAGE: &str = "Heal operation stopped successfully";
const HEAL_STOP_STILL_RUNNING_MESSAGE: &str =
Expand Down Expand Up @@ -100,6 +102,8 @@ struct HealStatusOutput {
heal_id: String,
healing: bool,
#[serde(skip_serializing_if = "Option::is_none")]
state: Option<HealRuntimeState>,
#[serde(skip_serializing_if = "Option::is_none")]
summary: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
Expand All @@ -126,6 +130,7 @@ impl From<&HealStatus> for HealStatusOutput {
Self {
heal_id: status.heal_id.clone(),
healing: status.healing,
state: status.state,
summary: status.summary.clone(),
detail: status.detail.clone(),
bucket: status.bucket.clone(),
Expand All @@ -147,6 +152,7 @@ impl From<&HealStatus> for HealStatusOutput {

fn has_heal_status_details(status: &HealStatus) -> bool {
status.healing
|| status.state.is_some()
|| !status.heal_id.is_empty()
|| status.summary.is_some()
|| status.detail.is_some()
Expand Down Expand Up @@ -199,6 +205,28 @@ fn heal_stop_output(status: &HealStatus) -> (ExitCode, HealOperationOutput) {
)
}

enum HealStatusIndicator {
Progress(&'static str),
Date(&'static str),
}

fn heal_status_indicator(status: &HealStatus) -> HealStatusIndicator {
match status.state {
Some(HealRuntimeState::Disabled) => HealStatusIndicator::Date("Disabled"),
Some(HealRuntimeState::Uninitialized) => HealStatusIndicator::Date("Uninitialized"),
Some(HealRuntimeState::Active) => HealStatusIndicator::Progress("In Progress"),
Some(HealRuntimeState::Idle) => HealStatusIndicator::Date("Idle"),
Some(HealRuntimeState::Unknown) | None => match status.summary.as_deref() {
Some("running") => HealStatusIndicator::Progress("In Progress"),
Some("finished") => HealStatusIndicator::Progress("Finished"),
Some("stopped") => HealStatusIndicator::Date("Stopped"),
Some("notFound") => HealStatusIndicator::Date("Not Found"),
_ if status.healing => HealStatusIndicator::Progress("In Progress"),
_ => HealStatusIndicator::Date("Idle"),
},
}
}

/// Execute a heal subcommand
pub async fn execute(cmd: HealCommands, formatter: &Formatter) -> ExitCode {
match cmd {
Expand Down Expand Up @@ -247,13 +275,9 @@ async fn execute_status(args: StatusArgs, formatter: &Formatter) -> ExitCode {
}

fn print_heal_status(status: &HealStatus, formatter: &Formatter) {
let healing_status = match status.summary.as_deref() {
Some("running") => formatter.style_size("In Progress"),
Some("finished") => formatter.style_size("Finished"),
Some("stopped") => formatter.style_date("Stopped"),
Some("notFound") => formatter.style_date("Not Found"),
_ if status.healing => formatter.style_size("In Progress"),
_ => formatter.style_date("Idle"),
let healing_status = match heal_status_indicator(status) {
HealStatusIndicator::Progress(label) => formatter.style_size(label),
HealStatusIndicator::Date(label) => formatter.style_date(label),
};

formatter.println(&format!(
Expand Down Expand Up @@ -600,6 +624,7 @@ mod tests {
let status = HealStatus {
heal_id: "heal-123".to_string(),
healing: true,
state: Some(HealRuntimeState::Active),
bucket: "test-bucket".to_string(),
object: "test/object.txt".to_string(),
scan_mode: Some(HealScanMode::Deep),
Expand All @@ -619,6 +644,7 @@ mod tests {
let output = HealStatusOutput::from(&status);
assert_eq!(output.heal_id, "heal-123");
assert!(output.healing);
assert_eq!(output.state, Some(HealRuntimeState::Active));
assert_eq!(output.bucket, "test-bucket");
assert_eq!(output.scan_mode, Some(HealScanMode::Deep));
assert_eq!(output.scan_cycle, 42);
Expand Down Expand Up @@ -646,5 +672,23 @@ mod tests {
heal_active_tasks: 1,
..Default::default()
}));

assert!(has_heal_status_details(&HealStatus {
state: Some(HealRuntimeState::Disabled),
..Default::default()
}));
}

#[test]
fn test_heal_status_indicator_reports_disabled_runtime() {
let status = HealStatus {
state: Some(HealRuntimeState::Disabled),
..Default::default()
};

assert!(matches!(
heal_status_indicator(&status),
HealStatusIndicator::Date("Disabled")
));
}
}
22 changes: 22 additions & 0 deletions crates/core/src/admin/cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,17 @@ impl std::str::FromStr for HealScanMode {
}
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum HealRuntimeState {
Disabled,
Uninitialized,
Idle,
Active,
#[serde(other)]
Unknown,
}

/// Request to start a heal operation
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
Expand Down Expand Up @@ -585,6 +596,9 @@ pub struct HealStatus {
#[serde(default)]
pub healing: bool,

#[serde(default, skip_serializing_if = "Option::is_none")]
pub state: Option<HealRuntimeState>,

/// Task summary for token-scoped manual heal status
#[serde(default, skip_serializing_if = "Option::is_none")]
pub summary: Option<String>,
Expand Down Expand Up @@ -1060,13 +1074,21 @@ mod tests {
let status = HealStatus::default();
assert!(status.heal_id.is_empty());
assert!(!status.healing);
assert!(status.state.is_none());
assert!(status.scan_mode.is_none());
assert_eq!(status.scan_cycle, 0);
assert_eq!(status.heal_queue_length, 0);
assert_eq!(status.heal_active_tasks, 0);
assert_eq!(status.items_scanned, 0);
}

#[test]
fn test_heal_runtime_state_unknown_value_is_preserved() {
let state: HealRuntimeState = serde_json::from_str(r#""future""#).unwrap();

assert_eq!(state, HealRuntimeState::Unknown);
}

#[test]
fn test_pool_status_deserialization() {
let json = r#"{"id":1,"cmdline":"/data/pool1/disk{1...4}","lastUpdate":"2026-05-06T00:00:00Z","status":"decommissioning","decommissionStatus":"running","rebalanceStatus":"none","totalSize":1000,"currentSize":600,"usedSize":400,"used":0.4,"decommissionInfo":{"startTime":"2026-05-06T00:00:01Z","startSize":100,"totalSize":1000,"currentSize":600,"complete":false,"failed":false,"canceled":false,"queued":true,"queuedBuckets":["bucket-a"],"decommissionedBuckets":["bucket-b"],"bucket":"bucket-a","prefix":"","object":"object.txt","stage":"migrate_object","objectsDecommissioned":2,"objectsDecommissionedFailed":1,"bytesDecommissioned":128,"bytesDecommissionedFailed":64,"waitingReason":"queued"}}"#;
Expand Down
9 changes: 5 additions & 4 deletions crates/core/src/admin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ mod types;

pub use cluster::{
BackendInfo, BackendType, BucketsInfo, ClusterInfo, DecommissionPoolStatus, DecommissionStatus,
DiskInfo, HealDriveInfo, HealDriveInfos, HealResultItem, HealScanMode, HealStartRequest,
HealStatus, HealTaskRequest, HealingDiskInfo, MemStats, ObjectsInfo, PoolDecommissionInfo,
PoolErasureSetInfo, PoolStatus, PoolTarget, RebalanceCleanupWarnings, RebalancePoolProgress,
RebalancePoolStatus, RebalanceStartResult, RebalanceStatus, ServerInfo, UsageInfo,
DiskInfo, HealDriveInfo, HealDriveInfos, HealResultItem, HealRuntimeState, HealScanMode,
HealStartRequest, HealStatus, HealTaskRequest, HealingDiskInfo, MemStats, ObjectsInfo,
PoolDecommissionInfo, PoolErasureSetInfo, PoolStatus, PoolTarget, RebalanceCleanupWarnings,
RebalancePoolProgress, RebalancePoolStatus, RebalanceStartResult, RebalanceStatus, ServerInfo,
UsageInfo,
};
pub use tier::{
TierAliyun, TierAzure, TierConfig, TierCreds, TierGCS, TierHuaweicloud, TierMinIO, TierR2,
Expand Down
18 changes: 14 additions & 4 deletions crates/s3/src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ use aws_sigv4::http_request::{
use aws_sigv4::sign::v4;
use rc_core::admin::{
AccessKeyInfo, AdminApi, BucketQuota, ClusterInfo, CreateServiceAccountRequest,
DecommissionPoolStatus, DecommissionStatus, Group, GroupStatus, HealScanMode, HealStartRequest,
HealStatus, HealTaskRequest, Policy, PolicyEntity, PolicyInfo, PoolStatus, PoolTarget,
RebalanceStartResult, RebalanceStatus, ServiceAccount, ServiceAccountCreateResponse,
UpdateGroupMembersRequest, User, UserStatus,
DecommissionPoolStatus, DecommissionStatus, Group, GroupStatus, HealRuntimeState, HealScanMode,
HealStartRequest, HealStatus, HealTaskRequest, Policy, PolicyEntity, PolicyInfo, PoolStatus,
PoolTarget, RebalanceStartResult, RebalanceStatus, ServiceAccount,
ServiceAccountCreateResponse, UpdateGroupMembersRequest, User, UserStatus,
};
use rc_core::{Alias, Error, Result};
use reqwest::header::{CONTENT_TYPE, HOST, HeaderMap, HeaderName, HeaderValue};
Expand Down Expand Up @@ -406,6 +406,8 @@ struct ServiceAccountInfo {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct BackgroundHealStatusResponse {
#[serde(default)]
state: Option<HealRuntimeState>,
#[serde(default)]
bitrot_start_time: Option<String>,
#[serde(default)]
Expand Down Expand Up @@ -487,6 +489,7 @@ impl From<BackgroundHealStatusResponse> for HealStatus {
heal_active_tasks: active_tasks,
..Default::default()
};
status.state = response.state;
if let Some(progress) = response.progress {
progress.apply_to_status(&mut status);
}
Expand Down Expand Up @@ -1444,6 +1447,7 @@ mod tests {
#[test]
fn test_background_heal_status_response_maps_to_heal_status() {
let status = HealStatus::from(BackgroundHealStatusResponse {
state: Some(HealRuntimeState::Active),
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 42,
current_scan_mode: Some(2),
Expand All @@ -1454,13 +1458,15 @@ mod tests {
});

assert!(status.healing);
assert_eq!(status.state, Some(HealRuntimeState::Active));
assert_eq!(status.started.as_deref(), Some("2026-04-19T10:00:00Z"));
assert_eq!(status.scan_mode, Some(HealScanMode::Deep));
assert_eq!(status.scan_cycle, 42);
assert_eq!(status.heal_queue_length, 3);
assert_eq!(status.heal_active_tasks, 1);

let idle = HealStatus::from(BackgroundHealStatusResponse {
state: Some(HealRuntimeState::Idle),
bitrot_start_time: None,
bitrot_start_cycle: 0,
current_scan_mode: Some(1),
Expand All @@ -1470,10 +1476,12 @@ mod tests {
progress: None,
});
assert!(!idle.healing);
assert_eq!(idle.state, Some(HealRuntimeState::Idle));
assert_eq!(idle.scan_mode, Some(HealScanMode::Normal));
assert!(idle.started.is_none());

let completed = HealStatus::from(BackgroundHealStatusResponse {
state: None,
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 42,
current_scan_mode: Some(1),
Expand All @@ -1487,6 +1495,7 @@ mod tests {
assert_eq!(completed.started.as_deref(), Some("2026-04-19T10:00:00Z"));

let legacy = HealStatus::from(BackgroundHealStatusResponse {
state: None,
bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()),
bitrot_start_cycle: 0,
current_scan_mode: None,
Expand All @@ -1498,6 +1507,7 @@ mod tests {
assert!(legacy.healing);

let active = HealStatus::from(BackgroundHealStatusResponse {
state: None,
bitrot_start_time: None,
bitrot_start_cycle: 0,
current_scan_mode: None,
Expand Down
11 changes: 11 additions & 0 deletions schemas/output_v2.json
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,17 @@
"type": "boolean",
"description": "Whether a heal operation is active"
},
"state": {
"type": "string",
"enum": [
"disabled",
"uninitialized",
"idle",
"active",
"unknown"
],
"description": "Runtime state reported by the heal service"
},
"summary": {
"type": "string",
"description": "Token-scoped manual heal task summary"
Expand Down
Loading