diff --git a/core/Cargo.lock b/core/Cargo.lock index 17a511f3710c..06315ca27143 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -7450,6 +7450,7 @@ dependencies = [ "reqsign-file-read-tokio", "serde", "serde_json", + "sha2 0.11.0", "tokio", "url", ] diff --git a/core/services/s3/Cargo.toml b/core/services/s3/Cargo.toml index 3ad6f87681e8..0143c1a304db 100644 --- a/core/services/s3/Cargo.toml +++ b/core/services/s3/Cargo.toml @@ -45,6 +45,7 @@ reqsign-aws-v4 = { version = "3.0.0", default-features = false } reqsign-core = { version = "3.0.1", default-features = false } reqsign-file-read-tokio = { version = "3.0.1", default-features = false } serde = { workspace = true, features = ["derive"] } +sha2 = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index 624351c6cd6f..644034798a91 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -568,6 +568,7 @@ impl S3Builder { /// /// Available options: /// - "crc32c" + /// - "sha256" /// - "md5" pub fn checksum_algorithm(mut self, checksum_algorithm: &str) -> Self { self.config.checksum_algorithm = Some(checksum_algorithm.to_string()); @@ -800,6 +801,7 @@ impl Builder for S3Builder { let checksum_algorithm = match config.checksum_algorithm.as_deref() { Some("crc32c") => Some(ChecksumAlgorithm::Crc32c), + Some("sha256") => Some(ChecksumAlgorithm::Sha256), Some("md5") => Some(ChecksumAlgorithm::Md5), None => None, v => { diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs index 0b3a7cafee85..4d580c5c1035 100644 --- a/core/services/s3/src/config.rs +++ b/core/services/s3/src/config.rs @@ -271,6 +271,7 @@ pub struct S3Config { /// /// Available options: /// - "crc32c" + /// - "sha256" /// - "md5" /// /// diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs index 77877b341b59..e51019e312eb 100644 --- a/core/services/s3/src/core.rs +++ b/core/services/s3/src/core.rs @@ -43,6 +43,8 @@ use reqsign_aws_v4::Credential; use reqsign_core::{Context, Signer}; use serde::Deserialize; use serde::Serialize; +use sha2::Digest as Sha2Digest; +use sha2::Sha256; use opendal_core::raw::*; use opendal_core::*; @@ -125,6 +127,13 @@ fn format_crc32c_iter(body: Buffer) -> String { BASE64_STANDARD.encode(crc.to_be_bytes()) } +fn format_sha256_iter(body: Buffer) -> String { + let mut hasher = Sha256::new(); + body.for_each(|b| hasher.update(&b)); + + BASE64_STANDARD.encode(hasher.finalize()) +} + impl Debug for S3Core { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("S3Core") @@ -296,6 +305,7 @@ impl S3Core { match self.checksum_algorithm { None => None, Some(ChecksumAlgorithm::Crc32c) => Some(format_crc32c_iter(body.clone())), + Some(ChecksumAlgorithm::Sha256) => Some(format_sha256_iter(body.clone())), Some(ChecksumAlgorithm::Md5) => Some(format_content_md5_iter(body.clone())), } } @@ -1406,6 +1416,8 @@ pub struct CompleteMultipartUploadRequestPart { pub etag: String, #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "Option::is_none")] pub checksum_crc32c: Option, + #[serde(rename = "ChecksumSHA256", skip_serializing_if = "Option::is_none")] + pub checksum_sha256: Option, } /// Output of `CompleteMultipartUpload` operation @@ -1557,6 +1569,7 @@ pub struct ListObjectVersionsOutputDeleteMarker { pub enum ChecksumAlgorithm { Crc32c, + Sha256, /// Mapping to the `Content-MD5` header from S3. Md5, } @@ -1564,6 +1577,7 @@ impl ChecksumAlgorithm { pub fn to_header_name(&self) -> HeaderName { match self { Self::Crc32c => HeaderName::from_static("x-amz-checksum-crc32c"), + Self::Sha256 => HeaderName::from_static("x-amz-checksum-sha256"), Self::Md5 => HeaderName::from_static("content-md5"), } } @@ -1575,6 +1589,7 @@ impl Display for ChecksumAlgorithm { "{}", match self { Self::Crc32c => "CRC32C", + Self::Sha256 => "SHA256", Self::Md5 => "MD5", } ) @@ -1588,6 +1603,20 @@ mod tests { use super::*; + #[test] + fn test_format_sha256_iter() { + // Base64-encoded SHA256 of "hello". + assert_eq!( + format_sha256_iter(Buffer::from("hello")), + "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ=" + ); + // Base64-encoded SHA256 of an empty input. + assert_eq!( + format_sha256_iter(Buffer::new()), + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + ); + } + /// This example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_Examples #[test] fn test_deserialize_initiate_multipart_upload_result() { diff --git a/core/services/s3/src/writer.rs b/core/services/s3/src/writer.rs index 8d0b9130a94e..5bb2f372869d 100644 --- a/core/services/s3/src/writer.rs +++ b/core/services/s3/src/writer.rs @@ -170,6 +170,13 @@ impl oio::MultipartWrite for S3Writer { part_number: p.part_number, etag: p.etag.clone(), checksum_crc32c: p.checksum.clone(), + ..Default::default() + }, + ChecksumAlgorithm::Sha256 => CompleteMultipartUploadRequestPart { + part_number: p.part_number, + etag: p.etag.clone(), + checksum_sha256: p.checksum.clone(), + ..Default::default() }, ChecksumAlgorithm::Md5 => CompleteMultipartUploadRequestPart { part_number: p.part_number,