Use existing reqwest client for AWS S3 requests (#5917)

This removes a lot of duplicate client dependency bloat for roughly
equivalent functionality.

Co-authored-by: Mathijs van Veluw <[email protected]>

Cargo.lock

@@ -184,7 +184,7 @@ dependencies = [
  "futures-lite",
  "parking",
  "polling",
- "rustix 1.0.7",
+ "rustix",
  "slab",
  "tracing",
  "windows-sys 0.59.0",
@@ -216,7 +216,7 @@ dependencies = [
  "cfg-if",
  "event-listener 5.4.0",
  "futures-lite",
- "rustix 1.0.7",
+ "rustix",
  "tracing",
 ]
 
@@ -232,7 +232,7 @@ dependencies = [
  "cfg-if",
  "futures-core",
  "futures-io",
- "rustix 1.0.7",
+ "rustix",
  "signal-hook-registry",
  "slab",
  "windows-sys 0.59.0",
@@ -373,29 +373,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "aws-lc-rs"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7"
-dependencies = [
- "aws-lc-sys",
- "zeroize",
-]
-
-[[package]]
-name = "aws-lc-sys"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079"
-dependencies = [
- "bindgen",
- "cc",
- "cmake",
- "dunce",
- "fs_extra",
-]
-
 [[package]]
 name = "aws-runtime"
 version = "1.5.8"
@@ -540,29 +517,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "aws-smithy-http-client"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15"
-dependencies = [
- "aws-smithy-async",
- "aws-smithy-runtime-api",
- "aws-smithy-types",
- "h2",
- "http 1.3.1",
- "hyper 1.6.0",
- "hyper-rustls",
- "hyper-util",
- "pin-project-lite",
- "rustls 0.23.28",
- "rustls-native-certs",
- "rustls-pki-types",
- "tokio",
- "tower",
- "tracing",
-]
-
 [[package]]
 name = "aws-smithy-json"
 version = "0.61.4"
@@ -599,7 +553,6 @@ checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
- "aws-smithy-http-client",
  "aws-smithy-observability",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -757,29 +710,6 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
 
-[[package]]
-name = "bindgen"
-version = "0.69.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
-dependencies = [
- "bitflags",
- "cexpr",
- "clang-sys",
- "itertools",
- "lazy_static",
- "lazycell",
- "log",
- "prettyplease",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash 1.1.0",
- "shlex",
- "syn",
- "which 4.4.2",
-]
-
 [[package]]
 name = "bitflags"
 version = "2.9.1"
@@ -937,15 +867,6 @@ dependencies = [
  "shlex",
 ]
 
-[[package]]
-name = "cexpr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
-dependencies = [
- "nom 7.1.3",
-]
-
 [[package]]
 name = "cfg-if"
 version = "1.0.1"
@@ -1014,26 +935,6 @@ dependencies = [
  "inout",
 ]
 
-[[package]]
-name = "clang-sys"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
-[[package]]
-name = "cmake"
-version = "0.1.54"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "codemap"
 version = "0.1.3"
@@ -1519,12 +1420,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "dunce"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
-
 [[package]]
 name = "either"
 version = "1.15.0"
@@ -1698,12 +1593,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fs_extra"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
-
 [[package]]
 name = "futures"
 version = "0.3.31"
@@ -2459,15 +2348,6 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.15"
@@ -2553,12 +2433,6 @@ dependencies = [
  "spin",
 ]
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "lettre"
 version = "0.11.17"
@@ -2597,16 +2471,6 @@ version = "0.2.174"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
 
-[[package]]
-name = "libloading"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
-dependencies = [
- "cfg-if",
- "windows-targets 0.53.2",
-]
-
 [[package]]
 name = "libm"
 version = "0.2.15"
@@ -2634,12 +2498,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.9.4"
@@ -3420,7 +3278,7 @@ dependencies = [
  "concurrent-queue",
  "hermit-abi",
  "pin-project-lite",
- "rustix 1.0.7",
+ "rustix",
  "tracing",
  "windows-sys 0.59.0",
 ]
@@ -3465,16 +3323,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "prettyplease"
-version = "0.2.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a"
-dependencies = [
- "proc-macro2",
- "syn",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.95"
@@ -3564,7 +3412,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash 2.1.1",
+ "rustc-hash",
  "rustls 0.23.28",
  "socket2",
  "thiserror 2.0.12",
@@ -3584,7 +3432,7 @@ dependencies = [
  "lru-slab",
  "rand 0.9.1",
  "ring",
- "rustc-hash 2.1.1",
+ "rustc-hash",
  "rustls 0.23.28",
  "rustls-pki-types",
  "slab",
@@ -4075,12 +3923,6 @@ version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.1"
@@ -4096,19 +3938,6 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -4118,7 +3947,7 @@ dependencies = [
  "bitflags",
  "errno",
  "libc",
- "linux-raw-sys 0.9.4",
+ "linux-raw-sys",
  "windows-sys 0.59.0",
 ]
 
@@ -4140,7 +3969,6 @@ version = "0.23.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643"
 dependencies = [
- "aws-lc-rs",
  "log",
  "once_cell",
  "ring",
@@ -4197,7 +4025,6 @@ version = "0.103.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
 dependencies = [
- "aws-lc-rs",
  "ring",
  "rustls-pki-types",
  "untrusted",
@@ -4649,7 +4476,7 @@ dependencies = [
  "fastrand",
  "getrandom 0.3.3",
  "once_cell",
- "rustix 1.0.7",
+ "rustix",
  "windows-sys 0.59.0",
 ]
 
@@ -5166,6 +4993,7 @@ dependencies = [
  "argon2",
  "aws-config",
  "aws-credential-types",
+ "aws-smithy-runtime-api",
  "bigdecimal",
  "bytes",
  "cached",
@@ -5190,6 +5018,7 @@ dependencies = [
  "handlebars",
  "hickory-resolver",
  "html5gum",
+ "http 1.3.1",
  "job_scheduler_ng",
  "jsonwebtoken",
  "lettre",
@@ -5228,7 +5057,7 @@ dependencies = [
  "url",
  "uuid",
  "webauthn-rs",
- "which 8.0.0",
+ "which",
  "yubico_ng",
 ]
 
@@ -5416,18 +5245,6 @@ dependencies = [
  "rustls-pki-types",
 ]
 
-[[package]]
-name = "which"
-version = "4.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix 0.38.44",
-]
-
 [[package]]
 name = "which"
 version = "8.0.0"
@@ -5435,7 +5252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d"
 dependencies = [
  "env_home",
- "rustix 1.0.7",
+ "rustix",
  "winsafe",
 ]
 

Cargo.toml

@@ -32,7 +32,7 @@ enable_mimalloc = ["dep:mimalloc"]
 # You also need to set an env variable `QUERY_LOGGER=1` to fully activate this so you do not have to re-compile
 # if you want to turn off the logging for a specific run.
 query_logger = ["dep:diesel_logger"]
-s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:anyhow", "dep:reqsign"]
+s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", "dep:anyhow", "dep:http", "dep:reqsign"]
 
 # Enable unstable features, requires nightly
 # Currently only used to enable rusts official ip support
@@ -180,12 +180,14 @@ rpassword = "7.4.0"
 grass_compiler = { version = "0.13.4", default-features = false }
 
 # File are accessed through Apache OpenDAL
-opendal = { version = "0.53.3", features = ["services-fs"] }
+opendal = { version = "0.53.3", features = ["services-fs"], default-features = false }
 
 # For retrieving AWS credentials, including temporary SSO credentials
 anyhow = { version = "1.0.98", optional = true }
-aws-config = { version = "1.8.0", features = ["behavior-version-latest"], optional = true }
+aws-config = { version = "1.8.0", features = ["behavior-version-latest", "rt-tokio", "credentials-process", "sso"], default-features = false, optional = true }
 aws-credential-types = { version = "1.2.3", optional = true }
+aws-smithy-runtime-api = { version = "1.8.1", optional = true }
+http = { version = "1.3.1", optional = true }
 reqsign = { version = "0.16.4", optional = true }
 
 # Strip debuginfo from the release builds

src/config.rs

@@ -1188,6 +1188,9 @@ fn opendal_operator_for_path(path: &str) -> Result<opendal::Operator, Error> {
 
 #[cfg(s3)]
 fn opendal_s3_operator_for_path(path: &str) -> Result<opendal::Operator, Error> {
+    use crate::http_client::aws::AwsReqwestConnector;
+    use aws_config::{default_provider::credentials::DefaultCredentialsChain, provider_config::ProviderConfig};
+
     // This is a custom AWS credential loader that uses the official AWS Rust
     // SDK config crate to load credentials. This ensures maximum compatibility
     // with AWS credential configurations. For example, OpenDAL doesn't support
@@ -1200,12 +1203,19 @@ fn opendal_s3_operator_for_path(path: &str) -> Result<opendal::Operator, Error>
             use aws_credential_types::provider::ProvideCredentials as _;
             use tokio::sync::OnceCell;
 
-            static DEFAULT_CREDENTIAL_CHAIN: OnceCell<
-                aws_config::default_provider::credentials::DefaultCredentialsChain,
-            > = OnceCell::const_new();
+            static DEFAULT_CREDENTIAL_CHAIN: OnceCell<DefaultCredentialsChain> = OnceCell::const_new();
 
             let chain = DEFAULT_CREDENTIAL_CHAIN
-                .get_or_init(|| aws_config::default_provider::credentials::DefaultCredentialsChain::builder().build())
+                .get_or_init(|| {
+                    let reqwest_client = reqwest::Client::builder().build().unwrap();
+                    let connector = AwsReqwestConnector {
+                        client: reqwest_client,
+                    };
+
+                    let conf = ProviderConfig::default().with_http_client(connector);
+
+                    DefaultCredentialsChain::builder().configure(conf).build()
+                })
                 .await;
 
             let creds = chain.provide_credentials().await?;

src/http_client.rs

@@ -244,3 +244,61 @@ impl Resolve for CustomDnsResolver {
         })
     }
 }
+
+#[cfg(s3)]
+pub(crate) mod aws {
+    use aws_smithy_runtime_api::client::{
+        http::{HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector},
+        orchestrator::HttpResponse,
+        result::ConnectorError,
+        runtime_components::RuntimeComponents,
+    };
+    use reqwest::Client;
+
+    // Adapter that wraps reqwest to be compatible with the AWS SDK
+    #[derive(Debug)]
+    pub(crate) struct AwsReqwestConnector {
+        pub(crate) client: Client,
+    }
+
+    impl HttpConnector for AwsReqwestConnector {
+        fn call(&self, request: aws_smithy_runtime_api::client::orchestrator::HttpRequest) -> HttpConnectorFuture {
+            // Convert the AWS-style request to a reqwest request
+            let client = self.client.clone();
+            let future = async move {
+                let method = reqwest::Method::from_bytes(request.method().as_bytes())
+                    .map_err(|e| ConnectorError::user(Box::new(e)))?;
+                let mut req_builder = client.request(method, request.uri().to_string());
+
+                for (name, value) in request.headers() {
+                    req_builder = req_builder.header(name, value);
+                }
+
+                if let Some(body_bytes) = request.body().bytes() {
+                    req_builder = req_builder.body(body_bytes.to_vec());
+                }
+
+                let response = req_builder.send().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
+
+                let status = response.status().into();
+                let bytes = response.bytes().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
+
+                Ok(HttpResponse::new(status, bytes.into()))
+            };
+
+            HttpConnectorFuture::new(Box::pin(future))
+        }
+    }
+
+    impl HttpClient for AwsReqwestConnector {
+        fn http_connector(
+            &self,
+            _settings: &HttpConnectorSettings,
+            _components: &RuntimeComponents,
+        ) -> SharedHttpConnector {
+            SharedHttpConnector::new(AwsReqwestConnector {
+                client: self.client.clone(),
+            })
+        }
+    }
+}