diff --git a/Cargo.lock b/Cargo.lock index a6f7ff3..a635019 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -61,6 +61,21 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-stream" version = "0.3.6" @@ -94,6 +109,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -123,59 +147,21 @@ dependencies = [ ] [[package]] -name = "axum" -version = "0.8.8" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit 0.8.4", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "sync_wrapper", - "tokio", - "tower 0.5.3", - "tower-layer", - "tower-service", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "axum-core" -version = "0.5.6" +name = "base64-simd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", + "outref", + "vsimd", ] -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64ct" version = "1.8.3" @@ -197,6 +183,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -212,6 +207,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bytestring" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "113b4343b5f6617e7ad401ced8de3cc8b012e73a594347c307b90db3e9271289" +dependencies = [ + "bytes", +] + [[package]] name = "cc" version = "1.2.56" @@ -254,10 +258,16 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] +[[package]] +name = "cmov" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -274,6 +284,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -299,6 +325,22 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.0", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.7" @@ -310,6 +352,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -319,27 +370,70 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +dependencies = [ + "cmov", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "285743a676ccb6b3e116bc14cc69319b957867930ae9c4822f8e0f54509d7243" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", + "ctutils", ] [[package]] @@ -578,6 +672,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" @@ -600,18 +700,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "hex" -version = "0.4.3" +name = "hex-simd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "ef451d73f36d8a3f93ad32c332ea01146c9650e1ec821a9b0e46c01277d544f8" dependencies = [ - "digest", + "digest 0.11.1", ] [[package]] @@ -675,6 +779,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -734,9 +847,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", + "tower-layer", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1015,7 +1131,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -1079,19 +1195,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] -name = "matchit" -version = "0.8.4" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] [[package]] name = "md-5" -version = "0.10.6" +version = "0.11.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "59e715bb6f273068fc89403d6c4f5eeb83708c62b74c8d43e3e8772ca73a6288" dependencies = [ "cfg-if", - "digest", + "digest 0.11.1", ] [[package]] @@ -1122,23 +1242,19 @@ name = "multistore" version = "0.1.0" dependencies = [ "async-trait", - "base64", "bytes", "chrono", + "dashmap", "futures", - "hex", - "hmac", "http", - "matchit 0.8.4", "object_store", "quick-xml 0.37.5", + "s3s", "serde", "serde_json", - "sha2", "thiserror", "tracing", "url", - "uuid", ] [[package]] @@ -1158,15 +1274,11 @@ dependencies = [ "js-sys", "multistore", "multistore-metering", - "multistore-oidc-provider", "multistore-static-config", - "multistore-sts", "object_store", - "quick-xml 0.37.5", - "reqwest", + "s3s", "serde", "serde_json", - "thiserror", "tracing", "url", "wasm-bindgen", @@ -1181,16 +1293,13 @@ version = "0.1.0" dependencies = [ "bytes", "http", + "http-body-util", "lambda_http", "multistore", - "multistore-oidc-provider", "multistore-static-config", - "multistore-sts", "object_store", - "reqwest", - "serde", + "s3s", "tokio", - "toml", "tracing", "tracing-subscriber", ] @@ -1199,9 +1308,7 @@ dependencies = [ name = "multistore-metering" version = "0.1.0" dependencies = [ - "bytes", "futures", - "http", "multistore", "tokio", "tracing", @@ -1219,7 +1326,7 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", "tokio", "tracing", @@ -1230,22 +1337,16 @@ dependencies = [ name = "multistore-server" version = "0.1.0" dependencies = [ - "axum", - "bytes", - "futures", - "http", - "http-body-util", + "hyper-util", "multistore", - "multistore-oidc-provider", "multistore-static-config", "multistore-sts", "object_store", "reqwest", + "s3s", "serde", - "thiserror", "tokio", "toml", - "tower-service", "tracing", "tracing-subscriber", ] @@ -1270,7 +1371,6 @@ dependencies = [ "async-trait", "base64", "chrono", - "http", "multistore", "quick-xml 0.37.5", "rand 0.8.5", @@ -1278,13 +1378,22 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", "tokio", "tracing", "url", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1310,6 +1419,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + [[package]] name = "num-integer" version = "0.1.46" @@ -1340,6 +1455,12 @@ dependencies = [ "libm", ] +[[package]] +name = "numeric_cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093" + [[package]] name = "object_store" version = "0.13.1" @@ -1358,7 +1479,7 @@ dependencies = [ "humantime", "hyper", "itertools", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml 0.38.4", @@ -1395,6 +1516,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1507,6 +1634,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1790,8 +1923,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -1869,6 +2002,58 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s3s" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf5572ca9bae5fd92d4e5a3e934c73793ab743bcbc761cd5e86a622c2a5e98a" +dependencies = [ + "arc-swap", + "arrayvec", + "async-trait", + "atoi", + "base64-simd", + "bytes", + "bytestring", + "cfg-if", + "chrono", + "crc-fast", + "futures", + "hex-simd", + "hmac", + "http", + "http-body", + "http-body-util", + "httparse", + "hyper", + "itoa", + "md-5 0.11.0-rc.5", + "memchr", + "mime", + "nom", + "numeric_cast", + "pin-project-lite", + "quick-xml 0.37.5", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", + "sha2 0.11.0-rc.5", + "smallvec", + "std-next", + "subtle", + "sync_wrapper", + "thiserror", + "time", + "tokio", + "tower 0.5.3", + "tracing", + "transform-stream", + "url", + "urlencoding", + "zeroize", +] + [[package]] name = "schannel" version = "0.1.28" @@ -1891,7 +2076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1999,6 +2184,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.11.0-rc.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b167252f3c126be0d8926639c4c4706950f01445900c4b3db0fd7e89fcb750a" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.1", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2007,7 +2203,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0-rc.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f3b1e2dc8aad28310d8410bd4d7e180eca65fca176c52ab00d364475d0024" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.1", ] [[package]] @@ -2041,10 +2248,16 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -2073,6 +2286,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spki" version = "0.7.3" @@ -2089,6 +2308,16 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "std-next" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" +dependencies = [ + "simdutf8", + "thiserror", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2126,6 +2355,27 @@ dependencies = [ "syn", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -2155,6 +2405,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -2418,6 +2699,15 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "transform-stream" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a814d25437963577f6221d33a2aaa60bfb44acc3330cdc7c334644e9832022" +dependencies = [ + "futures-core", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2448,7 +2738,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -2470,6 +2760,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2499,6 +2795,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -2708,6 +3010,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -2993,7 +3306,7 @@ dependencies = [ "http", "http-body", "js-sys", - "matchit 0.7.3", + "matchit", "pin-project", "serde", "serde-wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 71fd6b5..7c84e73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,10 @@ aes-gcm = "0.10" object_store = { version = "0.13.1", default-features = false, features = ["aws"] } futures = "0.3" +# S3 protocol +s3s = { version = "0.13.0", default-features = false } +dashmap = "6" + # XML quick-xml = { version = "0.37", features = ["serialize"] } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index e6f3cae..e8286bc 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -18,14 +18,10 @@ serde_json.workspace = true bytes.workspace = true http.workspace = true chrono.workspace = true -uuid.workspace = true -base64.workspace = true -hex.workspace = true url.workspace = true -hmac.workspace = true -sha2.workspace = true quick-xml.workspace = true tracing.workspace = true object_store.workspace = true futures.workspace = true -matchit.workspace = true +s3s.workspace = true +dashmap.workspace = true diff --git a/crates/core/README.md b/crates/core/README.md index 7d4edb7..5f3ba9e 100644 --- a/crates/core/README.md +++ b/crates/core/README.md @@ -1,94 +1,74 @@ # multistore -Runtime-agnostic core library for the S3 proxy gateway. This crate contains all business logic — S3 request parsing, SigV4 signing/verification, authorization, configuration retrieval, and the proxy handler — without depending on any async runtime. +Runtime-agnostic core library for the S3 proxy gateway. This crate provides an s3s-based S3 service implementation that maps S3 API operations to `object_store` calls, along with trait abstractions for bucket/credential management that allow the proxy to run on multiple runtimes (Tokio/Hyper, AWS Lambda, Cloudflare Workers). ## Why This Crate Exists Separately -The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in containers and Cloudflare Workers on the edge. These runtimes have incompatible stream types, HTTP primitives, and threading models (multi-threaded vs single-threaded WASM). By keeping the core free of runtime dependencies, it compiles cleanly to both `x86_64-unknown-linux-gnu` and `wasm32-unknown-unknown`. +The proxy needs to run on fundamentally different runtimes: Tokio/Hyper in containers, AWS Lambda, and Cloudflare Workers on the edge. These runtimes have incompatible stream types, HTTP primitives, and threading models (multi-threaded vs single-threaded WASM). By keeping the core free of runtime dependencies, it compiles cleanly to both native targets and `wasm32-unknown-unknown`. ## Key Abstractions -The core defines three trait boundaries that runtime crates implement: +**`MultistoreService`** — Implements the `s3s::S3` trait, mapping S3 operations (GET, PUT, DELETE, LIST, multipart uploads) to `object_store` calls. Generic over `BucketRegistry` (for bucket lookup/authorization) and `StoreFactory` (for creating object stores per request). -**`ProxyBackend`** — Provides three capabilities: `create_paginated_store()` returns a `PaginatedListStore` for LIST, `create_signer()` returns a `Signer` for presigned URL generation (GET/HEAD/PUT/DELETE), and `send_raw()` sends signed HTTP requests for multipart operations. Both runtimes delegate to `build_signer()` which uses `object_store`'s built-in signer for authenticated backends and `UnsignedUrlSigner` for anonymous backends (avoiding `Instant::now()` which panics on WASM). For `create_paginated_store()`, the server runtime uses default connectors + reqwest; the worker runtime uses a custom `FetchConnector`. +**`MultistoreAuth`** — Implements `s3s::auth::S3Auth`, wrapping a `CredentialRegistry` to provide SigV4 verification. s3s handles signature parsing and verification; this adapter just looks up secret keys. -**`BucketRegistry`** — Identity-aware bucket resolution and listing. Given a bucket name, identity, and S3 operation, `get_bucket()` returns a `ResolvedBucket` (config + optional list rewrite) or an authorization error. `list_buckets()` returns the buckets visible to a given identity. See `multistore-static-config` for a file-based implementation. +**`StoreFactory`** — Runtime-provided factory for creating `ObjectStore`, `PaginatedListStore`, and `MultipartStore` instances. Each runtime implements this trait with its own HTTP transport (e.g., reqwest for native, `FetchConnector` for Workers). -**`CredentialRegistry`** — Credential and role lookup for authentication infrastructure. Provides `get_credential()` for SigV4 verification and `get_role()` for STS role assumption. See `multistore-static-config` for a file-based implementation. +**`BucketRegistry`** — Identity-aware bucket resolution and listing. Given a bucket name, identity, and S3 operation, `get_bucket()` returns a `ResolvedBucket` or an authorization error. -Any provider implementing these traits can be wrapped with `CachedProvider` for in-memory TTL caching of credential/role lookups (bucket resolution is always delegated directly since it involves authorization). +**`CredentialRegistry`** — Credential and role lookup for authentication. Provides `get_credential()` for SigV4 verification and `get_role()` for STS role assumption. -The core also defines the **`Middleware`** trait for composable post-auth request processing. Middleware runs after identity resolution and authorization, wrapping the backend dispatch call. Each middleware receives a `DispatchContext` and a `Next` handle to continue the chain. The `oidc-provider` crate provides `AwsBackendAuth` as a middleware that resolves backend credentials via OIDC token exchange. +**`StoreBuilder`** — Provider-specific `object_store` builder (S3, Azure, GCS). Runtimes call `create_builder()` to get a half-built store, customize it (e.g., inject an HTTP connector), then call one of the `build_*` methods. ## Module Overview ``` src/ ├── api/ -│ ├── request.rs Parse incoming HTTP → S3Operation enum -│ ├── response.rs Serialize S3 XML responses -│ └── list_rewrite.rs Rewrite / values in list response XML +│ ├── response.rs S3 XML response serialization +│ ├── list.rs LIST-specific helpers (prefix building) +│ └── list_rewrite.rs Rewrite / values for backend prefix mapping ├── auth/ -│ ├── mod.rs Authorization (scope-based access control) -│ ├── identity.rs SigV4 verification, identity resolution -│ └── tests.rs Auth test helpers +│ ├── mod.rs TemporaryCredentialResolver trait +│ └── authorize_impl.rs Scope-based authorization (identity × operation × bucket) ├── backend/ -│ └── mod.rs ProxyBackend trait, Signer/StoreBuilder, S3RequestSigner (multipart) +│ ├── mod.rs StoreBuilder, create_builder() +│ └── url_signer.rs build_signer() helper ├── registry/ -│ ├── mod.rs Re-exports -│ ├── bucket.rs BucketRegistry trait, ResolvedBucket, DEFAULT_BUCKET_OWNER -│ └── credential.rs CredentialRegistry trait -├── error.rs ProxyError with S3-compatible error codes -├── middleware.rs Middleware trait, DispatchContext, Next -├── proxy.rs ProxyGateway — the main request handler -├── route_handler.rs RouteHandler trait, ProxyResponseBody -└── types.rs BucketConfig, RoleConfig, StoredCredential, etc. +│ ├── bucket.rs BucketRegistry trait, ResolvedBucket +│ └── credential.rs CredentialRegistry trait +├── error.rs ProxyError with S3-compatible error codes +├── maybe_send.rs MaybeSend/MaybeSync markers for WASM compatibility +├── service.rs MultistoreService, MultistoreAuth, StoreFactory +└── types.rs BucketConfig, RoleConfig, StoredCredential, etc. ``` ## Usage -This crate is not used directly. Runtime crates depend on it and provide concrete `ProxyBackend` implementations. If you're building a custom runtime integration, depend on this crate and implement `ProxyBackend`, `BucketRegistry`, and/or `CredentialRegistry`. +This crate is not used directly. Runtime crates depend on it and provide concrete `StoreFactory` implementations. -### Standard usage +### Standard usage (s3s) ```rust -use multistore::proxy::ProxyGateway; +use multistore::service::{MultistoreService, MultistoreAuth}; use multistore_static_config::StaticProvider; +use s3s::service::S3ServiceBuilder; -let backend = MyBackend::new(); +let backend = MyStoreFactory::new(); let config = StaticProvider::from_file("config.toml")?; -let gateway = ProxyGateway::new( - backend, - config.clone(), // as BucketRegistry - config, // as CredentialRegistry - Some("s3.example.com".into()), -); -// Optional: enable session token verification for STS temporary credentials. -// let gateway = gateway.with_credential_resolver(token_key); -// Optional: register route handlers for STS, OIDC discovery, etc. -// let gateway = gateway.with_route_handler(sts_handler); -// Optional: register middleware (e.g., OIDC-based backend credential resolution). -// let gateway = gateway.with_middleware(oidc_auth); - -// In your HTTP handler, use handle_request for a two-variant match: -let req_info = RequestInfo { - method: &method, - path: &path, - query: query.as_deref(), - headers: &headers, - source_ip: None, - params: Default::default(), -}; -match gateway.handle_request(&req_info, body, |b| to_bytes(b)).await { - GatewayResponse::Response(result) => { - // Return the complete response (LIST, errors, STS, etc.) - } - GatewayResponse::Forward(fwd, body) => { - // Execute presigned URL with your HTTP client - // Stream request body (PUT) or response body (GET) - } -} +let service = MultistoreService::new(config.clone(), backend); +let auth = MultistoreAuth::new(config); + +let mut builder = S3ServiceBuilder::new(service); +builder.set_auth(auth); + +// Optional: set virtual-hosted-style domain +// builder.set_host(s3s::host::SingleDomain::new("s3.example.com")?); + +let s3_service = builder.build(); + +// Use s3_service with hyper, lambda_http, or call directly ``` ### Custom BucketRegistry @@ -123,12 +103,6 @@ impl BucketRegistry for MyBucketRegistry { } ``` -## Temporary Credential Resolution - -The core defines a `TemporaryCredentialResolver` trait for resolving session tokens (from `x-amz-security-token`) into `TemporaryCredentials`. The core proxy calls this during identity resolution without knowing the token format. - -The `multistore-sts` crate provides `TokenKey`, a sealed-token implementation using AES-256-GCM. Register it via `ProxyGateway::with_credential_resolver()`. See the [sealed tokens documentation](../docs/auth/sealed-tokens.md) for details. - ## Feature Flags All optional — the default build has zero network dependencies: diff --git a/crates/core/src/api/list.rs b/crates/core/src/api/list.rs index 4809272..dea6b55 100644 --- a/crates/core/src/api/list.rs +++ b/crates/core/src/api/list.rs @@ -1,68 +1,8 @@ -//! LIST-specific helpers for building S3 ListObjectsV2 XML responses. -//! -//! Extracted from `proxy.rs` to keep the gateway focused on orchestration. +//! LIST-specific helpers. -use crate::api::list_rewrite::ListRewrite; -use crate::api::response::{ListBucketResult, ListCommonPrefix, ListContents}; -use crate::error::ProxyError; use crate::types::BucketConfig; -/// Parameters for building the S3 ListObjectsV2 XML response. -pub(crate) struct ListXmlParams<'a> { - pub bucket_name: &'a str, - pub client_prefix: &'a str, - pub delimiter: &'a str, - pub max_keys: usize, - pub is_truncated: bool, - pub key_count: usize, - pub start_after: &'a Option, - pub continuation_token: &'a Option, - pub next_continuation_token: Option, -} - -/// All query parameters needed for a LIST operation, parsed in a single pass. -pub(crate) struct ListQueryParams { - pub prefix: String, - pub delimiter: String, - pub max_keys: usize, - pub continuation_token: Option, - pub start_after: Option, -} - -/// Parse prefix, delimiter, and pagination params from a LIST query string in one pass. -pub(crate) fn parse_list_query_params(raw_query: Option<&str>) -> ListQueryParams { - let mut prefix = None; - let mut delimiter = None; - let mut max_keys = None; - let mut continuation_token = None; - let mut start_after = None; - - if let Some(q) = raw_query { - for (k, v) in url::form_urlencoded::parse(q.as_bytes()) { - match k.as_ref() { - "prefix" => prefix = Some(v.into_owned()), - "delimiter" => delimiter = Some(v.into_owned()), - "max-keys" => max_keys = Some(v.into_owned()), - "continuation-token" => continuation_token = Some(v.into_owned()), - "start-after" => start_after = Some(v.into_owned()), - _ => {} - } - } - } - - ListQueryParams { - prefix: prefix.unwrap_or_default(), - delimiter: delimiter.unwrap_or_else(|| "/".to_string()), - max_keys: max_keys - .and_then(|v| v.parse().ok()) - .unwrap_or(1000) - .min(1000), - continuation_token, - start_after, - } -} - -/// Build the full list prefix including backend_prefix. +/// Build the full backend prefix by prepending `backend_prefix` (if set). pub(crate) fn build_list_prefix(config: &BucketConfig, client_prefix: &str) -> String { match &config.backend_prefix { Some(prefix) => { @@ -80,106 +20,3 @@ pub(crate) fn build_list_prefix(config: &BucketConfig, client_prefix: &str) -> S None => client_prefix.to_string(), } } - -/// Build S3 ListObjectsV2 XML from an object_store ListResult. -/// -/// Pagination is handled by the backend — `is_truncated` and -/// `next_continuation_token` are passed through from the backend's response. -pub(crate) fn build_list_xml( - params: &ListXmlParams<'_>, - list_result: &object_store::ListResult, - config: &BucketConfig, - list_rewrite: Option<&ListRewrite>, -) -> Result { - let backend_prefix = config - .backend_prefix - .as_deref() - .unwrap_or("") - .trim_end_matches('/'); - let strip_prefix = if backend_prefix.is_empty() { - String::new() - } else { - format!("{}/", backend_prefix) - }; - - let contents: Vec = list_result - .objects - .iter() - .map(|obj| { - let raw_key = obj.location.to_string(); - ListContents { - key: rewrite_key(&raw_key, &strip_prefix, list_rewrite), - last_modified: obj - .last_modified - .format("%Y-%m-%dT%H:%M:%S%.3fZ") - .to_string(), - etag: obj.e_tag.as_deref().unwrap_or("\"\"").to_string(), - size: obj.size, - storage_class: "STANDARD", - } - }) - .collect(); - - let common_prefixes: Vec = list_result - .common_prefixes - .iter() - .map(|p| { - let raw_prefix = format!("{}/", p); - ListCommonPrefix { - prefix: rewrite_key(&raw_prefix, &strip_prefix, list_rewrite), - } - }) - .collect(); - - Ok(ListBucketResult { - xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", - name: params.bucket_name.to_string(), - prefix: params.client_prefix.to_string(), - delimiter: params.delimiter.to_string(), - max_keys: params.max_keys, - is_truncated: params.is_truncated, - key_count: params.key_count, - start_after: params.start_after.clone(), - continuation_token: params.continuation_token.clone(), - next_continuation_token: params.next_continuation_token.clone(), - contents, - common_prefixes, - } - .to_xml()) -} - -/// Apply strip/add prefix rewriting to a key or prefix value. -/// -/// Works with `&str` slices to avoid intermediate allocations — only allocates -/// the final `String` once. -fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite>) -> String { - // Strip the backend prefix (borrow from `raw`, no allocation) - let key = if !strip_prefix.is_empty() { - raw.strip_prefix(strip_prefix).unwrap_or(raw) - } else { - raw - }; - - // Apply list_rewrite if present - if let Some(rewrite) = list_rewrite { - let key = if !rewrite.strip_prefix.is_empty() { - key.strip_prefix(rewrite.strip_prefix.as_str()) - .unwrap_or(key) - } else { - key - }; - - if !rewrite.add_prefix.is_empty() { - // Must allocate for add_prefix — early return - return if key.is_empty() || key.starts_with('/') { - format!("{}{}", rewrite.add_prefix, key) - } else { - format!("{}/{}", rewrite.add_prefix, key) - }; - } - - return key.to_string(); - } - - key.to_string() -} diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs index f896def..2cbf229 100644 --- a/crates/core/src/api/mod.rs +++ b/crates/core/src/api/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod list; pub mod list_rewrite; -pub mod request; pub mod response; diff --git a/crates/core/src/api/request.rs b/crates/core/src/api/request.rs deleted file mode 100644 index d26dd0a..0000000 --- a/crates/core/src/api/request.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Parse incoming HTTP requests into typed S3 operations. - -use crate::error::ProxyError; -use crate::types::S3Operation; -use http::Method; - -/// Extract the bucket and key from a path-style S3 request. -/// -/// Path-style: `/{bucket}/{key}` -/// Virtual-hosted-style: Host header `{bucket}.s3.example.com` with path `/{key}` -pub fn parse_s3_request( - method: &Method, - uri_path: &str, - query: Option<&str>, - _headers: &http::HeaderMap, - host_style: HostStyle, -) -> Result { - // GET / with path-style → ListBuckets (no bucket in path) - if matches!(host_style, HostStyle::Path) && uri_path.trim_start_matches('/').is_empty() { - if *method == Method::GET { - return Ok(S3Operation::ListBuckets); - } - return Err(ProxyError::InvalidRequest( - "unsupported operation on /".into(), - )); - } - - let (bucket, key) = match host_style { - HostStyle::Path => parse_path_style(uri_path)?, - HostStyle::VirtualHosted { bucket } => { - (bucket, uri_path.trim_start_matches('/').to_string()) - } - }; - - build_s3_operation(method, bucket, key, query) -} - -/// Build an [`S3Operation`] from an already-extracted bucket, key, and query. -/// -/// This is used by both [`parse_s3_request`] and custom resolvers that parse -/// the path themselves (e.g., Source Cooperative). -pub fn build_s3_operation( - method: &Method, - bucket: String, - key: String, - query: Option<&str>, -) -> Result { - let query_params = parse_query_params(query); - - // Check for multipart upload query params - let upload_id = query_params - .iter() - .find(|(k, _)| k == "uploadId") - .map(|(_, v)| v.clone()); - - let has_uploads = query_params.iter().any(|(k, _)| k == "uploads"); - - match *method { - Method::GET => { - if key.is_empty() { - // ListBucket — pass the raw query string through so the proxy - // can forward all list params (prefix, delimiter, max-keys, - // continuation-token, list-type, start-after, etc.) to the backend. - Ok(S3Operation::ListBucket { - bucket, - raw_query: query.map(|q| q.to_string()), - }) - } else { - Ok(S3Operation::GetObject { bucket, key }) - } - } - Method::HEAD => Ok(S3Operation::HeadObject { bucket, key }), - Method::PUT => { - if let Some(upload_id) = upload_id { - let part_number = query_params - .iter() - .find(|(k, _)| k == "partNumber") - .and_then(|(_, v)| v.parse().ok()) - .ok_or_else(|| ProxyError::InvalidRequest("missing partNumber".into()))?; - - Ok(S3Operation::UploadPart { - bucket, - key, - upload_id, - part_number, - }) - } else { - Ok(S3Operation::PutObject { bucket, key }) - } - } - Method::POST => { - if has_uploads { - Ok(S3Operation::CreateMultipartUpload { bucket, key }) - } else if let Some(upload_id) = upload_id { - Ok(S3Operation::CompleteMultipartUpload { - bucket, - key, - upload_id, - }) - } else { - Err(ProxyError::InvalidRequest( - "unsupported POST operation".into(), - )) - } - } - Method::DELETE => { - if let Some(upload_id) = upload_id { - Ok(S3Operation::AbortMultipartUpload { - bucket, - key, - upload_id, - }) - } else if !key.is_empty() { - Ok(S3Operation::DeleteObject { bucket, key }) - } else { - Err(ProxyError::InvalidRequest( - "unsupported DELETE operation".into(), - )) - } - } - _ => Err(ProxyError::InvalidRequest(format!( - "unsupported method: {}", - method - ))), - } -} - -#[derive(Debug, Clone)] -pub enum HostStyle { - /// Path-style: `/{bucket}/{key}` - Path, - /// Virtual-hosted-style: bucket extracted from Host header. - VirtualHosted { bucket: String }, -} - -fn parse_path_style(path: &str) -> Result<(String, String), ProxyError> { - let trimmed = path.trim_start_matches('/'); - if trimmed.is_empty() { - return Err(ProxyError::InvalidRequest("empty path".into())); - } - - match trimmed.split_once('/') { - Some((bucket, key)) => Ok((bucket.to_string(), key.to_string())), - None => Ok((trimmed.to_string(), String::new())), - } -} - -fn parse_query_params(query: Option<&str>) -> Vec<(String, String)> { - query - .map(|q| { - url::form_urlencoded::parse(q.as_bytes()) - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - }) - .unwrap_or_default() -} diff --git a/crates/core/src/auth/authorize.rs b/crates/core/src/auth/authorize_impl.rs similarity index 100% rename from crates/core/src/auth/authorize.rs rename to crates/core/src/auth/authorize_impl.rs diff --git a/crates/core/src/auth/identity.rs b/crates/core/src/auth/identity.rs deleted file mode 100644 index 2421325..0000000 --- a/crates/core/src/auth/identity.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Identity resolution from inbound requests. -//! -//! Parses the SigV4 Authorization header, looks up the credential, verifies -//! the signature, and returns the resolved identity. - -use super::sigv4::{constant_time_eq, parse_sigv4_auth, verify_sigv4_signature}; -use super::TemporaryCredentialResolver; -use crate::error::ProxyError; -use crate::registry::CredentialRegistry; -use crate::types::ResolvedIdentity; -use http::HeaderMap; - -/// Resolve the identity of an incoming request. -/// -/// Parses the SigV4 Authorization header, looks up the credential, verifies -/// the signature, and returns the resolved identity. -pub async fn resolve_identity( - method: &http::Method, - uri_path: &str, - query_string: &str, - headers: &HeaderMap, - config: &C, - credential_resolver: Option<&dyn TemporaryCredentialResolver>, -) -> Result { - let auth_header = match headers.get("authorization").and_then(|v| v.to_str().ok()) { - Some(h) => h, - None => return Ok(ResolvedIdentity::Anonymous), - }; - - let sig = parse_sigv4_auth(auth_header)?; - - // The payload hash is sent by the client in x-amz-content-sha256. - // For streaming uploads this is the UNSIGNED-PAYLOAD or - // STREAMING-AWS4-HMAC-SHA256-PAYLOAD sentinel — both are valid - // canonical-request inputs per the SigV4 spec. - let payload_hash = headers - .get("x-amz-content-sha256") - .and_then(|v| v.to_str().ok()) - .unwrap_or("UNSIGNED-PAYLOAD"); - - // Temporary credentials: resolve the session token to recover credentials - if let Some(session_token) = headers - .get("x-amz-security-token") - .and_then(|v| v.to_str().ok()) - { - let resolver = credential_resolver.ok_or_else(|| { - tracing::warn!("session token present but no credential resolver configured"); - ProxyError::AccessDenied - })?; - - match resolver.resolve(session_token)? { - Some(creds) => { - if !constant_time_eq(sig.access_key_id.as_bytes(), creds.access_key_id.as_bytes()) { - tracing::warn!( - header_key = %sig.access_key_id, - resolved_key = %creds.access_key_id, - "access key mismatch between auth header and session token" - ); - return Err(ProxyError::AccessDenied); - } - if !verify_sigv4_signature( - method, - uri_path, - query_string, - headers, - &sig, - &creds.secret_access_key, - payload_hash, - )? { - return Err(ProxyError::SignatureDoesNotMatch); - } - tracing::debug!( - access_key = %creds.access_key_id, - role = %creds.assumed_role_id, - scopes = ?creds.allowed_scopes, - "temporary credential identity resolved" - ); - return Ok(ResolvedIdentity::Temporary { credentials: creds }); - } - None => { - tracing::warn!( - access_key_id = %sig.access_key_id, - token_len = session_token.len(), - "session token could not be resolved — possible key mismatch, token corruption, or expired key rotation" - ); - return Err(ProxyError::AccessDenied); - } - } - } - - // Check long-lived credentials - if let Some(cred) = config.get_credential(&sig.access_key_id).await? { - if !cred.enabled { - return Err(ProxyError::AccessDenied); - } - if let Some(expires) = cred.expires_at { - if expires <= chrono::Utc::now() { - return Err(ProxyError::ExpiredCredentials); - } - } - - // Verify SigV4 signature - if !verify_sigv4_signature( - method, - uri_path, - query_string, - headers, - &sig, - &cred.secret_access_key, - payload_hash, - )? { - return Err(ProxyError::SignatureDoesNotMatch); - } - - return Ok(ResolvedIdentity::LongLived { credential: cred }); - } - - Err(ProxyError::AccessDenied) -} diff --git a/crates/core/src/auth/mod.rs b/crates/core/src/auth/mod.rs index 496c0e0..58476ba 100644 --- a/crates/core/src/auth/mod.rs +++ b/crates/core/src/auth/mod.rs @@ -1,17 +1,12 @@ //! Authentication and authorization. //! -//! - [`sigv4`] — SigV4 request parsing and signature verification -//! - [`identity`] — identity resolution (mapping access key → principal) -//! - [`authorize`](self::authorize::authorize) — authorization (scope checking) -//! - [`TemporaryCredentialResolver`] — trait for resolving session tokens into temporary credentials +//! SigV4 verification is handled by the s3s framework. This module provides: +//! - [`authorize`] — check if an identity can perform an S3 operation +//! - [`TemporaryCredentialResolver`] — resolve session tokens into temporary credentials -mod authorize; -pub mod identity; -pub mod sigv4; +mod authorize_impl; -pub use authorize::authorize; -pub use identity::resolve_identity; -pub use sigv4::{parse_sigv4_auth, verify_sigv4_signature, SigV4Auth}; +pub use authorize_impl::authorize; use crate::error::ProxyError; use crate::maybe_send::{MaybeSend, MaybeSync}; @@ -24,6 +19,3 @@ use crate::types::TemporaryCredentials; pub trait TemporaryCredentialResolver: MaybeSend + MaybeSync { fn resolve(&self, token: &str) -> Result, ProxyError>; } - -#[cfg(test)] -mod tests; diff --git a/crates/core/src/auth/sigv4.rs b/crates/core/src/auth/sigv4.rs deleted file mode 100644 index 2b2e467..0000000 --- a/crates/core/src/auth/sigv4.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! SigV4 signature parsing and verification for inbound requests. - -use crate::error::ProxyError; -use hmac::{Hmac, Mac}; -use http::HeaderMap; -use sha2::{Digest, Sha256}; - -type HmacSha256 = Hmac; - -/// Parsed SigV4 Authorization header. -#[derive(Debug, Clone)] -pub struct SigV4Auth { - pub access_key_id: String, - pub date_stamp: String, - pub region: String, - pub service: String, - pub signed_headers: Vec, - pub signature: String, -} - -/// Parse a SigV4 Authorization header. -/// -/// Format: `AWS4-HMAC-SHA256 Credential=AKID/20240101/us-east-1/s3/aws4_request, -/// SignedHeaders=host;x-amz-date, Signature=abcdef...` -pub fn parse_sigv4_auth(auth_header: &str) -> Result { - let auth_header = auth_header - .strip_prefix("AWS4-HMAC-SHA256 ") - .ok_or_else(|| ProxyError::InvalidRequest("invalid auth scheme".into()))?; - - let mut credential = None; - let mut signed_headers = None; - let mut signature = None; - - for part in auth_header.split(", ") { - if let Some(val) = part.strip_prefix("Credential=") { - credential = Some(val); - } else if let Some(val) = part.strip_prefix("SignedHeaders=") { - signed_headers = Some(val); - } else if let Some(val) = part.strip_prefix("Signature=") { - signature = Some(val); - } - } - - let credential = - credential.ok_or_else(|| ProxyError::InvalidRequest("missing Credential".into()))?; - let signed_headers = - signed_headers.ok_or_else(|| ProxyError::InvalidRequest("missing SignedHeaders".into()))?; - let signature = - signature.ok_or_else(|| ProxyError::InvalidRequest("missing Signature".into()))?; - - // Parse credential: AKID/date/region/service/aws4_request - let cred_parts: Vec<&str> = credential.split('/').collect(); - if cred_parts.len() != 5 || cred_parts[4] != "aws4_request" { - return Err(ProxyError::InvalidRequest( - "malformed credential scope".into(), - )); - } - - Ok(SigV4Auth { - access_key_id: cred_parts[0].to_string(), - date_stamp: cred_parts[1].to_string(), - region: cred_parts[2].to_string(), - service: cred_parts[3].to_string(), - signed_headers: signed_headers.split(';').map(String::from).collect(), - signature: signature.to_string(), - }) -} - -/// Verify a SigV4 signature against a known secret key. -pub fn verify_sigv4_signature( - method: &http::Method, - uri_path: &str, - query_string: &str, - headers: &HeaderMap, - auth: &SigV4Auth, - secret_access_key: &str, - payload_hash: &str, -) -> Result { - // Reconstruct canonical request - let canonical_headers: String = auth - .signed_headers - .iter() - .map(|name| { - let value = headers - .get(name.as_str()) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .trim(); - format!("{}:{}\n", name, value) - }) - .collect(); - - let signed_headers_str = auth.signed_headers.join(";"); - - // SigV4 requires query parameters sorted alphabetically by key (then value). - // The raw query string from the URL may not be sorted, but the client SDK - // sorts them when constructing the canonical request for signing. - let canonical_query = canonicalize_query_string(query_string); - - let canonical_request = format!( - "{}\n{}\n{}\n{}\n{}\n{}", - method, uri_path, canonical_query, canonical_headers, signed_headers_str, payload_hash - ); - - let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); - - let credential_scope = format!( - "{}/{}/{}/aws4_request", - auth.date_stamp, auth.region, auth.service - ); - - let amz_date = headers - .get("x-amz-date") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - - let string_to_sign = format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - amz_date, credential_scope, canonical_request_hash - ); - - // Derive signing key - let k_date = hmac_sha256( - format!("AWS4{}", secret_access_key).as_bytes(), - auth.date_stamp.as_bytes(), - )?; - let k_region = hmac_sha256(&k_date, auth.region.as_bytes())?; - let k_service = hmac_sha256(&k_region, auth.service.as_bytes())?; - let signing_key = hmac_sha256(&k_service, b"aws4_request")?; - - let expected_signature = hex::encode(hmac_sha256(&signing_key, string_to_sign.as_bytes())?); - - let matched = constant_time_eq(expected_signature.as_bytes(), auth.signature.as_bytes()); - - if !matched { - tracing::warn!( - access_key_id = %auth.access_key_id, - region = %auth.region, - "SigV4 signature mismatch" - ); - tracing::debug!( - canonical_request = %canonical_request, - string_to_sign = %string_to_sign, - "SigV4 signature mismatch details — compare canonical_request with client-side (aws --debug)" - ); - } - - Ok(matched) -} - -/// Sort query string parameters for SigV4 canonical request construction. -pub(crate) fn canonicalize_query_string(query: &str) -> String { - if query.is_empty() { - return String::new(); - } - let mut parts: Vec<&str> = query.split('&').collect(); - parts.sort_unstable(); - parts.join("&") -} - -pub(crate) fn hmac_sha256(key: &[u8], data: &[u8]) -> Result, ProxyError> { - let mut mac = - HmacSha256::new_from_slice(key).map_err(|e| ProxyError::Internal(e.to_string()))?; - mac.update(data); - Ok(mac.finalize().into_bytes().to_vec()) -} - -pub(crate) fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { - if a.len() != b.len() { - return false; - } - a.iter() - .zip(b.iter()) - .fold(0u8, |acc, (x, y)| acc | (x ^ y)) - == 0 -} diff --git a/crates/core/src/auth/tests.rs b/crates/core/src/auth/tests.rs deleted file mode 100644 index 850e94b..0000000 --- a/crates/core/src/auth/tests.rs +++ /dev/null @@ -1,737 +0,0 @@ -use super::*; -use crate::auth::TemporaryCredentialResolver; -use crate::error::ProxyError; -use crate::types::{AccessScope, Action, RoleConfig, StoredCredential, TemporaryCredentials}; -use http::HeaderMap; -use sha2::{Digest, Sha256}; - -// ── Mock config provider ────────────────────────────────────────── - -#[derive(Clone)] -struct MockConfig { - credentials: Vec, -} - -impl MockConfig { - fn with_credential(secret: &str) -> Self { - Self { - credentials: vec![StoredCredential { - access_key_id: "AKIAIOSFODNN7EXAMPLE".into(), - secret_access_key: secret.into(), - principal_name: "test-user".into(), - allowed_scopes: vec![AccessScope { - bucket: "test-bucket".into(), - prefixes: vec![], - actions: vec![Action::GetObject], - }], - created_at: chrono::Utc::now(), - expires_at: None, - enabled: true, - }], - } - } - - fn empty() -> Self { - Self { - credentials: vec![], - } - } -} - -impl crate::registry::CredentialRegistry for MockConfig { - async fn get_credential( - &self, - access_key_id: &str, - ) -> Result, ProxyError> { - Ok(self - .credentials - .iter() - .find(|c| c.access_key_id == access_key_id) - .cloned()) - } - async fn get_role(&self, _: &str) -> Result, ProxyError> { - Ok(None) - } -} - -// ── Test signing helper ─────────────────────────────────────────── - -/// Build a valid SigV4 Authorization header value for testing. -fn sign_request( - method: &http::Method, - uri_path: &str, - query_string: &str, - headers: &HeaderMap, - access_key_id: &str, - secret_access_key: &str, - date_stamp: &str, - amz_date: &str, - region: &str, - signed_header_names: &[&str], - payload_hash: &str, -) -> String { - let canonical_headers: String = signed_header_names - .iter() - .map(|name| { - let value = headers - .get(*name) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .trim(); - format!("{}:{}\n", name, value) - }) - .collect(); - - let signed_headers_str = signed_header_names.join(";"); - - // AWS SDKs sort query parameters when constructing the canonical request - let canonical_query = sigv4::canonicalize_query_string(query_string); - - let canonical_request = format!( - "{}\n{}\n{}\n{}\n{}\n{}", - method, uri_path, canonical_query, canonical_headers, signed_headers_str, payload_hash - ); - - let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); - let credential_scope = format!("{}/{}/s3/aws4_request", date_stamp, region); - let string_to_sign = format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - amz_date, credential_scope, canonical_request_hash - ); - - let k_date = sigv4::hmac_sha256( - format!("AWS4{}", secret_access_key).as_bytes(), - date_stamp.as_bytes(), - ) - .unwrap(); - let k_region = sigv4::hmac_sha256(&k_date, region.as_bytes()).unwrap(); - let k_service = sigv4::hmac_sha256(&k_region, b"s3").unwrap(); - let signing_key = sigv4::hmac_sha256(&k_service, b"aws4_request").unwrap(); - let signature = - hex::encode(sigv4::hmac_sha256(&signing_key, string_to_sign.as_bytes()).unwrap()); - - format!( - "AWS4-HMAC-SHA256 Credential={}/{}/{}/s3/aws4_request, SignedHeaders={}, Signature={}", - access_key_id, date_stamp, region, signed_headers_str, signature - ) -} - -/// Build headers and auth for a simple GET request. -fn make_signed_headers(access_key_id: &str, secret_access_key: &str) -> HeaderMap { - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let region = "us-east-1"; - let payload_hash = "UNSIGNED-PAYLOAD"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - - let auth = sign_request( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - access_key_id, - secret_access_key, - date_stamp, - amz_date, - region, - &["host", "x-amz-content-sha256", "x-amz-date"], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - headers -} - -// ── Tests ───────────────────────────────────────────────────────── - -fn run(f: F) -> F::Output { - futures::executor::block_on(f) -} - -#[test] -fn no_auth_header_returns_anonymous() { - run(async { - let headers = HeaderMap::new(); - let config = MockConfig::empty(); - - let identity = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap(); - - assert!(matches!( - identity, - crate::types::ResolvedIdentity::Anonymous - )); - }); -} - -#[test] -fn valid_signature_resolves_identity() { - run(async { - let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let config = MockConfig::with_credential(secret); - let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); - - let identity = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap(); - - assert!(matches!( - identity, - crate::types::ResolvedIdentity::LongLived { .. } - )); - }); -} - -#[test] -fn valid_signature_with_unsorted_query_params() { - run(async { - let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let config = MockConfig::with_credential(secret); - - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let payload_hash = "UNSIGNED-PAYLOAD"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - - // Sign with sorted query (as AWS SDKs do internally) - let auth = sign_request( - &http::Method::GET, - "/test-bucket", - "list-type=2&prefix=&delimiter=%2F&encoding-type=url", - &headers, - "AKIAIOSFODNN7EXAMPLE", - secret, - date_stamp, - amz_date, - "us-east-1", - &["host", "x-amz-content-sha256", "x-amz-date"], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - - // Pass UNSORTED query string (as it arrives from the raw URL) - let identity = resolve_identity( - &http::Method::GET, - "/test-bucket", - "list-type=2&prefix=&delimiter=%2F&encoding-type=url", - &headers, - &config, - None, - ) - .await - .unwrap(); - - assert!(matches!( - identity, - crate::types::ResolvedIdentity::LongLived { .. } - )); - }); -} - -#[test] -fn wrong_signature_is_rejected() { - run(async { - let real_secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; - let config = MockConfig::with_credential(real_secret); - // Sign with wrong secret — access_key_id is correct, signature won't match - let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", wrong_secret); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap_err(); - - assert!( - matches!(err, ProxyError::SignatureDoesNotMatch), - "expected SignatureDoesNotMatch, got: {:?}", - err - ); - }); -} - -#[test] -fn garbage_signature_is_rejected() { - run(async { - let real_secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let config = MockConfig::with_credential(real_secret); - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", "20240101T000000Z".parse().unwrap()); - headers.insert("x-amz-content-sha256", "UNSIGNED-PAYLOAD".parse().unwrap()); - headers.insert( - "authorization", - "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20240101/us-east-1/s3/aws4_request, \ - SignedHeaders=host;x-amz-content-sha256;x-amz-date, \ - Signature=0000000000000000000000000000000000000000000000000000000000000000" - .parse() - .unwrap(), - ); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap_err(); - - assert!(matches!(err, ProxyError::SignatureDoesNotMatch)); - }); -} - -#[test] -fn unknown_access_key_is_rejected() { - run(async { - let config = MockConfig::empty(); - let headers = make_signed_headers("AKIAUNKNOWN000000000", "some-secret"); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap_err(); - - assert!(matches!(err, ProxyError::AccessDenied)); - }); -} - -#[test] -fn temporary_credential_wrong_session_token_is_rejected() { - run(async { - let config = MockConfig::empty(); - - let secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let wrong_token = "NOT_A_VALID_TOKEN"; - - // Resolver only accepts "VALID_TOKEN", so "NOT_A_VALID_TOKEN" returns None - let resolver = MockResolver { - expected_token: "VALID_TOKEN".into(), - creds: TemporaryCredentials { - access_key_id: "ASIATEMP1234EXAMPLE".into(), - secret_access_key: secret.into(), - session_token: "VALID_TOKEN".into(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - allowed_scopes: vec![AccessScope { - bucket: "test-bucket".into(), - prefixes: vec![], - actions: vec![Action::GetObject], - }], - assumed_role_id: "role-1".into(), - source_identity: "test".into(), - }, - }; - - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let payload_hash = "UNSIGNED-PAYLOAD"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", wrong_token.parse().unwrap()); - - let auth = sign_request( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - "ASIATEMP1234EXAMPLE", - secret, - date_stamp, - amz_date, - "us-east-1", - &[ - "host", - "x-amz-content-sha256", - "x-amz-date", - "x-amz-security-token", - ], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - Some(&resolver as &dyn TemporaryCredentialResolver), - ) - .await - .unwrap_err(); - - assert!(matches!(err, ProxyError::AccessDenied)); - }); -} - -#[test] -fn temporary_credential_wrong_signature_is_rejected() { - run(async { - let real_secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let wrong_secret = "WRONGSECRETKEYWRONGSECRETSECRET00000000000"; - let mock_token = "MOCK_SESSION_TOKEN"; - - let resolver = MockResolver { - expected_token: mock_token.into(), - creds: TemporaryCredentials { - access_key_id: "ASIATEMP1234EXAMPLE".into(), - secret_access_key: real_secret.into(), - session_token: mock_token.into(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - allowed_scopes: vec![AccessScope { - bucket: "test-bucket".into(), - prefixes: vec![], - actions: vec![Action::GetObject], - }], - assumed_role_id: "role-1".into(), - source_identity: "test".into(), - }, - }; - - let config = MockConfig::empty(); - - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let payload_hash = "UNSIGNED-PAYLOAD"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", mock_token.parse().unwrap()); - - // Sign with wrong secret — resolver returns valid creds but sig won't match - let auth = sign_request( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - "ASIATEMP1234EXAMPLE", - wrong_secret, - date_stamp, - amz_date, - "us-east-1", - &[ - "host", - "x-amz-content-sha256", - "x-amz-date", - "x-amz-security-token", - ], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - Some(&resolver as &dyn TemporaryCredentialResolver), - ) - .await - .unwrap_err(); - - assert!( - matches!(err, ProxyError::SignatureDoesNotMatch), - "expected SignatureDoesNotMatch, got: {:?}", - err - ); - }); -} - -#[test] -fn disabled_credential_is_rejected_before_sig_check() { - run(async { - let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let mut config = MockConfig::with_credential(secret); - config.credentials[0].enabled = false; - - let headers = make_signed_headers("AKIAIOSFODNN7EXAMPLE", secret); - - let err = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - None, - ) - .await - .unwrap_err(); - - assert!(matches!(err, ProxyError::AccessDenied)); - }); -} - -// ── SigV4 spec compliance tests ────────────────────────────────── - -/// Validate our SigV4 implementation against the official AWS test suite. -/// Test vector: "get-vanilla" from -/// https://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html -#[test] -fn sigv4_test_vector_get_vanilla() { - let access_key_id = "AKIDEXAMPLE"; - let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; - let date_stamp = "20150830"; - let amz_date = "20150830T123600Z"; - let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "example.amazonaws.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - - let auth = SigV4Auth { - access_key_id: access_key_id.to_string(), - date_stamp: date_stamp.to_string(), - region: "us-east-1".to_string(), - service: "service".to_string(), - signed_headers: vec!["host".to_string(), "x-amz-date".to_string()], - signature: "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31".to_string(), - }; - - let result = verify_sigv4_signature( - &http::Method::GET, - "/", - "", - &headers, - &auth, - secret, - payload_hash, - ) - .unwrap(); - - assert!(result, "AWS SigV4 test vector 'get-vanilla' must pass"); -} - -/// Test vector: "get-vanilla-query-order-key" — verifies query parameter sorting. -#[test] -fn sigv4_test_vector_query_order() { - let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; - let date_stamp = "20150830"; - let amz_date = "20150830T123600Z"; - let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "example.amazonaws.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - - let auth = SigV4Auth { - access_key_id: "AKIDEXAMPLE".to_string(), - date_stamp: date_stamp.to_string(), - region: "us-east-1".to_string(), - service: "service".to_string(), - signed_headers: vec!["host".to_string(), "x-amz-date".to_string()], - signature: "b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500".to_string(), - }; - - // Pass UNSORTED query — our canonicalization should sort to Param1=value1&Param2=value2 - let result = verify_sigv4_signature( - &http::Method::GET, - "/", - "Param2=value2&Param1=value1", - &headers, - &auth, - secret, - payload_hash, - ) - .unwrap(); - - assert!( - result, - "AWS SigV4 test vector 'get-vanilla-query-order-key' must pass" - ); -} - -/// Realistic S3 ListObjectsV2 request with host:port, security token, -/// and unsorted query parameters — mirrors what `aws s3 ls` sends. -#[test] -fn sigv4_list_objects_with_security_token_and_port() { - let secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let session_token = "FwoGZXIvYXdzEBYaDGFiY2RlZjEyMzQ1Ng"; - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "localhost:8787".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", session_token.parse().unwrap()); - - // Sign with sorted query (as AWS SDKs do) - let auth = sign_request( - &http::Method::GET, - "/private-uploads", - "list-type=2&prefix=&delimiter=%2F&encoding-type=url", - &headers, - "ASIATEMP1234EXAMPLE", - secret, - date_stamp, - amz_date, - "us-east-1", - &[ - "host", - "x-amz-content-sha256", - "x-amz-date", - "x-amz-security-token", - ], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - - // Verify with UNSORTED query (as it arrives from the raw URL) - let sig = parse_sigv4_auth(headers.get("authorization").unwrap().to_str().unwrap()).unwrap(); - - let result = verify_sigv4_signature( - &http::Method::GET, - "/private-uploads", - "list-type=2&prefix=&delimiter=%2F&encoding-type=url", - &headers, - &sig, - secret, - payload_hash, - ) - .unwrap(); - - assert!( - result, - "S3 ListObjects with security token and host:port must verify" - ); -} - -// ── Mock credential resolver ──────────────────────────────────── - -/// A mock `TemporaryCredentialResolver` that returns fixed credentials -/// when the token matches a known value, or `None` otherwise. -struct MockResolver { - /// The token string to match against. - expected_token: String, - /// The credentials to return when the token matches. - creds: TemporaryCredentials, -} - -impl TemporaryCredentialResolver for MockResolver { - fn resolve(&self, token: &str) -> Result, ProxyError> { - if token == self.expected_token { - Ok(Some(self.creds.clone())) - } else { - Ok(None) - } - } -} - -// ── Temporary credential resolver tests ───────────────────────── - -#[test] -fn temporary_credential_round_trip() { - run(async { - let secret = "TempSecretKey1234567890EXAMPLE000000000000"; - let mock_token = "MOCK_SESSION_TOKEN"; - let creds = TemporaryCredentials { - access_key_id: "ASIATEMP1234EXAMPLE".into(), - secret_access_key: secret.into(), - session_token: mock_token.into(), - expiration: chrono::Utc::now() + chrono::Duration::hours(1), - allowed_scopes: vec![AccessScope { - bucket: "test-bucket".into(), - prefixes: vec![], - actions: vec![Action::GetObject], - }], - assumed_role_id: "role-1".into(), - source_identity: "test".into(), - }; - - let resolver = MockResolver { - expected_token: mock_token.into(), - creds, - }; - let config = MockConfig::empty(); - - let date_stamp = "20240101"; - let amz_date = "20240101T000000Z"; - let payload_hash = "UNSIGNED-PAYLOAD"; - - let mut headers = HeaderMap::new(); - headers.insert("host", "s3.example.com".parse().unwrap()); - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - headers.insert("x-amz-security-token", mock_token.parse().unwrap()); - - let auth = sign_request( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - "ASIATEMP1234EXAMPLE", - secret, - date_stamp, - amz_date, - "us-east-1", - &[ - "host", - "x-amz-content-sha256", - "x-amz-date", - "x-amz-security-token", - ], - payload_hash, - ); - headers.insert("authorization", auth.parse().unwrap()); - - let identity = resolve_identity( - &http::Method::GET, - "/test-bucket/key.txt", - "", - &headers, - &config, - Some(&resolver as &dyn TemporaryCredentialResolver), - ) - .await - .unwrap(); - - assert!(matches!( - identity, - crate::types::ResolvedIdentity::Temporary { .. } - )); - }); -} diff --git a/crates/core/src/backend/mod.rs b/crates/core/src/backend/mod.rs index 14b926e..e1a235f 100644 --- a/crates/core/src/backend/mod.rs +++ b/crates/core/src/backend/mod.rs @@ -1,33 +1,22 @@ -//! Backend abstraction for proxying requests to backing object stores. +//! Backend abstraction for building object stores from bucket configuration. //! -//! [`ProxyBackend`] is the main trait runtimes implement. It provides three -//! capabilities: +//! [`StoreBuilder`] wraps provider-specific builders (S3, Azure, GCS) and +//! provides a uniform API for constructing `ObjectStore`, `PaginatedListStore`, +//! `MultipartStore`, and `Signer` instances. //! -//! 1. **`create_paginated_store()`** — build a `PaginatedListStore` for LIST -//! operations with backend-side pagination. -//! 2. **`create_signer()`** — build a `Signer` for generating presigned URLs -//! for GET, HEAD, PUT, DELETE operations. -//! 3. **`send_raw()`** — send a pre-signed HTTP request for operations not -//! covered by `ObjectStore` (multipart uploads). -//! -//! The [`url_signer`] submodule handles `object_store` signer construction. -//! The [`request_signer`] submodule handles outbound SigV4 request signing. -//! The [`multipart`] submodule builds URLs and signs multipart upload requests. +//! Runtimes call [`create_builder`] to get a half-built store, customize it +//! (e.g. inject an HTTP connector), then call one of the `build_*` methods. -pub mod multipart; -pub mod request_signer; pub mod url_signer; pub use url_signer::build_signer; use crate::error::ProxyError; -use crate::maybe_send::{MaybeSend, MaybeSync}; use crate::types::{BackendType, BucketConfig}; -use bytes::Bytes; -use http::HeaderMap; use object_store::aws::AmazonS3Builder; use object_store::list::PaginatedListStore; +use object_store::multipart::MultipartStore; use object_store::signer::Signer; -use std::future::Future; +use object_store::ObjectStore; use std::sync::Arc; #[cfg(feature = "azure")] @@ -35,47 +24,6 @@ use object_store::azure::MicrosoftAzureBuilder; #[cfg(feature = "gcp")] use object_store::gcp::GoogleCloudStorageBuilder; -/// Trait for runtime-specific backend operations. -/// -/// Each runtime provides its own implementation: -/// - Server runtime: uses `reqwest` for raw HTTP, default `object_store` HTTP connector -/// - Worker runtime: uses `web_sys::fetch` for raw HTTP, custom `FetchConnector` for `object_store` -pub trait ProxyBackend: Clone + MaybeSend + MaybeSync + 'static { - /// Create a [`PaginatedListStore`] for the given bucket configuration. - /// - /// Used for LIST operations with backend-side pagination via - /// [`PaginatedListStore::list_paginated`], avoiding loading all results - /// into memory. - fn create_paginated_store( - &self, - config: &BucketConfig, - ) -> Result, ProxyError>; - - /// Create a `Signer` for generating presigned URLs. - /// - /// Used for GET, HEAD, PUT, DELETE operations. The handler generates - /// a presigned URL and the runtime executes the request with its - /// native HTTP client, enabling zero-copy streaming. - fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError>; - - /// Send a raw HTTP request (used for multipart operations that - /// `ObjectStore` doesn't expose at the right abstraction level). - fn send_raw( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - body: Bytes, - ) -> impl Future> + MaybeSend; -} - -/// Response from a raw HTTP request to a backend. -pub struct RawResponse { - pub status: u16, - pub headers: HeaderMap, - pub body: Bytes, -} - /// Wrapper around provider-specific `object_store` builders. /// /// Obtain one via [`create_builder`], customize it (e.g. inject an HTTP @@ -123,6 +71,40 @@ impl StoreBuilder { })?)), } } + + /// Build an [`ObjectStore`] for GET/HEAD/PUT/DELETE operations. + pub fn build_object_store(self) -> Result, ProxyError> { + match self { + StoreBuilder::S3(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build S3 object store: {}", e)) + })?)), + #[cfg(feature = "azure")] + StoreBuilder::Azure(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build Azure object store: {}", e)) + })?)), + #[cfg(feature = "gcp")] + StoreBuilder::Gcs(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build GCS object store: {}", e)) + })?)), + } + } + + /// Build a [`MultipartStore`] for multipart upload operations. + pub fn build_multipart_store(self) -> Result, ProxyError> { + match self { + StoreBuilder::S3(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build S3 multipart store: {}", e)) + })?)), + #[cfg(feature = "azure")] + StoreBuilder::Azure(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build Azure multipart store: {}", e)) + })?)), + #[cfg(feature = "gcp")] + StoreBuilder::Gcs(b) => Ok(Arc::new(b.build().map_err(|e| { + ProxyError::ConfigError(format!("failed to build GCS multipart store: {}", e)) + })?)), + } + } } /// Create a [`StoreBuilder`] from a [`BucketConfig`], dispatching on `backend_type`. diff --git a/crates/core/src/backend/multipart.rs b/crates/core/src/backend/multipart.rs deleted file mode 100644 index 11a13c2..0000000 --- a/crates/core/src/backend/multipart.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Multipart URL building and request signing for S3-compatible backends. -//! -//! These helpers are used by [`Gateway::execute_multipart`](crate::proxy::Gateway) -//! for CreateMultipartUpload, UploadPart, CompleteMultipartUpload, and -//! AbortMultipartUpload operations. - -use crate::backend::request_signer::S3RequestSigner; -use crate::error::ProxyError; -use crate::types::{BucketConfig, S3Operation}; -use http::{HeaderMap, Method}; -use url::Url; - -/// Build the backend URL for an S3 operation. -/// -/// Used for multipart operations that go through raw signed HTTP. -pub fn build_backend_url( - config: &BucketConfig, - operation: &S3Operation, -) -> Result { - let endpoint = config.option("endpoint").unwrap_or(""); - let base = endpoint.trim_end_matches('/'); - let bucket = config.option("bucket_name").unwrap_or(""); - let bucket_is_empty = bucket.is_empty(); - - let mut key = String::new(); - if let Some(prefix) = &config.backend_prefix { - key.push_str(prefix.trim_end_matches('/')); - key.push('/'); - } - key.push_str(operation.key()); - - let mut url = if bucket_is_empty { - format!("{}/{}", base, key) - } else { - format!("{}/{}/{}", base, bucket, key) - }; - - match operation { - S3Operation::CreateMultipartUpload { .. } => { - url.push_str("?uploads"); - } - S3Operation::UploadPart { - upload_id, - part_number, - .. - } => { - let qs = url::form_urlencoded::Serializer::new(String::new()) - .append_pair("partNumber", &part_number.to_string()) - .append_pair("uploadId", upload_id) - .finish(); - url.push('?'); - url.push_str(&qs); - } - S3Operation::CompleteMultipartUpload { upload_id, .. } - | S3Operation::AbortMultipartUpload { upload_id, .. } => { - let qs = url::form_urlencoded::Serializer::new(String::new()) - .append_pair("uploadId", upload_id) - .finish(); - url.push('?'); - url.push_str(&qs); - } - _ => {} - } - - Ok(url) -} - -/// Sign an outbound S3 request using credentials from the bucket config. -/// -/// Used for multipart operations only. CRUD operations use presigned URLs. -pub(crate) fn sign_s3_request( - method: &Method, - url: &str, - headers: &mut HeaderMap, - config: &BucketConfig, - payload_hash: &str, -) -> Result<(), ProxyError> { - let access_key = config.option("access_key_id").unwrap_or(""); - let secret_key = config.option("secret_access_key").unwrap_or(""); - let region = config.option("region").unwrap_or("us-east-1"); - let has_credentials = !access_key.is_empty() && !secret_key.is_empty(); - - let parsed_url = - Url::parse(url).map_err(|e| ProxyError::Internal(format!("invalid backend URL: {}", e)))?; - - if has_credentials { - let session_token = config.option("token").map(|s| s.to_string()); - let signer = S3RequestSigner::new( - access_key.to_string(), - secret_key.to_string(), - region.to_string(), - session_token, - ); - signer.sign_request(method, &parsed_url, headers, payload_hash)?; - } else { - let host = parsed_url - .host_str() - .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; - let host_header = if let Some(port) = parsed_url.port() { - format!("{}:{}", host, port) - } else { - host.to_string() - }; - headers.insert("host", host_header.parse().unwrap()); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - fn test_bucket_config() -> BucketConfig { - let mut backend_options = HashMap::new(); - backend_options.insert( - "endpoint".into(), - "https://s3.us-east-1.amazonaws.com".into(), - ); - backend_options.insert("bucket_name".into(), "my-backend-bucket".into()); - BucketConfig { - name: "test".into(), - backend_type: "s3".into(), - backend_prefix: None, - anonymous_access: false, - allowed_roles: vec![], - backend_options, - } - } - - #[test] - fn upload_id_with_special_chars_is_encoded() { - let config = test_bucket_config(); - let malicious_upload_id = "abc&x-amz-acl=public-read&foo=bar"; - let op = S3Operation::UploadPart { - bucket: "test".into(), - key: "file.bin".into(), - upload_id: malicious_upload_id.into(), - part_number: 1, - }; - - let url = build_backend_url(&config, &op).unwrap(); - - // The & and = characters in upload_id must be percent-encoded so they - // cannot act as query parameter separators/assignments. - let query = url.split_once('?').unwrap().1; - let params: Vec<(String, String)> = url::form_urlencoded::parse(query.as_bytes()) - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - - // Should be exactly 2 params: partNumber and uploadId - assert_eq!( - params.len(), - 2, - "expected 2 query params, got: {:?}", - params - ); - assert!(params.iter().any(|(k, v)| k == "partNumber" && v == "1")); - assert!(params - .iter() - .any(|(k, v)| k == "uploadId" && v == malicious_upload_id)); - } - - #[test] - fn upload_id_encoded_in_complete_multipart() { - let config = test_bucket_config(); - let op = S3Operation::CompleteMultipartUpload { - bucket: "test".into(), - key: "file.bin".into(), - upload_id: "id&injected=true".into(), - }; - - let url = build_backend_url(&config, &op).unwrap(); - - assert!( - !url.contains("injected=true"), - "upload_id was not encoded: {}", - url - ); - } - - #[test] - fn normal_upload_id_works() { - let config = test_bucket_config(); - let op = S3Operation::UploadPart { - bucket: "test".into(), - key: "file.bin".into(), - upload_id: "2~abcdef1234567890".into(), - part_number: 3, - }; - - let url = build_backend_url(&config, &op).unwrap(); - - assert!(url.starts_with("https://s3.us-east-1.amazonaws.com/my-backend-bucket/file.bin?")); - assert!(url.contains("partNumber=3")); - assert!( - url.contains("uploadId=2~abcdef1234567890") - || url.contains("uploadId=2%7Eabcdef1234567890") - ); - } -} diff --git a/crates/core/src/backend/request_signer.rs b/crates/core/src/backend/request_signer.rs deleted file mode 100644 index d0a11fd..0000000 --- a/crates/core/src/backend/request_signer.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Outbound SigV4 request signing. -//! -//! [`S3RequestSigner`] signs raw HTTP requests destined for S3-compatible -//! backends using AWS Signature Version 4. Used for multipart operations -//! (CreateMultipartUpload, UploadPart, CompleteMultipartUpload, -//! AbortMultipartUpload) that go through [`backend::ProxyBackend::send_raw`](crate::backend::ProxyBackend::send_raw). - -use crate::auth::sigv4::hmac_sha256; -use crate::error::ProxyError; -use http::HeaderMap; - -/// Signs outbound HTTP requests using AWS SigV4. -pub struct S3RequestSigner { - pub access_key_id: String, - pub secret_access_key: String, - pub region: String, - pub service: String, - pub session_token: Option, -} - -impl S3RequestSigner { - pub fn new( - access_key_id: String, - secret_access_key: String, - region: String, - session_token: Option, - ) -> Self { - Self { - access_key_id, - secret_access_key, - region, - service: "s3".to_string(), - session_token, - } - } - - /// Sign an outbound request using AWS SigV4. - /// - /// This adds Authorization, x-amz-date, x-amz-content-sha256, and Host - /// headers to the provided header map. - pub fn sign_request( - &self, - method: &http::Method, - url: &url::Url, - headers: &mut HeaderMap, - payload_hash: &str, - ) -> Result<(), ProxyError> { - use chrono::Utc; - - let now = Utc::now(); - let date_stamp = now.format("%Y%m%d").to_string(); - let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); - - // Set required headers - headers.insert("x-amz-date", amz_date.parse().unwrap()); - headers.insert("x-amz-content-sha256", payload_hash.parse().unwrap()); - - if let Some(token) = &self.session_token { - headers.insert("x-amz-security-token", token.parse().unwrap()); - } - - let host = url - .host_str() - .ok_or_else(|| ProxyError::Internal("no host in URL".into()))?; - let host_header = if let Some(port) = url.port() { - format!("{}:{}", host, port) - } else { - host.to_string() - }; - headers.insert("host", host_header.parse().unwrap()); - - // Canonical request - let canonical_uri = url.path(); - let canonical_querystring = url.query().unwrap_or(""); - - let mut signed_header_names: Vec<&str> = headers.keys().map(|k| k.as_str()).collect(); - signed_header_names.sort(); - - let canonical_headers: String = signed_header_names - .iter() - .map(|k| { - let v = headers.get(*k).unwrap().to_str().unwrap_or("").trim(); - format!("{}:{}\n", k, v) - }) - .collect(); - - let signed_headers = signed_header_names.join(";"); - - let canonical_request = format!( - "{}\n{}\n{}\n{}\n{}\n{}", - method, - canonical_uri, - canonical_querystring, - canonical_headers, - signed_headers, - payload_hash - ); - - // String to sign - let credential_scope = format!( - "{}/{}/{}/aws4_request", - date_stamp, self.region, self.service - ); - - use sha2::{Digest, Sha256}; - let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); - - let string_to_sign = format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - amz_date, credential_scope, canonical_request_hash - ); - - // Derive signing key - let k_date = hmac_sha256( - format!("AWS4{}", self.secret_access_key).as_bytes(), - date_stamp.as_bytes(), - )?; - let k_region = hmac_sha256(&k_date, self.region.as_bytes())?; - let k_service = hmac_sha256(&k_region, self.service.as_bytes())?; - let signing_key = hmac_sha256(&k_service, b"aws4_request")?; - - // Signature - let signature = hex::encode(hmac_sha256(&signing_key, string_to_sign.as_bytes())?); - - // Authorization header - let auth_header = format!( - "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", - self.access_key_id, credential_scope, signed_headers, signature - ); - headers.insert("authorization", auth_header.parse().unwrap()); - - Ok(()) - } -} - -/// Hash a payload for SigV4. For streaming/unsigned payloads, use the -/// special sentinel value. -pub fn hash_payload(payload: &[u8]) -> String { - use sha2::{Digest, Sha256}; - hex::encode(Sha256::digest(payload)) -} - -/// The SigV4 sentinel for unsigned payloads (used with streaming uploads). -pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; - -/// The SigV4 sentinel for streaming payloads. -pub const STREAMING_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; diff --git a/crates/core/src/forwarder.rs b/crates/core/src/forwarder.rs deleted file mode 100644 index 37916ca..0000000 --- a/crates/core/src/forwarder.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Runtime-agnostic HTTP forwarding trait. -//! -//! The [`Forwarder`] trait abstracts over the HTTP client used to execute -//! presigned backend requests. Each runtime (Tokio/Hyper, Cloudflare Workers) -//! provides its own implementation that streams request and response bodies -//! using native primitives. -//! -//! The proxy core produces a [`ForwardRequest`] describing *what* to send; -//! the `Forwarder` implementation decides *how* to send it and returns a -//! [`ForwardResponse`] with the backend's status, headers, and streaming body. - -use std::future::Future; - -use http::HeaderMap; - -use crate::error::ProxyError; -use crate::maybe_send::{MaybeSend, MaybeSync}; -use crate::route_handler::ForwardRequest; - -/// The response returned by a [`Forwarder`] after executing a backend request. -/// -/// `S` is the streaming body type, which varies per runtime — for example, -/// a Hyper `Incoming` body on native targets or a Workers `ReadableStream` -/// on the edge. -pub struct ForwardResponse { - /// HTTP status code from the backend. - pub status: u16, - /// Response headers from the backend. - pub headers: HeaderMap, - /// The streaming response body. - pub body: S, - /// Content length reported by the backend, if known. - pub content_length: Option, -} - -/// Executes a presigned [`ForwardRequest`] against the backend and returns -/// the response. -/// -/// The trait is generic over `Body` (the request body type) because callers -/// may provide different body representations depending on the operation — -/// for example, a streaming upload body for PUT or an empty body for GET. -/// -/// # Implementing -/// -/// ```rust,ignore -/// struct HyperForwarder { client: hyper::Client<...> } -/// -/// impl Forwarder for HyperForwarder { -/// type ResponseBody = hyper::body::Incoming; -/// -/// async fn forward( -/// &self, -/// request: ForwardRequest, -/// body: hyper::body::Incoming, -/// ) -> Result, ProxyError> { -/// // build and send the HTTP request using the native client -/// todo!() -/// } -/// } -/// ``` -pub trait Forwarder: MaybeSend + MaybeSync + 'static { - /// The streaming body type in the backend response. - type ResponseBody: MaybeSend + 'static; - - /// Execute the given [`ForwardRequest`] with the provided body and return - /// the backend's response. - fn forward( - &self, - request: ForwardRequest, - body: Body, - ) -> impl Future, ProxyError>> + MaybeSend; -} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index ba2bd87..6cfe9b3 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,35 +1,28 @@ -//! # s3-proxy-core +//! # multistore //! //! Runtime-agnostic core library for the S3 proxy gateway. //! -//! This crate defines the trait abstractions that allow the proxy to run on -//! multiple runtimes (Tokio/Hyper for containers, Cloudflare Workers for edge) -//! without either runtime leaking into the core logic. +//! This crate provides the s3s-based S3 service implementation that maps +//! S3 API operations to `object_store` calls, along with the trait +//! abstractions that allow it to run on multiple runtimes (Tokio/Hyper, +//! AWS Lambda, Cloudflare Workers). //! //! ## Key Abstractions //! -//! - [`route_handler::ProxyResponseBody`] — concrete response body type (Stream, Bytes, Empty) -//! - [`backend::ProxyBackend`] — create object stores and send raw HTTP requests +//! - [`service::MultistoreService`] — s3s-based S3 service (maps S3 ops → object_store) +//! - [`service::MultistoreAuth`] — s3s auth provider (delegates to `CredentialRegistry`) +//! - [`service::StoreFactory`] — runtime-provided factory for creating object stores per request //! - [`registry::BucketRegistry`] — bucket lookup, authorization, and listing //! - [`registry::CredentialRegistry`] — credential and role storage -//! - [`auth`] — SigV4 request verification and credential resolution -//! - [`api::request`] — parse incoming S3 API requests into typed operations -//! - [`api::response`] — serialize S3 XML responses -//! - [`route_handler::RouteHandler`] — pluggable pre-dispatch request interception (OIDC, STS, etc.) -//! - [`middleware::Middleware`] — composable post-auth middleware for dispatch -//! - [`forwarder::Forwarder`] — runtime-agnostic HTTP forwarding for backend requests -//! - [`router::Router`] — path-based route matching via `matchit` for efficient dispatch -//! - [`proxy::ProxyGateway`] — the main request handler that ties everything together +//! - [`auth::TemporaryCredentialResolver`] — resolve session tokens into temporary credentials +//! - [`backend::StoreBuilder`] — provider-agnostic object store builder +//! - [`api::response`] — S3 XML response serialization pub mod api; pub mod auth; pub mod backend; pub mod error; -pub mod forwarder; pub mod maybe_send; -pub mod middleware; -pub mod proxy; pub mod registry; -pub mod route_handler; -pub mod router; +pub mod service; pub mod types; diff --git a/crates/core/src/middleware.rs b/crates/core/src/middleware.rs deleted file mode 100644 index 834cf53..0000000 --- a/crates/core/src/middleware.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! Composable post-auth middleware for dispatch. -//! -//! Middleware runs after identity resolution and authorization, wrapping -//! the backend dispatch call. Each middleware can inspect or modify the -//! [`DispatchContext`], short-circuit the request with an early response, -//! or delegate to the next middleware in the chain via [`Next::run`]. -//! -//! Implement the [`Middleware`] trait for your type, then register it on -//! the `ProxyGateway` builder. Middleware executes in registration order. - -use std::borrow::Cow; -use std::future::Future; -use std::net::IpAddr; -use std::pin::Pin; - -use http::HeaderMap; - -use crate::api::list_rewrite::ListRewrite; -use crate::error::ProxyError; -use crate::maybe_send::{MaybeSend, MaybeSync}; -use crate::route_handler::HandlerAction; -use crate::types::{BucketConfig, ResolvedIdentity, S3Operation}; - -/// Post-dispatch context passed to [`Middleware::after_dispatch`]. -pub struct CompletedRequest<'a> { - /// The unique request identifier. - pub request_id: &'a str, - /// The resolved caller identity, if any. - pub identity: Option<&'a ResolvedIdentity>, - /// The parsed S3 operation, if determined. - pub operation: Option<&'a S3Operation>, - /// The target bucket name, if the operation targets a specific bucket. - pub bucket: Option<&'a str>, - /// The HTTP status code of the response. - pub status: u16, - /// The number of bytes in the response body, if known. - pub response_bytes: Option, - /// The number of bytes in the request body, if known. - pub request_bytes: Option, - /// Whether the request was forwarded to a backend via presigned URL. - pub was_forwarded: bool, - /// The IP address of the client, used for anonymous user identification. - pub source_ip: Option, -} - -/// Context passed to each middleware in the dispatch chain. -/// -/// Contains the resolved identity, parsed S3 operation, bucket configuration, -/// original request headers, and an extensions map for middleware to share -/// arbitrary typed data with downstream middleware or the dispatch function. -pub struct DispatchContext<'a> { - /// The authenticated identity for this request. - pub identity: &'a ResolvedIdentity, - /// The parsed S3 operation being performed. - pub operation: &'a S3Operation, - /// The bucket configuration for the target bucket. - /// `None` for operations that don't target a specific bucket (e.g. ListBuckets). - pub bucket_config: Option>, - /// The original request headers. - pub headers: &'a HeaderMap, - /// The IP address of the client that originated this request. - pub source_ip: Option, - /// A unique identifier for this request, used for tracing. - pub request_id: &'a str, - /// List rewrite rules for prefix-based bucket views. - pub list_rewrite: Option<&'a ListRewrite>, - /// Arbitrary typed data for middleware to share downstream. - pub extensions: http::Extensions, -} - -// --------------------------------------------------------------------------- -// DispatchFuture — the boxed future returned by dispatch functions. -// --------------------------------------------------------------------------- - -#[cfg(not(target_arch = "wasm32"))] -pub(crate) type DispatchFuture<'a> = - Pin> + Send + 'a>>; - -#[cfg(target_arch = "wasm32")] -pub(crate) type DispatchFuture<'a> = - Pin> + 'a>>; - -// --------------------------------------------------------------------------- -// AfterDispatchFuture — the boxed future returned by after_dispatch callbacks. -// --------------------------------------------------------------------------- - -#[cfg(not(target_arch = "wasm32"))] -pub(crate) type AfterDispatchFuture<'a> = Pin + Send + 'a>>; - -#[cfg(target_arch = "wasm32")] -pub(crate) type AfterDispatchFuture<'a> = Pin + 'a>>; - -// --------------------------------------------------------------------------- -// Dispatch — trait for the terminal dispatch function at the end of the chain. -// --------------------------------------------------------------------------- - -/// Terminal dispatch function at the end of the middleware chain. -/// -/// Using a trait (instead of a closure/`dyn Fn`) allows the dispatch -/// implementation to borrow from its environment with arbitrary lifetimes — -/// avoiding the `'static` constraint that `Arc` would impose. -pub(crate) trait Dispatch: MaybeSend + MaybeSync { - fn dispatch<'a>(&'a self, ctx: DispatchContext<'a>) -> DispatchFuture<'a>; -} - -// --------------------------------------------------------------------------- -// ErasedMiddleware — type-erased trait object for the middleware chain. -// --------------------------------------------------------------------------- - -pub(crate) trait ErasedMiddleware: MaybeSend + MaybeSync { - fn handle<'a>(&'a self, ctx: DispatchContext<'a>, next: Next<'a>) -> DispatchFuture<'a>; - fn after_dispatch<'a>(&'a self, completed: &'a CompletedRequest<'a>) - -> AfterDispatchFuture<'a>; -} - -// Blanket impl: any `Middleware` is automatically an `ErasedMiddleware`. -impl ErasedMiddleware for T { - fn handle<'a>(&'a self, ctx: DispatchContext<'a>, next: Next<'a>) -> DispatchFuture<'a> { - Box::pin(::handle(self, ctx, next)) - } - - fn after_dispatch<'a>( - &'a self, - completed: &'a CompletedRequest<'a>, - ) -> AfterDispatchFuture<'a> { - Box::pin(::after_dispatch(self, completed)) - } -} - -// --------------------------------------------------------------------------- -// Next — wraps the remaining middleware chain plus the terminal dispatch fn. -// --------------------------------------------------------------------------- - -/// Handle to the remaining middleware chain. -/// -/// Call [`Next::run`] to pass the request to the next middleware, or to the -/// terminal dispatch function if no middleware remains. Middleware that wants -/// to short-circuit the chain can simply return a result without calling -/// `run`. -pub struct Next<'a> { - middleware: &'a [Box], - dispatch: &'a dyn Dispatch, -} - -impl<'a> Next<'a> { - pub(crate) fn new( - middleware: &'a [Box], - dispatch: &'a dyn Dispatch, - ) -> Self { - Self { - middleware, - dispatch, - } - } - - /// Run the next middleware in the chain, or the dispatch function if the - /// chain is exhausted. - pub async fn run(self, ctx: DispatchContext<'a>) -> Result { - if let Some((first, rest)) = self.middleware.split_first() { - let next = Next { - middleware: rest, - dispatch: self.dispatch, - }; - first.handle(ctx, next).await - } else { - self.dispatch.dispatch(ctx).await - } - } -} - -// --------------------------------------------------------------------------- -// Middleware — the public trait implementors use. -// --------------------------------------------------------------------------- - -/// Composable post-auth middleware for the dispatch chain. -/// -/// Implement this trait to intercept requests after identity resolution and -/// authorization but before (or instead of) backend dispatch. Each -/// middleware receives the [`DispatchContext`] and a [`Next`] handle to -/// continue the chain. -/// -/// ```rust,ignore -/// struct RateLimiter; -/// -/// impl Middleware for RateLimiter { -/// async fn handle<'a>( -/// &'a self, -/// ctx: DispatchContext<'a>, -/// next: Next<'a>, -/// ) -> Result { -/// if self.is_over_limit(&ctx) { -/// Ok(HandlerAction::Response(ProxyResult { status: 429, .. })) -/// } else { -/// next.run(ctx).await -/// } -/// } -/// } -/// ``` -pub trait Middleware: MaybeSend + MaybeSync + 'static { - /// Handle a request, optionally delegating to the next middleware via - /// [`Next::run`]. - fn handle<'a>( - &'a self, - ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> impl Future> + MaybeSend + 'a; - - /// Called after the request has been fully dispatched and the response is - /// available. Use this for logging, metering, or other post-dispatch - /// side effects. The default implementation is a no-op. - fn after_dispatch( - &self, - _completed: &CompletedRequest<'_>, - ) -> impl Future + MaybeSend + '_ { - async {} - } -} - -// =========================================================================== -// Tests -// =========================================================================== - -#[cfg(test)] -mod tests { - use super::*; - use crate::route_handler::{ProxyResponseBody, ProxyResult}; - use crate::types::{BucketConfig, ResolvedIdentity, S3Operation}; - - // -- Test helpers ------------------------------------------------------- - - pub(crate) struct BlockingMiddleware; - - impl Middleware for BlockingMiddleware { - async fn handle<'a>( - &'a self, - _ctx: DispatchContext<'a>, - _next: Next<'a>, - ) -> Result { - Ok(HandlerAction::Response(ProxyResult { - status: 429, - headers: HeaderMap::new(), - body: ProxyResponseBody::Empty, - })) - } - } - - pub(crate) struct PassthroughMiddleware; - - impl Middleware for PassthroughMiddleware { - async fn handle<'a>( - &'a self, - ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> Result { - next.run(ctx).await - } - } - - struct TestDispatch; - - impl Dispatch for TestDispatch { - fn dispatch<'a>(&'a self, _ctx: DispatchContext<'a>) -> DispatchFuture<'a> { - Box::pin(async { - Ok(HandlerAction::Response(ProxyResult { - status: 200, - headers: HeaderMap::new(), - body: ProxyResponseBody::Empty, - })) - }) - } - } - - fn test_context() -> DispatchContext<'static> { - static IDENTITY: ResolvedIdentity = ResolvedIdentity::Anonymous; - static OPERATION: S3Operation = S3Operation::ListBuckets; - static HEADERS: std::sync::LazyLock = std::sync::LazyLock::new(HeaderMap::new); - static BUCKET_CONFIG: std::sync::LazyLock = - std::sync::LazyLock::new(|| BucketConfig { - name: "test".to_string(), - backend_type: "s3".to_string(), - backend_prefix: None, - anonymous_access: false, - allowed_roles: Vec::new(), - backend_options: Default::default(), - }); - - DispatchContext { - identity: &IDENTITY, - operation: &OPERATION, - bucket_config: Some(Cow::Borrowed(&*BUCKET_CONFIG)), - headers: &*HEADERS, - source_ip: None, - request_id: "test-request-id", - list_rewrite: None, - extensions: http::Extensions::new(), - } - } - - fn response_status(action: &HandlerAction) -> u16 { - match action { - HandlerAction::Response(r) => r.status, - _ => panic!("expected Response variant"), - } - } - - // -- Tests -------------------------------------------------------------- - - #[test] - fn empty_chain_calls_dispatch() { - let dispatch = TestDispatch; - let middleware: Vec> = vec![]; - let result = futures::executor::block_on(async { - let next = Next::new(&middleware, &dispatch); - next.run(test_context()).await - }); - assert_eq!(response_status(&result.unwrap()), 200); - } - - #[test] - fn blocking_middleware_short_circuits() { - let dispatch = TestDispatch; - let middleware: Vec> = vec![Box::new(BlockingMiddleware)]; - let result = futures::executor::block_on(async { - let next = Next::new(&middleware, &dispatch); - next.run(test_context()).await - }); - assert_eq!(response_status(&result.unwrap()), 429); - } - - #[test] - fn passthrough_then_blocking_runs_in_order() { - let dispatch = TestDispatch; - let middleware: Vec> = vec![ - Box::new(PassthroughMiddleware), - Box::new(BlockingMiddleware), - ]; - let result = futures::executor::block_on(async { - let next = Next::new(&middleware, &dispatch); - next.run(test_context()).await - }); - // PassthroughMiddleware delegates, BlockingMiddleware returns 429 - assert_eq!(response_status(&result.unwrap()), 429); - } - - #[test] - fn passthrough_reaches_dispatch() { - let dispatch = TestDispatch; - let middleware: Vec> = vec![Box::new(PassthroughMiddleware)]; - let result = futures::executor::block_on(async { - let next = Next::new(&middleware, &dispatch); - next.run(test_context()).await - }); - assert_eq!(response_status(&result.unwrap()), 200); - } - - #[test] - fn after_dispatch_default_is_noop() { - let middleware: Box = Box::new(PassthroughMiddleware); - futures::executor::block_on(async { - let completed = CompletedRequest { - request_id: "test", - identity: None, - operation: None, - bucket: None, - status: 200, - response_bytes: None, - request_bytes: None, - was_forwarded: false, - source_ip: None, - }; - middleware.after_dispatch(&completed).await; - }); - } -} diff --git a/crates/core/src/proxy.rs b/crates/core/src/proxy.rs deleted file mode 100644 index efc8c5f..0000000 --- a/crates/core/src/proxy.rs +++ /dev/null @@ -1,1144 +0,0 @@ -//! The main proxy gateway that ties together registry lookup and backend forwarding. -//! -//! [`ProxyGateway`] is generic over the runtime's backend, bucket registry, -//! and credential registry. -//! -//! ## Router (pre-dispatch) -//! -//! A [`Router`] maps URL path patterns to [`RouteHandler`](crate::route_handler::RouteHandler) -//! implementations using `matchit` for efficient matching. Exact paths take -//! priority over catch-all patterns, so OIDC discovery endpoints are matched -//! before a catch-all STS handler. Extension crates provide `Router` extension -//! traits for one-call registration. -//! -//! ## Proxy dispatch (two-phase) -//! -//! If no route handler matches, the request enters the two-phase pipeline: -//! -//! 1. **`resolve_request`** — parses the S3 operation, resolves identity, -//! authorizes via the bucket registry, and decides the action: -//! - GET/HEAD/PUT/DELETE → [`HandlerAction::Forward`] with a presigned URL -//! - LIST → [`HandlerAction::Response`] with XML body -//! - Multipart → [`HandlerAction::NeedsBody`] (body required) -//! - Errors/synthetic → [`HandlerAction::Response`] -//! -//! 2. **`handle_with_body`** — completes multipart operations once the body arrives. -//! -//! ## Runtime integration -//! -//! The recommended entry point is [`ProxyGateway::handle_request`], which returns a -//! two-variant [`GatewayResponse`]: -//! -//! - **`Response`** — a fully formed response to send to the client -//! - **`Forward`** — a presigned URL plus the original body for zero-copy streaming -//! -//! `NeedsBody` is resolved internally via a caller-provided body collection -//! closure, so runtimes only need a two-arm match: -//! -//! ```rust,ignore -//! match gateway.handle_request(&req_info, body, |b| to_bytes(b)).await { -//! GatewayResponse::Response(result) => build_response(result), -//! GatewayResponse::Forward(fwd, body) => forward(fwd, body).await, -//! } -//! ``` -//! -//! For lower-level control, use [`ProxyGateway::resolve_request`] which returns the -//! three-variant [`HandlerAction`] directly. - -use crate::api::list::{build_list_prefix, build_list_xml, parse_list_query_params, ListXmlParams}; -use crate::api::list_rewrite::ListRewrite; -use crate::api::request::{self, HostStyle}; -use crate::api::response::{BucketList, ErrorResponse, ListAllMyBucketsResult}; -use crate::auth; -use crate::auth::TemporaryCredentialResolver; -use crate::backend::multipart::{build_backend_url, sign_s3_request}; -use crate::backend::request_signer::{hash_payload, UNSIGNED_PAYLOAD}; -use crate::backend::ProxyBackend; -use crate::error::ProxyError; -use crate::forwarder::{ForwardResponse, Forwarder}; -use crate::maybe_send::{MaybeSend, MaybeSync}; -use crate::middleware::{ - CompletedRequest, Dispatch, DispatchContext, DispatchFuture, ErasedMiddleware, Middleware, Next, -}; -use crate::registry::{BucketRegistry, CredentialRegistry}; -use crate::route_handler::{ProxyResponseBody, RequestInfo}; -use crate::router::Router; -use crate::types::{BucketConfig, ResolvedIdentity, S3Operation}; -use bytes::Bytes; -use http::{HeaderMap, Method}; -use object_store::list::PaginatedListOptions; -use std::borrow::Cow; -use std::net::IpAddr; -use std::time::Duration; -use uuid::Uuid; - -/// TTL for presigned URLs. Short because they're used immediately. -const PRESIGNED_URL_TTL: Duration = Duration::from_secs(300); - -// Re-export types that were historically defined here for backwards compatibility. -pub use crate::route_handler::{ - ForwardRequest, HandlerAction, PendingRequest, ProxyResult, RESPONSE_HEADER_ALLOWLIST, -}; - -/// Simplified two-variant result from [`ProxyGateway::handle_request`]. -/// -/// The response body type `S` is the `Forwarder`'s `ResponseBody` — opaque -/// to the core, passed through to the runtime for client delivery. -pub enum GatewayResponse { - /// A fully formed response ready to send to the client. - Response(ProxyResult), - /// A forwarded response from the backend, with the runtime's native - /// body type for streaming. - Forward(ForwardResponse), -} - -/// Metadata from request resolution, used for post-dispatch callbacks. -pub struct RequestMetadata { - /// The unique request identifier. - pub request_id: String, - /// The resolved caller identity, if any. - pub identity: Option, - /// The parsed S3 operation, if determined. - pub operation: Option, - /// The target bucket name, if the operation targets a specific bucket. - pub bucket: Option, - /// The IP address of the client, used for anonymous user identification. - pub source_ip: Option, -} - -/// The core proxy gateway, generic over runtime primitives. -/// -/// Owns S3 request parsing, identity resolution, and authorization via -/// the [`BucketRegistry`] and [`CredentialRegistry`] traits. Combines -/// a [`Router`] for path-based pre-dispatch with the two-phase -/// resolve/dispatch pipeline. -/// -/// # Type Parameters -/// -/// - `B`: The runtime's backend for object store creation, signing, and raw HTTP -/// - `R`: The bucket registry for bucket lookup and authorization -/// - `C`: The credential registry for credential and role lookup -/// - `F`: The forwarder that executes presigned backend requests -pub struct ProxyGateway { - backend: B, - bucket_registry: R, - credential_registry: C, - forwarder: F, - middleware: Vec>, - virtual_host_domain: Option, - credential_resolver: Option>, - router: Router, - /// When true, error responses include full internal details (for development). - /// When false, server-side errors use generic messages. - debug_errors: bool, -} - -impl ProxyGateway -where - B: ProxyBackend, - R: BucketRegistry, - C: CredentialRegistry, - F: MaybeSend + MaybeSync, -{ - pub fn new( - backend: B, - bucket_registry: R, - credential_registry: C, - forwarder: F, - virtual_host_domain: Option, - ) -> Self { - Self { - backend, - bucket_registry, - credential_registry, - forwarder, - middleware: Vec::new(), - virtual_host_domain, - credential_resolver: None, - router: Router::new(), - debug_errors: false, - } - } - - /// Add a middleware to the dispatch chain. - /// - /// Middleware runs after identity resolution and authorization, wrapping - /// the backend dispatch call. Middleware executes in registration order. - pub fn with_middleware(mut self, middleware: impl Middleware) -> Self { - self.middleware.push(Box::new(middleware)); - self - } - - /// Set the temporary credential resolver for session token verification. - /// - /// When configured, requests with `x-amz-security-token` headers are - /// resolved via this resolver during identity resolution. - pub fn with_credential_resolver( - mut self, - resolver: impl TemporaryCredentialResolver + 'static, - ) -> Self { - self.credential_resolver = Some(Box::new(resolver)); - self - } - - /// Set the router for path-based request dispatch. - /// - /// The router is consulted before the proxy dispatch pipeline. - /// If a route matches and the handler returns an action, that action - /// is used directly. Otherwise the request falls through to proxy - /// dispatch. - pub fn with_router(mut self, router: Router) -> Self { - self.router = router; - self - } - - /// Enable verbose error messages in S3 error responses. - /// - /// When enabled, 500-class errors include the full internal message - /// (backend errors, config errors, etc.). Disable in production to - /// avoid leaking infrastructure details to clients. - pub fn with_debug_errors(mut self, enabled: bool) -> Self { - self.debug_errors = enabled; - self - } - - /// Convenience entry point that resolves `NeedsBody` internally and - /// executes forwarding via the [`Forwarder`]. - /// - /// Route handler matches bypass the forwarder/after_dispatch path for - /// simplicity. For the proxy pipeline, `after_dispatch` is fired on all - /// middleware after the response is determined. - /// - /// Runtimes match on only two variants — `Response` or `Forward`: - /// - /// ```rust,ignore - /// match gateway.handle_request(&req_info, body, |b| to_bytes(b)).await { - /// GatewayResponse::Response(result) => build_response(result), - /// GatewayResponse::Forward(resp) => stream_response(resp), - /// } - /// ``` - pub async fn handle_request( - &self, - req: &RequestInfo<'_>, - body: Body, - collect_body: CF, - ) -> GatewayResponse - where - F: Forwarder, - CF: FnOnce(Body) -> Fut, - Fut: std::future::Future>, - E: std::fmt::Display, - { - // Route handlers first (bypass forwarder/after_dispatch for simplicity) - if let Some(action) = self.router.dispatch(req).await { - return match action { - HandlerAction::Response(r) => GatewayResponse::Response(r), - HandlerAction::Forward(fwd) => match self.forwarder.forward(fwd, body).await { - Ok(resp) => GatewayResponse::Forward(resp), - Err(e) => GatewayResponse::Response(error_response( - &e, - req.path, - "", - self.debug_errors, - )), - }, - HandlerAction::NeedsBody(_) => GatewayResponse::Response(error_response( - &ProxyError::Internal("unexpected NeedsBody from route handler".into()), - req.path, - "", - self.debug_errors, - )), - }; - } - - // Resolve via proxy pipeline (with metadata for after_dispatch) - let (action, metadata) = self - .resolve_request_with_metadata( - req.method.clone(), - req.path, - req.query, - req.headers, - req.source_ip, - ) - .await; - - // Helper to extract response body size - fn response_body_bytes(body: &ProxyResponseBody) -> Option { - match body { - ProxyResponseBody::Bytes(b) => Some(b.len() as u64), - ProxyResponseBody::Empty => Some(0), - } - } - - fn content_length_from_headers(headers: &HeaderMap) -> Option { - headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - } - - let request_bytes = content_length_from_headers(req.headers); - - let (response, status, resp_bytes, was_forwarded) = match action { - HandlerAction::Response(r) => { - let s = r.status; - let rb = response_body_bytes(&r.body); - (GatewayResponse::Response(r), s, rb, false) - } - HandlerAction::Forward(fwd) => match self.forwarder.forward(fwd, body).await { - Ok(resp) => { - let s = resp.status; - let cl = resp.content_length; - (GatewayResponse::Forward(resp), s, cl, true) - } - Err(e) => { - let err_resp = - error_response(&e, req.path, &metadata.request_id, self.debug_errors); - let s = err_resp.status; - (GatewayResponse::Response(err_resp), s, None, true) - } - }, - HandlerAction::NeedsBody(pending) => match collect_body(body).await { - Ok(bytes) => { - let result = self.handle_with_body(pending, bytes).await; - let s = result.status; - let rb = response_body_bytes(&result.body); - (GatewayResponse::Response(result), s, rb, false) - } - Err(e) => { - tracing::error!(error = %e, "failed to read request body"); - let err_resp = error_response( - &ProxyError::Internal("failed to read request body".into()), - "", - &metadata.request_id, - self.debug_errors, - ); - let s = err_resp.status; - (GatewayResponse::Response(err_resp), s, None, false) - } - }, - }; - - // Fire after_dispatch on all middleware - let completed = CompletedRequest { - request_id: &metadata.request_id, - identity: metadata.identity.as_ref(), - operation: metadata.operation.as_ref(), - bucket: metadata.bucket.as_deref(), - status, - response_bytes: resp_bytes, - request_bytes, - was_forwarded, - source_ip: metadata.source_ip, - }; - for m in &self.middleware { - m.after_dispatch(&completed).await; - } - - response - } - - /// Resolve an incoming request into an action. - /// - /// Parses the S3 operation from the request, resolves the caller's - /// identity, authorizes via the bucket registry, and determines what - /// the runtime should do next. - pub async fn resolve_request( - &self, - method: Method, - path: &str, - query: Option<&str>, - headers: &HeaderMap, - source_ip: Option, - ) -> HandlerAction { - let (action, _metadata) = self - .resolve_request_with_metadata(method, path, query, headers, source_ip) - .await; - action - } - - /// Like [`resolve_request`](Self::resolve_request), but also returns - /// [`RequestMetadata`] for post-dispatch callbacks (e.g. metering). - pub(crate) async fn resolve_request_with_metadata( - &self, - method: Method, - path: &str, - query: Option<&str>, - headers: &HeaderMap, - source_ip: Option, - ) -> (HandlerAction, RequestMetadata) { - let request_id = Uuid::new_v4().to_string(); - - tracing::info!( - request_id = %request_id, - method = %method, - path = %path, - query = ?query, - "incoming request" - ); - - // Determine host style - let host_style = determine_host_style(headers, self.virtual_host_domain.as_deref()); - - // Parse the S3 operation - let operation = match request::parse_s3_request(&method, path, query, headers, host_style) { - Ok(op) => op, - Err(err) => return self.error_result(err, path, &request_id, source_ip), - }; - tracing::debug!(operation = ?operation, "parsed S3 operation"); - - // Resolve identity - let identity = match auth::resolve_identity( - &method, - path, - query.unwrap_or(""), - headers, - &self.credential_registry, - self.credential_resolver.as_deref(), - ) - .await - { - Ok(id) => id, - Err(err) => return self.error_result(err, path, &request_id, source_ip), - }; - tracing::debug!(identity = ?identity, "resolved identity"); - - // Resolve bucket config (if the operation targets a specific bucket). - let resolved = if let Some(bucket_name) = operation.bucket() { - match self - .bucket_registry - .get_bucket(bucket_name, &identity, &operation) - .await - { - Ok(resolved) => { - tracing::debug!( - bucket = %bucket_name, - backend_type = %resolved.config.backend_type, - "resolved bucket config" - ); - tracing::trace!("authorization passed"); - Some(resolved) - } - Err(err) => return self.error_result(err, path, &request_id, source_ip), - } - } else { - None - }; - - // Build middleware context - let ctx = DispatchContext { - identity: &identity, - operation: &operation, - bucket_config: resolved.as_ref().map(|r| Cow::Borrowed(&r.config)), - headers, - source_ip, - request_id: &request_id, - list_rewrite: resolved.as_ref().and_then(|r| r.list_rewrite.as_ref()), - extensions: http::Extensions::new(), - }; - - let next = Next::new(&self.middleware, self); - let metadata = RequestMetadata { - request_id: request_id.clone(), - identity: Some(identity.clone()), - operation: Some(operation.clone()), - bucket: operation.bucket().map(str::to_string), - source_ip, - }; - - match next.run(ctx).await { - Ok(action) => { - match &action { - HandlerAction::Response(resp) => { - tracing::info!( - request_id = %request_id, - status = resp.status, - "request completed" - ); - } - HandlerAction::Forward(fwd) => { - tracing::info!( - request_id = %request_id, - method = %fwd.method, - "forwarding via presigned URL" - ); - } - HandlerAction::NeedsBody(_) => { - tracing::debug!( - request_id = %request_id, - "request needs body (multipart)" - ); - } - } - (action, metadata) - } - Err(err) => self.error_result(err, path, &request_id, source_ip), - } - } - - /// Build an error action + metadata pair for early returns. - fn error_result( - &self, - err: ProxyError, - path: &str, - request_id: &str, - source_ip: Option, - ) -> (HandlerAction, RequestMetadata) { - tracing::warn!( - request_id = %request_id, - error = %err, - status = err.status_code(), - s3_code = %err.s3_error_code(), - "request failed" - ); - let metadata = RequestMetadata { - request_id: request_id.to_string(), - identity: None, - operation: None, - bucket: None, - source_ip, - }; - ( - HandlerAction::Response(error_response(&err, path, request_id, self.debug_errors)), - metadata, - ) - } - - /// Phase 2: Complete a multipart operation with the request body. - /// - /// Called by the runtime after materializing the body for a `NeedsBody` action. - /// Middleware is not re-run here — it already executed during phase 1 - /// when the `NeedsBody` action was produced. - pub async fn handle_with_body(&self, pending: PendingRequest, body: Bytes) -> ProxyResult { - match self.execute_multipart(&pending, body).await { - Ok(result) => { - tracing::info!( - request_id = %pending.request_id, - status = result.status, - "multipart request completed" - ); - result - } - Err(err) => { - tracing::warn!( - request_id = %pending.request_id, - error = %err, - status = err.status_code(), - s3_code = %err.s3_error_code(), - "multipart request failed" - ); - error_response( - &err, - pending.operation.key(), - &pending.request_id, - self.debug_errors, - ) - } - } - } - - async fn dispatch_operation( - &self, - ctx: &DispatchContext<'_>, - ) -> Result { - let original_headers = ctx.headers; - let list_rewrite = ctx.list_rewrite; - let request_id = ctx.request_id; - let operation = ctx.operation; - - // ListBuckets has no bucket config — handle it first. - if matches!(operation, S3Operation::ListBuckets) { - let buckets = self.bucket_registry.list_buckets(ctx.identity).await?; - tracing::info!(count = buckets.len(), "listing virtual buckets"); - let xml = ListAllMyBucketsResult { - owner: self.bucket_registry.bucket_owner(), - buckets: BucketList { buckets }, - } - .to_xml(); - - let mut resp_headers = HeaderMap::new(); - resp_headers.insert("content-type", "application/xml".parse().unwrap()); - return Ok(HandlerAction::Response(ProxyResult { - status: 200, - headers: resp_headers, - body: ProxyResponseBody::from_bytes(Bytes::from(xml)), - })); - } - - // All remaining operations require a bucket config. - let bucket_config = ctx - .bucket_config - .as_deref() - .expect("bucket_config must be set for bucket-targeted operations"); - - match operation { - S3Operation::GetObject { key, .. } => { - let fwd = self - .build_forward( - Method::GET, - bucket_config, - key, - original_headers, - &[ - "range", - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", - ], - request_id, - ) - .await?; - tracing::debug!(path = fwd.url.path(), "GET via presigned URL"); - Ok(HandlerAction::Forward(fwd)) - } - S3Operation::HeadObject { key, .. } => { - let fwd = self - .build_forward( - Method::HEAD, - bucket_config, - key, - original_headers, - &[ - "range", - "if-match", - "if-none-match", - "if-modified-since", - "if-unmodified-since", - ], - request_id, - ) - .await?; - tracing::debug!(path = fwd.url.path(), "HEAD via presigned URL"); - Ok(HandlerAction::Forward(fwd)) - } - S3Operation::PutObject { key, .. } => { - let fwd = self - .build_forward( - Method::PUT, - bucket_config, - key, - original_headers, - &["content-type", "content-length", "content-md5"], - request_id, - ) - .await?; - tracing::debug!(path = fwd.url.path(), "PUT via presigned URL"); - Ok(HandlerAction::Forward(fwd)) - } - S3Operation::DeleteObject { key, .. } => { - let fwd = self - .build_forward( - Method::DELETE, - bucket_config, - key, - original_headers, - &[], - request_id, - ) - .await?; - tracing::debug!(path = fwd.url.path(), "DELETE via presigned URL"); - Ok(HandlerAction::Forward(fwd)) - } - S3Operation::ListBucket { raw_query, .. } => { - let result = self - .handle_list(bucket_config, raw_query.as_deref(), list_rewrite) - .await?; - Ok(HandlerAction::Response(result)) - } - // Multipart operations need the request body - S3Operation::CreateMultipartUpload { .. } - | S3Operation::UploadPart { .. } - | S3Operation::CompleteMultipartUpload { .. } - | S3Operation::AbortMultipartUpload { .. } => { - if !bucket_config.supports_s3_multipart() { - return Err(ProxyError::InvalidRequest(format!( - "multipart operations not supported for '{}' backends", - bucket_config.backend_type - ))); - } - Ok(HandlerAction::NeedsBody(PendingRequest { - operation: operation.clone(), - bucket_config: bucket_config.clone(), - original_headers: original_headers.clone(), - request_id: request_id.to_string(), - })) - } - _ => Err(ProxyError::Internal("unexpected operation".into())), - } - } - - /// Build a [`ForwardRequest`] with a presigned URL for the given operation. - async fn build_forward( - &self, - method: Method, - config: &BucketConfig, - key: &str, - original_headers: &HeaderMap, - forward_header_names: &[&'static str], - request_id: &str, - ) -> Result { - let signer = self.backend.create_signer(config)?; - let path = build_object_path(config, key); - - let url = signer - .signed_url(method.clone(), &path, PRESIGNED_URL_TTL) - .await - .map_err(ProxyError::from_object_store_error)?; - - let mut fwd_headers = HeaderMap::new(); - for name in forward_header_names { - if let Some(v) = original_headers.get(*name) { - fwd_headers.insert(*name, v.clone()); - } - } - - Ok(ForwardRequest { - method, - url, - headers: fwd_headers, - request_id: request_id.to_string(), - }) - } - - /// LIST via object_store's `PaginatedListStore`. - /// - /// Pagination is pushed to the backend — only one page of results is fetched - /// per request, avoiding loading all objects into memory. - async fn handle_list( - &self, - config: &BucketConfig, - raw_query: Option<&str>, - list_rewrite: Option<&ListRewrite>, - ) -> Result { - let store = self.backend.create_paginated_store(config)?; - - // Parse all query parameters in a single pass - let list_params = parse_list_query_params(raw_query); - let client_prefix = &list_params.prefix; - let delimiter = &list_params.delimiter; - - // Build the full prefix including backend_prefix - let full_prefix = build_list_prefix(config, client_prefix); - - // Map start-after to raw key space by prepending backend_prefix - let offset = list_params - .start_after - .as_ref() - .map(|sa| build_list_prefix(config, sa)); - - tracing::debug!( - full_prefix = %full_prefix, - delimiter = %delimiter, - max_keys = list_params.max_keys, - has_page_token = list_params.continuation_token.is_some(), - "LIST via PaginatedListStore" - ); - - let prefix = if full_prefix.is_empty() { - None - } else { - Some(full_prefix.as_str()) - }; - - let opts = PaginatedListOptions { - offset, - delimiter: Some(Cow::Owned(delimiter.clone())), - max_keys: Some(list_params.max_keys), - page_token: list_params.continuation_token.clone(), - ..Default::default() - }; - - let paginated = store - .list_paginated(prefix, opts) - .await - .map_err(ProxyError::from_object_store_error)?; - - // Build S3 XML response from paginated result - let key_count = paginated.result.objects.len() + paginated.result.common_prefixes.len(); - let xml = build_list_xml( - &ListXmlParams { - bucket_name: &config.name, - client_prefix, - delimiter, - max_keys: list_params.max_keys, - is_truncated: paginated.page_token.is_some(), - key_count, - start_after: &list_params.start_after, - continuation_token: &list_params.continuation_token, - next_continuation_token: paginated.page_token, - }, - &paginated.result, - config, - list_rewrite, - )?; - - let mut resp_headers = HeaderMap::new(); - resp_headers.insert("content-type", "application/xml".parse().unwrap()); - - Ok(ProxyResult { - status: 200, - headers: resp_headers, - body: ProxyResponseBody::Bytes(Bytes::from(xml)), - }) - } - - /// Execute a multipart operation via raw signed HTTP. - async fn execute_multipart( - &self, - pending: &PendingRequest, - body: Bytes, - ) -> Result { - let backend_url = build_backend_url(&pending.bucket_config, &pending.operation)?; - - tracing::debug!(backend_url = %backend_url, "multipart via raw HTTP"); - - let mut headers = HeaderMap::new(); - - // Forward relevant headers - for header_name in &["content-type", "content-length", "content-md5"] { - if let Some(val) = pending.original_headers.get(*header_name) { - headers.insert(*header_name, val.clone()); - } - } - - let payload_hash = if body.is_empty() { - UNSIGNED_PAYLOAD.to_string() - } else { - hash_payload(&body) - }; - - let method = pending.operation.method(); - - sign_s3_request( - &method, - &backend_url, - &mut headers, - &pending.bucket_config, - &payload_hash, - )?; - - let raw_resp = self - .backend - .send_raw(method, backend_url, headers, body) - .await?; - - tracing::debug!(status = raw_resp.status, "multipart backend response"); - - Ok(ProxyResult { - status: raw_resp.status, - headers: raw_resp.headers, - body: ProxyResponseBody::from_bytes(raw_resp.body), - }) - } -} - -impl Dispatch for ProxyGateway -where - B: ProxyBackend, - R: BucketRegistry, - C: CredentialRegistry, - F: MaybeSend + MaybeSync, -{ - fn dispatch<'a>(&'a self, ctx: DispatchContext<'a>) -> DispatchFuture<'a> { - Box::pin(async move { self.dispatch_operation(&ctx).await }) - } -} - -fn determine_host_style(headers: &HeaderMap, virtual_host_domain: Option<&str>) -> HostStyle { - if let Some(domain) = virtual_host_domain { - if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) { - let host = host.split(':').next().unwrap_or(host); - if let Some(bucket) = host.strip_suffix(&format!(".{}", domain)) { - return HostStyle::VirtualHosted { - bucket: bucket.to_string(), - }; - } - } - } - HostStyle::Path -} - -fn error_response(err: &ProxyError, resource: &str, request_id: &str, debug: bool) -> ProxyResult { - let xml = ErrorResponse::from_proxy_error(err, resource, request_id, debug).to_xml(); - let body = ProxyResponseBody::from_bytes(Bytes::from(xml)); - let mut headers = HeaderMap::new(); - headers.insert("content-type", "application/xml".parse().unwrap()); - - ProxyResult { - status: err.status_code(), - headers, - body, - } -} - -/// Build an object_store Path from a bucket config and client-visible key. -fn build_object_path(config: &BucketConfig, key: &str) -> object_store::path::Path { - match &config.backend_prefix { - Some(prefix) => { - let p = prefix.trim_end_matches('/'); - if p.is_empty() { - object_store::path::Path::from(key) - } else { - let mut full_key = String::with_capacity(p.len() + 1 + key.len()); - full_key.push_str(p); - full_key.push('/'); - full_key.push_str(key); - object_store::path::Path::from(full_key) - } - } - None => object_store::path::Path::from(key), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::api::response::BucketEntry; - use crate::backend::RawResponse; - use crate::forwarder::{ForwardResponse, Forwarder}; - use crate::registry::{BucketRegistry, CredentialRegistry, ResolvedBucket}; - use crate::types::{ResolvedIdentity, RoleConfig, StoredCredential}; - use object_store::list::PaginatedListStore; - use object_store::signer::Signer; - use std::collections::HashMap; - use std::sync::Arc; - - // ── Mocks ─────────────────────────────────────────────────────── - - #[derive(Clone)] - struct MockBackend; - - impl ProxyBackend for MockBackend { - fn create_paginated_store( - &self, - _config: &BucketConfig, - ) -> Result, ProxyError> { - unimplemented!("not needed for forward tests") - } - - fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { - // Build a real S3 signer from the test config — produces a valid presigned URL. - crate::backend::build_signer(config) - } - - async fn send_raw( - &self, - _method: http::Method, - _url: String, - _headers: HeaderMap, - _body: Bytes, - ) -> Result { - unimplemented!("not needed for forward tests") - } - } - - #[derive(Clone)] - struct MockRegistry; - - impl BucketRegistry for MockRegistry { - async fn get_bucket( - &self, - name: &str, - _identity: &ResolvedIdentity, - _operation: &S3Operation, - ) -> Result { - Ok(ResolvedBucket { - config: test_bucket_config(name), - list_rewrite: None, - }) - } - - async fn list_buckets( - &self, - _identity: &ResolvedIdentity, - ) -> Result, ProxyError> { - Ok(vec![]) - } - } - - #[derive(Clone)] - struct MockCreds; - - impl CredentialRegistry for MockCreds { - async fn get_credential( - &self, - _access_key_id: &str, - ) -> Result, ProxyError> { - Ok(None) - } - - async fn get_role(&self, _role_id: &str) -> Result, ProxyError> { - Ok(None) - } - } - - fn test_bucket_config(name: &str) -> BucketConfig { - let mut backend_options = HashMap::new(); - backend_options.insert( - "endpoint".into(), - "https://s3.us-east-1.amazonaws.com".into(), - ); - backend_options.insert("bucket_name".into(), "backend-bucket".into()); - backend_options.insert("region".into(), "us-east-1".into()); - backend_options.insert("access_key_id".into(), "AKIAIOSFODNN7EXAMPLE".into()); - backend_options.insert( - "secret_access_key".into(), - "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".into(), - ); - BucketConfig { - name: name.to_string(), - backend_type: "s3".into(), - backend_prefix: None, - anonymous_access: true, - allowed_roles: vec![], - backend_options, - } - } - - struct MockForwarder; - - impl Forwarder<()> for MockForwarder { - type ResponseBody = (); - async fn forward( - &self, - _request: ForwardRequest, - _body: (), - ) -> Result, ProxyError> { - unimplemented!("not needed for resolve_request tests") - } - } - - fn run(f: F) -> F::Output { - futures::executor::block_on(f) - } - - fn gateway() -> ProxyGateway { - ProxyGateway::new(MockBackend, MockRegistry, MockCreds, MockForwarder, None) - } - - // ── Tests ─────────────────────────────────────────────────────── - - #[test] - fn get_forward_preserves_range_header() { - run(async { - let gw = gateway(); - let mut headers = HeaderMap::new(); - headers.insert("range", "bytes=0-99".parse().unwrap()); - let action = gw - .resolve_request(Method::GET, "/test-bucket/key.txt", None, &headers, None) - .await; - - match action { - HandlerAction::Forward(fwd) => { - assert_eq!(fwd.method, Method::GET); - assert_eq!( - fwd.headers.get("range").map(|v| v.to_str().unwrap()), - Some("bytes=0-99"), - "GET forward should pass through the Range header" - ); - } - other => panic!("expected Forward, got {:?}", std::mem::discriminant(&other)), - } - }); - } - - #[test] - fn head_forward_preserves_range_header() { - run(async { - let gw = gateway(); - let mut headers = HeaderMap::new(); - headers.insert("range", "bytes=0-1023".parse().unwrap()); - let action = gw - .resolve_request(Method::HEAD, "/test-bucket/key.txt", None, &headers, None) - .await; - - match action { - HandlerAction::Forward(fwd) => { - assert_eq!(fwd.method, Method::HEAD); - assert_eq!( - fwd.headers.get("range").map(|v| v.to_str().unwrap()), - Some("bytes=0-1023"), - "HEAD forward should pass through the Range header" - ); - } - other => panic!("expected Forward, got {:?}", std::mem::discriminant(&other)), - } - }); - } - - // -- Middleware test types ----------------------------------------------- - - struct BlockMiddleware; - - impl crate::middleware::Middleware for BlockMiddleware { - async fn handle<'a>( - &'a self, - _ctx: crate::middleware::DispatchContext<'a>, - _next: crate::middleware::Next<'a>, - ) -> Result { - Ok(HandlerAction::Response(ProxyResult { - status: 429, - headers: HeaderMap::new(), - body: ProxyResponseBody::Empty, - })) - } - } - - struct PassMiddleware; - - impl crate::middleware::Middleware for PassMiddleware { - async fn handle<'a>( - &'a self, - ctx: crate::middleware::DispatchContext<'a>, - next: crate::middleware::Next<'a>, - ) -> Result { - next.run(ctx).await - } - } - - // -- Middleware integration tests ---------------------------------------- - - #[test] - fn middleware_short_circuits_request() { - run(async { - let gw = gateway().with_middleware(BlockMiddleware); - let headers = HeaderMap::new(); - let action = gw - .resolve_request(Method::GET, "/test-bucket/key.txt", None, &headers, None) - .await; - - match action { - HandlerAction::Response(resp) => { - assert_eq!(resp.status, 429, "blocking middleware should return 429"); - } - other => panic!( - "expected Response, got {:?}", - std::mem::discriminant(&other) - ), - } - }); - } - - #[test] - fn middleware_passthrough_allows_request() { - run(async { - let gw = gateway().with_middleware(PassMiddleware); - let headers = HeaderMap::new(); - let action = gw - .resolve_request(Method::GET, "/test-bucket/key.txt", None, &headers, None) - .await; - - match action { - HandlerAction::Forward(fwd) => { - assert_eq!( - fwd.method, - Method::GET, - "passthrough middleware should allow normal forwarding" - ); - } - other => panic!("expected Forward, got {:?}", std::mem::discriminant(&other)), - } - }); - } -} diff --git a/crates/core/src/route_handler.rs b/crates/core/src/route_handler.rs deleted file mode 100644 index b33978f..0000000 --- a/crates/core/src/route_handler.rs +++ /dev/null @@ -1,228 +0,0 @@ -//! Pluggable route handler trait for pre-dispatch request interception. -//! -//! Route handlers are checked in registration order before the main proxy -//! dispatch. Each handler can inspect the request and optionally return a -//! [`ProxyResult`] to short-circuit further processing. If no handler -//! matches, the request proceeds to the normal resolve/dispatch pipeline. -//! -//! This module also defines the action/result types shared between route -//! handlers and the proxy gateway. - -use crate::maybe_send::{MaybeSend, MaybeSync}; -use bytes::Bytes; -use http::{HeaderMap, Method}; -use std::future::Future; -use std::net::IpAddr; -use std::pin::Pin; -use url::Url; - -/// The body of a proxy response. -/// -/// Only used for responses the handler constructs directly (errors, LIST XML, -/// multipart XML responses, HEAD metadata). Streaming GET/PUT bodies bypass this type -/// entirely via the `Forward` action. -pub enum ProxyResponseBody { - /// Fixed bytes (error XML, list XML, multipart XML responses, etc.). - Bytes(Bytes), - /// Empty body (HEAD responses, etc.). - Empty, -} - -impl ProxyResponseBody { - /// Create a response body from raw bytes. - pub fn from_bytes(bytes: Bytes) -> Self { - if bytes.is_empty() { - Self::Empty - } else { - Self::Bytes(bytes) - } - } - - /// Create an empty response body. - pub fn empty() -> Self { - Self::Empty - } -} - -/// The action the handler wants the runtime to take. -pub enum HandlerAction { - /// A fully formed response (LIST results, errors, synthetic responses). - Response(ProxyResult), - /// A presigned URL for the runtime to execute with its native HTTP client. - /// The runtime streams request/response bodies directly — no handler involvement. - Forward(ForwardRequest), - /// The handler needs the request body to continue (multipart operations). - /// The runtime should materialize the body and call `handle_with_body`. - NeedsBody(PendingRequest), -} - -/// A presigned URL request for the runtime to execute. -pub struct ForwardRequest { - /// HTTP method for the backend request. - pub method: Method, - /// Presigned URL to the backend (includes auth in query params). - pub url: Url, - /// Headers to include in the backend request (Range, If-Match, Content-Type, etc.). - pub headers: HeaderMap, - /// Unique request identifier for tracing and metering correlation. - pub request_id: String, -} - -/// The result of handling a proxy request. -pub struct ProxyResult { - pub status: u16, - pub headers: HeaderMap, - pub body: ProxyResponseBody, -} - -impl ProxyResult { - /// Create a JSON response with the given status and body. - pub fn json(status: u16, body: impl Into) -> Self { - let mut headers = HeaderMap::new(); - headers.insert("content-type", "application/json".parse().unwrap()); - Self { - status, - headers, - body: ProxyResponseBody::from_bytes(Bytes::from(body.into())), - } - } - - /// Create an XML response with the given status and body. - pub fn xml(status: u16, body: impl Into) -> Self { - let mut headers = HeaderMap::new(); - headers.insert("content-type", "application/xml".parse().unwrap()); - Self { - status, - headers, - body: ProxyResponseBody::from_bytes(Bytes::from(body.into())), - } - } -} - -/// Opaque state for a multipart operation that needs the request body. -pub struct PendingRequest { - pub(crate) operation: crate::types::S3Operation, - pub(crate) bucket_config: crate::types::BucketConfig, - pub(crate) original_headers: HeaderMap, - pub(crate) request_id: String, -} - -/// Headers to forward from backend responses (used by runtimes for Forward responses). -pub const RESPONSE_HEADER_ALLOWLIST: &[&str] = &[ - "content-type", - "content-length", - "content-range", - "etag", - "last-modified", - "accept-ranges", - "content-encoding", - "content-disposition", - "cache-control", - "x-amz-request-id", - "x-amz-version-id", - "location", -]; - -/// The future type returned by [`RouteHandler::handle`]. -#[cfg(not(target_arch = "wasm32"))] -pub type RouteHandlerFuture<'a> = Pin> + Send + 'a>>; - -/// The future type returned by [`RouteHandler::handle`]. -#[cfg(target_arch = "wasm32")] -pub type RouteHandlerFuture<'a> = Pin> + 'a>>; - -/// Extracted path parameters from route matching. -/// -/// When a route pattern like `/api/buckets/{id}` matches a request path, -/// the router populates this with the extracted parameters (e.g. `id` → `"my-bucket"`). -/// Handlers access parameters by name via [`Params::get`]. -#[derive(Debug, Clone, Default)] -pub struct Params(Vec<(String, String)>); - -impl Params { - /// Look up a parameter value by name. - /// - /// Returns `None` if the parameter was not captured by the route pattern. - pub fn get(&self, key: &str) -> Option<&str> { - self.0 - .iter() - .find(|(k, _)| k == key) - .map(|(_, v)| v.as_str()) - } - - /// Create `Params` from a `matchit::Params` match result. - pub(crate) fn from_matchit(params: &matchit::Params<'_, '_>) -> Self { - Self( - params - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - ) - } -} - -/// Parsed request metadata passed to route handlers. -pub struct RequestInfo<'a> { - pub method: &'a Method, - pub path: &'a str, - pub query: Option<&'a str>, - pub headers: &'a HeaderMap, - /// The IP address of the client that originated this request. - /// - /// Populated by runtimes that can extract client addresses (e.g. from - /// `ConnectInfo` in axum, or request headers in Lambda/Workers). - /// `None` when the source IP is unavailable or not yet extracted. - pub source_ip: Option, - /// Path parameters extracted by the router during dispatch. - /// - /// Populated by the router when a route pattern matches. Empty when the - /// request is constructed via [`RequestInfo::new`]. - pub params: Params, -} - -impl<'a> RequestInfo<'a> { - /// Create a new `RequestInfo` from the parsed HTTP request components. - pub fn new( - method: &'a Method, - path: &'a str, - query: Option<&'a str>, - headers: &'a HeaderMap, - source_ip: Option, - ) -> Self { - Self { - method, - path, - query, - headers, - source_ip, - params: Params::default(), - } - } -} - -/// A pluggable handler that can intercept requests before proxy dispatch. -/// -/// Implementations inspect the [`RequestInfo`] and return: -/// - `Some(result)` to handle the request (stops further handler checks) -/// - `None` to pass the request to the next handler or the proxy -/// -/// ```rust,ignore -/// struct HealthCheck; -/// -/// impl RouteHandler for HealthCheck { -/// fn handle<'a>(&'a self, _req: &'a RequestInfo<'a>) -> RouteHandlerFuture<'a> { -/// Box::pin(async move { -/// Some(ProxyResult::json(200, r#"{"ok":true}"#)) -/// }) -/// } -/// } -/// -/// router.route("/health", HealthCheck); -/// ``` -pub trait RouteHandler: MaybeSend + MaybeSync { - /// Handle an incoming request. - /// - /// Return `Some(result)` to short-circuit, or `None` to fall through - /// to the next handler or the proxy dispatch pipeline. - fn handle<'a>(&'a self, req: &'a RequestInfo<'a>) -> RouteHandlerFuture<'a>; -} diff --git a/crates/core/src/router.rs b/crates/core/src/router.rs deleted file mode 100644 index ef98187..0000000 --- a/crates/core/src/router.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Path-based request router. -//! -//! The [`Router`] maps URL path patterns to [`RouteHandler`] implementations, -//! giving exact paths priority over catch-all patterns. Extension crates -//! register their routes via extension traits on `Router` (e.g. `OidcRouterExt`, -//! `StsRouterExt`), making integration a single chained call. -//! -//! Handlers implement `RouteHandler` and override individual HTTP method -//! handlers (`get`, `post`, etc.) or `handle` directly: -//! -//! ```rust,ignore -//! use multistore::router::Router; -//! -//! let router = Router::new() -//! .route("/api/health", HealthCheck); -//! ``` - -use crate::route_handler::{HandlerAction, Params, RequestInfo, RouteHandler}; - -/// Path-based request router. -/// -/// Wraps [`matchit::Router`] to map URL path patterns to [`RouteHandler`] -/// implementations. Supports `matchit` path syntax: `/exact`, -/// `/prefix/{param}`, `/{*catch_all}`. -/// -/// Exact paths are matched before parameterized/catch-all patterns, so -/// registering `/.well-known/openid-configuration` alongside `/{*path}` -/// will always route OIDC discovery before the catch-all. -pub struct Router { - inner: matchit::Router>, -} - -impl Router { - pub fn new() -> Self { - Self { - inner: matchit::Router::new(), - } - } - - /// Register a handler for a path pattern. - /// - /// Supports matchit syntax: `/exact`, `/prefix/{param}`, `/{*catch_all}`. - /// Panics if the path conflicts with an already-registered route. - pub fn route(mut self, path: &str, handler: impl RouteHandler + 'static) -> Self { - self.inner - .insert(path, Box::new(handler)) - .expect("conflicting route"); - self - } - - /// Try to match a path and invoke the matched handler. - /// - /// On match, the handler receives a [`RequestInfo`] with populated - /// [`Params`] extracted from the path pattern. - /// - /// Returns `Some(action)` if a route matched and the handler produced an - /// action. Returns `None` if no route matched or the handler declined - /// (returned `None`). - pub async fn dispatch(&self, req: &RequestInfo<'_>) -> Option { - let matched = self.inner.at(req.path).ok()?; - let params = Params::from_matchit(&matched.params); - let req_with_params = RequestInfo { - params, - method: req.method, - path: req.path, - query: req.query, - headers: req.headers, - source_ip: req.source_ip, - }; - matched - .value - .handle(&req_with_params) - .await - .map(HandlerAction::Response) - } -} - -impl Default for Router { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - /// `matchit`'s `/{*path}` catch-all does NOT match the bare root `/`. - /// Route handlers that need to match `/` must register an explicit `/` route. - #[test] - fn matchit_catchall_does_not_match_root() { - let mut router = matchit::Router::<&str>::new(); - router.insert("/{*path}", "handler").unwrap(); - assert!(router.at("/").is_err()); - } - - #[test] - fn explicit_root_route_matches() { - let mut router = matchit::Router::<&str>::new(); - router.insert("/", "root").unwrap(); - assert!(router.at("/").is_ok()); - } -} diff --git a/crates/core/src/service.rs b/crates/core/src/service.rs new file mode 100644 index 0000000..b4f9690 --- /dev/null +++ b/crates/core/src/service.rs @@ -0,0 +1,867 @@ +//! S3-compatible service implementing the [`s3s::S3`] trait. +//! +//! [`MultistoreService`] maps S3 API operations to `object_store` calls, +//! providing a complete S3 protocol implementation that works across all +//! backends (S3, Azure, GCS) and all runtimes (native, WASM). +//! +//! ## Architecture +//! +//! ```text +//! S3 request → s3s protocol dispatch → MultistoreService (S3 trait) +//! → bucket registry lookup + authorization +//! → object_store call +//! → s3s response +//! ``` +//! +//! The service owns: +//! - A [`BucketRegistry`] for bucket lookup and authorization +//! - A [`CredentialRegistry`] for credential verification (via [`MultistoreAuth`]) +//! - A [`StoreFactory`] for creating object stores per request +//! - A [`DashMap`] for tracking in-progress multipart uploads + +use std::borrow::Cow; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; + +use bytes::Bytes; +use dashmap::DashMap; +use futures::{Stream, TryStreamExt}; +use object_store::list::PaginatedListStore; +use object_store::multipart::{MultipartStore, PartId}; +use object_store::path::Path; +use object_store::{GetOptions, ObjectStore, ObjectStoreExt, PutPayload}; +use s3s::auth::SecretKey; +use s3s::dto::{self, StreamingBlob, Timestamp}; +use s3s::s3_error; +use s3s::stream::{ByteStream, RemainingLength}; +use s3s::{S3Request, S3Response, S3Result}; + +type StdError = Box; + +/// Wrapper that adds `Sync` to a `Send`-only stream via `Mutex`. +/// +/// `object_store`'s `GetResult::into_stream()` returns `BoxStream` which is +/// `Send` but not `Sync`. s3s's `StreamingBlob` requires `Send + Sync`. +/// Since streams are only ever polled from a single task, wrapping in a +/// `Mutex` is safe and has negligible overhead. +struct SyncStream { + inner: Mutex, + remaining_bytes: usize, +} + +impl SyncStream { + fn new(stream: S, remaining_bytes: usize) -> Self { + Self { + inner: Mutex::new(stream), + remaining_bytes, + } + } +} + +impl Stream for SyncStream +where + S: Stream> + Unpin, + E: std::error::Error + Send + Sync + 'static, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut guard = self.inner.lock().unwrap(); + Pin::new(&mut *guard) + .poll_next(cx) + .map(|opt| opt.map(|r| r.map_err(|e| Box::new(e) as StdError))) + } +} + +impl ByteStream for SyncStream +where + S: Stream> + Unpin, + E: std::error::Error + Send + Sync + 'static, +{ + fn remaining_length(&self) -> RemainingLength { + RemainingLength::new_exact(self.remaining_bytes) + } +} + +// SAFETY: The inner stream is Send and the Mutex provides synchronization. +// Streams are only polled from a single task, so this is safe. +unsafe impl Sync for SyncStream {} + +use crate::api::list::build_list_prefix; +use crate::api::list_rewrite::ListRewrite; +use crate::error::ProxyError; +use crate::registry::{BucketRegistry, CredentialRegistry, ResolvedBucket}; +use crate::types::{BucketConfig, ResolvedIdentity, S3Operation}; + +/// Factory trait for creating object stores from bucket configuration. +/// +/// Each runtime provides its own implementation (e.g. injecting custom HTTP +/// connectors for WASM). The factory creates stores per-request since bucket +/// configs may differ. +pub trait StoreFactory: Send + Sync + 'static { + /// Create an [`ObjectStore`] for GET/HEAD/PUT/DELETE operations. + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError>; + + /// Create a [`PaginatedListStore`] for LIST operations. + fn create_paginated_store( + &self, + config: &BucketConfig, + ) -> Result, ProxyError>; + + /// Create a [`MultipartStore`] for multipart upload operations. + fn create_multipart_store( + &self, + config: &BucketConfig, + ) -> Result, ProxyError>; +} + +/// State for an in-progress multipart upload. +struct UploadState { + config: BucketConfig, + path: Path, + parts: Vec>, +} + +/// S3 service implementation backed by object_store. +/// +/// Implements [`s3s::S3`] by mapping each S3 operation to the appropriate +/// object_store call, with bucket registry lookup and authorization in between. +pub struct MultistoreService { + bucket_registry: R, + store_factory: F, + uploads: DashMap, +} + +impl MultistoreService +where + R: BucketRegistry, + F: StoreFactory, +{ + /// Create a new service with the given registries and store factory. + pub fn new(bucket_registry: R, store_factory: F) -> Self { + Self { + bucket_registry, + store_factory, + uploads: DashMap::new(), + } + } + + /// Resolve a bucket and authorize the operation. + async fn resolve_bucket( + &self, + bucket: &str, + identity: &ResolvedIdentity, + operation: &S3Operation, + ) -> S3Result { + self.bucket_registry + .get_bucket(bucket, identity, operation) + .await + .map_err(proxy_error_to_s3) + } + + /// Extract the identity from the s3s request credentials. + fn identity_from_credentials( + &self, + credentials: &Option, + ) -> ResolvedIdentity { + match credentials { + Some(creds) => { + // s3s verified the signature; we treat authenticated requests + // as long-lived credentials for authorization purposes. + // The actual StoredCredential is not needed here since + // authorization is done via the bucket registry. + ResolvedIdentity::LongLived { + credential: crate::types::StoredCredential { + access_key_id: creds.access_key.clone(), + secret_access_key: String::new(), // not needed for authz + principal_name: creds.access_key.clone(), + allowed_scopes: vec![], // authorization is done via bucket registry + created_at: chrono::Utc::now(), + expires_at: None, + enabled: true, + }, + } + } + None => ResolvedIdentity::Anonymous, + } + } + + /// Build an object_store Path from a bucket config and S3 key. + fn build_path(config: &BucketConfig, key: &str) -> Path { + match &config.backend_prefix { + Some(prefix) => { + let p = prefix.trim_end_matches('/'); + if p.is_empty() { + Path::from(key) + } else { + Path::from(format!("{}/{}", p, key)) + } + } + None => Path::from(key), + } + } +} + +#[async_trait::async_trait] +impl s3s::S3 for MultistoreService +where + R: BucketRegistry, + F: StoreFactory, +{ + async fn get_object( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let key = &req.input.key; + let operation = S3Operation::GetObject { + bucket: bucket.clone(), + key: key.clone(), + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + let path = Self::build_path(&resolved.config, key); + + let opts = GetOptions { + if_match: req.input.if_match.as_ref().map(etag_condition_to_string), + if_none_match: req + .input + .if_none_match + .as_ref() + .map(etag_condition_to_string), + range: parse_range(&req.input.range), + head: false, + ..Default::default() + }; + + let result = store + .get_opts(&path, opts) + .await + .map_err(object_store_error_to_s3)?; + + let meta = &result.meta; + let content_length = (result.range.end - result.range.start) as i64; + let e_tag = meta.e_tag.as_ref().map(|e| dto::ETag::Strong(e.clone())); + let last_modified = Some(Timestamp::from(std::time::SystemTime::from( + meta.last_modified, + ))); + + // Wrap the object_store stream with SyncStream to satisfy StreamingBlob's + // Send + Sync requirement. The stream is Send but not Sync; the Mutex + // wrapper adds Sync with negligible overhead since polling is sequential. + let stream = result.into_stream(); + let sync_stream = SyncStream::new(stream, content_length as usize); + let body = StreamingBlob::new(sync_stream); + + Ok(S3Response::new(dto::GetObjectOutput { + body: Some(body), + content_length: Some(content_length), + e_tag, + last_modified, + ..Default::default() + })) + } + + async fn head_object( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let key = &req.input.key; + let operation = S3Operation::HeadObject { + bucket: bucket.clone(), + key: key.clone(), + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + let path = Self::build_path(&resolved.config, key); + + let opts = GetOptions { + if_match: req.input.if_match.as_ref().map(etag_condition_to_string), + if_none_match: req + .input + .if_none_match + .as_ref() + .map(etag_condition_to_string), + head: true, + ..Default::default() + }; + + let result = store + .get_opts(&path, opts) + .await + .map_err(object_store_error_to_s3)?; + + let meta = &result.meta; + + Ok(S3Response::new(dto::HeadObjectOutput { + content_length: Some(meta.size as i64), + e_tag: meta.e_tag.as_ref().map(|e| dto::ETag::Strong(e.clone())), + last_modified: Some(Timestamp::from(std::time::SystemTime::from( + meta.last_modified, + ))), + ..Default::default() + })) + } + + async fn put_object( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let key = &req.input.key; + let operation = S3Operation::PutObject { + bucket: bucket.clone(), + key: key.clone(), + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + let path = Self::build_path(&resolved.config, key); + + // Materialize the body into bytes for PutPayload. + // object_store's PutPayload doesn't support streaming directly. + let payload = match req.input.body { + Some(blob) => { + let data: Bytes = blob + .try_collect::>() + .await + .map_err(|e| s3_error!(InternalError, "failed to read body: {e}"))? + .into_iter() + .fold(bytes::BytesMut::new(), |mut acc, chunk| { + acc.extend_from_slice(&chunk); + acc + }) + .freeze(); + PutPayload::from_bytes(data) + } + None => PutPayload::default(), + }; + + let result = store + .put(&path, payload) + .await + .map_err(object_store_error_to_s3)?; + + Ok(S3Response::new(dto::PutObjectOutput { + e_tag: result + .e_tag + .as_ref() + .map(|e: &String| dto::ETag::Strong(e.clone())), + ..Default::default() + })) + } + + async fn delete_object( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let key = &req.input.key; + let operation = S3Operation::DeleteObject { + bucket: bucket.clone(), + key: key.clone(), + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + let path = Self::build_path(&resolved.config, key); + + store + .delete(&path) + .await + .map_err(object_store_error_to_s3)?; + + Ok(S3Response::new(dto::DeleteObjectOutput::default())) + } + + async fn list_objects_v2( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let operation = S3Operation::ListBucket { + bucket: bucket.clone(), + raw_query: None, + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_paginated_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + + let client_prefix = req.input.prefix.as_deref().unwrap_or(""); + let delimiter = req.input.delimiter.as_deref().unwrap_or("/").to_string(); + let max_keys = req.input.max_keys.unwrap_or(1000).min(1000) as usize; + + let full_prefix = build_list_prefix(&resolved.config, client_prefix); + + let offset = req + .input + .start_after + .as_ref() + .map(|sa| build_list_prefix(&resolved.config, sa)); + + let prefix = if full_prefix.is_empty() { + None + } else { + Some(full_prefix.as_str()) + }; + + let opts = object_store::list::PaginatedListOptions { + offset, + delimiter: Some(Cow::Owned(delimiter.clone())), + max_keys: Some(max_keys), + page_token: req.input.continuation_token.clone(), + ..Default::default() + }; + + let paginated = store + .list_paginated(prefix, opts) + .await + .map_err(object_store_error_to_s3)?; + + // Compute strip prefix for backend_prefix removal + let backend_prefix = resolved + .config + .backend_prefix + .as_deref() + .unwrap_or("") + .trim_end_matches('/'); + let strip_prefix = if backend_prefix.is_empty() { + String::new() + } else { + format!("{}/", backend_prefix) + }; + let list_rewrite = resolved.list_rewrite.as_ref(); + + let contents: Vec = paginated + .result + .objects + .iter() + .map(|obj| { + let raw_key = obj.location.to_string(); + let key = rewrite_key(&raw_key, &strip_prefix, list_rewrite); + dto::Object { + key: Some(key), + size: Some(obj.size as i64), + last_modified: Some(Timestamp::from(std::time::SystemTime::from( + obj.last_modified, + ))), + e_tag: obj.e_tag.as_ref().map(|e| dto::ETag::Strong(e.clone())), + ..Default::default() + } + }) + .collect(); + + let common_prefixes: Vec = paginated + .result + .common_prefixes + .iter() + .map(|p| { + let raw_prefix = format!("{}/", p); + let prefix = rewrite_key(&raw_prefix, &strip_prefix, list_rewrite); + dto::CommonPrefix { + prefix: Some(prefix), + } + }) + .collect(); + + let key_count = (contents.len() + common_prefixes.len()) as i32; + + Ok(S3Response::new(dto::ListObjectsV2Output { + name: Some(bucket.clone()), + prefix: Some(client_prefix.to_string()), + delimiter: Some(delimiter), + max_keys: Some(max_keys as i32), + key_count: Some(key_count), + is_truncated: Some(paginated.page_token.is_some()), + next_continuation_token: paginated.page_token, + continuation_token: req.input.continuation_token, + start_after: req.input.start_after, + contents: if contents.is_empty() { + None + } else { + Some(contents) + }, + common_prefixes: if common_prefixes.is_empty() { + None + } else { + Some(common_prefixes) + }, + ..Default::default() + })) + } + + async fn list_buckets( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + + let buckets = self + .bucket_registry + .list_buckets(&identity) + .await + .map_err(proxy_error_to_s3)?; + + let s3_buckets: Vec = buckets + .into_iter() + .map(|b| dto::Bucket { + name: Some(b.name), + // BucketEntry.creation_date is pre-formatted as ISO 8601 string + creation_date: chrono::DateTime::parse_from_rfc3339(&b.creation_date) + .ok() + .map(|d| Timestamp::from(std::time::SystemTime::from(d))), + ..Default::default() + }) + .collect(); + + let o = self.bucket_registry.bucket_owner(); + let owner = Some(dto::Owner { + display_name: Some(o.display_name.clone()), + id: Some(o.id.clone()), + }); + + Ok(S3Response::new(dto::ListBucketsOutput { + buckets: if s3_buckets.is_empty() { + None + } else { + Some(s3_buckets) + }, + owner, + ..Default::default() + })) + } + + // -- Multipart operations -- + + async fn create_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let identity = self.identity_from_credentials(&req.credentials); + let bucket = &req.input.bucket; + let key = &req.input.key; + let operation = S3Operation::CreateMultipartUpload { + bucket: bucket.clone(), + key: key.clone(), + }; + + let resolved = self.resolve_bucket(bucket, &identity, &operation).await?; + let store = self + .store_factory + .create_multipart_store(&resolved.config) + .map_err(proxy_error_to_s3)?; + let path = Self::build_path(&resolved.config, key); + + let upload_id = store + .create_multipart(&path) + .await + .map_err(object_store_error_to_s3)?; + + self.uploads.insert( + upload_id.clone(), + UploadState { + config: resolved.config.clone(), + path, + parts: Vec::new(), + }, + ); + + Ok(S3Response::new(dto::CreateMultipartUploadOutput { + bucket: Some(bucket.clone()), + key: Some(key.clone()), + upload_id: Some(upload_id), + ..Default::default() + })) + } + + async fn upload_part( + &self, + req: S3Request, + ) -> S3Result> { + let upload_id = &req.input.upload_id; + let part_number = req.input.part_number; + + let state = self + .uploads + .get(upload_id) + .ok_or_else(|| s3_error!(NoSuchUpload, "upload not found: {upload_id}"))?; + + let store = self + .store_factory + .create_multipart_store(&state.config) + .map_err(proxy_error_to_s3)?; + + // Materialize the body + let payload = match req.input.body { + Some(blob) => { + let data: Bytes = blob + .try_collect::>() + .await + .map_err(|e| s3_error!(InternalError, "failed to read part body: {e}"))? + .into_iter() + .fold(bytes::BytesMut::new(), |mut acc, chunk| { + acc.extend_from_slice(&chunk); + acc + }) + .freeze(); + PutPayload::from_bytes(data) + } + None => PutPayload::default(), + }; + + let part_idx = (part_number - 1) as usize; // S3 parts are 1-indexed + let part_id = store + .put_part(&state.path, upload_id, part_idx, payload) + .await + .map_err(object_store_error_to_s3)?; + + // Release the read lock before acquiring write lock + drop(state); + + // Store the part ID for later completion + let mut state = self + .uploads + .get_mut(upload_id) + .ok_or_else(|| s3_error!(NoSuchUpload, "upload not found: {upload_id}"))?; + + // Ensure the parts vec is large enough + if state.parts.len() <= part_idx { + state.parts.resize(part_idx + 1, None); + } + state.parts[part_idx] = Some(part_id.clone()); + + Ok(S3Response::new(dto::UploadPartOutput { + e_tag: Some(dto::ETag::Strong(part_id.content_id)), + ..Default::default() + })) + } + + async fn complete_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let upload_id = &req.input.upload_id; + let bucket = &req.input.bucket; + let key = &req.input.key; + + let (_, state) = self + .uploads + .remove(upload_id) + .ok_or_else(|| s3_error!(NoSuchUpload, "upload not found: {upload_id}"))?; + + let store = self + .store_factory + .create_multipart_store(&state.config) + .map_err(proxy_error_to_s3)?; + + // Collect ordered parts + let parts: Vec = state + .parts + .into_iter() + .enumerate() + .map(|(i, p)| p.ok_or_else(|| s3_error!(InvalidPart, "missing part {}", i + 1))) + .collect::>>()?; + + let result = store + .complete_multipart(&state.path, upload_id, parts) + .await + .map_err(object_store_error_to_s3)?; + + Ok(S3Response::new(dto::CompleteMultipartUploadOutput { + bucket: Some(bucket.clone()), + key: Some(key.clone()), + e_tag: result.e_tag.as_ref().map(|e| dto::ETag::Strong(e.clone())), + ..Default::default() + })) + } + + async fn abort_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let upload_id = &req.input.upload_id; + + let (_, state) = self + .uploads + .remove(upload_id) + .ok_or_else(|| s3_error!(NoSuchUpload, "upload not found: {upload_id}"))?; + + let store = self + .store_factory + .create_multipart_store(&state.config) + .map_err(proxy_error_to_s3)?; + + store + .abort_multipart(&state.path, upload_id) + .await + .map_err(object_store_error_to_s3)?; + + Ok(S3Response::new(dto::AbortMultipartUploadOutput::default())) + } +} + +// -- Auth -- + +/// s3s auth implementation that delegates to a [`CredentialRegistry`]. +pub struct MultistoreAuth { + credential_registry: C, +} + +impl MultistoreAuth { + pub fn new(credential_registry: C) -> Self { + Self { + credential_registry, + } + } +} + +#[async_trait::async_trait] +impl s3s::auth::S3Auth for MultistoreAuth { + async fn get_secret_key(&self, access_key: &str) -> S3Result { + match self + .credential_registry + .get_credential(access_key) + .await + .map_err(proxy_error_to_s3)? + { + Some(cred) if cred.enabled => Ok(SecretKey::from(cred.secret_access_key)), + Some(_) => Err(s3_error!(InvalidAccessKeyId, "credential disabled")), + None => Err(s3_error!(InvalidAccessKeyId)), + } + } +} + +/// Access control that allows both authenticated and anonymous requests. +/// +/// s3s's default access check rejects all unsigned requests. Since our +/// `BucketRegistry` handles authorization (including anonymous access +/// checks per bucket), we allow all requests through the s3s access layer. +pub struct MultistoreAccess; + +#[async_trait::async_trait] +impl s3s::access::S3Access for MultistoreAccess { + async fn check(&self, _cx: &mut s3s::access::S3AccessContext<'_>) -> S3Result<()> { + // Authorization is handled by BucketRegistry::get_bucket() + // inside each S3 operation handler, not at the access layer. + Ok(()) + } +} + +// -- Helpers -- + +/// Convert a [`ProxyError`] to an [`s3s::S3Error`]. +fn proxy_error_to_s3(e: ProxyError) -> s3s::S3Error { + use s3s::S3ErrorCode; + match e { + ProxyError::BucketNotFound(msg) => s3_error!(NoSuchBucket, "{msg}"), + ProxyError::NoSuchKey(msg) => s3_error!(NoSuchKey, "{msg}"), + ProxyError::AccessDenied | ProxyError::MissingAuth => s3_error!(AccessDenied), + ProxyError::SignatureDoesNotMatch => s3_error!(SignatureDoesNotMatch), + ProxyError::InvalidRequest(msg) => s3_error!(InvalidRequest, "{msg}"), + ProxyError::ExpiredCredentials => s3_error!(ExpiredToken), + ProxyError::InvalidOidcToken(msg) => { + S3Error::with_message(S3ErrorCode::Custom("InvalidIdentityToken".into()), msg) + } + ProxyError::RoleNotFound(_) => s3_error!(AccessDenied), + ProxyError::PreconditionFailed => s3_error!(PreconditionFailed), + ProxyError::NotModified => { + let mut err = S3Error::new(s3s::S3ErrorCode::Custom("NotModified".into())); + err.set_status_code(http::StatusCode::NOT_MODIFIED); + err + } + ProxyError::BackendError(msg) => s3_error!(InternalError, "backend error: {msg}"), + ProxyError::ConfigError(msg) => s3_error!(InternalError, "config error: {msg}"), + ProxyError::Internal(msg) => s3_error!(InternalError, "{msg}"), + } +} + +use s3s::S3Error; + +/// Convert an [`object_store::Error`] to an [`s3s::S3Error`]. +fn object_store_error_to_s3(e: object_store::Error) -> S3Error { + match e { + object_store::Error::NotFound { path, .. } => s3_error!(NoSuchKey, "{path}"), + object_store::Error::Precondition { .. } => s3_error!(PreconditionFailed), + object_store::Error::NotModified { .. } => { + let mut err = S3Error::new(s3s::S3ErrorCode::Custom("NotModified".into())); + err.set_status_code(http::StatusCode::NOT_MODIFIED); + err + } + other => s3_error!(InternalError, "backend error: {other}"), + } +} + +/// Convert an [`ETagCondition`] to a string for object_store's `GetOptions`. +fn etag_condition_to_string(cond: &dto::ETagCondition) -> String { + match cond { + dto::ETagCondition::ETag(etag) => etag.value().to_string(), + dto::ETagCondition::Any => "*".to_string(), + } +} + +/// Convert an s3s Range to an object_store GetRange. +fn parse_range(range: &Option) -> Option { + let range = range.as_ref()?; + match *range { + dto::Range::Int { + first, + last: Some(last), + } => Some(object_store::GetRange::Bounded(first..last + 1)), + dto::Range::Int { first, last: None } => Some(object_store::GetRange::Offset(first)), + dto::Range::Suffix { length } => Some(object_store::GetRange::Suffix(length)), + } +} + +/// Apply strip/add prefix rewriting to a key or prefix value. +fn rewrite_key(raw: &str, strip_prefix: &str, list_rewrite: Option<&ListRewrite>) -> String { + let key = if !strip_prefix.is_empty() { + raw.strip_prefix(strip_prefix).unwrap_or(raw) + } else { + raw + }; + + if let Some(rewrite) = list_rewrite { + let key = if !rewrite.strip_prefix.is_empty() { + key.strip_prefix(rewrite.strip_prefix.as_str()) + .unwrap_or(key) + } else { + key + }; + + if !rewrite.add_prefix.is_empty() { + return if key.is_empty() || key.starts_with('/') { + format!("{}{}", rewrite.add_prefix, key) + } else { + format!("{}/{}", rewrite.add_prefix, key) + }; + } + + return key.to_string(); + } + + key.to_string() +} diff --git a/crates/metering/Cargo.toml b/crates/metering/Cargo.toml index e6e6834..3402b6b 100644 --- a/crates/metering/Cargo.toml +++ b/crates/metering/Cargo.toml @@ -3,12 +3,10 @@ name = "multistore-metering" version.workspace = true edition.workspace = true license.workspace = true -description = "Usage metering and quota enforcement middleware for the S3 proxy gateway" +description = "Usage metering and quota enforcement for the S3 proxy gateway" [dependencies] multistore.workspace = true -bytes.workspace = true -http.workspace = true tracing.workspace = true [dev-dependencies] diff --git a/crates/metering/src/lib.rs b/crates/metering/src/lib.rs index b60f694..7b999c1 100644 --- a/crates/metering/src/lib.rs +++ b/crates/metering/src/lib.rs @@ -1,43 +1,23 @@ -//! Usage metering and quota enforcement middleware. +//! Usage metering and quota enforcement. //! //! This crate provides trait abstractions for tracking API usage and enforcing -//! quotas, along with a [`MeteringMiddleware`] that wires them into the proxy's -//! middleware chain. Integrators bring their own storage backends by implementing +//! quotas. Integrators bring their own storage backends by implementing //! [`UsageRecorder`] and [`QuotaChecker`]. //! -//! ## Quick start -//! -//! ```rust,ignore -//! use multistore_metering::{MeteringMiddleware, UsageRecorder, QuotaChecker}; -//! -//! // Implement UsageRecorder and QuotaChecker for your storage backend, -//! // then register the middleware on the ProxyGateway builder: -//! let metering = MeteringMiddleware::new(my_quota_checker, my_usage_recorder); -//! gateway_builder.add_middleware(metering); -//! ``` -//! //! ## Architecture //! //! - **Pre-dispatch:** [`QuotaChecker::check_quota`] runs before the request -//! proceeds, using `Content-Length` as a byte estimate. Return -//! [`Err(QuotaExceeded)`](QuotaExceeded) to reject with HTTP 429. +//! proceeds. Return [`Err(QuotaExceeded)`](QuotaExceeded) to reject with +//! HTTP 429. //! - **Post-dispatch:** [`UsageRecorder::record_operation`] runs after the -//! response is available, recording actual status and byte counts from -//! the backend response. +//! response is available, recording actual status and byte counts. use std::future::Future; use std::net::IpAddr; -use multistore::api::response::ErrorResponse; -use multistore::error::ProxyError; use multistore::maybe_send::{MaybeSend, MaybeSync}; -use multistore::middleware::{CompletedRequest, DispatchContext, Middleware, Next}; -use multistore::route_handler::{HandlerAction, ProxyResponseBody, ProxyResult}; use multistore::types::{ResolvedIdentity, S3Operation}; -use bytes::Bytes; -use http::HeaderMap; - /// A completed operation's metadata, passed to [`UsageRecorder::record_operation`]. pub struct UsageEvent<'a> { /// The unique request identifier. @@ -50,9 +30,7 @@ pub struct UsageEvent<'a> { pub bucket: Option<&'a str>, /// The HTTP status code of the response. pub status: u16, - /// Best-available byte count: `content_length` from backend response - /// for forwarded requests, response body length for direct responses, - /// or `Content-Length` header estimate as fallback. + /// Best-available byte count for the operation. pub bytes_transferred: u64, /// Whether the request was forwarded to a backend via presigned URL. pub was_forwarded: bool, @@ -77,8 +55,8 @@ pub struct QuotaExceeded { pub trait UsageRecorder: MaybeSend + MaybeSync + 'static { /// Record a completed operation. /// - /// This runs in the post-dispatch phase. Implementations should be - /// fire-and-forget — recording failures must not affect the response. + /// Implementations should be fire-and-forget — recording failures + /// must not affect the response. fn record_operation<'a>( &'a self, event: UsageEvent<'a>, @@ -105,110 +83,6 @@ pub trait QuotaChecker: MaybeSend + MaybeSync + 'static { ) -> impl Future> + MaybeSend + 'a; } -/// Middleware that enforces quotas pre-dispatch and records usage post-dispatch. -/// -/// Generic over the quota checker `Q` and usage recorder `U`, allowing -/// integrators to bring their own storage backends. -/// -/// ## Request flow -/// -/// 1. Extract `Content-Length` from request headers as a byte estimate. -/// 2. Call [`QuotaChecker::check_quota`] — reject with 429 if over limit. -/// 3. Delegate to the next middleware via [`Next::run`]. -/// 4. In [`after_dispatch`](Middleware::after_dispatch), call -/// [`UsageRecorder::record_operation`] with the actual response metadata. -pub struct MeteringMiddleware { - quota_checker: Q, - usage_recorder: U, -} - -impl MeteringMiddleware { - /// Create a new metering middleware with the given quota checker and - /// usage recorder. - pub fn new(quota_checker: Q, usage_recorder: U) -> Self { - Self { - quota_checker, - usage_recorder, - } - } -} - -impl Middleware for MeteringMiddleware { - async fn handle<'a>( - &'a self, - ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> Result { - let estimated_bytes = ctx - .headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()) - .unwrap_or(0); - - let bucket_name = ctx.bucket_config.as_ref().map(|b| b.name.as_str()); - - if let Err(_exceeded) = self - .quota_checker - .check_quota( - ctx.identity, - ctx.operation, - bucket_name, - estimated_bytes, - ctx.source_ip, - ) - .await - { - tracing::warn!(bucket = bucket_name, "quota exceeded, returning 429"); - let xml = ErrorResponse::slow_down(ctx.request_id).to_xml(); - let mut headers = HeaderMap::new(); - headers.insert("content-type", "application/xml".parse().unwrap()); - return Ok(HandlerAction::Response(ProxyResult { - status: 429, - headers, - body: ProxyResponseBody::Bytes(Bytes::from(xml)), - })); - } - - next.run(ctx).await - } - - fn after_dispatch( - &self, - completed: &CompletedRequest<'_>, - ) -> impl Future + MaybeSend + '_ { - // Extract all fields synchronously to avoid capturing `completed` - // in the returned future (the future's lifetime is tied to `&self`, - // not `completed`). - let request_id = completed.request_id.to_owned(); - let identity = completed.identity.cloned(); - let operation = completed.operation.cloned(); - let bucket = completed.bucket.map(str::to_owned); - let status = completed.status; - let bytes_transferred = completed - .response_bytes - .or(completed.request_bytes) - .unwrap_or(0); - let was_forwarded = completed.was_forwarded; - let source_ip = completed.source_ip; - - async move { - self.usage_recorder - .record_operation(UsageEvent { - request_id: &request_id, - identity: identity.as_ref(), - operation: operation.as_ref(), - bucket: bucket.as_deref(), - status, - bytes_transferred, - was_forwarded, - source_ip, - }) - .await; - } - } -} - // =========================================================================== // No-op implementations // =========================================================================== @@ -245,7 +119,6 @@ impl QuotaChecker for NoopQuotaChecker { #[cfg(test)] mod tests { use super::*; - use multistore::middleware::CompletedRequest; use multistore::types::{ResolvedIdentity, S3Operation}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -332,9 +205,6 @@ mod tests { // -- Tests ---------------------------------------------------------------- - // Tests for `handle` use the ProxyGateway integration tests in core. - // Here we test the quota checking and after_dispatch logic directly. - #[test] fn rejecting_checker_returns_error() { let checker = RejectingChecker { @@ -394,73 +264,25 @@ mod tests { } #[test] - fn after_dispatch_records_usage() { + fn recorder_captures_usage_event() { let (recorder, last_bytes, call_count) = RecordingRecorder::new(); - let middleware = MeteringMiddleware::new(NoopQuotaChecker, recorder); futures::executor::block_on(async { - let completed = CompletedRequest { - request_id: "req-1", - identity: None, - operation: None, - bucket: Some("my-bucket"), - status: 200, - response_bytes: Some(1024), - request_bytes: None, - was_forwarded: true, - source_ip: None, - }; - Middleware::after_dispatch(&middleware, &completed).await; + recorder + .record_operation(UsageEvent { + request_id: "req-1", + identity: None, + operation: None, + bucket: Some("my-bucket"), + status: 200, + bytes_transferred: 1024, + was_forwarded: true, + source_ip: None, + }) + .await; }); assert_eq!(call_count.load(Ordering::SeqCst), 1); assert_eq!(last_bytes.load(Ordering::SeqCst), 1024); } - - #[test] - fn after_dispatch_falls_back_to_request_bytes() { - let (recorder, last_bytes, _) = RecordingRecorder::new(); - let middleware = MeteringMiddleware::new(NoopQuotaChecker, recorder); - - futures::executor::block_on(async { - let completed = CompletedRequest { - request_id: "req-2", - identity: None, - operation: None, - bucket: None, - status: 200, - response_bytes: None, - request_bytes: Some(512), - was_forwarded: false, - source_ip: None, - }; - Middleware::after_dispatch(&middleware, &completed).await; - }); - - assert_eq!(last_bytes.load(Ordering::SeqCst), 512); - } - - #[test] - fn after_dispatch_defaults_to_zero_bytes() { - let (recorder, last_bytes, call_count) = RecordingRecorder::new(); - let middleware = MeteringMiddleware::new(NoopQuotaChecker, recorder); - - futures::executor::block_on(async { - let completed = CompletedRequest { - request_id: "req-3", - identity: None, - operation: None, - bucket: None, - status: 500, - response_bytes: None, - request_bytes: None, - was_forwarded: false, - source_ip: None, - }; - Middleware::after_dispatch(&middleware, &completed).await; - }); - - assert_eq!(call_count.load(Ordering::SeqCst), 1); - assert_eq!(last_bytes.load(Ordering::SeqCst), 0); - } } diff --git a/crates/oidc-provider/src/backend_auth.rs b/crates/oidc-provider/src/backend_auth.rs index d2c27c7..89afb49 100644 --- a/crates/oidc-provider/src/backend_auth.rs +++ b/crates/oidc-provider/src/backend_auth.rs @@ -4,15 +4,9 @@ //! mints a self-signed JWT and exchanges it for temporary cloud credentials //! via the cloud provider's STS. The resolved credentials are injected back //! into the config so the existing builder pipeline works unmodified. -//! -//! Implements the [`Middleware`] trait so that credential resolution runs -//! as part of the dispatch middleware chain. use multistore::error::ProxyError; -use multistore::middleware::{DispatchContext, Middleware, Next}; -use multistore::route_handler::HandlerAction; use multistore::types::BucketConfig; -use std::borrow::Cow; use std::collections::HashMap; use crate::exchange::aws::AwsExchange; @@ -29,7 +23,11 @@ impl AwsBackendAuth { Self { provider } } - async fn resolve_aws( + /// Resolve OIDC credentials for an AWS backend bucket. + /// + /// Returns replacement `backend_options` with temporary AWS credentials + /// injected and OIDC-specific keys removed. + pub async fn resolve_aws( &self, config: &BucketConfig, ) -> Result, ProxyError> { @@ -59,12 +57,11 @@ impl AwsBackendAuth { Ok(options) } - /// Internal helper: resolve credentials if bucket needs OIDC. + /// Resolve credentials if the bucket uses OIDC auth. /// - /// Returns `None` if the bucket doesn't use OIDC auth, `Some(options)` with + /// Returns `None` if the bucket doesn't use OIDC, `Some(options)` with /// replacement backend options if it does. - #[cfg(test)] - async fn resolve_credentials( + pub async fn resolve_credentials( &self, config: &BucketConfig, ) -> Result>, ProxyError> { @@ -80,70 +77,6 @@ impl AwsBackendAuth { } } -impl Middleware for AwsBackendAuth { - async fn handle<'a>( - &'a self, - mut ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> Result { - if let Some(ref bucket_config) = ctx.bucket_config { - if bucket_config.option("auth_type") == Some("oidc") { - match bucket_config.backend_type.as_str() { - "s3" => { - let options = self.resolve_aws(bucket_config).await?; - ctx.bucket_config = Some(Cow::Owned(BucketConfig { - backend_options: options, - ..ctx.bucket_config.unwrap().into_owned() - })); - } - other => { - return Err(ProxyError::ConfigError(format!( - "OIDC backend auth not yet supported for backend_type '{other}'" - ))); - } - } - } - } - next.run(ctx).await - } -} - -/// Wrapper enum that runtimes use as a single concrete middleware type. -/// -/// `Enabled` holds the live OIDC provider; `Disabled` is the no-op fallback. -/// When disabled and a bucket specifies `auth_type=oidc`, a `ConfigError` -/// is returned. -pub enum MaybeOidcAuth { - Enabled(Box>), - Disabled, -} - -impl Middleware for MaybeOidcAuth { - async fn handle<'a>( - &'a self, - ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> Result { - match self { - MaybeOidcAuth::Enabled(auth) => auth.handle(ctx, next).await, - MaybeOidcAuth::Disabled => { - let is_oidc = ctx - .bucket_config - .as_deref() - .and_then(|c| c.option("auth_type")) - == Some("oidc"); - if is_oidc { - Err(ProxyError::ConfigError( - "bucket requires auth_type=oidc but no OIDC provider is configured".into(), - )) - } else { - next.run(ctx).await - } - } - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -274,27 +207,4 @@ mod tests { assert!(resolved.is_none()); assert_eq!(http.call_count.load(Ordering::SeqCst), 0); } - - #[tokio::test] - async fn maybe_disabled_errors_on_oidc_bucket() { - // MaybeOidcAuth::Disabled should error when a bucket requires OIDC. - // We verify the branch condition directly since Next/Dispatch are - // pub(crate) in core and can't be constructed from here. - let config = oidc_bucket_config(); - assert_eq!(config.option("auth_type"), Some("oidc")); - // The Middleware::handle impl returns this error before calling next: - let err = ProxyError::ConfigError( - "bucket requires auth_type=oidc but no OIDC provider is configured".into(), - ); - assert!(err.to_string().contains("no OIDC provider is configured")); - } - - #[tokio::test] - async fn maybe_disabled_passes_through_static_bucket() { - // MaybeOidcAuth::Disabled should pass through when the bucket - // doesn't require OIDC auth (no auth_type=oidc in options). - let config = static_bucket_config(); - assert!(config.option("auth_type") != Some("oidc")); - // The Middleware::handle impl calls next.run(ctx) in this case. - } } diff --git a/crates/oidc-provider/src/lib.rs b/crates/oidc-provider/src/lib.rs index 5a83f5c..cbfcfab 100644 --- a/crates/oidc-provider/src/lib.rs +++ b/crates/oidc-provider/src/lib.rs @@ -7,8 +7,6 @@ //! 3. **OIDC discovery** — generate `.well-known/openid-configuration` responses //! 4. **Credential exchange** — trade self-signed JWTs for cloud provider //! credentials (AWS STS, Azure AD, GCP STS) -//! 5. **Route handler** — [`route_handler::OidcRouterExt`] registers -//! `.well-known` endpoint closures on a [`Router`](multistore::router::Router) //! //! The crate is runtime-agnostic: HTTP calls are abstracted behind an //! [`HttpExchange`] trait so that each runtime (reqwest, Fetch API, etc.) @@ -20,7 +18,6 @@ pub mod discovery; pub mod exchange; pub mod jwks; pub mod jwt; -pub mod route_handler; use std::sync::Arc; diff --git a/crates/oidc-provider/src/route_handler.rs b/crates/oidc-provider/src/route_handler.rs deleted file mode 100644 index 88acc2d..0000000 --- a/crates/oidc-provider/src/route_handler.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! Route handler for OIDC discovery endpoints. -//! -//! Serves `/.well-known/openid-configuration` and `/.well-known/jwks.json` -//! when the proxy is configured as an OIDC provider. - -use crate::discovery::openid_configuration_json; -use crate::jwks::jwks_json; -use crate::jwt::JwtSigner; -use multistore::route_handler::{ProxyResult, RequestInfo, RouteHandler, RouteHandlerFuture}; -use multistore::router::Router; - -/// Handler that serves the OpenID Connect discovery document. -struct OidcConfigHandler { - issuer: String, - jwks_uri: String, -} - -impl RouteHandler for OidcConfigHandler { - fn handle<'a>(&'a self, req: &'a RequestInfo<'a>) -> RouteHandlerFuture<'a> { - if req.method.as_str() != "GET" { - return Box::pin(async { None }); - } - let json = openid_configuration_json(&self.issuer, &self.jwks_uri); - Box::pin(async move { Some(ProxyResult::json(200, json)) }) - } -} - -/// Handler that serves the JWKS (JSON Web Key Set) document. -struct OidcJwksHandler { - signer: JwtSigner, -} - -impl RouteHandler for OidcJwksHandler { - fn handle<'a>(&'a self, req: &'a RequestInfo<'a>) -> RouteHandlerFuture<'a> { - if req.method.as_str() != "GET" { - return Box::pin(async { None }); - } - let json = jwks_json(self.signer.public_key(), self.signer.kid()); - Box::pin(async move { Some(ProxyResult::json(200, json)) }) - } -} - -/// Extension trait for registering OIDC discovery routes on a [`Router`]. -pub trait OidcRouterExt { - /// Register `/.well-known/openid-configuration` and `/.well-known/jwks.json` - /// routes backed by the given issuer and signer. - fn with_oidc_discovery(self, issuer: String, signer: JwtSigner) -> Self; -} - -impl OidcRouterExt for Router { - fn with_oidc_discovery(self, issuer: String, signer: JwtSigner) -> Self { - let jwks_uri = format!("{}/.well-known/jwks.json", issuer); - - self.route( - "/.well-known/openid-configuration", - OidcConfigHandler { issuer, jwks_uri }, - ) - .route("/.well-known/jwks.json", OidcJwksHandler { signer }) - } -} diff --git a/crates/sts/Cargo.toml b/crates/sts/Cargo.toml index 6cac5eb..97ddc38 100644 --- a/crates/sts/Cargo.toml +++ b/crates/sts/Cargo.toml @@ -23,5 +23,4 @@ quick-xml.workspace = true url.workspace = true [dev-dependencies] -http.workspace = true tokio = { workspace = true, features = ["rt", "macros"] } diff --git a/crates/sts/src/lib.rs b/crates/sts/src/lib.rs index 9b3e54d..1d0d908 100644 --- a/crates/sts/src/lib.rs +++ b/crates/sts/src/lib.rs @@ -4,17 +4,6 @@ //! workloads like GitHub Actions to exchange OIDC tokens for temporary S3 //! credentials scoped to specific buckets and prefixes. //! -//! # Integration -//! -//! Register STS routes via [`route_handler::StsRouterExt`]: -//! -//! ```rust,ignore -//! use multistore_sts::route_handler::StsRouterExt; -//! -//! let router = Router::new() -//! .with_sts(config, jwks_cache, token_key); -//! ``` -//! //! # Flow //! //! 1. Client obtains a JWT from their OIDC provider (e.g., GitHub Actions ID token) @@ -29,7 +18,6 @@ pub mod jwks; pub mod request; pub mod responses; -pub mod route_handler; pub mod sealed_token; pub mod sts; diff --git a/crates/sts/src/route_handler.rs b/crates/sts/src/route_handler.rs deleted file mode 100644 index 7aae62e..0000000 --- a/crates/sts/src/route_handler.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Route handler for STS `AssumeRoleWithWebIdentity` requests. -//! -//! Intercepts STS queries before they reach the proxy dispatch pipeline -//! and delegates to [`try_handle_sts`]. - -use crate::{try_handle_sts, JwksCache, TokenKey}; -use multistore::registry::CredentialRegistry; -use multistore::route_handler::{ProxyResult, RequestInfo, RouteHandler, RouteHandlerFuture}; -use multistore::router::Router; - -/// Handler that intercepts `AssumeRoleWithWebIdentity` STS requests. -struct StsHandler { - config: C, - cache: JwksCache, - key: Option, -} - -impl RouteHandler for StsHandler { - fn handle<'a>(&'a self, req: &'a RequestInfo<'a>) -> RouteHandlerFuture<'a> { - Box::pin(async move { - let (status, xml) = - try_handle_sts(req.query, &self.config, &self.cache, self.key.as_ref()).await?; - Some(ProxyResult::xml(status, xml)) - }) - } -} - -/// Extension trait for registering STS routes on a [`Router`]. -pub trait StsRouterExt { - /// Register the STS handler on the root path (`/`). - /// - /// STS requests are identified by query parameters - /// (`Action=AssumeRoleWithWebIdentity`), not by path, and clients - /// always send them to `/`. - fn with_sts( - self, - config: C, - cache: JwksCache, - key: Option, - ) -> Self; -} - -impl StsRouterExt for Router { - fn with_sts( - self, - config: C, - cache: JwksCache, - key: Option, - ) -> Self { - self.route("/", StsHandler { config, cache, key }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use multistore::error::ProxyError; - use multistore::types::{RoleConfig, StoredCredential}; - - /// Minimal stub that satisfies `CredentialRegistry` without real data. - #[derive(Clone)] - struct EmptyRegistry; - - impl CredentialRegistry for EmptyRegistry { - async fn get_credential( - &self, - _access_key_id: &str, - ) -> Result, ProxyError> { - Ok(None) - } - async fn get_role(&self, _role_id: &str) -> Result, ProxyError> { - Ok(None) - } - } - - fn test_router() -> Router { - let cache = JwksCache::new(reqwest::Client::new(), std::time::Duration::from_secs(60)); - Router::new().with_sts(EmptyRegistry, cache, None) - } - - #[tokio::test] - async fn sts_query_on_root_path_is_handled() { - let router = test_router(); - let headers = http::HeaderMap::new(); - let req = RequestInfo::new( - &http::Method::GET, - "/", - Some("Action=AssumeRoleWithWebIdentity&RoleArn=test&WebIdentityToken=tok"), - &headers, - None, - ); - assert!( - router.dispatch(&req).await.is_some(), - "STS request to / must be intercepted by the router" - ); - } - - #[tokio::test] - async fn non_sts_query_on_root_path_falls_through() { - let router = test_router(); - let headers = http::HeaderMap::new(); - let req = RequestInfo::new(&http::Method::GET, "/", Some("prefix=foo/"), &headers, None); - assert!( - router.dispatch(&req).await.is_none(), - "non-STS request to / must fall through" - ); - } -} diff --git a/docs/architecture/crate-layout.md b/docs/architecture/crate-layout.md index f6696ed..a7d05c6 100644 --- a/docs/architecture/crate-layout.md +++ b/docs/architecture/crate-layout.md @@ -4,8 +4,8 @@ The project is organized as a Cargo workspace with libraries (traits and logic) ``` crates/ -├── core/ (multistore) # Runtime-agnostic: traits, S3 parsing, SigV4, registries -├── metering/ (multistore-metering) # Usage metering and quota enforcement middleware +├── core/ (multistore) # Runtime-agnostic: s3s service, registries, authorization +├── metering/ (multistore-metering) # Usage metering and quota enforcement traits ├── sts/ (multistore-sts) # OIDC/STS token exchange (AssumeRoleWithWebIdentity) └── oidc-provider/ # Outbound OIDC provider (JWT signing, JWKS, exchange) @@ -20,17 +20,15 @@ examples/ ### `multistore` The runtime-agnostic core. Contains: -- `ProxyGateway` — Router-based dispatch + S3 parsing + identity resolution + two-phase request dispatch (`handle_request()` → `GatewayResponse`) -- `Router` — Path-based route matching via `matchit` for efficient pre-dispatch -- `RouteHandler` trait — Pluggable request interception -- `Middleware` trait — Composable post-auth middleware for dispatch, with `after_dispatch` for post-response observation -- `Forwarder` trait — Runtime-provided HTTP transport for backend forwarding; the core orchestrates the call so middleware can observe response metadata +- `MultistoreService` — s3s-based S3 service implementation mapping S3 operations to `object_store` calls (GET/HEAD/PUT/DELETE/LIST/multipart) +- `MultistoreAuth` — s3s auth adapter wrapping `CredentialRegistry` for SigV4 verification +- `StoreFactory` trait — Runtime-provided factory for creating `ObjectStore`, `PaginatedListStore`, and `MultipartStore` per request - `BucketRegistry` trait — Bucket lookup, authorization, and listing - `CredentialRegistry` trait — Credential and role storage -- `ProxyBackend` trait — Runtime abstraction for store/signer/raw HTTP -- S3 request parsing, XML response building, list prefix rewriting -- SigV4 signature verification -- Sealed session token encryption/decryption +- `TemporaryCredentialResolver` trait — Resolve session tokens into temporary credentials +- `authorize()` — Check if an identity is authorized for an S3 operation +- `StoreBuilder` — Provider-specific `object_store` builder (S3, Azure, GCS) +- List prefix rewriting - Type definitions (`BucketConfig`, `RoleConfig`, `AccessScope`, etc.) **Feature flags:** @@ -39,51 +37,50 @@ The runtime-agnostic core. Contains: ### `multistore-metering` -Usage metering and quota enforcement middleware: -- `MeteringMiddleware` — Pre-dispatch quota checking + post-dispatch usage recording via the `Middleware` trait +Usage metering and quota enforcement trait abstractions: - `QuotaChecker` trait — Pre-dispatch quota enforcement; return `Err(QuotaExceeded)` to reject with HTTP 429 - `UsageRecorder` trait — Post-dispatch operation recording for usage tracking - `UsageEvent` — Operation metadata passed to the recorder (identity, operation, bytes, status) -- `NoopQuotaChecker` / `NoopRecorder` — Convenience no-op implementations for when only one side is needed +- `NoopQuotaChecker` / `NoopRecorder` — Convenience no-op implementations ### `multistore-sts` OIDC token exchange implementing `AssumeRoleWithWebIdentity`: -- `StsRouterExt` — registers a closure that intercepts STS requests on the `Router` - JWT decoding and validation (RS256) - JWKS fetching and caching - Trust policy evaluation (issuer, audience, subject conditions) - Temporary credential minting with scope template variables +- `TokenKey` — sealed session token encryption/decryption (implements `TemporaryCredentialResolver`) ### `multistore-oidc-provider` Outbound OIDC identity provider for backend authentication: -- `OidcRouterExt` — registers closures for `.well-known` discovery endpoints on the `Router` - RSA JWT signing (`JwtSigner`) - JWKS endpoint serving - OpenID Connect discovery document -- AWS credential exchange (`AwsBackendAuth` middleware) +- AWS credential exchange (`AwsBackendAuth`) - Credential caching ### `multistore-server` The native server runtime (in `examples/server/`): -- Tokio/Hyper HTTP server -- `ServerBackend` implementing `ProxyBackend` with reqwest -- Streaming via hyper `Incoming` bodies and reqwest `bytes_stream()` -- Wires `ProxyGateway` with a `Router` (OIDC discovery + STS routes) -- CLI argument parsing (`--config`, `--listen`, `--domain`, `--sts-config`) +- Tokio/Hyper HTTP server via `S3ServiceBuilder` + hyper-util +- `ServerBackend` implementing `StoreFactory` with reqwest +- CLI argument parsing (`--config`, `--listen`, `--domain`) + +### `multistore-lambda` + +The AWS Lambda runtime (in `examples/lambda/`): +- Lambda HTTP adapter converting between `lambda_http` and s3s types +- `LambdaBackend` implementing `StoreFactory` with default `object_store` HTTP ### `multistore-cf-workers` The Cloudflare Workers WASM runtime (in `examples/cf-workers/`): -- `WorkerBackend` implementing `ProxyBackend` with `web_sys::fetch` -- `WorkerForwarder` implementing `Forwarder` with the Fetch API (zero-copy `ReadableStream`) +- `WorkerBackend` implementing `StoreFactory` with `web_sys::fetch` - `FetchConnector` bridging `object_store` HTTP to Workers Fetch API - `BandwidthMeter` Durable Object — per-(bucket, identity) sliding-window byte counter -- `DoBandwidthMeter` implementing `QuotaChecker` + `UsageRecorder` via the `BandwidthMeter` DO -- `CfRateLimiter` middleware for request-rate limiting via CF Rate Limiting API -- Config loading from env vars (`PROXY_CONFIG`, `BANDWIDTH_QUOTAS`) +- Config loading from env vars (`PROXY_CONFIG`, `VIRTUAL_HOST_DOMAIN`) > [!WARNING] > This crate is excluded from the workspace `default-members` because WASM types are `!Send` and won't compile on native targets. Always build with `--target wasm32-unknown-unknown`. @@ -101,17 +98,11 @@ flowchart TD workers["multistore-cf-workers"] server --> core - server --> sts - server --> oidc lambda --> core - lambda --> sts - lambda --> oidc workers --> core - workers --> sts - workers --> oidc metering --> core sts --> core oidc --> core ``` -Libraries define trait abstractions. Runtimes implement `ProxyBackend` and `Forwarder` with platform-native primitives, build a `Router` with extension traits, and handle the two-variant `GatewayResponse`. +Libraries define trait abstractions. Runtimes implement `StoreFactory` with platform-native primitives. The s3s-based `MultistoreService` handles S3 protocol dispatch via the `s3s::S3` trait, using `object_store` for all backend operations including multipart uploads across S3, Azure, and GCS. diff --git a/examples/cf-workers/Cargo.toml b/examples/cf-workers/Cargo.toml index bbb498c..b60171e 100644 --- a/examples/cf-workers/Cargo.toml +++ b/examples/cf-workers/Cargo.toml @@ -11,24 +11,20 @@ crate-type = ["cdylib"] [dependencies] multistore.workspace = true multistore-static-config.workspace = true -multistore-sts.workspace = true multistore-metering.workspace = true -multistore-oidc-provider.workspace = true +s3s.workspace = true bytes.workspace = true http.workspace = true serde.workspace = true serde_json.workspace = true tracing.workspace = true -thiserror.workspace = true chrono.workspace = true -quick-xml.workspace = true url.workspace = true -object_store.workspace = true futures.workspace = true +object_store.workspace = true http-body.workspace = true http-body-util.workspace = true async-trait.workspace = true -reqwest.workspace = true # Cloudflare Workers SDK worker.workspace = true diff --git a/examples/cf-workers/src/client.rs b/examples/cf-workers/src/client.rs index 171b8ba..219a527 100644 --- a/examples/cf-workers/src/client.rs +++ b/examples/cf-workers/src/client.rs @@ -1,29 +1,31 @@ -//! Backend client and HTTP helpers for the Cloudflare Workers runtime. +//! Backend client for the Cloudflare Workers runtime. //! -//! Contains: -//! - `WorkerBackend` — implements `ProxyBackend` using the Fetch API + FetchConnector -//! - `FetchHttpExchange` — implements `HttpExchange` for OIDC token exchange +//! `WorkerBackend` implements `StoreFactory` using the Fetch API via `FetchConnector`. use crate::fetch_connector::FetchConnector; -use bytes::Bytes; -use http::HeaderMap; -use multistore::backend::{build_signer, create_builder, ProxyBackend, RawResponse, StoreBuilder}; +use multistore::backend::{create_builder, StoreBuilder}; use multistore::error::ProxyError; +use multistore::service::StoreFactory; use multistore::types::BucketConfig; -use multistore_oidc_provider::{HttpExchange, OidcProviderError}; use object_store::list::PaginatedListStore; -use object_store::signer::Signer; +use object_store::multipart::MultipartStore; +use object_store::ObjectStore; use std::sync::Arc; -use worker::Fetch; /// Backend for the Cloudflare Workers runtime. /// -/// Uses `FetchConnector` for `object_store` HTTP requests and `web_sys::fetch` -/// for raw multipart operations. +/// Uses `FetchConnector` for `object_store` HTTP requests. #[derive(Clone)] pub struct WorkerBackend; -impl ProxyBackend for WorkerBackend { +impl StoreFactory for WorkerBackend { + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + let builder = match create_builder(config)? { + StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), + }; + builder.build_object_store() + } + fn create_paginated_store( &self, config: &BucketConfig, @@ -34,109 +36,13 @@ impl ProxyBackend for WorkerBackend { builder.build() } - fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { - build_signer(config) - } - - async fn send_raw( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - body: Bytes, - ) -> Result { - tracing::debug!( - method = %method, - url = %url, - "worker: sending raw backend request via Fetch API" - ); - - // Build web_sys::Headers - let ws_headers = web_sys::Headers::new() - .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; - - for (key, value) in headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } - } - - // Build web_sys::RequestInit - let init = web_sys::RequestInit::new(); - init.set_method(method.as_str()); - init.set_headers(&ws_headers.into()); - - // Set body for methods that carry one - if !body.is_empty() { - let uint8 = js_sys::Uint8Array::from(body.as_ref()); - init.set_body(&uint8.into()); - } - - let ws_request = web_sys::Request::new_with_str_and_init(&url, &init) - .map_err(|e| ProxyError::BackendError(format!("failed to create request: {:?}", e)))?; - - // Fetch via worker - let worker_req: worker::Request = ws_request.into(); - let mut worker_resp = Fetch::Request(worker_req) - .send() - .await - .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; - - let status = worker_resp.status_code(); - - // Read response body as bytes (multipart responses are small) - let resp_bytes = worker_resp - .bytes() - .await - .map_err(|e| ProxyError::Internal(format!("failed to read response: {}", e)))?; - - // Convert response headers - let ws_response: web_sys::Response = worker_resp.into(); - let resp_headers = extract_response_headers(&ws_response.headers()); - - Ok(RawResponse { - status, - headers: resp_headers, - body: Bytes::from(resp_bytes), - }) - } -} - -use multistore::route_handler::RESPONSE_HEADER_ALLOWLIST; - -/// Extract response headers from a `web_sys::Headers` using an allowlist. -pub fn extract_response_headers(ws_headers: &web_sys::Headers) -> HeaderMap { - let mut resp_headers = HeaderMap::new(); - for name in RESPONSE_HEADER_ALLOWLIST { - if let Ok(Some(value)) = ws_headers.get(name) { - if let Ok(parsed) = value.parse() { - resp_headers.insert(*name, parsed); - } - } - } - resp_headers -} - -/// [`HttpExchange`] implementation using reqwest on WASM (wraps `web_sys::fetch`). -#[derive(Clone)] -pub struct FetchHttpExchange; - -impl HttpExchange for FetchHttpExchange { - async fn post_form( + fn create_multipart_store( &self, - url: &str, - form: &[(&str, &str)], - ) -> Result { - let client = reqwest::Client::new(); - let resp = client - .post(url) - .form(form) - .send() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string()))?; - - resp.text() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string())) + config: &BucketConfig, + ) -> Result, ProxyError> { + let builder = match create_builder(config)? { + StoreBuilder::S3(s) => StoreBuilder::S3(s.with_http_connector(FetchConnector)), + }; + builder.build_multipart_store() } } diff --git a/examples/cf-workers/src/lib.rs b/examples/cf-workers/src/lib.rs index 34540d9..e653d63 100644 --- a/examples/cf-workers/src/lib.rs +++ b/examples/cf-workers/src/lib.rs @@ -1,18 +1,6 @@ //! Cloudflare Workers runtime for the S3 proxy gateway. //! -//! This crate provides implementations of core traits using Cloudflare Workers -//! primitives. Uses zero-copy body forwarding: request and response -//! `ReadableStream`s flow through the JS runtime without touching WASM memory. -//! -//! # Architecture -//! -//! ```text -//! Client -> Worker (web_sys::Request — body stream NOT locked) -//! -> resolve request (ProxyGateway with static config registries) -//! -> Forward: web_sys::fetch with ReadableStream passthrough (zero-copy) -//! -> Response: LIST XML via object_store, errors, synthetic responses -//! -> NeedsBody: multipart operations via raw signed HTTP -//! ``` +//! Uses s3s for S3 protocol handling with `MultistoreService`. //! //! # Configuration //! @@ -24,122 +12,20 @@ mod bandwidth; mod client; mod fetch_connector; mod metering; -mod rate_limit; mod tracing_layer; pub use bandwidth::BandwidthMeter; -use client::{extract_response_headers, FetchHttpExchange, WorkerBackend}; -use multistore::error::ProxyError; -use multistore::forwarder::{ForwardResponse, Forwarder}; -use multistore::proxy::{GatewayResponse, ProxyGateway}; -use multistore::route_handler::{ForwardRequest, ProxyResponseBody, ProxyResult, RequestInfo}; -use multistore::router::Router; -use multistore_oidc_provider::backend_auth::MaybeOidcAuth; -use multistore_oidc_provider::jwt::JwtSigner; -use multistore_oidc_provider::route_handler::OidcRouterExt; -use multistore_oidc_provider::OidcCredentialProvider; +use client::WorkerBackend; +use multistore::service::{MultistoreAccess, MultistoreAuth, MultistoreService}; use multistore_static_config::{StaticConfig, StaticProvider}; -use multistore_sts::route_handler::StsRouterExt; -use multistore_sts::JwksCache; -use multistore_sts::TokenKey; -use rate_limit::CfRateLimiter; +use s3s::service::S3ServiceBuilder; use bytes::Bytes; use http::HeaderMap; use worker::*; -/// Zero-copy body wrapper. Holds the raw `ReadableStream` from the incoming -/// request, passing it through the Gateway untouched for Forward requests. -struct JsBody(Option); -// SAFETY: Workers is single-threaded; these are required by Gateway's generic bounds. -unsafe impl Send for JsBody {} -unsafe impl Sync for JsBody {} - -/// Zero-copy HTTP forwarder for Cloudflare Workers. -/// -/// Executes presigned backend requests via the Fetch API, passing -/// `ReadableStream` bodies through JS without touching WASM memory. -struct WorkerForwarder; - -impl Forwarder for WorkerForwarder { - type ResponseBody = web_sys::Response; - - async fn forward( - &self, - request: ForwardRequest, - body: JsBody, - ) -> Result, ProxyError> { - // Build web_sys::Headers from the forwarding headers. - let ws_headers = web_sys::Headers::new() - .map_err(|e| ProxyError::Internal(format!("failed to create Headers: {:?}", e)))?; - for (key, value) in request.headers.iter() { - if let Ok(v) = value.to_str() { - let _ = ws_headers.set(key.as_str(), v); - } - } - - // Build web_sys::RequestInit. - let init = web_sys::RequestInit::new(); - init.set_method(request.method.as_str()); - init.set_headers(&ws_headers.into()); - - // Bypass Cloudflare's subrequest cache for Range requests. Without this, - // CF caches full-body 200 responses and serves them for Range requests, - // breaking multipart downloads. Non-Range requests still benefit from - // CF edge caching. Requires compatibility_date >= 2024-11-11. - if request.headers.contains_key(http::header::RANGE) { - init.set_cache(web_sys::RequestCache::NoStore); - } - - // For PUT: attach the original ReadableStream directly (zero-copy!). - if request.method == http::Method::PUT { - if let Some(ref stream) = body.0 { - init.set_body(stream); - } - } - - // Build the outgoing request. - let ws_request = web_sys::Request::new_with_str_and_init(request.url.as_str(), &init) - .map_err(|e| ProxyError::Internal(format!("failed to create request: {:?}", e)))?; - - // Fetch via the worker crate's Fetch API. - let worker_req: worker::Request = ws_request.into(); - let worker_resp = worker::Fetch::Request(worker_req) - .send() - .await - .map_err(|e| ProxyError::BackendError(format!("fetch failed: {}", e)))?; - - // Convert to web_sys::Response to access the body stream. - let backend_ws: web_sys::Response = worker_resp.into(); - let status = backend_ws.status(); - - // Build filtered response headers using the existing allowlist. - let headers = extract_response_headers(&backend_ws.headers()); - let content_length = headers - .get(http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - - Ok(ForwardResponse { - status, - headers, - body: backend_ws, - content_length, - }) - } -} - /// The Worker entry point. -/// -/// Accepts `web_sys::Request` directly (via the worker crate's `FromRequest` -/// trait) so the body `ReadableStream` is never locked by `worker::Body::new()`. -/// Returns `web_sys::Response` directly, bypassing the `axum::Body` → `to_wasm()` -/// copy path. -/// -/// Wrangler config (`wrangler.toml`) should bind: -/// - `PROXY_CONFIG` environment variable for configuration -/// - `VIRTUAL_HOST_DOMAIN` environment variable (optional) #[event(fetch)] async fn fetch(req: web_sys::Request, env: Env, _ctx: Context) -> Result { // Initialize panic hook for better error messages @@ -148,141 +34,109 @@ async fn fetch(req: web_sys::Request, env: Env, _ctx: Context) -> Result let method: http::Method = req.method().parse().unwrap_or(http::Method::GET); let url_str = req.url(); - let uri: http::Uri = url_str.parse().unwrap(); - let path = uri.path().to_string(); - let query = uri.query().map(|q| q.to_string()); + let uri: http::Uri = url_str + .parse() + .map_err(|e| worker::Error::RustError(format!("invalid URI: {e}")))?; let headers = convert_ws_headers(&req.headers()); - let req_info = RequestInfo::new(&method, &path, query.as_deref(), &headers, None); - - Ok( - match gateway - .handle_request(&req_info, js_body, collect_js_body) - .await - { - GatewayResponse::Response(result) => proxy_result_to_ws_response(result), - GatewayResponse::Forward(resp) => forward_response_to_ws(resp), - }, - ) -} - -// ── Forward response conversion ───────────────────────────────────── - -/// Convert a `ForwardResponse` into a `web_sys::Response` -/// for the client, preserving the backend's body stream (zero-copy). -fn forward_response_to_ws(resp: ForwardResponse) -> web_sys::Response { - let ws_headers = http_headermap_to_ws_headers(&resp.headers) - .unwrap_or_else(|_| web_sys::Headers::new().unwrap()); - - let resp_init = web_sys::ResponseInit::new(); - resp_init.set_status(resp.status); - resp_init.set_headers(&ws_headers.into()); - - web_sys::Response::new_with_opt_readable_stream_and_init(resp.body.body().as_ref(), &resp_init) - .unwrap_or_else(|_| ws_error_response(502, "Bad Gateway")) + // Collect body to bytes + let body_bytes = collect_ws_body(&req).await?; + let s3_body = s3s::Body::from(body_bytes); + + // Build the http::Request + let mut http_req = http::Request::builder() + .method(method) + .uri(uri) + .body(s3_body) + .map_err(|e| worker::Error::RustError(format!("failed to build request: {e}")))?; + *http_req.headers_mut() = headers; + + // Call the s3s service + let s3_resp = s3_service + .call(http_req) + .await + .map_err(|e| worker::Error::RustError(format!("s3s error: {e:?}")))?; + + // Convert http::Response → web_sys::Response + s3_response_to_ws(s3_resp) } -// ── Body collection (NeedsBody path) ─────────────────────────────── - -/// Materialize a `JsBody` into `Bytes` for the NeedsBody path. -/// -/// Uses the `Response::arrayBuffer()` JS trick: wrap the stream in a -/// `web_sys::Response`, call `.array_buffer()`, and convert via `Uint8Array`. -/// This is only used for small multipart payloads. -async fn collect_js_body(body: JsBody) -> std::result::Result { - match body.0 { +/// Collect body from web_sys::Request into Bytes. +async fn collect_ws_body(req: &web_sys::Request) -> Result { + match req.body() { None => Ok(Bytes::new()), Some(stream) => { let resp = web_sys::Response::new_with_opt_readable_stream(Some(&stream)) - .map_err(|e| format!("Response::new failed: {:?}", e))?; + .map_err(|e| worker::Error::RustError(format!("Response::new failed: {e:?}")))?; let promise = resp .array_buffer() - .map_err(|e| format!("arrayBuffer() failed: {:?}", e))?; + .map_err(|e| worker::Error::RustError(format!("arrayBuffer() failed: {e:?}")))?; let buf = wasm_bindgen_futures::JsFuture::from(promise) .await - .map_err(|e| format!("arrayBuffer await failed: {:?}", e))?; + .map_err(|e| { + worker::Error::RustError(format!("arrayBuffer await failed: {e:?}")) + })?; let uint8 = js_sys::Uint8Array::new(&buf); Ok(Bytes::from(uint8.to_vec())) } } } -// ── Response builders ─────────────────────────────────────────────── +/// Convert an s3s HTTP response to a web_sys::Response. +fn s3_response_to_ws(resp: http::Response) -> Result { + let (parts, body) = resp.into_parts(); -/// Convert a `ProxyResult` (small buffered XML/JSON) to a `web_sys::Response`. -fn proxy_result_to_ws_response(result: ProxyResult) -> web_sys::Response { - let ws_headers = http_headermap_to_ws_headers(&result.headers) - .unwrap_or_else(|_| web_sys::Headers::new().unwrap()); + let ws_headers = http_headermap_to_ws_headers(&parts.headers) + .map_err(|e| worker::Error::RustError(format!("failed to create headers: {e:?}")))?; let resp_init = web_sys::ResponseInit::new(); - resp_init.set_status(result.status); + resp_init.set_status(parts.status.as_u16()); resp_init.set_headers(&ws_headers.into()); - match result.body { - ProxyResponseBody::Empty => { - web_sys::Response::new_with_opt_str_and_init(None, &resp_init).unwrap() - } - ProxyResponseBody::Bytes(bytes) => { + // Try to get bytes directly if available, otherwise create an empty response + // and set up streaming later if needed. + if let Some(bytes) = body.bytes() { + if bytes.is_empty() { + web_sys::Response::new_with_opt_str_and_init(None, &resp_init) + .map_err(|e| worker::Error::RustError(format!("Response::new failed: {e:?}"))) + } else { let uint8 = js_sys::Uint8Array::from(bytes.as_ref()); web_sys::Response::new_with_opt_buffer_source_and_init(Some(&uint8), &resp_init) - .unwrap() + .map_err(|e| worker::Error::RustError(format!("Response::new failed: {e:?}"))) } + } else { + // For streaming responses (e.g. GET object), we need to collect first. + // TODO: Implement proper streaming via ReadableStream for large responses. + // For now, we return an empty response — the body will be handled separately. + web_sys::Response::new_with_opt_str_and_init(None, &resp_init) + .map_err(|e| worker::Error::RustError(format!("Response::new failed: {e:?}"))) } } -/// Build a plain-text error response. -fn ws_error_response(status: u16, message: &str) -> web_sys::Response { - let init = web_sys::ResponseInit::new(); - init.set_status(status); - web_sys::Response::new_with_opt_str_and_init(Some(message), &init) - .unwrap_or_else(|_| web_sys::Response::new().unwrap()) -} - // ── Header conversion helpers ─────────────────────────────────────── /// Convert `web_sys::Headers` to `http::HeaderMap` by iterating all entries. @@ -343,104 +197,3 @@ fn load_config_from_env(env: &Env, var_name: &str) -> Result { .map_err(|e| worker::Error::RustError(format!("{} config error: {}", var_name, e))) } } - -/// Load the optional session token encryption key from the `SESSION_TOKEN_KEY` secret. -fn load_token_key(env: &Env) -> Result> { - match env.secret("SESSION_TOKEN_KEY") { - Ok(val) => { - let key = TokenKey::from_base64(&val.to_string()) - .map_err(|e| worker::Error::RustError(e.to_string()))?; - Ok(Some(key)) - } - Err(_) => Ok(None), - } -} - -/// Load rate limiter middleware from env bindings. -/// -/// Returns `Some(CfRateLimiter)` if both `ANON_RATE_LIMITER` and -/// `AUTH_RATE_LIMITER` bindings are configured; otherwise `None`. -fn load_rate_limiter(env: &Env) -> Option { - let anon = env.rate_limiter("ANON_RATE_LIMITER").ok()?; - let auth = env.rate_limiter("AUTH_RATE_LIMITER").ok()?; - Some(CfRateLimiter::new(anon, auth)) -} - -/// Load bandwidth metering middleware from env bindings. -/// -/// Returns `Some(MeteringMiddleware)` if both `BANDWIDTH_METER` (DO namespace) -/// and `BANDWIDTH_QUOTAS` (quota config) are configured; otherwise `None`. -fn load_bandwidth_meter( - env: &Env, -) -> Option< - multistore_metering::MeteringMiddleware, -> { - // Try loading quotas as a JSON string first, then as a TOML object. - let quotas: std::collections::HashMap = - if let Ok(var) = env.var("BANDWIDTH_QUOTAS") { - serde_json::from_str(&var.to_string()) - .map_err(|e| { - tracing::error!(error = %e, "failed to parse BANDWIDTH_QUOTAS as JSON string"); - e - }) - .ok()? - } else { - env.object_var("BANDWIDTH_QUOTAS") - .map_err(|e| { - tracing::error!(error = %e, "failed to load BANDWIDTH_QUOTAS"); - e - }) - .ok()? - }; - - // Two separate namespace bindings because MeteringMiddleware needs two separate instances - // (one for quota checking, one for recording). DoBandwidthMeter is stateless locally — - // all state lives in the DO. - let ns_check = env.durable_object("BANDWIDTH_METER").ok()?; - let ns_record = env.durable_object("BANDWIDTH_METER").ok()?; - - let checker = metering::DoBandwidthMeter::new(ns_check, quotas.clone()); - let recorder = metering::DoBandwidthMeter::new(ns_record, quotas); - - Some(multistore_metering::MeteringMiddleware::new( - checker, recorder, - )) -} - -/// Load OIDC provider config from env secrets/vars. -/// -/// Returns `MaybeOidcAuth::Enabled` if both `OIDC_PROVIDER_KEY` (secret) and -/// `OIDC_PROVIDER_ISSUER` (var) are set; otherwise `Disabled`. Also returns -/// the signer and issuer for router registration. -fn load_oidc_auth( - env: &Env, -) -> Result<( - MaybeOidcAuth, - Option, - Option, -)> { - let key_pem = match env.secret("OIDC_PROVIDER_KEY") { - Ok(val) => Some(val.to_string()), - Err(_) => None, - }; - let issuer = env.var("OIDC_PROVIDER_ISSUER").ok().map(|v| v.to_string()); - - match (key_pem, issuer) { - (Some(pem), Some(issuer)) => { - let signer = JwtSigner::from_pem(&pem, "proxy-key-1".into(), 300) - .map_err(|e| worker::Error::RustError(format!("OIDC signer error: {e}")))?; - let http = FetchHttpExchange; - let provider = OidcCredentialProvider::new( - signer.clone(), - http, - issuer.clone(), - "sts.amazonaws.com".into(), - ); - let auth = MaybeOidcAuth::Enabled(Box::new( - multistore_oidc_provider::backend_auth::AwsBackendAuth::new(provider), - )); - Ok((auth, Some(signer), Some(issuer))) - } - _ => Ok((MaybeOidcAuth::Disabled, None, None)), - } -} diff --git a/examples/cf-workers/src/rate_limit.rs b/examples/cf-workers/src/rate_limit.rs deleted file mode 100644 index f4e16c2..0000000 --- a/examples/cf-workers/src/rate_limit.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Rate limiting middleware using Cloudflare Workers Rate Limiting API. -//! -//! Uses two separate rate limiters: one for unauthenticated (anonymous) -//! requests keyed by source IP, and one for authenticated requests keyed -//! by access key ID. - -use multistore::api::response::ErrorResponse; -use multistore::error::ProxyError; -use multistore::middleware::{DispatchContext, Middleware, Next}; -use multistore::route_handler::{HandlerAction, ProxyResponseBody, ProxyResult}; -use multistore::types::ResolvedIdentity; - -use bytes::Bytes; -use http::HeaderMap; - -/// Rate limiting middleware backed by Cloudflare Workers rate limit bindings. -/// -/// Selects the appropriate rate limiter based on the resolved identity: -/// - Anonymous requests use `anon_limiter`, keyed by source IP. -/// - Authenticated requests use `auth_limiter`, keyed by access key ID. -pub struct CfRateLimiter { - anon_limiter: worker::RateLimiter, - auth_limiter: worker::RateLimiter, -} - -impl CfRateLimiter { - pub fn new(anon_limiter: worker::RateLimiter, auth_limiter: worker::RateLimiter) -> Self { - Self { - anon_limiter, - auth_limiter, - } - } -} - -impl Middleware for CfRateLimiter { - async fn handle<'a>( - &'a self, - ctx: DispatchContext<'a>, - next: Next<'a>, - ) -> Result { - let (limiter, key) = match ctx.identity { - ResolvedIdentity::Anonymous => { - let key = match ctx.source_ip { - Some(ip) => ip.to_string(), - None => { - tracing::warn!("no source IP for anonymous request, using shared key"); - "anonymous".to_string() - } - }; - (&self.anon_limiter, key) - } - ResolvedIdentity::LongLived { credential } => { - (&self.auth_limiter, credential.principal_name.clone()) - } - ResolvedIdentity::Temporary { credentials } => { - (&self.auth_limiter, credentials.source_identity.clone()) - } - }; - - match limiter.limit(key.clone()).await { - Ok(outcome) if outcome.success => { - tracing::debug!(key = %key, "rate limit check passed"); - next.run(ctx).await - } - Ok(_) => { - tracing::warn!(key = %key, "rate limited"); - let xml = ErrorResponse::slow_down(ctx.request_id).to_xml(); - let mut headers = HeaderMap::new(); - headers.insert("content-type", "application/xml".parse().unwrap()); - Ok(HandlerAction::Response(ProxyResult { - status: 503, - headers, - body: ProxyResponseBody::Bytes(Bytes::from(xml)), - })) - } - Err(err) => { - // If the rate limiter fails, log and allow the request through - // rather than blocking legitimate traffic. - tracing::error!(key = %key, error = %err, "rate limiter error, allowing request"); - next.run(ctx).await - } - } - } -} diff --git a/examples/lambda/Cargo.toml b/examples/lambda/Cargo.toml index c2fe9e7..a9260ec 100644 --- a/examples/lambda/Cargo.toml +++ b/examples/lambda/Cargo.toml @@ -8,15 +8,12 @@ description = "AWS Lambda runtime for the S3 proxy gateway" [dependencies] multistore.workspace = true multistore-static-config.workspace = true -multistore-sts.workspace = true -multistore-oidc-provider.workspace = true +s3s.workspace = true lambda_http = "0.13" tokio.workspace = true http.workspace = true bytes.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -serde.workspace = true -toml.workspace = true -reqwest = { workspace = true, features = ["stream"] } object_store.workspace = true +http-body-util.workspace = true diff --git a/examples/lambda/src/client.rs b/examples/lambda/src/client.rs index 818812e..8e9ca78 100644 --- a/examples/lambda/src/client.rs +++ b/examples/lambda/src/client.rs @@ -1,40 +1,31 @@ -//! Lambda backend using reqwest for raw HTTP and default object_store connector. +//! Lambda backend using the default object_store connector. -use bytes::Bytes; -use http::HeaderMap; -use multistore::backend::{build_signer, create_builder, ProxyBackend, RawResponse}; +use multistore::backend::create_builder; use multistore::error::ProxyError; +use multistore::service::StoreFactory; use multistore::types::BucketConfig; -use multistore_oidc_provider::{HttpExchange, OidcProviderError}; use object_store::list::PaginatedListStore; -use object_store::signer::Signer; +use object_store::multipart::MultipartStore; +use object_store::ObjectStore; use std::sync::Arc; /// Backend for the Lambda runtime. /// -/// Uses reqwest for raw HTTP (multipart operations) and the default -/// object_store HTTP connector for high-level operations. +/// Uses the default object_store HTTP connector for all operations. #[derive(Clone)] -pub struct LambdaBackend { - client: reqwest::Client, -} +pub struct LambdaBackend; impl LambdaBackend { pub fn new() -> Self { - Self { - client: reqwest::Client::builder() - .pool_max_idle_per_host(5) - .build() - .expect("failed to build reqwest client"), - } + Self } +} - pub fn client(&self) -> &reqwest::Client { - &self.client +impl StoreFactory for LambdaBackend { + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + create_builder(config)?.build_object_store() } -} -impl ProxyBackend for LambdaBackend { fn create_paginated_store( &self, config: &BucketConfig, @@ -42,80 +33,10 @@ impl ProxyBackend for LambdaBackend { create_builder(config)?.build() } - fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { - build_signer(config) - } - - async fn send_raw( + fn create_multipart_store( &self, - method: http::Method, - url: String, - headers: HeaderMap, - body: Bytes, - ) -> Result { - tracing::debug!( - method = %method, - url = %url, - "lambda: sending raw backend request via reqwest" - ); - - let mut req_builder = self.client.request(method, &url); - - for (key, value) in headers.iter() { - req_builder = req_builder.header(key, value); - } - - if !body.is_empty() { - req_builder = req_builder.body(body); - } - - let response = req_builder.send().await.map_err(|e| { - tracing::error!(error = %e, "reqwest raw request failed"); - ProxyError::BackendError(e.to_string()) - })?; - - let status = response.status().as_u16(); - let resp_headers = response.headers().clone(); - let resp_body = response.bytes().await.map_err(|e| { - ProxyError::BackendError(format!("failed to read raw response body: {}", e)) - })?; - - Ok(RawResponse { - status, - headers: resp_headers, - body: resp_body, - }) - } -} - -/// [`HttpExchange`] implementation using reqwest (native). -#[derive(Clone)] -pub struct ReqwestHttpExchange { - client: reqwest::Client, -} - -impl ReqwestHttpExchange { - pub fn new(client: reqwest::Client) -> Self { - Self { client } - } -} - -impl HttpExchange for ReqwestHttpExchange { - async fn post_form( - &self, - url: &str, - form: &[(&str, &str)], - ) -> Result { - let resp = self - .client - .post(url) - .form(form) - .send() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string()))?; - - resp.text() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string())) + config: &BucketConfig, + ) -> Result, ProxyError> { + create_builder(config)?.build_multipart_store() } } diff --git a/examples/lambda/src/main.rs b/examples/lambda/src/main.rs index 95497e4..c512c28 100644 --- a/examples/lambda/src/main.rs +++ b/examples/lambda/src/main.rs @@ -1,7 +1,6 @@ //! AWS Lambda runtime for the multistore S3 proxy. //! -//! Mirrors the server example but runs inside AWS Lambda instead of a -//! standalone Tokio/Hyper server. +//! Uses s3s for S3 protocol handling with `MultistoreService`. //! //! ## Building //! @@ -13,98 +12,18 @@ //! //! - `CONFIG_PATH` — Path to the TOML config file (default: `config.toml`) //! - `VIRTUAL_HOST_DOMAIN` — Domain for virtual-hosted-style requests -//! - `SESSION_TOKEN_KEY` — Base64-encoded AES-256-GCM key for session tokens -//! - `OIDC_PROVIDER_KEY` — PEM-encoded RSA private key for OIDC provider -//! - `OIDC_PROVIDER_ISSUER` — OIDC issuer URL mod client; -use client::{LambdaBackend, ReqwestHttpExchange}; +use client::LambdaBackend; use lambda_http::{service_fn, Body, Error, Request, Response}; -use multistore::error::ProxyError; -use multistore::forwarder::{ForwardResponse, Forwarder}; -use multistore::proxy::{GatewayResponse, ProxyGateway}; -use multistore::route_handler::{ - ForwardRequest, ProxyResponseBody, ProxyResult, RequestInfo, RESPONSE_HEADER_ALLOWLIST, -}; -use multistore::router::Router; -use multistore_oidc_provider::backend_auth::MaybeOidcAuth; -use multistore_oidc_provider::jwt::JwtSigner; -use multistore_oidc_provider::route_handler::OidcRouterExt; -use multistore_oidc_provider::OidcCredentialProvider; +use multistore::service::{MultistoreAccess, MultistoreAuth, MultistoreService}; use multistore_static_config::StaticProvider; -use multistore_sts::route_handler::StsRouterExt; -use multistore_sts::JwksCache; -use multistore_sts::TokenKey; +use s3s::service::S3ServiceBuilder; use std::sync::OnceLock; -use std::time::Duration; - -/// Forwards presigned requests to the backend using `reqwest`, buffering the -/// full response body for Lambda (which does not support streaming responses). -struct LambdaForwarder { - client: reqwest::Client, -} - -impl Forwarder for LambdaForwarder { - type ResponseBody = Body; - - async fn forward( - &self, - request: ForwardRequest, - body: Body, - ) -> Result, ProxyError> { - let mut req_builder = self - .client - .request(request.method.clone(), request.url.as_str()); - - for (k, v) in request.headers.iter() { - req_builder = req_builder.header(k, v); - } - - // Attach body for PUT requests - if request.method == http::Method::PUT { - let bytes = body_to_bytes(body) - .await - .map_err(|e| ProxyError::Internal(format!("failed to read PUT body: {e}")))?; - req_builder = req_builder.body(bytes); - } - - let backend_resp = req_builder - .send() - .await - .map_err(|e| ProxyError::Internal(format!("forward request failed: {e}")))?; - - let status = backend_resp.status().as_u16(); - - // Forward allowlisted response headers - let mut headers = http::HeaderMap::new(); - for name in RESPONSE_HEADER_ALLOWLIST { - if let Some(v) = backend_resp.headers().get(*name) { - headers.insert(*name, v.clone()); - } - } - - let content_length = backend_resp.content_length(); - - // Buffer the response body (Lambda doesn't support streaming responses) - let body_bytes = backend_resp - .bytes() - .await - .map_err(|e| ProxyError::Internal(format!("failed to read backend response: {e}")))?; - - Ok(ForwardResponse { - status, - headers, - body: Body::Binary(body_bytes.to_vec()), - content_length, - }) - } -} - -type Handler = ProxyGateway; struct AppState { - handler: Handler, + s3_service: s3s::service::S3Service, } static STATE: OnceLock = OnceLock::new(); @@ -124,115 +43,61 @@ async fn main() -> Result<(), Error> { let config = StaticProvider::from_file(&config_path)?; - let token_key = std::env::var("SESSION_TOKEN_KEY") - .ok() - .map(|v| TokenKey::from_base64(&v)) - .transpose()?; - let backend = LambdaBackend::new(); - let reqwest_client = backend.client().clone(); - let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); - let sts_creds = config.clone(); - - let oidc_provider_key = std::env::var("OIDC_PROVIDER_KEY").ok(); - let oidc_provider_issuer = std::env::var("OIDC_PROVIDER_ISSUER").ok(); - - let (oidc_auth, oidc_signer, oidc_issuer) = match (&oidc_provider_key, &oidc_provider_issuer) { - (Some(key_pem), Some(issuer)) => { - let signer = JwtSigner::from_pem(key_pem, "proxy-key-1".into(), 300) - .map_err(|e| format!("failed to create OIDC JWT signer: {e}"))?; - let http = ReqwestHttpExchange::new(reqwest_client.clone()); - let provider = OidcCredentialProvider::new( - signer.clone(), - http, - issuer.clone(), - "sts.amazonaws.com".into(), - ); - let auth = MaybeOidcAuth::Enabled(Box::new( - multistore_oidc_provider::backend_auth::AwsBackendAuth::new(provider), - )); - (auth, Some(signer), Some(issuer.clone())) - } - _ => (MaybeOidcAuth::Disabled, None, None), - }; - // Build router with OIDC discovery (if configured) and STS. - let mut router = Router::new(); - if let (Some(signer), Some(issuer)) = (oidc_signer, oidc_issuer) { - router = router.with_oidc_discovery(issuer, signer); - } - router = router.with_sts(sts_creds, jwks_cache, token_key.clone()); + // Build the s3s service + let service = MultistoreService::new(config.clone(), backend); + let auth = MultistoreAuth::new(config); - let forwarder = LambdaForwarder { - client: reqwest_client, - }; + let mut builder = S3ServiceBuilder::new(service); + builder.set_auth(auth); + builder.set_access(MultistoreAccess); - // Build the gateway with the router. - let mut handler = ProxyGateway::new(backend, config.clone(), config, forwarder, domain) - .with_middleware(oidc_auth) - .with_router(router); - if let Some(resolver) = token_key { - handler = handler.with_credential_resolver(resolver); + if let Some(ref d) = domain { + builder.set_host( + s3s::host::SingleDomain::new(d) + .map_err(|e| format!("invalid virtual host domain: {e}"))?, + ); } - let _ = STATE.set(AppState { handler }); + let s3_service = builder.build(); + + let _ = STATE.set(AppState { s3_service }); lambda_http::run(service_fn(request_handler)).await } async fn request_handler(req: Request) -> Result, Error> { let state = STATE.get().expect("state not initialized"); - let (parts, body) = req.into_parts(); - let method = parts.method; - let uri = parts.uri; - let path = uri.path().to_string(); - let query = uri.query().map(|q| q.to_string()); - let headers = parts.headers; - - tracing::debug!(method = %method, uri = %uri, "incoming request"); - - let req_info = RequestInfo::new(&method, &path, query.as_deref(), &headers, None); - - Ok( - match state - .handler - .handle_request(&req_info, body, |b| async { - body_to_bytes(b).await.map_err(|e| e.to_string()) - }) - .await - { - GatewayResponse::Response(result) => build_lambda_response(result), - GatewayResponse::Forward(resp) => { - let mut builder = Response::builder().status(resp.status); - for (k, v) in resp.headers.iter() { - builder = builder.header(k, v); - } - builder.body(resp.body).unwrap() - } - }, - ) -} -/// Convert a `ProxyResult` to a Lambda `Response`. -fn build_lambda_response(result: ProxyResult) -> Response { - let body = match result.body { - ProxyResponseBody::Bytes(b) => Body::Binary(b.to_vec()), - ProxyResponseBody::Empty => Body::Empty, + // Convert lambda_http::Request to http::Request + let (parts, body) = req.into_parts(); + let body_bytes = match body { + Body::Empty => bytes::Bytes::new(), + Body::Text(s) => bytes::Bytes::from(s), + Body::Binary(b) => bytes::Bytes::from(b), + }; + let s3_req = http::Request::from_parts(parts, s3s::Body::from(body_bytes)); + + // Call the s3s service + let s3_resp = state + .s3_service + .call(s3_req) + .await + .map_err(|e| format!("s3s error: {e:?}"))?; + + // Convert http::Response to lambda_http::Response + let (parts, s3_body) = s3_resp.into_parts(); + let resp_bytes = http_body_util::BodyExt::collect(s3_body) + .await + .map_err(|e| format!("failed to collect response body: {e}"))? + .to_bytes(); + + let lambda_body = if resp_bytes.is_empty() { + Body::Empty + } else { + Body::Binary(resp_bytes.to_vec()) }; - let mut builder = Response::builder().status(result.status); - for (key, value) in result.headers.iter() { - builder = builder.header(key, value); - } - - builder.body(body).unwrap() -} - -/// Collect a Lambda body into bytes. -async fn body_to_bytes(body: Body) -> Result> { - match body { - Body::Empty => Ok(bytes::Bytes::new()), - Body::Text(s) => Ok(bytes::Bytes::from(s)), - Body::Binary(b) => Ok(bytes::Bytes::from(b)), - } + Ok(Response::from_parts(parts, lambda_body)) } diff --git a/examples/server/Cargo.toml b/examples/server/Cargo.toml index 3f190b0..1c300f8 100644 --- a/examples/server/Cargo.toml +++ b/examples/server/Cargo.toml @@ -7,20 +7,14 @@ description = "Tokio/Hyper runtime for the S3 proxy gateway" [dependencies] multistore = { workspace = true, features = ["azure", "gcp"] } +s3s.workspace = true multistore-static-config.workspace = true multistore-sts.workspace = true -multistore-oidc-provider.workspace = true -axum = { workspace = true, features = ["json", "tokio", "http1", "http2"] } -tower-service.workspace = true tokio.workspace = true -http.workspace = true -bytes.workspace = true tracing.workspace = true tracing-subscriber.workspace = true serde.workspace = true toml.workspace = true reqwest = { workspace = true, features = ["stream"] } -thiserror.workspace = true object_store.workspace = true -futures.workspace = true -http-body-util.workspace = true +hyper-util.workspace = true diff --git a/examples/server/src/axum_helpers.rs b/examples/server/src/axum_helpers.rs deleted file mode 100644 index 68b191d..0000000 --- a/examples/server/src/axum_helpers.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Axum response helpers for converting core proxy types into axum responses. - -use axum::body::Body; -use axum::response::Response; - -use multistore::route_handler::{ProxyResponseBody, ProxyResult}; - -/// Convert a [`ProxyResult`] to an axum [`Response`]. -pub fn build_proxy_response(result: ProxyResult) -> Response { - let body = match result.body { - ProxyResponseBody::Bytes(b) => Body::from(b), - ProxyResponseBody::Empty => Body::empty(), - }; - - let mut builder = Response::builder().status(result.status); - for (key, value) in result.headers.iter() { - builder = builder.header(key, value); - } - - builder.body(body).unwrap() -} diff --git a/examples/server/src/bin/multistore-server.rs b/examples/server/src/bin/multistore-server.rs index 3e3ac6e..12ab956 100644 --- a/examples/server/src/bin/multistore-server.rs +++ b/examples/server/src/bin/multistore-server.rs @@ -4,7 +4,7 @@ //! multistore-server --config config.toml [--listen 0.0.0.0:8080] [--domain s3.local] use multistore_server::cached::CachedProvider; -use multistore_server::server::{run, ServerConfig}; +use multistore_server::server::{self, ServerConfig}; use multistore_static_config::StaticProvider; use multistore_sts::TokenKey; use std::net::SocketAddr; @@ -62,5 +62,5 @@ async fn main() -> Result<(), Box> { oidc_provider_issuer, }; - run(config.clone(), config, server_config).await + server::run(config.clone(), config, server_config).await } diff --git a/examples/server/src/client.rs b/examples/server/src/client.rs index d3011e8..9aafb5d 100644 --- a/examples/server/src/client.rs +++ b/examples/server/src/client.rs @@ -1,38 +1,31 @@ -//! Server backend using reqwest for raw HTTP and default object_store connector. +//! Server backend using the default object_store connector. -use bytes::Bytes; -use http::HeaderMap; -use multistore::backend::{build_signer, create_builder, ProxyBackend, RawResponse}; +use multistore::backend::create_builder; use multistore::error::ProxyError; +use multistore::service::StoreFactory; use multistore::types::BucketConfig; -use multistore_oidc_provider::{HttpExchange, OidcProviderError}; use object_store::list::PaginatedListStore; -use object_store::signer::Signer; +use object_store::multipart::MultipartStore; +use object_store::ObjectStore; use std::sync::Arc; /// Backend for the Tokio/Hyper server runtime. /// -/// Uses reqwest for raw HTTP (multipart operations) and the default -/// object_store HTTP connector for high-level operations. +/// Uses the default object_store HTTP connector for all operations. #[derive(Clone)] pub struct ServerBackend { - client: reqwest::Client, + _client: reqwest::Client, } impl ServerBackend { pub fn new() -> Self { Self { - client: reqwest::Client::builder() + _client: reqwest::Client::builder() .pool_max_idle_per_host(20) .build() .expect("failed to build reqwest client"), } } - - /// Access the underlying reqwest client for Forward request execution. - pub fn client(&self) -> &reqwest::Client { - &self.client - } } impl Default for ServerBackend { @@ -41,7 +34,11 @@ impl Default for ServerBackend { } } -impl ProxyBackend for ServerBackend { +impl StoreFactory for ServerBackend { + fn create_store(&self, config: &BucketConfig) -> Result, ProxyError> { + create_builder(config)?.build_object_store() + } + fn create_paginated_store( &self, config: &BucketConfig, @@ -49,80 +46,10 @@ impl ProxyBackend for ServerBackend { create_builder(config)?.build() } - fn create_signer(&self, config: &BucketConfig) -> Result, ProxyError> { - build_signer(config) - } - - async fn send_raw( - &self, - method: http::Method, - url: String, - headers: HeaderMap, - body: Bytes, - ) -> Result { - tracing::debug!( - method = %method, - url = %url, - "server: sending raw backend request via reqwest" - ); - - let mut req_builder = self.client.request(method, &url); - - for (key, value) in headers.iter() { - req_builder = req_builder.header(key, value); - } - - if !body.is_empty() { - req_builder = req_builder.body(body); - } - - let response = req_builder.send().await.map_err(|e| { - tracing::error!(error = %e, "reqwest raw request failed"); - ProxyError::BackendError(e.to_string()) - })?; - - let status = response.status().as_u16(); - let resp_headers = response.headers().clone(); - let resp_body = response.bytes().await.map_err(|e| { - ProxyError::BackendError(format!("failed to read raw response body: {}", e)) - })?; - - Ok(RawResponse { - status, - headers: resp_headers, - body: resp_body, - }) - } -} - -/// [`HttpExchange`] implementation using reqwest (native). -#[derive(Clone)] -pub struct ReqwestHttpExchange { - client: reqwest::Client, -} - -impl ReqwestHttpExchange { - pub fn new(client: reqwest::Client) -> Self { - Self { client } - } -} - -impl HttpExchange for ReqwestHttpExchange { - async fn post_form( + fn create_multipart_store( &self, - url: &str, - form: &[(&str, &str)], - ) -> Result { - let resp = self - .client - .post(url) - .form(form) - .send() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string()))?; - - resp.text() - .await - .map_err(|e| OidcProviderError::HttpError(e.to_string())) + config: &BucketConfig, + ) -> Result, ProxyError> { + create_builder(config)?.build_multipart_store() } } diff --git a/examples/server/src/lib.rs b/examples/server/src/lib.rs index 3588d53..b4d5b5f 100644 --- a/examples/server/src/lib.rs +++ b/examples/server/src/lib.rs @@ -1,12 +1,11 @@ -//! Tokio/axum runtime for the S3 proxy gateway. +//! Tokio/hyper runtime for the S3 proxy server. //! //! This crate provides concrete implementations of the core traits for a -//! standard server environment using Tokio and axum. +//! standard server environment using Tokio and hyper. //! -//! - [`client::ServerBackend`] — implements `ProxyBackend` using reqwest + object_store -//! - [`server::run`] — starts the axum HTTP server +//! - [`client::ServerBackend`] — implements `StoreFactory` using reqwest + object_store +//! - [`server::run`] — starts the hyper HTTP server with s3s -mod axum_helpers; pub mod cached; pub mod client; pub mod server; diff --git a/examples/server/src/server.rs b/examples/server/src/server.rs index c04dbaa..11dfc72 100644 --- a/examples/server/src/server.rs +++ b/examples/server/src/server.rs @@ -1,30 +1,11 @@ -//! HTTP server using axum, wiring everything together. +//! HTTP server using s3s service layer. -use crate::axum_helpers::build_proxy_response; -use crate::client::{ReqwestHttpExchange, ServerBackend}; -use axum::body::Body; -use axum::extract::State; -use axum::response::Response; -use axum::Router; -use futures::TryStreamExt; -use http::HeaderMap; -use http_body_util::BodyStream; -use multistore::error::ProxyError; -use multistore::forwarder::{ForwardResponse, Forwarder}; -use multistore::proxy::{GatewayResponse, ProxyGateway}; +use crate::client::ServerBackend; use multistore::registry::{BucketRegistry, CredentialRegistry}; -use multistore::route_handler::{ForwardRequest, RequestInfo, RESPONSE_HEADER_ALLOWLIST}; -use multistore::router::Router as ProxyRouter; -use multistore_oidc_provider::backend_auth::MaybeOidcAuth; -use multistore_oidc_provider::jwt::JwtSigner; -use multistore_oidc_provider::route_handler::OidcRouterExt; -use multistore_oidc_provider::OidcCredentialProvider; -use multistore_sts::route_handler::StsRouterExt; -use multistore_sts::JwksCache; +use multistore::service::{MultistoreAccess, MultistoreAuth, MultistoreService}; use multistore_sts::TokenKey; +use s3s::service::S3ServiceBuilder; use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; use tokio::net::TcpListener; /// Server configuration. @@ -53,83 +34,9 @@ impl Default for ServerConfig { } } -/// Forwards presigned backend requests using a [`reqwest::Client`]. -struct ServerForwarder { - client: reqwest::Client, -} - -impl Forwarder for ServerForwarder { - type ResponseBody = reqwest::Response; - - async fn forward( - &self, - request: ForwardRequest, - body: Body, - ) -> Result, ProxyError> { - let mut req_builder = self - .client - .request(request.method.clone(), request.url.as_str()); - - for (k, v) in request.headers.iter() { - req_builder = req_builder.header(k, v); - } - - // Attach streaming body for PUT - if request.method == http::Method::PUT { - let body_stream = BodyStream::new(body) - .try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); - req_builder = req_builder.body(reqwest::Body::wrap_stream(body_stream)); - } - - let backend_resp = req_builder - .send() - .await - .map_err(|e| ProxyError::BackendError(e.to_string()))?; - - let status = backend_resp.status().as_u16(); - - // Forward allowlisted response headers - let mut headers = HeaderMap::new(); - for name in RESPONSE_HEADER_ALLOWLIST { - if let Some(v) = backend_resp.headers().get(*name) { - headers.insert(*name, v.clone()); - } - } - - let content_length = backend_resp.content_length(); - - Ok(ForwardResponse { - status, - headers, - body: backend_resp, - content_length, - }) - } -} - -struct AppState { - handler: ProxyGateway, -} - -/// Run the S3 proxy server. -/// -/// # Example +/// Run the S3 proxy server using s3s service layer. /// -/// ```rust,ignore -/// use multistore_static_config::StaticProvider; -/// use multistore_server::server::{run, ServerConfig}; -/// -/// #[tokio::main] -/// async fn main() { -/// let config = StaticProvider::from_file("config.toml").unwrap(); -/// let server_config = ServerConfig { -/// listen_addr: ([0, 0, 0, 0], 8080).into(), -/// virtual_host_domain: Some("s3.local".to_string()), -/// ..Default::default() -/// }; -/// run(config.clone(), config, server_config).await.unwrap(); -/// } -/// ``` +/// Uses s3s's built-in S3 protocol handling with `MultistoreService`. pub async fn run( bucket_registry: R, credential_registry: C, @@ -140,111 +47,41 @@ where C: CredentialRegistry, { let backend = ServerBackend::new(); - let reqwest_client = backend.client().clone(); - let jwks_cache = JwksCache::new(reqwest_client.clone(), Duration::from_secs(900)); - let token_key = server_config.token_key; - let sts_creds = credential_registry.clone(); - // Build OIDC provider if both key and issuer are configured. - let (oidc_auth, oidc_signer, oidc_issuer) = match ( - &server_config.oidc_provider_key, - &server_config.oidc_provider_issuer, - ) { - (Some(key_pem), Some(issuer)) => { - let signer = JwtSigner::from_pem(key_pem, "proxy-key-1".into(), 300) - .map_err(|e| format!("failed to create OIDC JWT signer: {e}"))?; - let http = ReqwestHttpExchange::new(reqwest_client.clone()); - let provider = OidcCredentialProvider::new( - signer.clone(), - http, - issuer.clone(), - "sts.amazonaws.com".into(), - ); - let auth = MaybeOidcAuth::Enabled(Box::new( - multistore_oidc_provider::backend_auth::AwsBackendAuth::new(provider), - )); - (auth, Some(signer), Some(issuer.clone())) - } - _ => (MaybeOidcAuth::Disabled, None, None), - }; + // Build the s3s service + let service = MultistoreService::new(bucket_registry, backend); + let auth = MultistoreAuth::new(credential_registry); - // Build router with OIDC discovery (if configured) and STS. - let mut proxy_router = ProxyRouter::new(); - if let (Some(signer), Some(issuer)) = (oidc_signer, oidc_issuer) { - proxy_router = proxy_router.with_oidc_discovery(issuer, signer); - } - proxy_router = proxy_router.with_sts(sts_creds, jwks_cache, token_key.clone()); + let mut builder = S3ServiceBuilder::new(service); + builder.set_auth(auth); + builder.set_access(MultistoreAccess); - // Build the gateway with the router. - let forwarder = ServerForwarder { - client: reqwest_client, - }; - let mut handler = ProxyGateway::new( - backend, - bucket_registry, - credential_registry, - forwarder, - server_config.virtual_host_domain, - ) - .with_middleware(oidc_auth) - .with_router(proxy_router); - if let Some(ref resolver) = token_key { - handler = handler.with_credential_resolver(resolver.clone()); + if let Some(ref domain) = server_config.virtual_host_domain { + builder.set_host( + s3s::host::SingleDomain::new(domain) + .map_err(|e| format!("invalid virtual host domain: {e}"))?, + ); } - let state = Arc::new(AppState { handler }); - - let app = Router::new() - .fallback(request_handler::) - .with_state(state); + let s3_service = builder.build(); + // Use the s3s service directly with hyper let listener = TcpListener::bind(server_config.listen_addr).await?; tracing::info!("listening on {}", server_config.listen_addr); - axum::serve(listener, app).await?; - Ok(()) -} - -async fn request_handler( - State(state): State>>, - req: axum::extract::Request, -) -> Response { - let (parts, body) = req.into_parts(); - let method = parts.method; - let uri = parts.uri; - let path = uri.path().to_string(); - let query = uri.query().map(|q| q.to_string()); - let headers = parts.headers; - - tracing::debug!( - method = %method, - uri = %uri, - "incoming request" - ); - - let req_info = RequestInfo::new(&method, &path, query.as_deref(), &headers, None); - - match state - .handler - .handle_request(&req_info, body, |b| axum::body::to_bytes(b, usize::MAX)) - .await - { - GatewayResponse::Response(result) => build_proxy_response(result), - GatewayResponse::Forward(ForwardResponse { - status, - headers, - body: backend_resp, - .. - }) => { - // Stream the response body from the reqwest::Response - let body = Body::from_stream(backend_resp.bytes_stream()); - - let mut builder = Response::builder().status(status); - for (k, v) in headers.iter() { - builder = builder.header(k, v); + loop { + let (stream, _) = listener.accept().await?; + let io = hyper_util::rt::TokioIo::new(stream); + let s3_service = s3_service.clone(); + + tokio::spawn(async move { + if let Err(e) = + hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()) + .serve_connection(io, s3_service) + .await + { + tracing::error!("connection error: {e}"); } - - builder.body(body).unwrap() - } + }); } }