Skip to content

Commit 8df4e9f

Browse files
ggcrggcr
committed
ListBuckets implemented
1 parent 104f4c3 commit 8df4e9f

5 files changed

Lines changed: 180 additions & 30 deletions

File tree

‎README.md‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
> [!WARNING]
44
> **WIP** Currently supporting:
55
>
6-
> - ListObjectsV2
6+
> - ListBuckets, ListObjectsV2
77
> - CreateBucket, DeleteBucket, HeadBucket
88
> - GetObject, PutObject, DeleteObject, HeadObject
99
>
@@ -41,6 +41,28 @@ int main() {
4141
}
4242
```
4343

44+
List all buckets:
45+
46+
```cpp
47+
#include <s3cpp/s3.h>
48+
49+
int main() {
50+
S3Client client("access_key", "secret_key");
51+
52+
auto result = client.ListBuckets();
53+
54+
if (!result) {
55+
std::println("Error: {}", result.error().Message);
56+
return 1;
57+
}
58+
59+
for (const auto& bucket : result->Buckets) {
60+
std::println("Bucket: {}, Created: {}", bucket.Name, bucket.CreationDate);
61+
}
62+
return 0;
63+
}
64+
```
65+
4466
List objects in a bucket:
4567

4668
```cpp

‎src/s3cpp/s3.cpp‎

Lines changed: 103 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include "s3cpp/httpclient.h"
22
#include <expected>
3+
#include <print>
34
#include <s3cpp/s3.h>
45

56
std::expected<ListObjectsResult, Error> S3Client::ListObjects(const std::string& bucket, const ListObjectsInput& options) {
@@ -40,12 +41,49 @@ std::expected<ListObjectsResult, Error> S3Client::ListObjects(const std::string&
4041
const std::vector<XMLNode>& XMLBody = Parser.parse(res.body());
4142

4243
if (res.is_ok()) {
43-
return deserializeListBucketResult(XMLBody, maxKeys);
44+
return deserializeListObjectsResult(XMLBody, maxKeys);
4445
}
4546
return std::unexpected<Error>(deserializeError(XMLBody));
4647
}
4748

48-
std::expected<ListObjectsResult, Error> S3Client::deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys) {
49+
std::expected<ListAllMyBucketsResult, Error> S3Client::ListBuckets(const ListBucketsInput& options) {
50+
std::string url = (addressing_style_ == S3AddressingStyle::VirtualHosted)
51+
? std::format("https://{}/", endpoint_)
52+
: std::format("http://{}/", endpoint_);
53+
54+
// Build URL with query parameters
55+
bool firstParam = true;
56+
if (options.BucketRegion.has_value()) {
57+
url += std::format("{}bucket-region={}", firstParam ? "?" : "&", options.BucketRegion.value());
58+
firstParam = false;
59+
}
60+
if (options.ContinuationToken.has_value()) {
61+
url += std::format("{}continuation-token={}", firstParam ? "?" : "&", options.ContinuationToken.value());
62+
firstParam = false;
63+
}
64+
if (options.MaxBuckets.has_value()) {
65+
url += std::format("{}max-buckets={}", firstParam ? "?" : "&", options.MaxBuckets.value());
66+
firstParam = false;
67+
}
68+
if (options.Prefix.has_value()) {
69+
url += std::format("{}prefix={}", firstParam ? "?" : "&", options.Prefix.value());
70+
firstParam = false;
71+
}
72+
73+
HttpRequest req = Client.get(url).header("Host", endpoint_);
74+
75+
Signer.sign(req);
76+
HttpResponse res = req.execute();
77+
78+
const std::vector<XMLNode>& XMLBody = Parser.parse(res.body());
79+
80+
if (res.is_ok()) {
81+
return deserializeListBucketsResult(XMLBody, options.MaxBuckets);
82+
}
83+
return std::unexpected<Error>(deserializeError(XMLBody));
84+
}
85+
86+
std::expected<ListObjectsResult, Error> S3Client::deserializeListObjectsResult(const std::vector<XMLNode>& nodes, const int maxKeys) {
4987
ListObjectsResult result;
5088
result.Contents.reserve(maxKeys);
5189
result.CommonPrefixes.reserve(maxKeys);
@@ -155,6 +193,69 @@ std::expected<ListObjectsResult, Error> S3Client::deserializeListBucketResult(co
155193
return result;
156194
}
157195

196+
std::expected<ListAllMyBucketsResult, Error> S3Client::deserializeListBucketsResult(const std::vector<XMLNode>& nodes, std::optional<int> maxBuckets) {
197+
ListAllMyBucketsResult result;
198+
if (maxBuckets.has_value())
199+
result.Buckets.reserve(maxBuckets.value());
200+
result.Buckets.push_back(Bucket {});
201+
202+
int bucketsIdx = 0;
203+
204+
// To keep track when we need to append an element
205+
std::vector<std::string_view> seenBuckets;
206+
207+
for (const auto& node : nodes) {
208+
/* Sigh... no reflection */
209+
210+
// Check if we've seen this tag before in the current object
211+
if (node.tag.contains("ListAllMyBucketsResult.Buckets.")) {
212+
if (std::find(seenBuckets.begin(), seenBuckets.end(), node.tag) != seenBuckets.end()) {
213+
result.Buckets.push_back(Bucket {});
214+
seenBuckets.clear();
215+
bucketsIdx++;
216+
}
217+
}
218+
219+
if (node.tag == "ListAllMyBucketsResult.Buckets.Bucket.BucketArn") {
220+
result.Buckets[bucketsIdx].BucketARN = std::move(node.value);
221+
} else if (node.tag == "ListAllMyBucketsResult.Buckets.Bucket.BucketRegion") {
222+
result.Buckets[bucketsIdx].BucketRegion = std::move(node.value);
223+
} else if (node.tag == "ListAllMyBucketsResult.Buckets.Bucket.CreationDate") {
224+
result.Buckets[bucketsIdx].CreationDate = std::move(node.value);
225+
} else if (node.tag == "ListAllMyBucketsResult.Buckets.Bucket.Name") {
226+
result.Buckets[bucketsIdx].Name = std::move(node.value);
227+
} else if (node.tag == "ListAllMyBucketsResult.Owner.DisplayName") {
228+
result.Owner.DisplayName = std::move(node.value);
229+
} else if (node.tag == "ListAllMyBucketsResult.Owner.ID") {
230+
result.Owner.ID = std::move(node.value);
231+
} else if (node.tag == "ListAllMyBucketsResult.ContinuationToken") {
232+
result.ContinuationToken = std::move(node.value);
233+
} else if (node.tag == "ListAllMyBucketsResult.Prefix") {
234+
result.Prefix = std::move(node.value);
235+
} else {
236+
// Detect and parse error
237+
// Note(cristian): This fallback should not be needed as we have
238+
// the HTTP status codes for this, however, I like it
239+
if (node.tag.substr(0, 6) == "Error.") {
240+
return std::unexpected<Error>(deserializeError(nodes));
241+
}
242+
throw std::runtime_error(std::format("No case for ListAllMyBucketsResult response found for: {}", node.tag));
243+
}
244+
245+
// Add already seen fields
246+
if (node.tag.contains("ListAllMyBucketsResult.Buckets")) {
247+
seenBuckets.push_back(node.tag);
248+
}
249+
}
250+
251+
// Remove the initial empty object if it was never populated
252+
if (!result.Buckets.empty() && result.Buckets[0].Name.empty()) {
253+
result.Buckets.erase(result.Buckets.begin());
254+
}
255+
256+
return result;
257+
}
258+
158259
std::expected<std::string, Error> S3Client::GetObject(const std::string& bucket, const std::string& key, const GetObjectInput& options) {
159260
std::string url = buildURL(bucket) + std::format("/{}", key);
160261

‎src/s3cpp/s3.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ class S3Client {
3232

3333
// S3 operations: Goal is to support CRUD and stay minimal
3434
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket, const ListObjectsInput& options = {});
35+
std::expected<ListAllMyBucketsResult, Error> ListBuckets(const ListBucketsInput& options = {});
3536
std::expected<std::string, Error> GetObject(const std::string& bucket, const std::string& key, const GetObjectInput& options = {});
3637
std::expected<PutObjectResult, Error> PutObject(const std::string& bucket, const std::string& key, const std::string& body, const PutObjectInput& options = {});
3738
std::expected<DeleteObjectResult, Error> DeleteObject(const std::string& bucket, const std::string& key, const DeleteObjectInput& options = {});
3839
std::expected<CreateBucketResult, Error> CreateBucket(const std::string& bucket, const CreateBucketConfiguration& configuration = {}, const CreateBucketInput& options = {});
3940
std::expected<void, Error> DeleteBucket(const std::string& bucket, const DeleteBucketInput& options = {});
4041
std::expected<HeadBucketResult, Error> HeadBucket(const std::string& bucket, const HeadBucketInput& options = {});
4142
std::expected<HeadObjectResult, Error> HeadObject(const std::string& bucket, const std::string& key, const HeadObjectInput& options = {});
42-
// - HeadObject
4343

4444
// S3 responses
4545

@@ -53,7 +53,8 @@ class S3Client {
5353
*
5454
* Otherwise; wait until C++26 to introduce reflection
5555
*/
56-
std::expected<ListObjectsResult, Error> deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys);
56+
std::expected<ListObjectsResult, Error> deserializeListObjectsResult(const std::vector<XMLNode>& nodes, const int maxKeys);
57+
std::expected<ListAllMyBucketsResult, Error> deserializeListBucketsResult(const std::vector<XMLNode>& nodes, std::optional<int> maxBuckets);
5758
std::expected<PutObjectResult, Error> deserializePutObjectResult(const std::map<std::string, std::string, LowerCaseCompare>& headers);
5859
std::expected<DeleteObjectResult, Error> deserializeDeleteObjectResult(const std::map<std::string, std::string, LowerCaseCompare>& headers);
5960
std::expected<CreateBucketResult, Error> deserializeCreateBucketResult(const std::map<std::string, std::string, LowerCaseCompare>& headers);

‎src/s3cpp/types.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,30 @@ struct ListObjectsResult {
7777
std::string StartAfter;
7878
};
7979

80+
struct ListBucketsInput {
81+
std::optional<std::string> BucketRegion;
82+
std::optional<std::string> ContinuationToken;
83+
std::optional<int> MaxBuckets;
84+
std::optional<std::string> Prefix;
85+
};
86+
87+
struct Bucket {
88+
std::string BucketARN;
89+
std::string BucketRegion;
90+
std::string CreationDate;
91+
std::string Name;
92+
};
93+
94+
struct ListAllMyBucketsResult {
95+
std::vector<Bucket> Buckets;
96+
struct Owner_ {
97+
std::string DisplayName;
98+
std::string ID;
99+
} Owner;
100+
std::string ContinuationToken;
101+
std::string Prefix;
102+
};
103+
80104
struct PutObjectInput {
81105
std::optional<std::string> CacheControl;
82106
std::optional<std::string> ContentDisposition;

‎test/s3_test.cpp‎

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#include "gtest/gtest.h"
2-
#include <random>
32
#include <s3cpp/s3.h>
43
#include <string>
54

@@ -15,9 +14,9 @@ class S3 : public ::testing::Test {
1514
return;
1615
}
1716

18-
// Upload 10k files
17+
// Upload 1001 files
1918
if (client.ListObjects("my-bucket")->Contents.empty()) {
20-
for (int i = 1; i <= 10'000; i++) {
19+
for (int i = 1; i <= 1'001; i++) {
2120
const std::string key = std::format("path/to/file_{}.txt", i);
2221
const std::string body = std::format("This is test file number {}", i);
2322
auto putObjRes = client.PutObject("my-bucket", key, body);
@@ -30,22 +29,9 @@ class S3 : public ::testing::Test {
3029
}
3130
};
3231

33-
std::string generateRandomBucketName(const std::string& prefix = "test-bucket") {
34-
static std::random_device rd;
35-
static std::mt19937 gen(rd());
36-
static std::uniform_int_distribution<> dis(0, 35);
37-
38-
const char* chars = "abcdefghijklmnopqrstuvwxyz0123456789";
39-
std::string suffix;
40-
for (int i = 0; i < 8; ++i) {
41-
suffix += chars[dis(gen)];
42-
}
43-
return prefix + "-" + suffix;
44-
}
45-
4632
TEST_F(S3, ListObjectsBucket) {
4733
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
48-
// Assuming the bucket has the 10K objects
34+
// Assuming the bucket has 1001 objects
4935
// Once we implement PutObject we will do this ourselves with s3cpp
5036
std::expected<ListObjectsResult, Error> res = client.ListObjects("my-bucket");
5137
if (!res)
@@ -114,7 +100,7 @@ TEST_F(S3, ListObjectsCheckFields) {
114100

115101
TEST_F(S3, ListObjectsCheckLenKeys) {
116102
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
117-
// has 10K objects - limit is 1000 keys
103+
// has 1001 objects - limit is 1000 keys
118104
std::expected<ListObjectsResult, Error> res = client.ListObjects("my-bucket", { .Prefix = "path/to/" });
119105
if (!res)
120106
GTEST_FAIL();
@@ -123,7 +109,7 @@ TEST_F(S3, ListObjectsCheckLenKeys) {
123109

124110
TEST_F(S3, ListObjectsPaginator) {
125111
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
126-
// has 10K objects - fetch 100 per page
112+
// has 1001 objects - fetch 100 per page
127113
ListObjectsPaginator paginator(client, "my-bucket", "path/to/", 100);
128114

129115
int totalObjects = 0;
@@ -144,8 +130,8 @@ TEST_F(S3, ListObjectsPaginator) {
144130
}
145131
}
146132

147-
EXPECT_EQ(totalObjects, 10000);
148-
EXPECT_EQ(pageCount, 100);
133+
EXPECT_EQ(totalObjects, 1001);
134+
EXPECT_EQ(pageCount, 11);
149135
}
150136

151137
TEST_F(S3, GetObjectExists) {
@@ -247,7 +233,7 @@ TEST_F(S3, PutObjectTxt) {
247233

248234
TEST_F(S3, CreateBucket) {
249235
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
250-
std::string bucketName = generateRandomBucketName("test-bucket-s3cpp");
236+
std::string bucketName = "test-bucket-s3cpp";
251237
CreateBucketConfiguration config;
252238
CreateBucketInput options;
253239

@@ -270,7 +256,7 @@ TEST_F(S3, CreateBucket) {
270256

271257
TEST_F(S3, CreateBucketWithLocationConstraint) {
272258
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
273-
std::string bucketName = generateRandomBucketName("test-bucket-location");
259+
std::string bucketName = "test-bucket-location";
274260
CreateBucketConfiguration config;
275261
config.LocationConstraint = "us-west-2";
276262
CreateBucketInput options;
@@ -294,7 +280,7 @@ TEST_F(S3, CreateBucketWithLocationConstraint) {
294280

295281
TEST_F(S3, CreateBucketWithTags) {
296282
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
297-
std::string bucketName = generateRandomBucketName("test-bucket-tags");
283+
std::string bucketName = "test-bucket-tags";
298284
CreateBucketConfiguration config;
299285
config.Tags = {
300286
{ "Environment", "Test" },
@@ -415,7 +401,7 @@ TEST_F(S3, DeleteBucketAndElementsWithPaginator) {
415401
TEST_F(S3, HeadBucketExists) {
416402
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
417403

418-
// Premise is that "my-bucket" already exists and contains 10k objects
404+
// Premise is that "my-bucket" already exists and contains 1001 objects
419405
auto res = client.HeadBucket("my-bucket");
420406
if (!res)
421407
FAIL() << std::format("HeadBucket request failed for an existing bucket. Code={}, Message={}", res.error().Code, res.error().Message);
@@ -453,3 +439,19 @@ TEST_F(S3, HeadObjectNotExists) {
453439

454440
EXPECT_EQ(res.error().Code, "NoSuchKey");
455441
}
442+
443+
TEST_F(S3, ListBuckets) {
444+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
445+
446+
std::expected<ListAllMyBucketsResult, Error> res = client.ListBuckets();
447+
if (res) {
448+
EXPECT_EQ(res->Buckets.size(), 5);
449+
std::vector<std::string> bucket_names;
450+
for (const auto& Bucket : res->Buckets) {
451+
bucket_names.push_back(Bucket.Name);
452+
}
453+
EXPECT_TRUE(std::find(bucket_names.begin(), bucket_names.end(), "my-bucket") != bucket_names.end());
454+
} else {
455+
FAIL() << std::format("ListBuckets request failed. Code={}, Message={}", res.error().Code, res.error().Message);
456+
}
457+
}

0 commit comments

Comments
 (0)