JCLOUDS-1644: Create AWS S3 buckets with ownership and public access block

AWS changed the defaults when creating buckets to prevent public-read
and other canned ACLs.  Background:
https://stackoverflow.com/a/76102067/2800111
This commit is contained in:
Andrew Gaul
2025-01-19 18:50:07 -08:00
parent bdfac92184
commit dd6cb51454
11 changed files with 258 additions and 5 deletions
@@ -66,8 +66,10 @@ import org.jclouds.s3.binders.BindCannedAclToRequest;
import org.jclouds.s3.binders.BindIterableAsPayloadToDeleteRequest;
import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
import org.jclouds.s3.binders.BindObjectMetadataToRequest;
import org.jclouds.s3.binders.BindOwnershipControlsToXMLPayload;
import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
import org.jclouds.s3.binders.BindPayerToXmlPayload;
import org.jclouds.s3.binders.BindPublicAccessBlockConfigurationToXMLPayload;
import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.BucketLogging;
@@ -79,6 +81,7 @@ import org.jclouds.s3.domain.ListMultipartUploadResponse;
import org.jclouds.s3.domain.ListMultipartUploadsResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.Payer;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
import org.jclouds.s3.filters.RequestAuthorizeSignature;
@@ -813,4 +816,19 @@ public interface S3Client extends Closeable {
@QueryParam("delimiter") @Nullable String delimiter, @QueryParam("max-uploads") @Nullable Integer maxUploads,
@QueryParam("key-marker") @Nullable String keyMarker, @QueryParam("prefix") @Nullable String prefix,
@QueryParam("upload-id-marker") @Nullable String uploadIdMarker);
@Named("PutBucketOwnershipControls")
@PUT
@Path("/")
@QueryParams(keys = "ownershipControls")
void putBucketOwnershipControls(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
// BucketOwnerPreferred | ObjectWriter | BucketOwnerEnforced
@BinderParam(BindOwnershipControlsToXMLPayload.class) String objectOwnership);
@Named("PutPublicAccessBlock")
@PUT
@Path("/")
@QueryParams(keys = "publicAccessBlock")
void putPublicAccessBlock(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
@BinderParam(BindPublicAccessBlockConfigurationToXMLPayload.class) PublicAccessBlockConfiguration configuration);
}
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.binders;
import static org.jclouds.s3.binders.XMLHelper.asString;
import static org.jclouds.s3.binders.XMLHelper.createDocument;
import static org.jclouds.s3.binders.XMLHelper.elem;
import static org.jclouds.s3.binders.XMLHelper.elemWithText;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import javax.xml.parsers.FactoryConfigurationError;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.reference.S3Constants;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import com.google.common.base.Throwables;
@Singleton
public final class BindOwnershipControlsToXMLPayload implements Binder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
String from = (String) payload;
try {
request.setPayload(generatePayload(from));
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
return request;
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new RuntimeException("error transforming acl: " + from, e);
}
}
protected String generatePayload(String objectOwnership)
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
Document document = createDocument();
Element rootNode = elem(document, "OwnershipControls", document);
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
Element ruleNode = elem(rootNode, "Rule", document);
elemWithText(ruleNode, "ObjectOwnership", objectOwnership, document);
return asString(document);
}
}
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.binders;
import static org.jclouds.s3.binders.XMLHelper.asString;
import static org.jclouds.s3.binders.XMLHelper.createDocument;
import static org.jclouds.s3.binders.XMLHelper.elem;
import static org.jclouds.s3.binders.XMLHelper.elemWithText;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import javax.xml.parsers.FactoryConfigurationError;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import org.jclouds.http.HttpRequest;
import org.jclouds.rest.Binder;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.reference.S3Constants;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import com.google.common.base.Throwables;
@Singleton
public final class BindPublicAccessBlockConfigurationToXMLPayload implements Binder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
PublicAccessBlockConfiguration configuration = (PublicAccessBlockConfiguration) payload;
try {
request.setPayload(generatePayload(configuration));
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
return request;
} catch (Exception e) {
Throwables.propagateIfPossible(e);
throw new RuntimeException("error transforming configuration: " + configuration, e);
}
}
protected String generatePayload(PublicAccessBlockConfiguration configuration)
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
Document document = createDocument();
Element rootNode = elem(document, "PublicAccessBlockConfiguration", document);
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
elemWithText(rootNode, "BlockPublicAcls", String.valueOf(configuration.blockPublicAcls()), document);
elemWithText(rootNode, "IgnorePublicAcls", String.valueOf(configuration.ignorePublicAcls()), document);
elemWithText(rootNode, "BlockPublicPolicy", String.valueOf(configuration.blockPublicPolicy()), document);
elemWithText(rootNode, "RestrictPublicBuckets", String.valueOf(configuration.restrictPublicBuckets()), document);
return asString(document);
}
}
@@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.domain;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.Beta;
@AutoValue
@Beta
public abstract class PublicAccessBlockConfiguration {
public abstract boolean blockPublicAcls();
public abstract boolean ignorePublicAcls();
public abstract boolean blockPublicPolicy();
public abstract boolean restrictPublicBuckets();
public static PublicAccessBlockConfiguration create(boolean blockPublicAcls, boolean ignorePublicAcls, boolean blockPublicPolicy, boolean restrictPublicBuckets) {
return new AutoValue_PublicAccessBlockConfiguration(blockPublicAcls, ignorePublicAcls, blockPublicPolicy, restrictPublicBuckets);
}
}
@@ -163,11 +163,16 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
}
protected void allowPublicReadable(String containerName) {
}
@Test(groups = {"fails-on-s3proxy"})
public void testCopyCannedAccessPolicyPublic() throws Exception {
String containerName = getContainerName();
String destinationContainer = getContainerName();
try {
allowPublicReadable(destinationContainer);
addBlobToContainer(containerName, sourceKey);
validateContent(containerName, sourceKey);
@@ -193,6 +198,7 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
final String publicReadWriteObjectKey = "public-read-write-acl";
final String containerName = getContainerName();
try {
allowPublicReadable(containerName);
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(publicReadWriteObjectKey);
object.setPayload("");
@@ -295,6 +301,7 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
final String publicReadObjectKey = "public-read-acl";
final String containerName = getContainerName();
try {
allowPublicReadable(containerName);
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(publicReadObjectKey);
object.setPayload("");
@@ -715,6 +722,7 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
public void testUpdateBucketCannedACL() throws Exception {
String containerName = getContainerName();
try {
allowPublicReadable(containerName);
getApi().updateBucketCannedACL(containerName, CannedAccessPolicy.PUBLIC_READ);
AccessControlList acl = getApi().getBucketACL(containerName);
assertThat(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ)).isTrue();
@@ -730,6 +738,7 @@ public class S3ClientLiveTest extends BaseBlobStoreIntegrationTest {
public void testUpdateObjectCannedACL() throws Exception {
String containerName = getContainerName();
try {
allowPublicReadable(containerName);
String key = "testUpdateObjectCannedACL";
S3Object object = getApi().newS3Object();
object.getMetadata().setKey(key);
@@ -25,14 +25,12 @@ import static org.jclouds.s3.domain.AccessControlList.Permission.READ;
import static org.jclouds.s3.domain.AccessControlList.Permission.READ_ACP;
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE;
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE_ACP;
import static org.jclouds.s3.domain.CannedAccessPolicy.PUBLIC_READ;
import static org.jclouds.s3.domain.Payer.BUCKET_OWNER;
import static org.jclouds.s3.domain.Payer.REQUESTER;
import static org.jclouds.s3.options.ListBucketOptions.Builder.afterMarker;
import static org.jclouds.s3.options.ListBucketOptions.Builder.delimiter;
import static org.jclouds.s3.options.ListBucketOptions.Builder.maxResults;
import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
@@ -53,6 +51,7 @@ import org.jclouds.s3.domain.AccessControlList.EmailAddressGrantee;
import org.jclouds.s3.domain.AccessControlList.Grant;
import org.jclouds.s3.domain.BucketLogging;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.S3Object;
import org.jclouds.util.Strings2;
@@ -154,7 +153,9 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest {
public void testPublicReadAccessPolicy() throws Exception {
String bucketName = getScratchContainerName();
try {
getApi().putBucketInRegion(null, bucketName, withBucketAcl(PUBLIC_READ));
getApi().putBucketInRegion(/*region=*/ null, bucketName);
allowPublicReadable(bucketName);
getApi().updateBucketCannedACL(bucketName, CannedAccessPolicy.PUBLIC_READ);
AccessControlList acl = getApi().getBucketACL(bucketName);
assertTrue(acl.hasPermission(ALL_USERS, READ), acl.toString());
// TODO: I believe that the following should work based on the above acl assertion passing.
@@ -209,11 +210,15 @@ public class BucketsLiveTest extends BaseBlobStoreIntegrationTest {
}
}
protected void allowPublicReadable(String containerName) {
}
@Test(groups = {"fails-on-s3proxy"})
public void testBucketLogging() throws Exception {
final String bucketName = getContainerName();
final String targetBucket = getContainerName();
try {
allowPublicReadable(targetBucket);
assertNull(getApi().getBucketLogging(bucketName));
setupAclForBucketLoggingTarget(targetBucket);
@@ -729,12 +729,16 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true));
}
protected void allowPublicReadable(String containerName) {
}
@Test(groups = { "integration", "live" })
public void testSetBlobAccess() throws Exception {
BlobStore blobStore = view.getBlobStore();
String containerName = getContainerName();
String blobName = "set-access-blob-name";
try {
allowPublicReadable(containerName);
addBlobToContainer(containerName, blobName, blobName, MediaType.TEXT_PLAIN);
assertThat(blobStore.getBlobAccess(containerName, blobName)).isEqualTo(BlobAccess.PRIVATE);
@@ -778,6 +782,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest {
public void testPutBlobAccessMultipart() throws Exception {
BlobStore blobStore = view.getBlobStore();
String containerName = getContainerName();
allowPublicReadable(containerName);
ByteSource byteSource = TestUtils.randomByteSource().slice(0, getMinimumMultipartBlobSize());
Payload payload = Payloads.newByteSourcePayload(byteSource);
payload.getContentMetadata().setContentLength(byteSource.size());
@@ -29,6 +29,7 @@ import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions;
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.ContainerAccess;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
@@ -47,7 +48,9 @@ import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions;
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.s3.domain.BucketMetadata;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
@@ -58,6 +61,7 @@ import com.google.common.base.Supplier;
public class AWSS3BlobStore extends S3BlobStore {
private final BlobToObject blob2Object;
private final AWSS3Client awsSync;
@Inject
AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@@ -70,6 +74,7 @@ public class AWSS3BlobStore extends S3BlobStore {
super(context, blobUtils, defaultLocation, locations, slicer, sync, convertBucketsToStorageMetadata,
container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
blob2ObjectMetadata, object2BlobMd, fetchBlobMetadataProvider);
this.awsSync = sync;
this.blob2Object = blob2Object;
}
@@ -102,6 +107,30 @@ public class AWSS3BlobStore extends S3BlobStore {
// JCLOUDS-334 for details.
return false;
}
// AWS blocks creating buckets with public-read canned ACL by default since 25 April 2023. Instead create a bucket, override the block, and set the ACL.
if (options.isPublicRead()) {
boolean created = super.createContainerInLocation(location, container, new CreateContainerOptions());
if (!created) {
return false;
}
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
awsSync.updateBucketCannedACL(container, CannedAccessPolicy.PUBLIC_READ);
return true;
}
return super.createContainerInLocation(location, container, options);
}
@Override
public void setContainerAccess(String container, ContainerAccess access) {
CannedAccessPolicy acl = CannedAccessPolicy.PRIVATE;
if (access == ContainerAccess.PUBLIC_READ) {
acl = CannedAccessPolicy.PUBLIC_READ;
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
}
awsSync.updateBucketCannedACL(container, acl);
}
}
@@ -37,6 +37,7 @@ import org.jclouds.s3.S3ClientLiveTest;
import org.jclouds.s3.domain.ListBucketResponse;
import org.jclouds.s3.domain.ObjectMetadata;
import org.jclouds.s3.domain.ObjectMetadata.StorageClass;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.domain.S3Object;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
@@ -56,6 +57,13 @@ public class AWSS3ClientLiveTest extends S3ClientLiveTest {
return view.unwrapApi(AWSS3Client.class);
}
@Override
protected void allowPublicReadable(String containerName) {
getApi().putBucketOwnershipControls(containerName, "ObjectWriter");
getApi().putPublicAccessBlock(containerName, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
}
@BeforeClass(groups = { "integration", "live" })
@Override
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
@@ -16,7 +16,9 @@
*/
package org.jclouds.aws.s3.blobstore.integration;
import org.jclouds.s3.S3Client;
import org.jclouds.s3.blobstore.integration.S3BlobIntegrationLiveTest;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.testng.annotations.Test;
import org.testng.SkipException;
@@ -26,6 +28,14 @@ public class AWSS3BlobIntegrationLiveTest extends S3BlobIntegrationLiveTest {
provider = "aws-s3";
}
@Override
protected void allowPublicReadable(String containerName) {
S3Client client = view.unwrapApi(S3Client.class);
client.putBucketOwnershipControls(containerName, "ObjectWriter");
client.putPublicAccessBlock(containerName, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
}
@Override
public void testCopyIfModifiedSinceNegative() throws Exception {
throw new SkipException("S3 supports copyIfModifiedSince but test uses time in the future which Amazon does not support");
@@ -16,7 +16,6 @@
*/
package org.jclouds.aws.s3.services;
import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@@ -25,6 +24,7 @@ import org.jclouds.s3.domain.AccessControlList;
import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI;
import org.jclouds.s3.domain.AccessControlList.Permission;
import org.jclouds.s3.domain.CannedAccessPolicy;
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
import org.jclouds.s3.services.BucketsLiveTest;
import org.testng.annotations.Test;
@@ -36,6 +36,13 @@ public class AWSBucketsLiveTest extends BucketsLiveTest {
provider = "aws-s3";
}
@Override
protected void allowPublicReadable(String containerName) {
getApi().putBucketOwnershipControls(containerName, "ObjectWriter");
getApi().putPublicAccessBlock(containerName, PublicAccessBlockConfiguration.create(
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
}
public void testDefaultBucketLocation() throws Exception {
String bucketName = getContainerName();
@@ -53,7 +60,9 @@ public class AWSBucketsLiveTest extends BucketsLiveTest {
public void testEu() throws Exception {
final String bucketName = getScratchContainerName();
try {
getApi().putBucketInRegion(Region.EU_WEST_1, bucketName + "eu", withBucketAcl(CannedAccessPolicy.PUBLIC_READ));
getApi().putBucketInRegion(Region.EU_WEST_1, bucketName);
allowPublicReadable(bucketName);
getApi().updateBucketCannedACL(bucketName, CannedAccessPolicy.PUBLIC_READ);
assertConsistencyAware(new Runnable() {
public void run() {
try {