Merge pull request #8125 from eugenp/revert-8119-BAEL-3275-2

Revert "BAEL-3275: Using blocking queue for pub-sub"
This commit is contained in:
Eric Martin
2019-10-31 20:43:47 -05:00
committed by GitHub
parent db85c8f275
commit 3225470df5
20543 changed files with 1642750 additions and 0 deletions
@@ -0,0 +1,120 @@
package com.baeldung.dynamodb;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ResourceInUseException;
import com.baeldung.dynamodb.entity.ProductInfo;
import com.baeldung.dynamodb.repository.ProductInfoRepository;
import com.baeldung.dynamodb.rule.LocalDbCreationRule;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
public class ProductInfoRepositoryIntegrationTest {
@ClassRule
public static LocalDbCreationRule dynamoDB = new LocalDbCreationRule();
private static DynamoDBMapper dynamoDBMapper;
private static AmazonDynamoDB amazonDynamoDB;
private ProductInfoRepository repository;
private static final String DYNAMODB_ENDPOINT = "amazon.dynamodb.endpoint";
private static final String AWS_ACCESSKEY = "amazon.aws.accesskey";
private static final String AWS_SECRETKEY = "amazon.aws.secretkey";
private static final String EXPECTED_COST = "20";
private static final String EXPECTED_PRICE = "50";
@BeforeClass
public static void setupClass() {
Properties testProperties = loadFromFileInClasspath("test.properties")
.filter(properties -> !isEmpty(properties.getProperty(AWS_ACCESSKEY)))
.filter(properties -> !isEmpty(properties.getProperty(AWS_SECRETKEY)))
.filter(properties -> !isEmpty(properties.getProperty(DYNAMODB_ENDPOINT)))
.orElseThrow(() -> new RuntimeException("Unable to get all of the required test property values"));
String amazonAWSAccessKey = testProperties.getProperty(AWS_ACCESSKEY);
String amazonAWSSecretKey = testProperties.getProperty(AWS_SECRETKEY);
String amazonDynamoDBEndpoint = testProperties.getProperty(DYNAMODB_ENDPOINT);
amazonDynamoDB = new AmazonDynamoDBClient(new BasicAWSCredentials(amazonAWSAccessKey, amazonAWSSecretKey));
amazonDynamoDB.setEndpoint(amazonDynamoDBEndpoint);
dynamoDBMapper = new DynamoDBMapper(amazonDynamoDB);
}
@Before
public void setup() {
try {
repository = new ProductInfoRepository();
repository.setMapper(dynamoDBMapper);
CreateTableRequest tableRequest = dynamoDBMapper.generateCreateTableRequest(ProductInfo.class);
tableRequest.setProvisionedThroughput(new ProvisionedThroughput(1L, 1L));
amazonDynamoDB.createTable(tableRequest);
} catch (ResourceInUseException e) {
// Do nothing, table already created
}
// TODO How to handle different environments. i.e. AVOID deleting all entries in ProductInfo on table
dynamoDBMapper.batchDelete((List<ProductInfo>) repository.findAll());
}
@Test
public void givenItemWithExpectedCost_whenRunFindAll_thenItemIsFound() {
ProductInfo productInfo = new ProductInfo(EXPECTED_COST, EXPECTED_PRICE);
repository.save(productInfo);
List<ProductInfo> result = (List<ProductInfo>) repository.findAll();
assertThat(result.size(), is(greaterThan(0)));
assertThat(result.get(0).getCost(), is(equalTo(EXPECTED_COST)));
}
private static boolean isEmpty(String inputString) {
return inputString == null || "".equals(inputString);
}
private static Optional<Properties> loadFromFileInClasspath(String fileName) {
InputStream stream = null;
try {
Properties config = new Properties();
Path configLocation = Paths.get(ClassLoader.getSystemResource(fileName).toURI());
stream = Files.newInputStream(configLocation);
config.load(stream);
return Optional.of(config);
} catch (Exception e) {
return Optional.empty();
} finally {
if (stream != null) {
try {
stream.close();
} catch (IOException e) {
}
}
}
}
}
@@ -0,0 +1,35 @@
package com.baeldung.dynamodb.rule;
import com.amazonaws.services.dynamodbv2.local.main.ServerRunner;
import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer;
import org.junit.rules.ExternalResource;
public class LocalDbCreationRule extends ExternalResource {
protected DynamoDBProxyServer server;
public LocalDbCreationRule() {
System.setProperty("sqlite4java.library.path", "native-libs");
}
@Override
protected void before() throws Exception {
String port = "8000";
this.server = ServerRunner.createServerFromCommandLineArgs(new String[]{"-inMemory", "-port", port});
server.start();
}
@Override
protected void after() {
this.stopUnchecked(server);
}
protected void stopUnchecked(DynamoDBProxyServer dynamoDbServer) {
try {
dynamoDbServer.stop();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
@@ -0,0 +1,351 @@
package com.baeldung.jets3t;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jets3t.service.S3Service;
import org.jets3t.service.ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.StorageObject;
import org.jets3t.service.security.AWSCredentials;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.*;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.stream.Collectors;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static junit.framework.TestCase.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class JetS3tLiveTest {
private Log log = LogFactory.getLog(JetS3tLiveTest.class);
private static final String BucketName = "baeldung-barfoo";
private static final String TestString = "test string";
private static final String TestStringName = "string object";
private static final String TgtBucket = "baeldung-tgtbucket";
private static S3Service s3Service;
@BeforeClass
public static void connectS3() throws Exception {
// Replace with your keys
String awsAccessKey = "your access key";
String awsSecretKey = "your secret key";
// Create credentials
AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
// Create service
s3Service = new RestS3Service(awsCredentials);
}
@Test
public void givenCreate_AndDeleteBucket_CountGoesUpThenDown() throws Exception {
// List buckets, get a count
S3Bucket[] myBuckets = s3Service.listAllBuckets();
int count = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
// Create a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// List again
myBuckets = s3Service.listAllBuckets();
int newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
// We should have one more
assertEquals((count + 1), newCount);
// Delete so next test doesn't fail
deleteBucket();
// Check the count again, just for laughs
myBuckets = s3Service.listAllBuckets();
newCount = Arrays.stream(myBuckets).map(S3Bucket::getName).collect(Collectors.toList()).size();
assertEquals(count, newCount);
}
private S3Bucket createBucket() throws Exception {
S3Bucket bucket = s3Service.createBucket(BucketName);
log.info(bucket);
return bucket;
}
private void deleteBucket() throws ServiceException {
s3Service.deleteBucket(BucketName);
}
@Test
public void givenString_Uploaded_StringInfoIsAvailable() throws Exception {
// Create a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// Upload a string
uploadStringData();
// Get the details
StorageObject objectDetailsOnly = s3Service.getObjectDetails(BucketName, TestStringName);
log.info("Content type: " + objectDetailsOnly.getContentType() + " length: " + objectDetailsOnly.getContentLength());
// Delete it
deleteObject(TestStringName);
// For next test
deleteBucket();
}
private void uploadStringData() throws Exception {
S3Object stringObject = new S3Object(TestStringName, TestString);
s3Service.putObject(BucketName, stringObject);
log.info("Content type:" + stringObject.getContentType());
}
private void deleteObject(String objectName) throws ServiceException {
s3Service.deleteObject(BucketName, objectName);
}
@Test
public void givenStringUploaded_StringIsDownloaded() throws Exception {
// Get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
uploadStringData();
// Download
S3Object stringObject = s3Service.getObject(BucketName, TestStringName);
// Process stream into a string
String downloadedString = new BufferedReader(new InputStreamReader(stringObject.getDataInputStream())).lines().collect(Collectors.joining("\n"));
// Verify
assertTrue(TestString.equals(downloadedString));
// Clean up for next test
deleteObject(TestStringName);
deleteBucket();
}
@Test
public void givenBinaryFileUploaded_FileIsDownloaded() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// Put a binary file
S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
s3Service.putObject(BucketName, fileObject);
// Print info about type and name
log.info("Content type:" + fileObject.getContentType());
log.info("File object name is " + fileObject.getName());
// Download
S3Object newFileObject = s3Service.getObject(BucketName, "test.jpg");
// Save to a different name
File newFile = new File("src/test/resources/newtest.jpg");
Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/newtest.jpg");
assertTrue(origMD5.equals(newMD5));
// Clean up
deleteObject("test.jpg");
deleteBucket();
}
// Get MD5 hash for a file
private String getFileMD5(String filename) throws IOException {
try (FileInputStream fis = new FileInputStream(new File(filename))) {
return DigestUtils.md5Hex(fis);
}
}
@Test
public void givenStreamDataUploaded_StreamDataIsDownloaded() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
ArrayList<Integer> numbers = new ArrayList<>();
numbers.add(2);
numbers.add(3);
numbers.add(5);
numbers.add(7);
// Serialize ArrayList
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
ObjectOutputStream objectOutputStream = new ObjectOutputStream(bytes);
objectOutputStream.writeObject(numbers);
// Wrap bytes
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes.toByteArray());
// Create and populate object
S3Object streamObject = new S3Object("stream");
streamObject.setDataInputStream(byteArrayInputStream);
streamObject.setContentLength(byteArrayInputStream.available());
streamObject.setContentType("binary/octet-stream");
// Put it
s3Service.putObject(BucketName, streamObject);
// Get it
S3Object newStreamObject = s3Service.getObject(BucketName, "stream");
// Convert back to ArrayList
ObjectInputStream objectInputStream = new ObjectInputStream(newStreamObject.getDataInputStream());
ArrayList<Integer> newNumbers = (ArrayList<Integer>)objectInputStream.readObject();
assertEquals(2, (int)newNumbers.get(0));
assertEquals(3, (int)newNumbers.get(1));
assertEquals(5, (int)newNumbers.get(2));
assertEquals(7, (int)newNumbers.get(3));
// Clean up
deleteObject("stream");
deleteBucket();
}
@Test
public void whenFileCopied_CopyIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// Put a binary file
S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
s3Service.putObject(BucketName, fileObject);
// Copy it
S3Object targetObject = new S3Object("testcopy.jpg");
s3Service.copyObject(BucketName, "test.jpg", BucketName, targetObject, false);
// Download
S3Object newFileObject = s3Service.getObject(BucketName, "testcopy.jpg");
// Save to a different name
File newFile = new File("src/test/resources/testcopy.jpg");
Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/testcopy.jpg");
assertTrue(origMD5.equals(newMD5));
// Clean up
deleteObject("test.jpg");
deleteObject("testcopy.jpg");
deleteBucket();
}
@Test
public void whenFileRenamed_NewNameIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// Put a binary file
S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
s3Service.putObject(BucketName, fileObject);
// Copy it
s3Service.renameObject(BucketName, "test.jpg", new S3Object("spidey.jpg"));
// Download
S3Object newFileObject = s3Service.getObject(BucketName, "spidey.jpg");
// Save to a different name
File newFile = new File("src/test/resources/spidey.jpg");
Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
assertTrue(origMD5.equals(newMD5));
// Clean up
deleteObject("test.jpg");
deleteObject("spidey.jpg");
deleteBucket();
}
@Test
public void whenFileMoved_NewInstanceIsSame() throws Exception {
// get a bucket
S3Bucket bucket = createBucket();
assertNotNull(bucket);
// create another bucket
S3Bucket tgtBucket = s3Service.createBucket(TgtBucket);
// Put a binary file
S3Object fileObject = new S3Object(new File("src/test/resources/test.jpg"));
s3Service.putObject(BucketName, fileObject);
// Copy it
s3Service.moveObject(BucketName, "test.jpg", TgtBucket,
new S3Object("spidey.jpg"), false);
// Download
S3Object newFileObject = s3Service.getObject(TgtBucket, "spidey.jpg");
// Save to a different name
File newFile = new File("src/test/resources/spidey.jpg");
Files.copy(newFileObject.getDataInputStream(), newFile.toPath(), REPLACE_EXISTING);
// Get hashes and compare
String origMD5 = getFileMD5("src/test/resources/test.jpg");
String newMD5 = getFileMD5("src/test/resources/spidey.jpg");
assertTrue(origMD5.equals(newMD5));
// Clean up
deleteBucket();
s3Service.deleteObject(TgtBucket, "spidey.jpg");
s3Service.deleteBucket(TgtBucket);
}
}
@@ -0,0 +1,113 @@
package com.baeldung.s3;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import org.junit.Before;
import org.junit.Test;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
import com.amazonaws.services.s3.model.PutObjectResult;
public class AWSS3ServiceIntegrationTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "key_name";
private static final String BUCKET_NAME2 = "bucket_name2";
private static final String KEY_NAME2 = "key_name2";
private AmazonS3 s3;
private AWSS3Service service;
@Before
public void setUp() {
s3 = mock(AmazonS3.class);
service = new AWSS3Service(s3);
}
@Test
public void whenInitializingAWSS3Service_thenNotNull() {
assertThat(new AWSS3Service()).isNotNull();
}
@Test
public void whenVerifyingIfS3BucketExist_thenCorrect() {
service.doesBucketExist(BUCKET_NAME);
verify(s3).doesBucketExist(BUCKET_NAME);
}
@Test
public void whenVerifyingCreationOfS3Bucket_thenCorrect() {
service.createBucket(BUCKET_NAME);
verify(s3).createBucket(BUCKET_NAME);
}
@Test
public void whenVerifyingListBuckets_thenCorrect() {
service.listBuckets();
verify(s3).listBuckets();
}
@Test
public void whenDeletingBucket_thenCorrect() {
service.deleteBucket(BUCKET_NAME);
verify(s3).deleteBucket(BUCKET_NAME);
}
@Test
public void whenVerifyingPutObject_thenCorrect() {
File file = mock(File.class);
PutObjectResult result = mock(PutObjectResult.class);
when(s3.putObject(anyString(), anyString(), (File) any())).thenReturn(result);
assertThat(service.putObject(BUCKET_NAME, KEY_NAME, file)).isEqualTo(result);
verify(s3).putObject(BUCKET_NAME, KEY_NAME, file);
}
@Test
public void whenVerifyingListObjects_thenCorrect() {
service.listObjects(BUCKET_NAME);
verify(s3).listObjects(BUCKET_NAME);
}
@Test
public void whenVerifyingGetObject_thenCorrect() {
service.getObject(BUCKET_NAME, KEY_NAME);
verify(s3).getObject(BUCKET_NAME, KEY_NAME);
}
@Test
public void whenVerifyingCopyObject_thenCorrect() {
CopyObjectResult result = mock(CopyObjectResult.class);
when(s3.copyObject(anyString(), anyString(), anyString(), anyString())).thenReturn(result);
assertThat(service.copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2)).isEqualTo(result);
verify(s3).copyObject(BUCKET_NAME, KEY_NAME, BUCKET_NAME2, KEY_NAME2);
}
@Test
public void whenVerifyingDeleteObject_thenCorrect() {
service.deleteObject(BUCKET_NAME, KEY_NAME);
verify(s3).deleteObject(BUCKET_NAME, KEY_NAME);
}
@Test
public void whenVerifyingDeleteObjects_thenCorrect() {
DeleteObjectsRequest request = mock(DeleteObjectsRequest.class);
DeleteObjectsResult result = mock(DeleteObjectsResult.class);
when(s3.deleteObjects((DeleteObjectsRequest)any())).thenReturn(result);
assertThat(service.deleteObjects(request)).isEqualTo(result);
verify(s3).deleteObjects(request);
}
}
@@ -0,0 +1,61 @@
package com.baeldung.s3;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class MultipartUploadLiveTest {
private static final String BUCKET_NAME = "bucket_name";
private static final String KEY_NAME = "picture.jpg";
private AmazonS3 amazonS3;
private TransferManager tm;
private ProgressListener progressListener;
@Before
public void setup() {
amazonS3 = mock(AmazonS3.class);
tm = TransferManagerBuilder
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold((long) (5 * 1024 * 1025))
.withExecutorFactory(() -> Executors.newFixedThreadPool(5))
.build();
progressListener =
progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
}
@Test
public void whenUploadingFileWithTransferManager_thenVerifyUploadRequested() {
File file = mock(File.class);
PutObjectResult s3Result = mock(PutObjectResult.class);
when(amazonS3.putObject(anyString(), anyString(), (File) any())).thenReturn(s3Result);
when(file.getName()).thenReturn(KEY_NAME);
PutObjectRequest request = new PutObjectRequest(BUCKET_NAME, KEY_NAME, file);
request.setGeneralProgressListener(progressListener);
Upload upload = tm.upload(request);
assertThat(upload).isNotNull();
verify(amazonS3).putObject(request);
}
}