JCLOUDS-1368: Correct use of slicing algorithm

MultipartUploadSlicingAlgorithm creates multiple equal-sized parts and
a remaining amount.  BaseBlobStore.putMultipartBlob used this
interface incorrectly, which could create more parts than intended
since the remaining size could be larger than the part size.  This
manifested with Google Cloud Storage which only allows 32 parts.
This commit is contained in:
Andrew Gaul
2018-01-04 14:52:10 -08:00
parent 2220996605
commit 382c9201e3
2 changed files with 14 additions and 6 deletions
@@ -356,14 +356,24 @@ public abstract class BaseBlobStore implements BlobStore {
MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
try {
long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
// TODO: inject MultipartUploadSlicingAlgorithm to override default part size
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
long partSize = algorithm.calculateChunkSize(contentLength);
int partNumber = 1;
for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
// TODO: for InputStream payloads, this buffers all parts in-memory!
while (partNumber < algorithm.getParts()) {
Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize);
BlobUploader b =
new BlobUploader(mpu, partNumber++, payload);
parts.add(executor.submit(b));
algorithm.addCopied(partSize);
}
if (algorithm.getRemaining() != 0) {
Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining());
BlobUploader b =
new BlobUploader(mpu, partNumber, payload);
parts.add(executor.submit(b));
}
return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
} catch (RuntimeException re) {
@@ -132,8 +132,7 @@ public final class MultipartUploadSlicingAlgorithm {
this.copied = copied;
}
@VisibleForTesting
protected int getParts() {
public int getParts() {
return parts;
}
@@ -141,7 +140,7 @@ public final class MultipartUploadSlicingAlgorithm {
return ++part;
}
protected void addCopied(long copied) {
public void addCopied(long copied) {
this.copied += copied;
}
@@ -156,8 +155,7 @@ public final class MultipartUploadSlicingAlgorithm {
return chunkSize;
}
@VisibleForTesting
protected long getRemaining() {
public long getRemaining() {
return remaining;
}