AtaraxiaDev/cryptomator-android
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Improve chunked upload in S3

Browse Source
This commit is contained in:
Julian Raufelder 2021-04-29 16:47:23 +02:00 committed by Manuel Jenny
parent c332ad91e4
commit 06c23abc62
No known key found for this signature in database
GPG Key ID: 1C80FE62B2BEAA18
1 changed files with 28 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
data/src/main/java/org/cryptomator/data/cloud/s3/S3Impl.java
View File

@ -110,14 +110,14 @@ class S3Impl {
return S3CloudNodeFactory.file(parent, name, size, parent.getKey() + name); return S3CloudNodeFactory.file(parent, name, size, parent.getKey() + name);
} }
public S3Folder folder(S3Folder parent, String name) { public S3Folder folder(S3Folder parent, String name) {
return S3CloudNodeFactory.folder(parent, name, parent.getKey() + name); return S3CloudNodeFactory.folder(parent, name, parent.getKey() + name);
} }
public void bucketExists() throws BackendException { public void bucketExists() throws BackendException {
if (!client().doesBucketExist(cloud.s3Bucket())) { if (!client().doesBucketExist(cloud.s3Bucket())) {
throw new NoSuchBucketException(cloud.s3Bucket()); throw new NoSuchBucketException(cloud.s3Bucket());
} }
} }
public boolean exists(S3Node node) { public boolean exists(S3Node node) {
@ -125,25 +125,18 @@ class S3Impl {
ListObjectsV2Result result = client().listObjectsV2(cloud.s3Bucket(), key); ListObjectsV2Result result = client().listObjectsV2(cloud.s3Bucket(), key);
if (result.getObjectSummaries().size() > 0) { return result.getObjectSummaries().size() > 0;
return true;
} else {
return false;
}
} }
public List<S3Node> list(S3Folder folder) throws IOException, BackendException { public List<S3Node> list(S3Folder folder) throws IOException, BackendException {
List<S3Node> result = new ArrayList<>(); List<S3Node> result = new ArrayList<>();
ListObjectsV2Request request = new ListObjectsV2Request() ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(cloud.s3Bucket()).withPrefix(folder.getKey()).withDelimiter(DELIMITER);
.withBucketName(cloud.s3Bucket())
.withPrefix(folder.getKey())
.withDelimiter(DELIMITER);
ListObjectsV2Result listObjects = client().listObjectsV2(request); ListObjectsV2Result listObjects = client().listObjectsV2(request);
for(String prefix : listObjects.getCommonPrefixes()) { for (String prefix : listObjects.getCommonPrefixes()) {
// add folders // add folders
result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(prefix))); result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(prefix)));
} }
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) { for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
@ -200,7 +193,7 @@ class S3Impl {
} else { } else {
CopyObjectResult result = client().copyObject(cloud.s3Bucket(), source.getPath(), cloud.s3Bucket(), target.getPath()); CopyObjectResult result = client().copyObject(cloud.s3Bucket(), source.getPath(), cloud.s3Bucket(), target.getPath());
client().deleteObject(cloud.s3Bucket(), source.getPath()); client().deleteObject(cloud.s3Bucket(), source.getPath());
return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File)source).getSize(), Optional.of(result.getLastModifiedDate())); return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File) source).getSize(), Optional.of(result.getLastModifiedDate()));
} }
} }
@ -219,17 +212,18 @@ class S3Impl {
uploadChunkedFile(file, data, progressAware, result, size); uploadChunkedFile(file, data, progressAware, result, size);
} }
progressAware.onProgress(Progress.completed(UploadState.upload(file)));
try { try {
return S3CloudNodeFactory.file(file.getParent(), file.getName(), result.get()); ObjectMetadata objectMetadata = result.get();
objectMetadata = objectMetadata == null ? client().getObjectMetadata(cloud.s3Bucket(), file.getPath()) : objectMetadata;
progressAware.onProgress(Progress.completed(UploadState.upload(file)));
return S3CloudNodeFactory.file(file.getParent(), file.getName(), objectMetadata);
} catch (ExecutionException | InterruptedException e) { } catch (ExecutionException | InterruptedException e) {
throw new FatalBackendException(e); throw new FatalBackendException(e);
} }
} }
private void uploadFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<ObjectMetadata> result, final long size) // private void uploadFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<ObjectMetadata> result, final long size) //
throws IOException { throws IOException {
AtomicLong bytesTransferred = new AtomicLong(0); AtomicLong bytesTransferred = new AtomicLong(0);
ProgressListener listener = progressEvent -> { ProgressListener listener = progressEvent -> {
@ -252,13 +246,12 @@ class S3Impl {
private void uploadChunkedFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<ObjectMetadata> result, final long size) // private void uploadChunkedFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<ObjectMetadata> result, final long size) //
throws IOException { throws IOException {
AtomicLong bytesTransferred = new AtomicLong(0);
TransferUtility tu = TransferUtility TransferUtility tu = TransferUtility //
.builder() .builder() //
.s3Client(client()) .s3Client(client()) //
.context(context) .context(context) //
.defaultBucket(cloud.s3Bucket()) .defaultBucket(cloud.s3Bucket()) //
.build(); .build();
TransferListener transferListener = new TransferListener() { TransferListener transferListener = new TransferListener() {
@ -266,19 +259,17 @@ class S3Impl {
public void onStateChanged(int id, TransferState state) { public void onStateChanged(int id, TransferState state) {
if (state.equals(TransferState.COMPLETED)) { if (state.equals(TransferState.COMPLETED)) {
progressAware.onProgress(Progress.completed(UploadState.upload(file))); progressAware.onProgress(Progress.completed(UploadState.upload(file)));
ObjectMetadata om = client().getObjectMetadata(cloud.s3Bucket(), file.getPath()); result.complete(null);
result.complete(om);
} }
} }
@Override @Override
public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) { public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) {
bytesTransferred.set(bytesTransferred.get()+bytesCurrent);
progressAware.onProgress( // progressAware.onProgress( //
progress(UploadState.upload(file)) // progress(UploadState.upload(file)) //
.between(0) // .between(0) //
.and(bytesTotal) // .and(size) //
.withValue(bytesTransferred.get())); .withValue(bytesCurrent));
} }
@Override @Override
@ -287,10 +278,7 @@ class S3Impl {
} }
}; };
UploadOptions uploadOptions = UploadOptions UploadOptions uploadOptions = UploadOptions.builder().transferListener(transferListener).build();
.builder()
.transferListener(transferListener)
.build();
tu.upload(file.getPath(), data.open(context), uploadOptions); tu.upload(file.getPath(), data.open(context), uploadOptions);
} }
@ -339,10 +327,10 @@ class S3Impl {
bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred()); bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred());
progressAware.onProgress( // progressAware.onProgress( //
progress(DownloadState.download(file)) // progress(DownloadState.download(file)) //
.between(0) // .between(0) //
.and(file.getSize().orElse(Long.MAX_VALUE)) // .and(file.getSize().orElse(Long.MAX_VALUE)) //
.withValue(bytesTransferred.get())); .withValue(bytesTransferred.get()));
}; };
GetObjectRequest request = new GetObjectRequest(cloud.s3Bucket(), file.getPath()); GetObjectRequest request = new GetObjectRequest(cloud.s3Bucket(), file.getPath());
@ -381,7 +369,7 @@ class S3Impl {
} }
public String currentAccount() { public String currentAccount() {
Owner currentAccount = client() // Owner currentAccount = client() //
.getS3AccountOwner(); .getS3AccountOwner();
return currentAccount.getDisplayName(); return currentAccount.getDisplayName();
} }