Switch from aws-android-sdk-s3 to minio-java

This commit is contained in:
Julian Raufelder 2021-05-11 14:56:04 +02:00
parent 33eb4ce735
commit 5e0f88bcff
No known key found for this signature in database
GPG Key ID: 17EE71F6634E381D
7 changed files with 325 additions and 358 deletions

View File

@ -53,8 +53,6 @@ ext {
// do not update to 1.4.0 until minsdk is 7.x (or desugaring works better) otherwise it will crash on 6.x
cryptolibVersion = '2.0.0-rc1'
awsAndroidSdkS3 = '2.23.0'
dropboxVersion = '4.0.0'
googleApiServicesVersion = 'v3-rev197-1.25.0'
@ -63,6 +61,9 @@ ext {
msgraphVersion = '2.10.0'
minIoVersion = '8.2.1'
staxVersion = '1.2.0' // needed for minIO
commonsCodecVersion = '1.15'
recyclerViewFastScrollVersion = '2.0.1'
@ -107,7 +108,6 @@ ext {
androidxViewpager : "androidx.viewpager:viewpager:${androidxViewpagerVersion}",
androidxSwiperefresh : "androidx.swiperefreshlayout:swiperefreshlayout:${androidxSwiperefreshVersion}",
androidxPreference : "androidx.preference:preference:${androidxPreferenceVersion}",
awsAndroidS3 : "com.amazonaws:aws-android-sdk-s3:${awsAndroidSdkS3}",
documentFile : "androidx.documentfile:documentfile:${androidxDocumentfileVersion}",
recyclerView : "androidx.recyclerview:recyclerview:${androidxRecyclerViewVersion}",
androidxTestCore : "androidx.test:core:${androidxTestCoreVersion}",
@ -132,9 +132,10 @@ ext {
junitParams : "org.junit.jupiter:junit-jupiter-params:${jUnitVersion}",
junit4 : "org.junit.jupiter:junit-jupiter:${jUnit4Version}",
junit4Engine : "org.junit.vintage:junit-vintage-engine:${jUnitVersion}",
msgraph : "com.microsoft.graph:microsoft-graph:${msgraphVersion}",
minIo : "io.minio:minio:${minIoVersion}",
mockito : "org.mockito:mockito-core:${mockitoVersion}",
mockitoInline : "org.mockito:mockito-inline:${mockitoInlineVersion}",
msgraph : "com.microsoft.graph:microsoft-graph:${msgraphVersion}",
multidex : "androidx.multidex:multidex:${multidexVersion}",
okHttp : "com.squareup.okhttp3:okhttp:${okHttpVersion}",
okHttpDigest : "com.burgstaller:okhttp-digest:${okHttpDigestVersion}",
@ -142,6 +143,7 @@ ext {
rxJava : "io.reactivex.rxjava2:rxjava:${rxJavaVersion}",
rxAndroid : "io.reactivex.rxjava2:rxandroid:${rxAndroidVersion}",
rxBinding : "com.jakewharton.rxbinding2:rxbinding:${rxBindingVersion}",
stax : "stax:stax:${staxVersion}",
testingSupportLib : "com.android.support.test:testing-support-lib:${testingSupportLibVersion}",
timber : "com.jakewharton.timber:timber:${timberVersion}",
velocity : "org.apache.velocity:velocity:${velocityVersion}",

View File

@ -110,10 +110,12 @@ dependencies {
implementation dependencies.jsonWebTokenJson
// cloud
implementation dependencies.awsAndroidS3
implementation dependencies.dropbox
implementation dependencies.msgraph
implementation dependencies.stax
compile dependencies.minIo
playstoreImplementation dependencies.googlePlayServicesAuth
apkstoreImplementation dependencies.googlePlayServicesAuth

View File

@ -1,51 +1,97 @@
package org.cryptomator.data.cloud.s3;
import android.content.Context;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import com.amazonaws.Request;
import com.amazonaws.Response;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.handlers.RequestHandler2;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import org.cryptomator.data.cloud.okhttplogging.HttpLoggingInterceptor;
import org.cryptomator.domain.S3Cloud;
import org.cryptomator.util.SharedPreferencesHandler;
import org.cryptomator.util.crypto.CredentialCryptor;
import org.cryptomator.util.file.LruFileCacheUtil;
import java.util.concurrent.TimeUnit;
import io.minio.MinioClient;
import okhttp3.Cache;
import okhttp3.CacheControl;
import okhttp3.Interceptor;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import timber.log.Timber;
import static org.cryptomator.data.util.NetworkTimeout.CONNECTION;
import static org.cryptomator.data.util.NetworkTimeout.READ;
import static org.cryptomator.data.util.NetworkTimeout.WRITE;
import static org.cryptomator.util.file.LruFileCacheUtil.Cache.S3;
class S3ClientFactory {
private AmazonS3 apiClient;
private MinioClient apiClient;
public AmazonS3 getClient(S3Cloud cloud, Context context) {
private static Interceptor httpLoggingInterceptor(Context context) {
return new HttpLoggingInterceptor(message -> Timber.tag("OkHttp").d(message), context);
}
public MinioClient getClient(S3Cloud cloud, Context context) {
if (apiClient == null) {
apiClient = createApiClient(cloud, context);
}
return apiClient;
}
private AmazonS3 createApiClient(S3Cloud cloud, Context context) {
Region region = Region.getRegion(Regions.DEFAULT_REGION);
String endpoint = null;
private MinioClient createApiClient(S3Cloud cloud, Context context) {
final SharedPreferencesHandler sharedPreferencesHandler = new SharedPreferencesHandler(context);
MinioClient.Builder minioClientBuilder = MinioClient.builder();
if (cloud.s3Endpoint() != null) {
minioClientBuilder.endpoint(cloud.s3Endpoint());
} else {
minioClientBuilder.endpoint("https://s3.amazonaws.com");
}
if (cloud.s3Region() != null) {
region = Region.getRegion(cloud.s3Region());
} else if (cloud.s3Endpoint() != null) {
endpoint = cloud.s3Endpoint();
minioClientBuilder.region(cloud.s3Region());
}
AmazonS3Client client = new AmazonS3Client(new BasicAWSCredentials(decrypt(cloud.accessKey(), context), decrypt(cloud.secretKey(), context)), region);
OkHttpClient.Builder httpClientBuilder = new OkHttpClient() //
.newBuilder() //
.connectTimeout(CONNECTION.getTimeout(), CONNECTION.getUnit()) //
.readTimeout(READ.getTimeout(), READ.getUnit()) //
.writeTimeout(WRITE.getTimeout(), WRITE.getUnit()) //
.addInterceptor(httpLoggingInterceptor(context));
if (endpoint != null) {
client.setEndpoint(cloud.s3Endpoint());
if (sharedPreferencesHandler.useLruCache()) {
final Cache cache = new Cache(new LruFileCacheUtil(context).resolve(S3), sharedPreferencesHandler.lruCacheSize());
httpClientBuilder.cache(cache).addInterceptor(provideOfflineCacheInterceptor(context));
}
client.addRequestHandler(new LoggingAwareRequestHandler());
return minioClientBuilder.credentials(decrypt(cloud.accessKey(), context), decrypt(cloud.secretKey(), context)).httpClient(httpClientBuilder.build()).build();
}
return client;
private static Interceptor provideOfflineCacheInterceptor(final Context context) {
return chain -> {
Request request = chain.request();
if (isNetworkAvailable(context)) {
final CacheControl cacheControl = new CacheControl.Builder() //
.maxAge(0, TimeUnit.DAYS) //
.build();
request = request.newBuilder() //
.cacheControl(cacheControl) //
.build();
}
return chain.proceed(request);
};
}
private static boolean isNetworkAvailable(final Context context) {
ConnectivityManager connectivityManager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
NetworkInfo activeNetworkInfo = connectivityManager.getActiveNetworkInfo();
return activeNetworkInfo != null && activeNetworkInfo.isConnected();
}
private String decrypt(String password, Context context) {
@ -53,35 +99,4 @@ class S3ClientFactory {
.getInstance(context) //
.decrypt(password);
}
private static class LoggingAwareRequestHandler extends RequestHandler2 {
@Override
public void beforeRequest(Request<?> request) {
Timber.tag("S3Client").d("Sending request (%s) %s", request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), request.toString());
}
@Override
public void afterResponse(Request<?> request, Response<?> response) {
Timber.tag("S3Client").d( //
"Response received (%s) with status %s (%s)", //
request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), //
response.getHttpResponse().getStatusText(), //
response.getHttpResponse().getStatusCode());
}
@Override
public void afterError(Request<?> request, Response<?> response, Exception e) {
if (response != null) {
Timber.tag("S3Client").e( //
e, //
"Error occurred (%s) with status %s (%s)", //
request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), //
response.getHttpResponse().getStatusText(), //
response.getHttpResponse().getStatusCode());
} else {
Timber.tag("S3Client").e(e, "Error occurred (%s)", request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano());
}
}
}
}

View File

@ -2,8 +2,6 @@ package org.cryptomator.data.cloud.s3;
import android.content.Context;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import org.cryptomator.data.cloud.InterceptingCloudContentRepository;
import org.cryptomator.domain.S3Cloud;
import org.cryptomator.domain.exception.BackendException;
@ -23,6 +21,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import io.minio.errors.ErrorResponseException;
import static org.cryptomator.util.ExceptionUtil.contains;
class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Cloud, S3Node, S3Folder, S3File> {
@ -42,9 +42,9 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
}
private void throwNoSuchBucketExceptionIfRequired(Exception e) throws NoSuchBucketException {
if (e instanceof AmazonS3Exception) {
String errorCode = ((AmazonS3Exception)e).getErrorCode();
if(S3CloudApiExceptions.isNoSuchBucketException(errorCode)) {
if (e instanceof ErrorResponseException) {
String errorCode = ((ErrorResponseException) e).errorResponse().code();
if (S3CloudApiExceptions.isNoSuchBucketException(errorCode)) {
throw new NoSuchBucketException(cloud.s3Bucket());
}
}
@ -57,8 +57,8 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
}
private void throwWrongCredentialsExceptionIfRequired(Exception e) {
if (e instanceof AmazonS3Exception) {
String errorCode = ((AmazonS3Exception) e).getErrorCode();
if (e instanceof ErrorResponseException) {
String errorCode = ((ErrorResponseException) e).errorResponse().code();
if (S3CloudApiExceptions.isAccessProblem(errorCode)) {
throw new WrongCredentialsException(cloud);
}
@ -79,7 +79,7 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
@Override
public S3Folder resolve(S3Cloud cloud, String path) throws BackendException {
return this.cloud.resolve(path);
return this.cloud.resolve(path);
}
@Override
@ -107,7 +107,7 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
@Override
public boolean exists(S3Node node) throws BackendException {
return cloud.exists(node);
return cloud.exists(node);
}
@Override
@ -158,7 +158,7 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
@Override
public void read(S3File file, Optional<File> encryptedTmpFile, OutputStream data, ProgressAware<DownloadState> progressAware) throws BackendException {
try {
cloud.read(file, encryptedTmpFile, data, progressAware);
cloud.read(file, data, progressAware);
} catch (IOException e) {
throw new FatalBackendException(e);
}
@ -175,7 +175,7 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository<S3Clou
@Override
public String checkAuthenticationAndRetrieveCurrentAccount(S3Cloud cloud) throws BackendException {
return this.cloud.checkAuthenticationAndRetrieveCurrentAccount();
return this.cloud.checkAuthentication();
}
@Override

View File

@ -1,8 +1,5 @@
package org.cryptomator.data.cloud.s3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.cryptomator.util.Optional;
import java.util.Date;
@ -11,16 +8,10 @@ class S3CloudNodeFactory {
private static final String DELIMITER = "/";
public static S3File file(S3Folder parent, S3ObjectSummary file) {
String name = getNameFromKey(file.getKey());
return new S3File(parent, name, getNodePath(parent, name), Optional.ofNullable(file.getSize()), Optional.ofNullable(file.getLastModified()));
public static S3File file(S3Folder parent, String name) {
return new S3File(parent, name, getNodePath(parent, name), Optional.empty(), Optional.empty());
}
public static S3File file(S3Folder parent, String name, ObjectMetadata file) {
return new S3File(parent, name, getNodePath(parent, name), Optional.ofNullable(file.getContentLength()), Optional.ofNullable(file.getLastModified()));
}
public static S3File file(S3Folder parent, String name, Optional<Long> size) {
return new S3File(parent, name, getNodePath(parent, name), size, Optional.empty());
}
@ -33,11 +24,6 @@ class S3CloudNodeFactory {
return new S3File(parent, name, getNodePath(parent, name), size, lastModified);
}
public static S3Folder folder(S3Folder parent, S3ObjectSummary folder) {
String name = getNameFromKey(folder.getKey());
return new S3Folder(parent, name, getNodePath(parent, name));
}
public static S3Folder folder(S3Folder parent, String name) {
return new S3Folder(parent, name, getNodePath(parent, name));
}
@ -53,17 +39,9 @@ class S3CloudNodeFactory {
public static String getNameFromKey(String key) {
String name = key;
if (key.endsWith(DELIMITER)) {
name = key.substring(0, key.length() -1);
name = key.substring(0, key.length() - 1);
}
return name.contains(DELIMITER) ? name.substring(name.lastIndexOf(DELIMITER) + 1) : name;
}
public static S3Node from(S3Folder parent, S3ObjectSummary objectSummary) {
if (objectSummary.getKey().endsWith(DELIMITER)) {
return folder(parent, objectSummary);
} else {
return file(parent, objectSummary);
}
}
}

View File

@ -2,27 +2,9 @@ package org.cryptomator.data.cloud.s3;
import android.content.Context;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.mobileconnectors.s3.transferutility.TransferListener;
import com.amazonaws.mobileconnectors.s3.transferutility.TransferState;
import com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility;
import com.amazonaws.mobileconnectors.s3.transferutility.UploadOptions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.Owner;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.tomclaw.cache.DiskLruCache;
import org.cryptomator.data.util.CopyStream;
import org.cryptomator.data.util.TransferredBytesAwareInputStream;
import org.cryptomator.data.util.TransferredBytesAwareOutputStream;
import org.cryptomator.domain.S3Cloud;
import org.cryptomator.domain.exception.BackendException;
import org.cryptomator.domain.exception.CloudNodeAlreadyExistsException;
@ -30,7 +12,6 @@ import org.cryptomator.domain.exception.FatalBackendException;
import org.cryptomator.domain.exception.ForbiddenException;
import org.cryptomator.domain.exception.NoSuchBucketException;
import org.cryptomator.domain.exception.NoSuchCloudFileException;
import org.cryptomator.domain.exception.UnauthorizedException;
import org.cryptomator.domain.exception.authentication.WrongCredentialsException;
import org.cryptomator.domain.usecases.ProgressAware;
import org.cryptomator.domain.usecases.cloud.DataSource;
@ -38,31 +19,39 @@ import org.cryptomator.domain.usecases.cloud.DownloadState;
import org.cryptomator.domain.usecases.cloud.Progress;
import org.cryptomator.domain.usecases.cloud.UploadState;
import org.cryptomator.util.Optional;
import org.cryptomator.util.SharedPreferencesHandler;
import org.cryptomator.util.concurrent.CompletableFuture;
import org.cryptomator.util.file.LruFileCacheUtil;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicLong;
import timber.log.Timber;
import io.minio.BucketExistsArgs;
import io.minio.CopyObjectArgs;
import io.minio.CopySource;
import io.minio.GetObjectArgs;
import io.minio.GetObjectResponse;
import io.minio.ListObjectsArgs;
import io.minio.MinioClient;
import io.minio.ObjectWriteResponse;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import io.minio.RemoveObjectsArgs;
import io.minio.Result;
import io.minio.StatObjectArgs;
import io.minio.StatObjectResponse;
import io.minio.errors.ErrorResponseException;
import io.minio.messages.DeleteError;
import io.minio.messages.DeleteObject;
import io.minio.messages.Item;
import static org.cryptomator.domain.usecases.cloud.Progress.progress;
import static org.cryptomator.util.file.LruFileCacheUtil.Cache.S3;
import static org.cryptomator.util.file.LruFileCacheUtil.retrieveFromLruCache;
import static org.cryptomator.util.file.LruFileCacheUtil.storeToLruCache;
class S3Impl {
private static final long CHUNKED_UPLOAD_MAX_SIZE = 100L << 20;
private static final String DELIMITER = "/";
private final S3ClientFactory clientFactory = new S3ClientFactory();
@ -70,9 +59,6 @@ class S3Impl {
private final RootS3Folder root;
private final Context context;
private final SharedPreferencesHandler sharedPreferencesHandler;
private DiskLruCache diskLruCache;
S3Impl(Context context, S3Cloud cloud) {
if (cloud.accessKey() == null || cloud.secretKey() == null) {
throw new WrongCredentialsException(cloud);
@ -81,10 +67,9 @@ class S3Impl {
this.context = context;
this.cloud = cloud;
this.root = new RootS3Folder(cloud);
this.sharedPreferencesHandler = new SharedPreferencesHandler(context);
}
private AmazonS3 client() {
private MinioClient client() {
return clientFactory.getClient(cloud, context);
}
@ -118,31 +103,40 @@ class S3Impl {
return S3CloudNodeFactory.folder(parent, name, parent.getKey() + name);
}
public boolean exists(S3Node node) {
public boolean exists(S3Node node) throws BackendException {
String key = node.getKey();
ListObjectsV2Result result = client().listObjectsV2(cloud.s3Bucket(), key);
return result.getObjectSummaries().size() > 0;
try {
client().statObject(StatObjectArgs.builder().bucket(cloud.s3Bucket()).object(key).build());
} catch (ErrorResponseException e) {
if (e.errorResponse().code().equals("NoSuchKey")) {
return false;
}
throw new FatalBackendException(e);
} catch (Exception ex) {
handleApiError(ex, node.getPath());
}
return true;
}
public List<S3Node> list(S3Folder folder) throws IOException, BackendException {
List<S3Node> result = new ArrayList<>();
ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(cloud.s3Bucket()).withPrefix(folder.getKey()).withDelimiter(DELIMITER);
ListObjectsV2Result listObjects = client().listObjectsV2(request);
for (String prefix : listObjects.getCommonPrefixes()) {
// add folders
result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(prefix)));
}
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
// add files but skip parent folder
if (!objectSummary.getKey().equals(listObjects.getPrefix())) {
result.add(S3CloudNodeFactory.file(folder, objectSummary));
ListObjectsArgs request = ListObjectsArgs.builder().bucket(cloud.s3Bucket()).prefix(folder.getKey()).delimiter(DELIMITER).build();
Iterable<Result<Item>> listObjects = client().listObjects(request);
for (Result<Item> object : listObjects) {
try {
Item item = object.get();
if (item.isDir()) {
result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(item.objectName())));
} else {
S3File file = S3CloudNodeFactory.file(folder, S3CloudNodeFactory.getNameFromKey(item.objectName()), Optional.of(item.size()), Optional.of(Date.from(item.lastModified().toInstant())));
result.add(file);
}
} catch (Exception ex) {
handleApiError(ex, folder.getPath());
}
}
return result;
}
@ -154,16 +148,17 @@ class S3Impl {
);
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0);
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
try {
PutObjectRequest putObjectRequest = new PutObjectRequest(cloud.s3Bucket(), folder.getKey(), emptyContent, metadata);
client().putObject(putObjectRequest);
} catch(AmazonS3Exception ex) {
handleApiError(ex, folder.getName());
PutObjectArgs putObjectArgs = PutObjectArgs //
.builder() //
.bucket(cloud.s3Bucket()) //
.object(folder.getKey()) //
.stream(new ByteArrayInputStream(new byte[0]), 0, -1) //
.build();
client().putObject(putObjectArgs);
} catch (Exception ex) {
handleApiError(ex, folder.getPath());
}
return S3CloudNodeFactory.folder(folder.getParent(), folder.getName());
@ -175,28 +170,58 @@ class S3Impl {
}
if (source instanceof S3Folder) {
ListObjectsV2Result listObjects = client().listObjectsV2(cloud.s3Bucket(), source.getPath());
List<S3Node> nodes = list((S3Folder) source);
if (listObjects.getObjectSummaries().size() > 0) {
List<DeleteObject> objectsToDelete = new LinkedList<>();
List<DeleteObjectsRequest.KeyVersion> objectsToDelete = new ArrayList<>();
for (S3Node node : nodes) {
objectsToDelete.add(new DeleteObject(node.getKey()));
for (S3ObjectSummary summary : listObjects.getObjectSummaries()) {
objectsToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
String destinationKey = summary.getKey().replace(source.getPath(), target.getPath());
client().copyObject(cloud.s3Bucket(), summary.getKey(), cloud.s3Bucket(), destinationKey);
String targetKey;
if (node instanceof S3Folder) {
targetKey = S3CloudNodeFactory.folder((S3Folder) target, node.getName()).getKey();
} else {
targetKey = S3CloudNodeFactory.file((S3Folder) target, node.getName()).getKey();
}
CopySource copySource = CopySource.builder().bucket(cloud.s3Bucket()).object(node.getPath()).build();
CopyObjectArgs copyObjectArgs = CopyObjectArgs.builder().bucket(cloud.s3Bucket()).object(targetKey).source(copySource).build();
try {
client().copyObject(copyObjectArgs);
} catch (Exception ex) {
handleApiError(ex, source.getPath());
}
client().deleteObjects(new DeleteObjectsRequest(cloud.s3Bucket()).withKeys(objectsToDelete));
} else {
throw new NoSuchCloudFileException(source.getPath());
}
RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket(cloud.s3Bucket()).objects(objectsToDelete).build();
for (Result<DeleteError> result : client().removeObjects(removeObjectsArgs)) {
try {
result.get();
} catch (Exception ex) {
handleApiError(ex, source.getPath());
}
}
return S3CloudNodeFactory.folder(target.getParent(), target.getName());
} else {
CopyObjectResult result = client().copyObject(cloud.s3Bucket(), source.getPath(), cloud.s3Bucket(), target.getPath());
client().deleteObject(cloud.s3Bucket(), source.getPath());
return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File) source).getSize(), Optional.of(result.getLastModifiedDate()));
CopySource copySource = CopySource.builder().bucket(cloud.s3Bucket()).object(source.getPath()).build();
CopyObjectArgs copyObjectArgs = CopyObjectArgs.builder().bucket(cloud.s3Bucket()).object(target.getPath()).source(copySource).build();
try {
ObjectWriteResponse result = client().copyObject(copyObjectArgs);
delete(source);
Date lastModified = result.headers().getDate("Last-Modified");
return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File) source).getSize(), Optional.ofNullable(lastModified));
} catch (Exception ex) {
handleApiError(ex, source.getPath());
}
}
throw new IllegalStateException();
}
public S3File write(S3File file, DataSource data, final ProgressAware<UploadState> progressAware, boolean replace, long size) throws IOException, BackendException {
@ -206,212 +231,157 @@ class S3Impl {
progressAware.onProgress(Progress.started(UploadState.upload(file)));
final CompletableFuture<Optional<ObjectMetadata>> result = new CompletableFuture<>();
try {
if (size <= CHUNKED_UPLOAD_MAX_SIZE) {
uploadFile(file, data, progressAware, result, size);
} else {
uploadChunkedFile(file, data, progressAware, result, size);
}
} catch(AmazonS3Exception ex) {
handleApiError(ex, file.getName());
}
try {
Optional<ObjectMetadata> objectMetadataOptional = result.get();
ObjectMetadata objectMetadata = objectMetadataOptional.orElseGet(() -> client().getObjectMetadata(cloud.s3Bucket(), file.getPath()));
progressAware.onProgress(Progress.completed(UploadState.upload(file)));
return S3CloudNodeFactory.file(file.getParent(), file.getName(), objectMetadata);
} catch (ExecutionException | InterruptedException e) {
throw new FatalBackendException(e);
}
}
private void uploadFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<Optional<ObjectMetadata>> result, final long size) //
throws IOException {
AtomicLong bytesTransferred = new AtomicLong(0);
ProgressListener listener = progressEvent -> {
bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred());
progressAware.onProgress( //
progress(UploadState.upload(file)) //
.between(0) //
.and(size) //
.withValue(bytesTransferred.get()));
};
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(data.size(context).get());
PutObjectRequest request = new PutObjectRequest(cloud.s3Bucket(), file.getPath(), data.open(context), metadata);
request.setGeneralProgressListener(listener);
result.complete(Optional.of(client().putObject(request).getMetadata()));
}
private void uploadChunkedFile(final S3File file, DataSource data, final ProgressAware<UploadState> progressAware, CompletableFuture<Optional<ObjectMetadata>> result, final long size) //
throws IOException {
TransferUtility tu = TransferUtility //
.builder() //
.s3Client(client()) //
.context(context) //
.defaultBucket(cloud.s3Bucket()) //
.build();
TransferListener transferListener = new TransferListener() {
try (TransferredBytesAwareDataSource out = new TransferredBytesAwareDataSource(data) {
@Override
public void onStateChanged(int id, TransferState state) {
if (state.equals(TransferState.COMPLETED)) {
progressAware.onProgress(Progress.completed(UploadState.upload(file)));
result.complete(Optional.empty());
}
}
@Override
public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) {
public void bytesTransferred(long transferred) {
progressAware.onProgress( //
progress(UploadState.upload(file)) //
.between(0) //
.and(size) //
.withValue(bytesCurrent));
.withValue(transferred));
}
@Override
public void onError(int id, Exception ex) {
result.fail(ex);
}) {
try {
PutObjectArgs putObjectArgs = PutObjectArgs.builder().bucket(cloud.s3Bucket()).object(file.getKey()).stream(out.open(context), data.size(context).get(), -1).build();
client().putObject(putObjectArgs);
StatObjectResponse statObjectResponse = client().statObject(StatObjectArgs //
.builder() //
.bucket(cloud.s3Bucket()) //
.object(file.getPath()) //
.build());
progressAware.onProgress(Progress.completed(UploadState.upload(file)));
return S3CloudNodeFactory.file(file.getParent(), file.getName(), Optional.of(statObjectResponse.size()), Optional.of(Date.from(statObjectResponse.lastModified().toInstant())));
} catch (Exception ex) {
handleApiError(ex, file.getPath());
}
};
UploadOptions uploadOptions = UploadOptions.builder().transferListener(transferListener).build();
tu.upload(file.getPath(), data.open(context), uploadOptions);
}
public void read(S3File file, Optional<File> encryptedTmpFile, OutputStream data, final ProgressAware<DownloadState> progressAware) throws IOException, BackendException {
progressAware.onProgress(Progress.started(DownloadState.download(file)));
Optional<String> cacheKey = Optional.empty();
Optional<File> cacheFile = Optional.empty();
ListObjectsV2Result listObjects;
if (sharedPreferencesHandler.useLruCache() && createLruCache(sharedPreferencesHandler.lruCacheSize())) {
listObjects = client().listObjectsV2(cloud.s3Bucket(), file.getKey());
if (listObjects.getObjectSummaries().size() != 1) {
throw new NoSuchCloudFileException(file.getKey());
}
S3ObjectSummary summary = listObjects.getObjectSummaries().get(0);
cacheKey = Optional.of(summary.getKey() + summary.getETag());
File cachedFile = diskLruCache.get(cacheKey.get());
cacheFile = cachedFile != null ? Optional.of(cachedFile) : Optional.empty();
}
if (sharedPreferencesHandler.useLruCache() && cacheFile.isPresent()) {
try {
retrieveFromLruCache(cacheFile.get(), data);
} catch (IOException e) {
Timber.tag("S3Impl").w(e, "Error while retrieving content from Cache, get from web request");
writeToData(file, data, encryptedTmpFile, cacheKey, progressAware);
}
} else {
writeToData(file, data, encryptedTmpFile, cacheKey, progressAware);
throw new IllegalStateException();
}
public void read(S3File file, OutputStream data, final ProgressAware<DownloadState> progressAware) throws IOException, BackendException {
progressAware.onProgress(Progress.started(DownloadState.download(file)));
GetObjectArgs getObjectArgs = GetObjectArgs.builder().bucket(cloud.s3Bucket()).object(file.getPath()).build();
try (GetObjectResponse response = client().getObject(getObjectArgs); //
TransferredBytesAwareOutputStream out = new TransferredBytesAwareOutputStream(data) {
@Override
public void bytesTransferred(long transferred) {
progressAware.onProgress( //
progress(DownloadState.download(file)) //
.between(0) //
.and(file.getSize().orElse(Long.MAX_VALUE)) //
.withValue(transferred));
}
}) {
CopyStream.copyStreamToStream(response, out);
} catch (Exception ex) {
handleApiError(ex, file.getPath());
}
progressAware.onProgress(Progress.completed(DownloadState.download(file)));
}
private void writeToData(final S3File file, //
final OutputStream data, //
final Optional<File> encryptedTmpFile, //
final Optional<String> cacheKey, //
final ProgressAware<DownloadState> progressAware) throws IOException, BackendException {
AtomicLong bytesTransferred = new AtomicLong(0);
ProgressListener listener = progressEvent -> {
bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred());
progressAware.onProgress( //
progress(DownloadState.download(file)) //
.between(0) //
.and(file.getSize().orElse(Long.MAX_VALUE)) //
.withValue(bytesTransferred.get()));
};
GetObjectRequest request = new GetObjectRequest(cloud.s3Bucket(), file.getPath());
request.setGeneralProgressListener(listener);
try {
S3Object s3Object = client().getObject(request);
CopyStream.copyStreamToStream(s3Object.getObjectContent(), data);
if (sharedPreferencesHandler.useLruCache() && encryptedTmpFile.isPresent() && cacheKey.isPresent()) {
try {
storeToLruCache(diskLruCache, cacheKey.get(), encryptedTmpFile.get());
} catch (IOException e) {
Timber.tag("S3Impl").e(e, "Failed to write downloaded file in LRU cache");
}
}
} catch(AmazonS3Exception ex) {
handleApiError(ex, file.getName());
}
}
public void delete(S3Node node) throws IOException, BackendException {
if (node instanceof S3Folder) {
List<S3ObjectSummary> summaries = client().listObjectsV2(cloud.s3Bucket(), node.getPath()).getObjectSummaries();
List<KeyVersion> keys = new ArrayList<>();
for (S3ObjectSummary summary : summaries) {
keys.add(new KeyVersion(summary.getKey()));
List<DeleteObject> objectsToDelete = new LinkedList<>();
ListObjectsArgs request = ListObjectsArgs.builder().bucket(cloud.s3Bucket()).prefix(node.getKey()).recursive(true).delimiter(DELIMITER).build();
Iterable<Result<Item>> listObjects = client().listObjects(request);
for (Result<Item> object : listObjects) {
try {
Item item = object.get();
objectsToDelete.add(new DeleteObject(item.objectName()));
} catch (Exception e) {
handleApiError(e, node.getPath());
}
}
DeleteObjectsRequest request = new DeleteObjectsRequest(cloud.s3Bucket());
request.withKeys(keys);
RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket(cloud.s3Bucket()).objects(objectsToDelete).build();
Iterable<Result<DeleteError>> results = client().removeObjects(removeObjectsArgs);
for (Result<DeleteError> result : results) {
try {
DeleteError error = result.get();
System.out.println("Error in deleting object " + error.objectName() + "; " + error.message());
} catch (Exception e) {
handleApiError(e, node.getPath());
}
}
client().deleteObjects(request);
} else {
client().deleteObject(cloud.s3Bucket(), node.getPath());
}
}
public String checkAuthenticationAndRetrieveCurrentAccount() throws NoSuchBucketException {
if (!client().doesBucketExist(cloud.s3Bucket())) {
throw new NoSuchBucketException(cloud.s3Bucket());
}
Owner currentAccount = client() //
.getS3AccountOwner();
return currentAccount.getDisplayName();
}
private boolean createLruCache(int cacheSize) {
if (diskLruCache == null) {
RemoveObjectArgs removeObjectArgs = RemoveObjectArgs.builder().bucket(cloud.s3Bucket()).object(node.getKey()).build();
try {
diskLruCache = DiskLruCache.create(new LruFileCacheUtil(context).resolve(S3), cacheSize);
} catch (IOException e) {
Timber.tag("S3Impl").e(e, "Failed to setup LRU cache");
return false;
client().removeObject(removeObjectArgs);
} catch (Exception e) {
handleApiError(e, "");
}
}
return true;
}
private void handleApiError(AmazonS3Exception ex, String name) throws BackendException {
String errorCode = ex.getErrorCode();
if (S3CloudApiExceptions.isAccessProblem(errorCode)) {
throw new ForbiddenException();
} else if (S3CloudApiErrorCodes.NO_SUCH_BUCKET.getValue().equals(errorCode)) {
throw new NoSuchBucketException(name);
} else if (S3CloudApiErrorCodes.NO_SUCH_KEY.getValue().equals(errorCode)) {
throw new NoSuchCloudFileException(name);
} else {
public String checkAuthentication() throws NoSuchBucketException, BackendException {
try {
if (!client().bucketExists(BucketExistsArgs.builder().bucket(cloud.s3Bucket()).build())) {
throw new NoSuchBucketException(cloud.s3Bucket());
}
} catch (Exception e) {
handleApiError(e, "");
}
return "";
}
private void handleApiError(Exception ex, String name) throws BackendException {
if (ex instanceof ErrorResponseException) {
String errorCode = ((ErrorResponseException) ex).errorResponse().code();
if (S3CloudApiExceptions.isAccessProblem(errorCode)) {
throw new ForbiddenException();
} else if (S3CloudApiErrorCodes.NO_SUCH_BUCKET.getValue().equals(errorCode)) {
throw new NoSuchBucketException(name);
} else if (S3CloudApiErrorCodes.NO_SUCH_KEY.getValue().equals(errorCode)) {
throw new NoSuchCloudFileException(name);
} else {
throw new FatalBackendException(ex);
}
} else {
throw new FatalBackendException(ex);
}
}
private static abstract class TransferredBytesAwareDataSource implements DataSource {
private final DataSource data;
TransferredBytesAwareDataSource(DataSource data) {
this.data = data;
}
@Override
public Optional<Long> size(Context context) {
return data.size(context);
}
@Override
public InputStream open(Context context) throws IOException {
return new TransferredBytesAwareInputStream(data.open(context)) {
@Override
public void bytesTransferred(long transferred) {
S3Impl.TransferredBytesAwareDataSource.this.bytesTransferred(transferred);
}
};
}
@Override
public void close() throws IOException {
data.close();
}
public abstract void bytesTransferred(long transferred);
@Override
public DataSource decorate(DataSource delegate) {
return delegate;
}
}
}

View File

@ -99,6 +99,13 @@
android:action="android.intent.action.VIEW"
android:data="https://github.com/microsoftgraph/msgraph-sdk-android" />
</Preference>
<Preference
android:summary="Apache License v2"
android:title="MinIO Client SDK for Java">
<intent
android:action="android.intent.action.VIEW"
android:data="https://github.com/minio/minio-java" />
</Preference>
<Preference
android:summary="Apache License v2"
android:title="OkHttp (Square, Inc.)">
@ -134,13 +141,6 @@
android:action="android.intent.action.VIEW"
android:data="https://github.com/ReactiveX/RxJava" />
</Preference>
<Preference
android:summary="Apache License v2"
android:title="AWS SDK for Android">
<intent
android:action="android.intent.action.VIEW"
android:data="https://github.com/aws-amplify/aws-sdk-android" />
</Preference>
<Preference
android:summary="Apache License v2"
android:title="Subsampling Scale Image View">