diff --git a/buildsystem/dependencies.gradle b/buildsystem/dependencies.gradle index bee1b83d..c17c0d55 100644 --- a/buildsystem/dependencies.gradle +++ b/buildsystem/dependencies.gradle @@ -53,8 +53,6 @@ ext { // do not update to 1.4.0 until minsdk is 7.x (or desugaring works better) otherwise it will crash on 6.x cryptolibVersion = '2.0.0-rc1' - awsAndroidSdkS3 = '2.23.0' - dropboxVersion = '4.0.0' googleApiServicesVersion = 'v3-rev197-1.25.0' @@ -63,6 +61,9 @@ ext { msgraphVersion = '2.10.0' + minIoVersion = '8.2.1' + staxVersion = '1.2.0' // needed for minIO + commonsCodecVersion = '1.15' recyclerViewFastScrollVersion = '2.0.1' @@ -95,8 +96,6 @@ ext { jsonWebTokenApiVersion = '0.11.2' - - dependencies = [ android : "com.google.android:android:${androidVersion}", androidAnnotations : "androidx.annotation:annotation:${androidSupportAnnotationsVersion}", @@ -107,7 +106,6 @@ ext { androidxViewpager : "androidx.viewpager:viewpager:${androidxViewpagerVersion}", androidxSwiperefresh : "androidx.swiperefreshlayout:swiperefreshlayout:${androidxSwiperefreshVersion}", androidxPreference : "androidx.preference:preference:${androidxPreferenceVersion}", - awsAndroidS3 : "com.amazonaws:aws-android-sdk-s3:${awsAndroidSdkS3}", documentFile : "androidx.documentfile:documentfile:${androidxDocumentfileVersion}", recyclerView : "androidx.recyclerview:recyclerview:${androidxRecyclerViewVersion}", androidxTestCore : "androidx.test:core:${androidxTestCoreVersion}", @@ -132,9 +130,10 @@ ext { junitParams : "org.junit.jupiter:junit-jupiter-params:${jUnitVersion}", junit4 : "org.junit.jupiter:junit-jupiter:${jUnit4Version}", junit4Engine : "org.junit.vintage:junit-vintage-engine:${jUnitVersion}", - msgraph : "com.microsoft.graph:microsoft-graph:${msgraphVersion}", + minIo : "io.minio:minio:${minIoVersion}", mockito : "org.mockito:mockito-core:${mockitoVersion}", mockitoInline : "org.mockito:mockito-inline:${mockitoInlineVersion}", + msgraph : "com.microsoft.graph:microsoft-graph:${msgraphVersion}", multidex : "androidx.multidex:multidex:${multidexVersion}", okHttp : "com.squareup.okhttp3:okhttp:${okHttpVersion}", okHttpDigest : "com.burgstaller:okhttp-digest:${okHttpDigestVersion}", @@ -142,6 +141,7 @@ ext { rxJava : "io.reactivex.rxjava2:rxjava:${rxJavaVersion}", rxAndroid : "io.reactivex.rxjava2:rxandroid:${rxAndroidVersion}", rxBinding : "com.jakewharton.rxbinding2:rxbinding:${rxBindingVersion}", + stax : "stax:stax:${staxVersion}", testingSupportLib : "com.android.support.test:testing-support-lib:${testingSupportLibVersion}", timber : "com.jakewharton.timber:timber:${timberVersion}", velocity : "org.apache.velocity:velocity:${velocityVersion}", diff --git a/data/build.gradle b/data/build.gradle index 137c58da..4d884510 100644 --- a/data/build.gradle +++ b/data/build.gradle @@ -78,7 +78,7 @@ android { } greendao { - schemaVersion 7 + schemaVersion 8 } configurations.all { @@ -110,10 +110,12 @@ dependencies { implementation dependencies.jsonWebTokenJson // cloud - implementation dependencies.awsAndroidS3 implementation dependencies.dropbox implementation dependencies.msgraph + implementation dependencies.stax + compile dependencies.minIo + playstoreImplementation dependencies.googlePlayServicesAuth apkstoreImplementation dependencies.googlePlayServicesAuth diff --git a/data/src/main/java/org/cryptomator/data/cloud/s3/S3ClientFactory.java b/data/src/main/java/org/cryptomator/data/cloud/s3/S3ClientFactory.java index ed0c1838..e46659a3 100644 --- a/data/src/main/java/org/cryptomator/data/cloud/s3/S3ClientFactory.java +++ b/data/src/main/java/org/cryptomator/data/cloud/s3/S3ClientFactory.java @@ -1,51 +1,93 @@ package org.cryptomator.data.cloud.s3; import android.content.Context; +import android.net.ConnectivityManager; +import android.net.NetworkInfo; -import com.amazonaws.Request; -import com.amazonaws.Response; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.handlers.RequestHandler2; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; - +import org.cryptomator.data.cloud.okhttplogging.HttpLoggingInterceptor; import org.cryptomator.domain.S3Cloud; +import org.cryptomator.util.SharedPreferencesHandler; import org.cryptomator.util.crypto.CredentialCryptor; +import org.cryptomator.util.file.LruFileCacheUtil; +import java.util.concurrent.TimeUnit; + +import io.minio.MinioClient; +import okhttp3.Cache; +import okhttp3.CacheControl; +import okhttp3.Interceptor; +import okhttp3.OkHttpClient; +import okhttp3.Request; import timber.log.Timber; +import static org.cryptomator.data.util.NetworkTimeout.CONNECTION; +import static org.cryptomator.data.util.NetworkTimeout.READ; +import static org.cryptomator.data.util.NetworkTimeout.WRITE; +import static org.cryptomator.util.file.LruFileCacheUtil.Cache.S3; + class S3ClientFactory { - private AmazonS3 apiClient; + private MinioClient apiClient; - public AmazonS3 getClient(S3Cloud cloud, Context context) { + private static Interceptor httpLoggingInterceptor(Context context) { + return new HttpLoggingInterceptor(message -> Timber.tag("OkHttp").d(message), context); + } + + public MinioClient getClient(S3Cloud cloud, Context context) { if (apiClient == null) { apiClient = createApiClient(cloud, context); } return apiClient; } - private AmazonS3 createApiClient(S3Cloud cloud, Context context) { - Region region = Region.getRegion(Regions.DEFAULT_REGION); - String endpoint = null; + private MinioClient createApiClient(S3Cloud cloud, Context context) { + final SharedPreferencesHandler sharedPreferencesHandler = new SharedPreferencesHandler(context); - if (cloud.s3Region() != null) { - region = Region.getRegion(cloud.s3Region()); - } else if (cloud.s3Endpoint() != null) { - endpoint = cloud.s3Endpoint(); + MinioClient.Builder minioClientBuilder = MinioClient.builder(); + + minioClientBuilder.endpoint(cloud.s3Endpoint()); + minioClientBuilder.region(cloud.s3Region()); + + OkHttpClient.Builder httpClientBuilder = new OkHttpClient() // + .newBuilder() // + .connectTimeout(CONNECTION.getTimeout(), CONNECTION.getUnit()) // + .readTimeout(READ.getTimeout(), READ.getUnit()) // + .writeTimeout(WRITE.getTimeout(), WRITE.getUnit()) // + .addInterceptor(httpLoggingInterceptor(context)); + + if (sharedPreferencesHandler.useLruCache()) { + final Cache cache = new Cache(new LruFileCacheUtil(context).resolve(S3), sharedPreferencesHandler.lruCacheSize()); + httpClientBuilder.cache(cache).addInterceptor(provideOfflineCacheInterceptor(context)); } - AmazonS3Client client = new AmazonS3Client(new BasicAWSCredentials(decrypt(cloud.accessKey(), context), decrypt(cloud.secretKey(), context)), region); + return minioClientBuilder // + .credentials(decrypt(cloud.accessKey(), context), decrypt(cloud.secretKey(), context)) // + .httpClient(httpClientBuilder.build()) // + .build(); + } - if (endpoint != null) { - client.setEndpoint(cloud.s3Endpoint()); - } + private static Interceptor provideOfflineCacheInterceptor(final Context context) { + return chain -> { + Request request = chain.request(); - client.addRequestHandler(new LoggingAwareRequestHandler()); + if (isNetworkAvailable(context)) { + final CacheControl cacheControl = new CacheControl.Builder() // + .maxAge(0, TimeUnit.DAYS) // + .build(); - return client; + request = request.newBuilder() // + .cacheControl(cacheControl) // + .build(); + } + + return chain.proceed(request); + }; + } + + private static boolean isNetworkAvailable(final Context context) { + ConnectivityManager connectivityManager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE); + NetworkInfo activeNetworkInfo = connectivityManager.getActiveNetworkInfo(); + return activeNetworkInfo != null && activeNetworkInfo.isConnected(); } private String decrypt(String password, Context context) { @@ -53,35 +95,4 @@ class S3ClientFactory { .getInstance(context) // .decrypt(password); } - - private static class LoggingAwareRequestHandler extends RequestHandler2 { - - @Override - public void beforeRequest(Request request) { - Timber.tag("S3Client").d("Sending request (%s) %s", request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), request.toString()); - } - - @Override - public void afterResponse(Request request, Response response) { - Timber.tag("S3Client").d( // - "Response received (%s) with status %s (%s)", // - request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), // - response.getHttpResponse().getStatusText(), // - response.getHttpResponse().getStatusCode()); - } - - @Override - public void afterError(Request request, Response response, Exception e) { - if (response != null) { - Timber.tag("S3Client").e( // - e, // - "Error occurred (%s) with status %s (%s)", // - request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano(), // - response.getHttpResponse().getStatusText(), // - response.getHttpResponse().getStatusCode()); - } else { - Timber.tag("S3Client").e(e, "Error occurred (%s)", request.getAWSRequestMetrics().getTimingInfo().getStartTimeNano()); - } - } - } } diff --git a/data/src/main/java/org/cryptomator/data/cloud/s3/S3CloudContentRepository.java b/data/src/main/java/org/cryptomator/data/cloud/s3/S3CloudContentRepository.java index a6cfa94c..a9e3a54d 100644 --- a/data/src/main/java/org/cryptomator/data/cloud/s3/S3CloudContentRepository.java +++ b/data/src/main/java/org/cryptomator/data/cloud/s3/S3CloudContentRepository.java @@ -2,12 +2,11 @@ package org.cryptomator.data.cloud.s3; import android.content.Context; -import com.amazonaws.services.s3.model.AmazonS3Exception; - import org.cryptomator.data.cloud.InterceptingCloudContentRepository; import org.cryptomator.domain.S3Cloud; import org.cryptomator.domain.exception.BackendException; import org.cryptomator.domain.exception.FatalBackendException; +import org.cryptomator.domain.exception.ForbiddenException; import org.cryptomator.domain.exception.NetworkConnectionException; import org.cryptomator.domain.exception.NoSuchBucketException; import org.cryptomator.domain.exception.authentication.WrongCredentialsException; @@ -23,6 +22,8 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; +import io.minio.errors.ErrorResponseException; + import static org.cryptomator.util.ExceptionUtil.contains; class S3CloudContentRepository extends InterceptingCloudContentRepository { @@ -42,9 +43,9 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository encryptedTmpFile, OutputStream data, ProgressAware progressAware) throws BackendException { try { - cloud.read(file, encryptedTmpFile, data, progressAware); + cloud.read(file, data, progressAware); } catch (IOException e) { throw new FatalBackendException(e); } @@ -175,7 +178,7 @@ class S3CloudContentRepository extends InterceptingCloudContentRepository size) { return new S3File(parent, name, getNodePath(parent, name), size, Optional.empty()); } @@ -33,11 +24,6 @@ class S3CloudNodeFactory { return new S3File(parent, name, getNodePath(parent, name), size, lastModified); } - public static S3Folder folder(S3Folder parent, S3ObjectSummary folder) { - String name = getNameFromKey(folder.getKey()); - return new S3Folder(parent, name, getNodePath(parent, name)); - } - public static S3Folder folder(S3Folder parent, String name) { return new S3Folder(parent, name, getNodePath(parent, name)); } @@ -47,23 +33,15 @@ class S3CloudNodeFactory { } private static String getNodePath(S3Folder parent, String name) { - return parent.getKey() + name; + return parent.getPath() + "/" + name; } public static String getNameFromKey(String key) { String name = key; if (key.endsWith(DELIMITER)) { - name = key.substring(0, key.length() -1); + name = key.substring(0, key.length() - 1); } return name.contains(DELIMITER) ? name.substring(name.lastIndexOf(DELIMITER) + 1) : name; } - public static S3Node from(S3Folder parent, S3ObjectSummary objectSummary) { - if (objectSummary.getKey().endsWith(DELIMITER)) { - return folder(parent, objectSummary); - } else { - return file(parent, objectSummary); - } - } - } diff --git a/data/src/main/java/org/cryptomator/data/cloud/s3/S3File.java b/data/src/main/java/org/cryptomator/data/cloud/s3/S3File.java index ce3520f8..99b45ea9 100644 --- a/data/src/main/java/org/cryptomator/data/cloud/s3/S3File.java +++ b/data/src/main/java/org/cryptomator/data/cloud/s3/S3File.java @@ -8,6 +8,8 @@ import java.util.Date; class S3File implements CloudFile, S3Node { + private static final String DELIMITER = "/"; + private final S3Folder parent; private final String name; private final String path; @@ -39,6 +41,9 @@ class S3File implements CloudFile, S3Node { @Override public String getKey() { + if (path.startsWith(DELIMITER)) { + return path.substring(DELIMITER.length()); + } return path; } diff --git a/data/src/main/java/org/cryptomator/data/cloud/s3/S3Folder.java b/data/src/main/java/org/cryptomator/data/cloud/s3/S3Folder.java index 591c6dea..bdfd8fc8 100644 --- a/data/src/main/java/org/cryptomator/data/cloud/s3/S3Folder.java +++ b/data/src/main/java/org/cryptomator/data/cloud/s3/S3Folder.java @@ -34,6 +34,9 @@ class S3Folder implements CloudFolder, S3Node { @Override public String getKey() { + if (path.startsWith(DELIMITER)) { + return path.substring(DELIMITER.length()) + DELIMITER; + } return path + DELIMITER; } diff --git a/data/src/main/java/org/cryptomator/data/cloud/s3/S3Impl.java b/data/src/main/java/org/cryptomator/data/cloud/s3/S3Impl.java index 05a72fea..85bc07c1 100644 --- a/data/src/main/java/org/cryptomator/data/cloud/s3/S3Impl.java +++ b/data/src/main/java/org/cryptomator/data/cloud/s3/S3Impl.java @@ -2,27 +2,9 @@ package org.cryptomator.data.cloud.s3; import android.content.Context; -import com.amazonaws.event.ProgressListener; -import com.amazonaws.mobileconnectors.s3.transferutility.TransferListener; -import com.amazonaws.mobileconnectors.s3.transferutility.TransferState; -import com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility; -import com.amazonaws.mobileconnectors.s3.transferutility.UploadOptions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.Owner; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.tomclaw.cache.DiskLruCache; - import org.cryptomator.data.util.CopyStream; +import org.cryptomator.data.util.TransferredBytesAwareInputStream; +import org.cryptomator.data.util.TransferredBytesAwareOutputStream; import org.cryptomator.domain.S3Cloud; import org.cryptomator.domain.exception.BackendException; import org.cryptomator.domain.exception.CloudNodeAlreadyExistsException; @@ -30,7 +12,6 @@ import org.cryptomator.domain.exception.FatalBackendException; import org.cryptomator.domain.exception.ForbiddenException; import org.cryptomator.domain.exception.NoSuchBucketException; import org.cryptomator.domain.exception.NoSuchCloudFileException; -import org.cryptomator.domain.exception.UnauthorizedException; import org.cryptomator.domain.exception.authentication.WrongCredentialsException; import org.cryptomator.domain.usecases.ProgressAware; import org.cryptomator.domain.usecases.cloud.DataSource; @@ -38,31 +19,40 @@ import org.cryptomator.domain.usecases.cloud.DownloadState; import org.cryptomator.domain.usecases.cloud.Progress; import org.cryptomator.domain.usecases.cloud.UploadState; import org.cryptomator.util.Optional; -import org.cryptomator.util.SharedPreferencesHandler; -import org.cryptomator.util.concurrent.CompletableFuture; -import org.cryptomator.util.file.LruFileCacheUtil; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedList; import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicLong; +import io.minio.BucketExistsArgs; +import io.minio.CopyObjectArgs; +import io.minio.CopySource; +import io.minio.GetObjectArgs; +import io.minio.GetObjectResponse; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.ObjectWriteResponse; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteError; +import io.minio.messages.DeleteObject; +import io.minio.messages.Item; import timber.log.Timber; import static org.cryptomator.domain.usecases.cloud.Progress.progress; -import static org.cryptomator.util.file.LruFileCacheUtil.Cache.S3; -import static org.cryptomator.util.file.LruFileCacheUtil.retrieveFromLruCache; -import static org.cryptomator.util.file.LruFileCacheUtil.storeToLruCache; class S3Impl { - private static final long CHUNKED_UPLOAD_MAX_SIZE = 100L << 20; private static final String DELIMITER = "/"; private final S3ClientFactory clientFactory = new S3ClientFactory(); @@ -70,9 +60,6 @@ class S3Impl { private final RootS3Folder root; private final Context context; - private final SharedPreferencesHandler sharedPreferencesHandler; - private DiskLruCache diskLruCache; - S3Impl(Context context, S3Cloud cloud) { if (cloud.accessKey() == null || cloud.secretKey() == null) { throw new WrongCredentialsException(cloud); @@ -81,10 +68,9 @@ class S3Impl { this.context = context; this.cloud = cloud; this.root = new RootS3Folder(cloud); - this.sharedPreferencesHandler = new SharedPreferencesHandler(context); } - private AmazonS3 client() { + private MinioClient client() { return clientFactory.getClient(cloud, context); } @@ -118,31 +104,48 @@ class S3Impl { return S3CloudNodeFactory.folder(parent, name, parent.getKey() + name); } - public boolean exists(S3Node node) { + public boolean exists(S3Node node) throws BackendException { String key = node.getKey(); + try { + if(!(node instanceof RootS3Folder)) { + client().statObject(StatObjectArgs.builder().bucket(cloud.s3Bucket()).object(key).build()); + return true; + } else { + // stat requests throws an IllegalStateException if key is empty string + ListObjectsArgs request = ListObjectsArgs.builder().bucket(cloud.s3Bucket()).prefix(key).delimiter(DELIMITER).build(); + return client().listObjects(request).iterator().hasNext(); + } + } catch (ErrorResponseException e) { + if (S3CloudApiErrorCodes.NO_SUCH_KEY.getValue().equals(e.errorResponse().code())) { + return false; + } + throw new FatalBackendException(e); + } catch (Exception ex) { + handleApiError(ex, node.getPath()); + } - ListObjectsV2Result result = client().listObjectsV2(cloud.s3Bucket(), key); - - return result.getObjectSummaries().size() > 0; + throw new FatalBackendException(new IllegalStateException("Exception thrown but not handled?")); } public List list(S3Folder folder) throws IOException, BackendException { List result = new ArrayList<>(); - ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(cloud.s3Bucket()).withPrefix(folder.getKey()).withDelimiter(DELIMITER); - - ListObjectsV2Result listObjects = client().listObjectsV2(request); - for (String prefix : listObjects.getCommonPrefixes()) { - // add folders - result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(prefix))); - } - - for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) { - // add files but skip parent folder - if (!objectSummary.getKey().equals(listObjects.getPrefix())) { - result.add(S3CloudNodeFactory.file(folder, objectSummary)); + ListObjectsArgs request = ListObjectsArgs.builder().bucket(cloud.s3Bucket()).prefix(folder.getKey()).delimiter(DELIMITER).build(); + Iterable> listObjects = client().listObjects(request); + for (Result object : listObjects) { + try { + Item item = object.get(); + if (item.isDir()) { + result.add(S3CloudNodeFactory.folder(folder, S3CloudNodeFactory.getNameFromKey(item.objectName()))); + } else { + S3File file = S3CloudNodeFactory.file(folder, S3CloudNodeFactory.getNameFromKey(item.objectName()), Optional.of(item.size()), Optional.of(Date.from(item.lastModified().toInstant()))); + result.add(file); + } + } catch (Exception ex) { + handleApiError(ex, folder.getPath()); } } + return result; } @@ -154,16 +157,17 @@ class S3Impl { ); } - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(0); - - InputStream emptyContent = new ByteArrayInputStream(new byte[0]); - try { - PutObjectRequest putObjectRequest = new PutObjectRequest(cloud.s3Bucket(), folder.getKey(), emptyContent, metadata); - client().putObject(putObjectRequest); - } catch(AmazonS3Exception ex) { - handleApiError(ex, folder.getName()); + PutObjectArgs putObjectArgs = PutObjectArgs // + .builder() // + .bucket(cloud.s3Bucket()) // + .object(folder.getKey()) // + .stream(new ByteArrayInputStream(new byte[0]), 0, -1) // + .build(); + + client().putObject(putObjectArgs); + } catch (Exception ex) { + handleApiError(ex, folder.getPath()); } return S3CloudNodeFactory.folder(folder.getParent(), folder.getName()); @@ -175,28 +179,58 @@ class S3Impl { } if (source instanceof S3Folder) { - ListObjectsV2Result listObjects = client().listObjectsV2(cloud.s3Bucket(), source.getPath()); + List nodes = list((S3Folder) source); - if (listObjects.getObjectSummaries().size() > 0) { + List objectsToDelete = new LinkedList<>(); - List objectsToDelete = new ArrayList<>(); + for (S3Node node : nodes) { + objectsToDelete.add(new DeleteObject(node.getKey())); - for (S3ObjectSummary summary : listObjects.getObjectSummaries()) { - objectsToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); - String destinationKey = summary.getKey().replace(source.getPath(), target.getPath()); - - client().copyObject(cloud.s3Bucket(), summary.getKey(), cloud.s3Bucket(), destinationKey); + String targetKey; + if (node instanceof S3Folder) { + targetKey = S3CloudNodeFactory.folder((S3Folder) target, node.getName()).getKey(); + } else { + targetKey = S3CloudNodeFactory.file((S3Folder) target, node.getName()).getKey(); + } + + CopySource copySource = CopySource.builder().bucket(cloud.s3Bucket()).object(node.getKey()).build(); + + CopyObjectArgs copyObjectArgs = CopyObjectArgs.builder().bucket(cloud.s3Bucket()).object(targetKey).source(copySource).build(); + try { + client().copyObject(copyObjectArgs); + } catch (Exception ex) { + handleApiError(ex, source.getPath()); } - client().deleteObjects(new DeleteObjectsRequest(cloud.s3Bucket()).withKeys(objectsToDelete)); - } else { - throw new NoSuchCloudFileException(source.getPath()); } + + RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket(cloud.s3Bucket()).objects(objectsToDelete).build(); + + for (Result result : client().removeObjects(removeObjectsArgs)) { + try { + result.get(); + } catch (Exception ex) { + handleApiError(ex, source.getPath()); + } + } + return S3CloudNodeFactory.folder(target.getParent(), target.getName()); } else { - CopyObjectResult result = client().copyObject(cloud.s3Bucket(), source.getPath(), cloud.s3Bucket(), target.getPath()); - client().deleteObject(cloud.s3Bucket(), source.getPath()); - return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File) source).getSize(), Optional.of(result.getLastModifiedDate())); + CopySource copySource = CopySource.builder().bucket(cloud.s3Bucket()).object(source.getKey()).build(); + CopyObjectArgs copyObjectArgs = CopyObjectArgs.builder().bucket(cloud.s3Bucket()).object(target.getKey()).source(copySource).build(); + try { + ObjectWriteResponse result = client().copyObject(copyObjectArgs); + + delete(source); + + Date lastModified = result.headers().getDate("Last-Modified"); + + return S3CloudNodeFactory.file(target.getParent(), target.getName(), ((S3File) source).getSize(), Optional.ofNullable(lastModified)); + } catch (Exception ex) { + handleApiError(ex, source.getPath()); + } } + + throw new FatalBackendException(new IllegalStateException("Exception thrown but not handled?")); } public S3File write(S3File file, DataSource data, final ProgressAware progressAware, boolean replace, long size) throws IOException, BackendException { @@ -206,212 +240,165 @@ class S3Impl { progressAware.onProgress(Progress.started(UploadState.upload(file))); - final CompletableFuture> result = new CompletableFuture<>(); - - try { - if (size <= CHUNKED_UPLOAD_MAX_SIZE) { - uploadFile(file, data, progressAware, result, size); - } else { - uploadChunkedFile(file, data, progressAware, result, size); - } - } catch(AmazonS3Exception ex) { - handleApiError(ex, file.getName()); - } - - try { - Optional objectMetadataOptional = result.get(); - ObjectMetadata objectMetadata = objectMetadataOptional.orElseGet(() -> client().getObjectMetadata(cloud.s3Bucket(), file.getPath())); - progressAware.onProgress(Progress.completed(UploadState.upload(file))); - return S3CloudNodeFactory.file(file.getParent(), file.getName(), objectMetadata); - } catch (ExecutionException | InterruptedException e) { - throw new FatalBackendException(e); - } - - } - - private void uploadFile(final S3File file, DataSource data, final ProgressAware progressAware, CompletableFuture> result, final long size) // - throws IOException { - AtomicLong bytesTransferred = new AtomicLong(0); - ProgressListener listener = progressEvent -> { - bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred()); - progressAware.onProgress( // - progress(UploadState.upload(file)) // - .between(0) // - .and(size) // - .withValue(bytesTransferred.get())); - }; - - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(data.size(context).get()); - - PutObjectRequest request = new PutObjectRequest(cloud.s3Bucket(), file.getPath(), data.open(context), metadata); - request.setGeneralProgressListener(listener); - - result.complete(Optional.of(client().putObject(request).getMetadata())); - } - - private void uploadChunkedFile(final S3File file, DataSource data, final ProgressAware progressAware, CompletableFuture> result, final long size) // - throws IOException { - - TransferUtility tu = TransferUtility // - .builder() // - .s3Client(client()) // - .context(context) // - .defaultBucket(cloud.s3Bucket()) // - .build(); - - TransferListener transferListener = new TransferListener() { + try (TransferredBytesAwareDataSource out = new TransferredBytesAwareDataSource(data) { @Override - public void onStateChanged(int id, TransferState state) { - if (state.equals(TransferState.COMPLETED)) { - progressAware.onProgress(Progress.completed(UploadState.upload(file))); - result.complete(Optional.empty()); - } - } - - @Override - public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) { + public void bytesTransferred(long transferred) { progressAware.onProgress( // progress(UploadState.upload(file)) // .between(0) // .and(size) // - .withValue(bytesCurrent)); + .withValue(transferred)); } + }) { + try { + PutObjectArgs putObjectArgs = PutObjectArgs.builder().bucket(cloud.s3Bucket()).object(file.getKey()).stream(out.open(context), data.size(context).get(), -1).build(); + ObjectWriteResponse objectWriteResponse = client().putObject(putObjectArgs); - @Override - public void onError(int id, Exception ex) { - result.fail(ex); + Date lastModified = objectWriteResponse.headers().getDate("Last-Modified"); + + if(lastModified == null) { + StatObjectResponse statObjectResponse = client().statObject(StatObjectArgs // + .builder() // + .bucket(cloud.s3Bucket()) // + .object(file.getKey()) // + .build()); + + lastModified = Date.from(statObjectResponse.lastModified().toInstant()); + } + + progressAware.onProgress(Progress.completed(UploadState.upload(file))); + return S3CloudNodeFactory.file(file.getParent(), file.getName(), Optional.of(size), Optional.of(lastModified)); + } catch (Exception ex) { + handleApiError(ex, file.getPath()); } - }; - - UploadOptions uploadOptions = UploadOptions.builder().transferListener(transferListener).build(); - - tu.upload(file.getPath(), data.open(context), uploadOptions); - } - - public void read(S3File file, Optional encryptedTmpFile, OutputStream data, final ProgressAware progressAware) throws IOException, BackendException { - progressAware.onProgress(Progress.started(DownloadState.download(file))); - - Optional cacheKey = Optional.empty(); - Optional cacheFile = Optional.empty(); - - ListObjectsV2Result listObjects; - - if (sharedPreferencesHandler.useLruCache() && createLruCache(sharedPreferencesHandler.lruCacheSize())) { - listObjects = client().listObjectsV2(cloud.s3Bucket(), file.getKey()); - if (listObjects.getObjectSummaries().size() != 1) { - throw new NoSuchCloudFileException(file.getKey()); - } - S3ObjectSummary summary = listObjects.getObjectSummaries().get(0); - cacheKey = Optional.of(summary.getKey() + summary.getETag()); - - File cachedFile = diskLruCache.get(cacheKey.get()); - cacheFile = cachedFile != null ? Optional.of(cachedFile) : Optional.empty(); } - if (sharedPreferencesHandler.useLruCache() && cacheFile.isPresent()) { - try { - retrieveFromLruCache(cacheFile.get(), data); - } catch (IOException e) { - Timber.tag("S3Impl").w(e, "Error while retrieving content from Cache, get from web request"); - writeToData(file, data, encryptedTmpFile, cacheKey, progressAware); - } - } else { - writeToData(file, data, encryptedTmpFile, cacheKey, progressAware); + throw new FatalBackendException(new IllegalStateException("Exception thrown but not handled?")); + } + + public void read(S3File file, OutputStream data, final ProgressAware progressAware) throws IOException, BackendException { + progressAware.onProgress(Progress.started(DownloadState.download(file))); + + GetObjectArgs getObjectArgs = GetObjectArgs.builder().bucket(cloud.s3Bucket()).object(file.getKey()).build(); + + try (GetObjectResponse response = client().getObject(getObjectArgs); // + TransferredBytesAwareOutputStream out = new TransferredBytesAwareOutputStream(data) { + @Override + public void bytesTransferred(long transferred) { + progressAware.onProgress( // + progress(DownloadState.download(file)) // + .between(0) // + .and(file.getSize().orElse(Long.MAX_VALUE)) // + .withValue(transferred)); + } + }) { + CopyStream.copyStreamToStream(response, out); + } catch (Exception ex) { + handleApiError(ex, file.getPath()); } progressAware.onProgress(Progress.completed(DownloadState.download(file))); } - private void writeToData(final S3File file, // - final OutputStream data, // - final Optional encryptedTmpFile, // - final Optional cacheKey, // - final ProgressAware progressAware) throws IOException, BackendException { - AtomicLong bytesTransferred = new AtomicLong(0); - ProgressListener listener = progressEvent -> { - bytesTransferred.set(bytesTransferred.get() + progressEvent.getBytesTransferred()); - - progressAware.onProgress( // - progress(DownloadState.download(file)) // - .between(0) // - .and(file.getSize().orElse(Long.MAX_VALUE)) // - .withValue(bytesTransferred.get())); - }; - - GetObjectRequest request = new GetObjectRequest(cloud.s3Bucket(), file.getPath()); - request.setGeneralProgressListener(listener); - - try { - S3Object s3Object = client().getObject(request); - - CopyStream.copyStreamToStream(s3Object.getObjectContent(), data); - - if (sharedPreferencesHandler.useLruCache() && encryptedTmpFile.isPresent() && cacheKey.isPresent()) { - try { - storeToLruCache(diskLruCache, cacheKey.get(), encryptedTmpFile.get()); - } catch (IOException e) { - Timber.tag("S3Impl").e(e, "Failed to write downloaded file in LRU cache"); - } - } - } catch(AmazonS3Exception ex) { - handleApiError(ex, file.getName()); - } - - } - public void delete(S3Node node) throws IOException, BackendException { if (node instanceof S3Folder) { - List summaries = client().listObjectsV2(cloud.s3Bucket(), node.getPath()).getObjectSummaries(); - List keys = new ArrayList<>(); - for (S3ObjectSummary summary : summaries) { - keys.add(new KeyVersion(summary.getKey())); + List objectsToDelete = new LinkedList<>(); + + ListObjectsArgs request = ListObjectsArgs.builder().bucket(cloud.s3Bucket()).prefix(node.getKey()).recursive(true).delimiter(DELIMITER).build(); + Iterable> listObjects = client().listObjects(request); + for (Result object : listObjects) { + try { + Item item = object.get(); + objectsToDelete.add(new DeleteObject(item.objectName())); + } catch (Exception e) { + handleApiError(e, node.getPath()); + } } - DeleteObjectsRequest request = new DeleteObjectsRequest(cloud.s3Bucket()); - request.withKeys(keys); + RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket(cloud.s3Bucket()).objects(objectsToDelete).build(); + Iterable> results = client().removeObjects(removeObjectsArgs); + for (Result result : results) { + try { + DeleteError error = result.get(); + Timber.tag("S3Impl").e("Error in deleting object " + error.objectName() + "; " + error.message()); + } catch (Exception e) { + handleApiError(e, node.getPath()); + } + } - client().deleteObjects(request); } else { - client().deleteObject(cloud.s3Bucket(), node.getPath()); - } - } - - public String checkAuthenticationAndRetrieveCurrentAccount() throws NoSuchBucketException { - if (!client().doesBucketExist(cloud.s3Bucket())) { - throw new NoSuchBucketException(cloud.s3Bucket()); - } - - Owner currentAccount = client() // - .getS3AccountOwner(); - - return currentAccount.getDisplayName(); - } - - private boolean createLruCache(int cacheSize) { - if (diskLruCache == null) { + RemoveObjectArgs removeObjectArgs = RemoveObjectArgs.builder().bucket(cloud.s3Bucket()).object(node.getKey()).build(); try { - diskLruCache = DiskLruCache.create(new LruFileCacheUtil(context).resolve(S3), cacheSize); - } catch (IOException e) { - Timber.tag("S3Impl").e(e, "Failed to setup LRU cache"); - return false; + client().removeObject(removeObjectArgs); + } catch (Exception e) { + handleApiError(e, ""); } } - - return true; } - private void handleApiError(AmazonS3Exception ex, String name) throws BackendException { - String errorCode = ex.getErrorCode(); - if (S3CloudApiExceptions.isAccessProblem(errorCode)) { - throw new ForbiddenException(); - } else if (S3CloudApiErrorCodes.NO_SUCH_BUCKET.getValue().equals(errorCode)) { - throw new NoSuchBucketException(name); - } else if (S3CloudApiErrorCodes.NO_SUCH_KEY.getValue().equals(errorCode)) { - throw new NoSuchCloudFileException(name); - } else { + public String checkAuthentication() throws NoSuchBucketException, BackendException { + try { + if (!client().bucketExists(BucketExistsArgs.builder().bucket(cloud.s3Bucket()).build())) { + throw new NoSuchBucketException(cloud.s3Bucket()); + } + } catch (Exception e) { + handleApiError(e, ""); + } + + return ""; + } + + private void handleApiError(Exception ex, String name) throws BackendException { + if (ex instanceof ErrorResponseException) { + String errorCode = ((ErrorResponseException) ex).errorResponse().code(); + if (S3CloudApiExceptions.isAccessProblem(errorCode)) { + throw new ForbiddenException(); + } else if (S3CloudApiErrorCodes.NO_SUCH_BUCKET.getValue().equals(errorCode)) { + throw new NoSuchBucketException(name); + } else if (S3CloudApiErrorCodes.NO_SUCH_KEY.getValue().equals(errorCode)) { + throw new NoSuchCloudFileException(name); + } else { + throw new FatalBackendException(ex); + } + } else { throw new FatalBackendException(ex); } } + + private static abstract class TransferredBytesAwareDataSource implements DataSource { + + private final DataSource data; + + TransferredBytesAwareDataSource(DataSource data) { + this.data = data; + } + + @Override + public Optional size(Context context) { + return data.size(context); + } + + @Override + public InputStream open(Context context) throws IOException { + return new TransferredBytesAwareInputStream(data.open(context)) { + @Override + public void bytesTransferred(long transferred) { + S3Impl.TransferredBytesAwareDataSource.this.bytesTransferred(transferred); + } + }; + } + + @Override + public void close() throws IOException { + data.close(); + } + + public abstract void bytesTransferred(long transferred); + + @Override + public DataSource decorate(DataSource delegate) { + return delegate; + } + } } diff --git a/data/src/main/java/org/cryptomator/data/db/DatabaseUpgrades.java b/data/src/main/java/org/cryptomator/data/db/DatabaseUpgrades.java index 59fef0f2..dc1c1d70 100644 --- a/data/src/main/java/org/cryptomator/data/db/DatabaseUpgrades.java +++ b/data/src/main/java/org/cryptomator/data/db/DatabaseUpgrades.java @@ -25,7 +25,8 @@ class DatabaseUpgrades { Upgrade3To4 upgrade3To4, // Upgrade4To5 upgrade4To5, // Upgrade5To6 upgrade5To6, // - Upgrade6To7 upgrade6To7) { + Upgrade6To7 upgrade6To7, // + Upgrade7To8 upgrade7To8) { availableUpgrades = defineUpgrades( // upgrade0To1, // @@ -34,7 +35,8 @@ class DatabaseUpgrades { upgrade3To4, // upgrade4To5, // upgrade5To6, // - upgrade6To7); + upgrade6To7, // + upgrade7To8); } private static Comparator reverseOrder() { diff --git a/data/src/main/java/org/cryptomator/data/db/Sql.java b/data/src/main/java/org/cryptomator/data/db/Sql.java index 1fc488a4..e5d71208 100644 --- a/data/src/main/java/org/cryptomator/data/db/Sql.java +++ b/data/src/main/java/org/cryptomator/data/db/Sql.java @@ -498,27 +498,26 @@ class Sql { public static class SqlDeleteBuilder { - private final String table; - private String whereClause; - private String[] whereArgs; + private final String tableName; - public SqlDeleteBuilder(String table) { - this.table = table; + private final StringBuilder whereClause = new StringBuilder(); + private final List whereArgs = new ArrayList<>(); + + public SqlDeleteBuilder(String tableName) { + this.tableName = tableName; } - public SqlDeleteBuilder whereClause(String whereClause) { - this.whereClause = whereClause; - return this; - } - - public SqlDeleteBuilder whereArgs(String[] whereArgs) { - this.whereArgs = whereArgs; + public SqlDeleteBuilder where(String column, Criterion criterion) { + if (whereClause.length() > 0) { + whereClause.append(" AND "); + } + criterion.appendTo(column, whereClause, whereArgs); return this; } public void executeOn(Database wrapped) { SQLiteDatabase db = unwrap(wrapped); - db.delete(table, whereClause, whereArgs); + db.delete(tableName, whereClause.toString(), whereArgs.toArray(new String[whereArgs.size()])); } } diff --git a/data/src/main/java/org/cryptomator/data/db/Upgrade7To8.kt b/data/src/main/java/org/cryptomator/data/db/Upgrade7To8.kt new file mode 100644 index 00000000..5f1a65cc --- /dev/null +++ b/data/src/main/java/org/cryptomator/data/db/Upgrade7To8.kt @@ -0,0 +1,32 @@ +package org.cryptomator.data.db + +import org.greenrobot.greendao.database.Database +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +internal class Upgrade7To8 @Inject constructor() : DatabaseUpgrade(7, 8) { + + override fun internalApplyTo(db: Database, origin: Int) { + db.beginTransaction() + try { + dropS3Vaults(db) + dropS3Clouds(db) + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + + private fun dropS3Vaults(db: Database) { + Sql.deleteFrom("VAULT_ENTITY") // + .where("CLOUD_TYPE", Sql.eq("S3")) + .executeOn(db) + } + + private fun dropS3Clouds(db: Database) { + Sql.deleteFrom("CLOUD_ENTITY") // + .where("TYPE", Sql.eq("S3")) + .executeOn(db) + } +} diff --git a/presentation/src/main/java/org/cryptomator/presentation/presenter/S3AddOrChangePresenter.kt b/presentation/src/main/java/org/cryptomator/presentation/presenter/S3AddOrChangePresenter.kt index 20fe2282..e066851d 100644 --- a/presentation/src/main/java/org/cryptomator/presentation/presenter/S3AddOrChangePresenter.kt +++ b/presentation/src/main/java/org/cryptomator/presentation/presenter/S3AddOrChangePresenter.kt @@ -35,7 +35,7 @@ class S3AddOrChangePresenter @Inject internal constructor( // if (displayName.isEmpty()) { statusMessage = getString(R.string.screen_s3_settings_msg_display_name_not_empty) } - if (endpoint.isNullOrEmpty() && region.isNullOrEmpty()) { + if (endpoint.isNullOrEmpty() || region.isNullOrEmpty()) { statusMessage = getString(R.string.screen_s3_settings_msg_endpoint_and_region_not_empty) } diff --git a/presentation/src/main/java/org/cryptomator/presentation/ui/fragment/S3AddOrChangeFragment.kt b/presentation/src/main/java/org/cryptomator/presentation/ui/fragment/S3AddOrChangeFragment.kt index d80feb76..8805a072 100644 --- a/presentation/src/main/java/org/cryptomator/presentation/ui/fragment/S3AddOrChangeFragment.kt +++ b/presentation/src/main/java/org/cryptomator/presentation/ui/fragment/S3AddOrChangeFragment.kt @@ -2,7 +2,6 @@ package org.cryptomator.presentation.ui.fragment import android.os.Bundle import android.view.inputmethod.EditorInfo -import com.google.android.material.switchmaterial.SwitchMaterial import org.cryptomator.generator.Fragment import org.cryptomator.presentation.R import org.cryptomator.presentation.model.S3CloudModel @@ -13,10 +12,9 @@ import kotlinx.android.synthetic.main.fragment_setup_s3.accessKeyEditText import kotlinx.android.synthetic.main.fragment_setup_s3.bucketEditText import kotlinx.android.synthetic.main.fragment_setup_s3.createCloudButton import kotlinx.android.synthetic.main.fragment_setup_s3.displayNameEditText -import kotlinx.android.synthetic.main.fragment_setup_s3.regionOrEndpointEditText -import kotlinx.android.synthetic.main.fragment_setup_s3.regionOrEndpointEditTextLayout +import kotlinx.android.synthetic.main.fragment_setup_s3.endpointEditText +import kotlinx.android.synthetic.main.fragment_setup_s3.regionEditText import kotlinx.android.synthetic.main.fragment_setup_s3.secretKeyEditText -import kotlinx.android.synthetic.main.fragment_setup_s3.toggleCustomS3 import timber.log.Timber @Fragment(R.layout.fragment_setup_s3) @@ -40,17 +38,6 @@ class S3AddOrChangeFragment : BaseFragment() { } showEditableCloudContent(s3CloudModel) - - toggleCustomS3.setOnClickListener { switch -> - regionOrEndpointEditText.text?.clear() - toggleUseAmazonS3((switch as SwitchMaterial).isChecked) - } - } - - private fun toggleUseAmazonS3(checked: Boolean) = if (checked) { - regionOrEndpointEditTextLayout.setHint(R.string.screen_s3_settings_region_label) - } else { - regionOrEndpointEditTextLayout.setHint(R.string.screen_s3_settings_endpoint_label) } private fun showEditableCloudContent(s3CloudModel: S3CloudModel?) { @@ -60,16 +47,9 @@ class S3AddOrChangeFragment : BaseFragment() { accessKeyEditText.setText(decrypt(s3CloudModel.accessKey())) secretKeyEditText.setText(decrypt(s3CloudModel.secretKey())) bucketEditText.setText(s3CloudModel.s3Bucket()) - - if (it.s3Endpoint().isNotEmpty()) { - toggleCustomS3.isChecked = false - regionOrEndpointEditText.setText(s3CloudModel.s3Endpoint()) - regionOrEndpointEditTextLayout.setHint(R.string.screen_s3_settings_endpoint_label) - } else { - regionOrEndpointEditText.setText(s3CloudModel.s3Region()) - regionOrEndpointEditTextLayout.setHint(R.string.screen_s3_settings_region_label) - } - } ?: regionOrEndpointEditTextLayout.setHint(R.string.screen_s3_settings_region_label) + endpointEditText.setText(s3CloudModel.s3Endpoint()) + regionEditText.setText(s3CloudModel.s3Region()) + } } private fun decrypt(text: String?): String { @@ -91,11 +71,14 @@ class S3AddOrChangeFragment : BaseFragment() { val bucket = bucketEditText.text.toString().trim() val displayName = displayNameEditText.text.toString().trim() - if (toggleCustomS3.isChecked) { - s3AddOrChangePresenter.checkUserInput(accessKey, secretKey, bucket, null, regionOrEndpointEditText.text.toString().trim(), cloudId, displayName) - } else { - s3AddOrChangePresenter.checkUserInput(accessKey, secretKey, bucket, regionOrEndpointEditText.text.toString().trim(), null, cloudId, displayName) - } + s3AddOrChangePresenter.checkUserInput( // + accessKey, // + secretKey, // + bucket, // + endpointEditText.text.toString().trim(), // + regionEditText.text.toString().trim(), // + cloudId, // + displayName) } fun hideKeyboard() { diff --git a/presentation/src/main/res/layout/fragment_setup_s3.xml b/presentation/src/main/res/layout/fragment_setup_s3.xml index deeb56e9..9fd9cc1d 100644 --- a/presentation/src/main/res/layout/fragment_setup_s3.xml +++ b/presentation/src/main/res/layout/fragment_setup_s3.xml @@ -79,27 +79,34 @@ - + + + + +