diff --git a/app/src/main/java/fr/free/nrw/commons/contributions/Contribution.kt b/app/src/main/java/fr/free/nrw/commons/contributions/Contribution.kt index f51d4df5c..be763fe8a 100644 --- a/app/src/main/java/fr/free/nrw/commons/contributions/Contribution.kt +++ b/app/src/main/java/fr/free/nrw/commons/contributions/Contribution.kt @@ -5,6 +5,7 @@ import android.os.Parcelable import androidx.room.Embedded import androidx.room.Entity import androidx.room.PrimaryKey +import fr.free.nrw.commons.CommonsApplication import fr.free.nrw.commons.Media import fr.free.nrw.commons.auth.SessionManager import fr.free.nrw.commons.upload.UploadItem @@ -13,7 +14,8 @@ import fr.free.nrw.commons.upload.WikidataPlace import fr.free.nrw.commons.upload.WikidataPlace.Companion.from import fr.free.nrw.commons.upload.structure.depictions.DepictedItem import kotlinx.parcelize.Parcelize -import java.util.* +import java.io.File +import java.util.Date @Entity(tableName = "contribution") @Parcelize @@ -117,4 +119,19 @@ data class Contribution constructor( descriptions.filter { it.descriptionText.isNotEmpty() } .joinToString(separator = "") { "{{${it.languageCode}|1=${it.descriptionText}}}" } } + + val fileKey : String? get() = chunkInfo?.uploadResult?.filekey + val localUriPath: File? get() = localUri?.path?.let { File(it) } + + fun isCompleted(): Boolean { + return chunkInfo != null && chunkInfo!!.totalChunks == chunkInfo!!.indexOfNextChunkToUpload + } + + fun isPaused(): Boolean { + return CommonsApplication.pauseUploads[pageId] ?: false + } + + fun unpause() { + CommonsApplication.pauseUploads[pageId] = false + } } diff --git a/app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt b/app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt index 9c9d70637..a77798201 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt +++ b/app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt @@ -52,7 +52,7 @@ class CountingRequestBody( } } - interface Listener { + fun interface Listener { /** * Will be triggered when write progresses * @param bytesWritten diff --git a/app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java b/app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java index 14154a66d..a08a486e3 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java @@ -1,6 +1,7 @@ package fr.free.nrw.commons.upload; import android.content.Context; +import android.net.Uri; import fr.free.nrw.commons.location.LatLng; import io.reactivex.Observable; import java.io.BufferedInputStream; @@ -21,9 +22,11 @@ import timber.log.Timber; @Singleton public class FileUtilsWrapper { - @Inject - public FileUtilsWrapper() { + private final Context context; + @Inject + public FileUtilsWrapper(final Context context) { + this.context = context; } public String getFileExt(String fileName) { @@ -42,11 +45,18 @@ public class FileUtilsWrapper { return FileUtils.getGeolocationOfFile(filePath, inAppPictureLocation); } + public String getMimeType(File file) { + return getMimeType(Uri.parse(file.getPath())); + } + + public String getMimeType(Uri uri) { + return FileUtils.getMimeType(context, uri); + } /** * Takes a file as input and returns an Observable of files with the specified chunk size */ - public List getFileChunks(Context context, File file, final int chunkSize) + public List getFileChunks(File file, final int chunkSize) throws IOException { final byte[] buffer = new byte[chunkSize]; @@ -56,7 +66,7 @@ public class FileUtilsWrapper { final List buffers = new ArrayList<>(); int size; while ((size = bis.read(buffer)) > 0) { - buffers.add(writeToFile(context, Arrays.copyOf(buffer, size), file.getName(), + buffers.add(writeToFile(Arrays.copyOf(buffer, size), file.getName(), getFileExt(file.getName()))); } return buffers; @@ -66,7 +76,7 @@ public class FileUtilsWrapper { /** * Create a temp file containing the passed byte data. */ - private File writeToFile(Context context, final byte[] data, final String fileName, + private File writeToFile(final byte[] data, final String fileName, String fileExtension) throws IOException { final File file = File.createTempFile(fileName, fileExtension, context.getCacheDir()); diff --git a/app/src/main/java/fr/free/nrw/commons/upload/PageContentsCreator.java b/app/src/main/java/fr/free/nrw/commons/upload/PageContentsCreator.java index e31d84274..46d4c015f 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/PageContentsCreator.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/PageContentsCreator.java @@ -17,7 +17,7 @@ import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import timber.log.Timber; -class PageContentsCreator { +public class PageContentsCreator { //{{According to Exif data|2009-01-09}} private static final String TEMPLATE_DATE_ACC_TO_EXIF = "{{According to Exif data|%s}}"; diff --git a/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java b/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java index fb633937a..e69de29bb 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java @@ -1,236 +0,0 @@ -package fr.free.nrw.commons.upload; - -import static fr.free.nrw.commons.di.NetworkingModule.NAMED_COMMONS_CSRF; - -import android.content.Context; -import android.net.Uri; -import androidx.annotation.Nullable; -import com.google.gson.Gson; -import fr.free.nrw.commons.CommonsApplication; -import fr.free.nrw.commons.contributions.ChunkInfo; -import fr.free.nrw.commons.contributions.Contribution; -import fr.free.nrw.commons.upload.worker.UploadWorker.NotificationUpdateProgressListener; -import io.reactivex.Observable; -import io.reactivex.disposables.CompositeDisposable; -import java.io.File; -import java.io.IOException; -import java.net.URLEncoder; -import java.util.Date; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; -import okhttp3.MediaType; -import okhttp3.MultipartBody; -import okhttp3.RequestBody; -import fr.free.nrw.commons.auth.csrf.CsrfTokenClient; -import fr.free.nrw.commons.wikidata.mwapi.MwException; -import timber.log.Timber; - -@Singleton -public class UploadClient { - - private final int CHUNK_SIZE = 512 * 1024; // 512 KB - - //This is maximum duration for which a stash is persisted on MediaWiki - // https://www.mediawiki.org/wiki/Manual:$wgUploadStashMaxAge - private final int MAX_CHUNK_AGE = 6 * 3600 * 1000; // 6 hours - - private final UploadInterface uploadInterface; - private final CsrfTokenClient csrfTokenClient; - private final PageContentsCreator pageContentsCreator; - private final FileUtilsWrapper fileUtilsWrapper; - private final Gson gson; - - private final CompositeDisposable compositeDisposable = new CompositeDisposable(); - - @Inject - public UploadClient(final UploadInterface uploadInterface, - @Named(NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient, - final PageContentsCreator pageContentsCreator, - final FileUtilsWrapper fileUtilsWrapper, final Gson gson) { - this.uploadInterface = uploadInterface; - this.csrfTokenClient = csrfTokenClient; - this.pageContentsCreator = pageContentsCreator; - this.fileUtilsWrapper = fileUtilsWrapper; - this.gson = gson; - } - - /** - * Upload file to stash in chunks of specified size. Uploading files in chunks will make - * handling of large files easier. Also, it will be useful in supporting pause/resume of - * uploads - */ - public Observable uploadFileToStash( - final Context context, final String filename, final Contribution contribution, - final NotificationUpdateProgressListener notificationUpdater) throws IOException { - if (contribution.getChunkInfo() != null - && contribution.getChunkInfo().getTotalChunks() == contribution.getChunkInfo() - .getIndexOfNextChunkToUpload()) { - return Observable.just(new StashUploadResult(StashUploadState.SUCCESS, - contribution.getChunkInfo().getUploadResult().getFilekey())); - } - - CommonsApplication.pauseUploads.put(contribution.getPageId(), false); - - final File file = new File(contribution.getLocalUri().getPath()); - final List fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE); - - final int totalChunks = fileChunks.size(); - - final MediaType mediaType = MediaType - .parse(FileUtils.getMimeType(context, Uri.parse(file.getPath()))); - - final AtomicReference chunkInfo = new AtomicReference<>(); - if (isStashValid(contribution)) { - chunkInfo.set(contribution.getChunkInfo()); - - Timber.d("Chunk: Next Chunk: %s, Total Chunks: %s", - contribution.getChunkInfo().getIndexOfNextChunkToUpload(), - contribution.getChunkInfo().getTotalChunks()); - } - - final AtomicInteger index = new AtomicInteger(); - final AtomicBoolean failures = new AtomicBoolean(); - - compositeDisposable.add(Observable.fromIterable(fileChunks).forEach(chunkFile -> { - if (CommonsApplication.pauseUploads.get(contribution.getPageId()) || failures.get()) { - return; - } - - if (chunkInfo.get() != null && index.get() < chunkInfo.get() - .getIndexOfNextChunkToUpload()) { - index.incrementAndGet(); - Timber.d("Chunk: Increment and return: %s", index.get()); - return; - } - index.getAndIncrement(); - final int offset = - chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getOffset() : 0; - - Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset); - final String filekey = - chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getFilekey() : null; - - final RequestBody requestBody = RequestBody - .create(mediaType, chunkFile); - final CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody, - notificationUpdater::onProgress, offset, - file.length()); - - compositeDisposable.add(uploadChunkToStash(filename, - file.length(), - offset, - filekey, - countingRequestBody).subscribe(uploadResult -> { - Timber.d("Chunk: Received Chunk number: %s, offset: %s", index.get(), - uploadResult.getOffset()); - chunkInfo.set( - new ChunkInfo(uploadResult, index.get(), totalChunks)); - notificationUpdater.onChunkUploaded(contribution, chunkInfo.get()); - }, throwable -> { - Timber.e(throwable, "Received error in chunk upload"); - failures.set(true); - })); - })); - - if (CommonsApplication.pauseUploads.get(contribution.getPageId())) { - Timber.d("Upload stash paused %s", contribution.getPageId()); - return Observable.just(new StashUploadResult(StashUploadState.PAUSED, null)); - } else if (failures.get()) { - Timber.d("Upload stash contains failures %s", contribution.getPageId()); - return Observable.just(new StashUploadResult(StashUploadState.FAILED, null)); - } else if (chunkInfo.get() != null) { - Timber.d("Upload stash success %s", contribution.getPageId()); - return Observable.just(new StashUploadResult(StashUploadState.SUCCESS, - chunkInfo.get().getUploadResult().getFilekey())); - } else { - Timber.d("Upload stash failed %s", contribution.getPageId()); - return Observable.just(new StashUploadResult(StashUploadState.FAILED, null)); - } - } - - /** - * Stash is valid for 6 hours. This function checks the validity of stash - * - * @param contribution - * @return - */ - private boolean isStashValid(Contribution contribution) { - return contribution.getChunkInfo() != null && - contribution.getDateModified() - .after(new Date(System.currentTimeMillis() - MAX_CHUNK_AGE)); - } - - /** - * Uploads a file chunk to stash - * - * @param filename The name of the file being uploaded - * @param fileSize The total size of the file - * @param offset The offset returned by the previous chunk upload - * @param fileKey The filekey returned by the previous chunk upload - * @param countingRequestBody Request body with chunk file - * @return - */ - Observable uploadChunkToStash(final String filename, - final long fileSize, - final long offset, - final String fileKey, - final CountingRequestBody countingRequestBody) { - final MultipartBody.Part filePart; - try { - filePart = MultipartBody.Part - .createFormData("chunk", URLEncoder.encode(filename, "utf-8"), countingRequestBody); - - return uploadInterface.uploadFileToStash(toRequestBody(filename), - toRequestBody(String.valueOf(fileSize)), - toRequestBody(String.valueOf(offset)), - toRequestBody(fileKey), - toRequestBody(csrfTokenClient.getTokenBlocking()), - filePart) - .map(UploadResponse::getUpload); - } catch (final Throwable throwable) { - Timber.e(throwable, "Failed to upload chunk to stash"); - return Observable.error(throwable); - } - } - - /** - * Converts string value to request body - */ - @Nullable - private RequestBody toRequestBody(@Nullable final String value) { - return value == null ? null : RequestBody.create(okhttp3.MultipartBody.FORM, value); - } - - - public Observable uploadFileFromStash( - final Contribution contribution, - final String uniqueFileName, - final String fileKey) { - try { - return uploadInterface - .uploadFileFromStash(csrfTokenClient.getTokenBlocking(), - pageContentsCreator.createFrom(contribution), - CommonsApplication.DEFAULT_EDIT_SUMMARY, - uniqueFileName, - fileKey).map(uploadResponse -> { - final UploadResponse uploadResult = gson - .fromJson(uploadResponse, UploadResponse.class); - if (uploadResult.getUpload() == null) { - final MwException exception = gson - .fromJson(uploadResponse, MwException.class); - Timber.e(exception, "Error in uploading file from stash"); - throw new Exception(exception.getErrorCode()); - } - return uploadResult.getUpload(); - }); - } catch (final Throwable throwable) { - Timber.e(throwable, "Exception occurred in uploading file from stash"); - return Observable.error(throwable); - } - } -} diff --git a/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.kt b/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.kt new file mode 100644 index 000000000..e082501d8 --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/upload/UploadClient.kt @@ -0,0 +1,259 @@ +package fr.free.nrw.commons.upload + +import com.google.gson.Gson +import com.google.gson.JsonObject +import fr.free.nrw.commons.CommonsApplication +import fr.free.nrw.commons.auth.csrf.CsrfTokenClient +import fr.free.nrw.commons.contributions.ChunkInfo +import fr.free.nrw.commons.contributions.Contribution +import fr.free.nrw.commons.upload.worker.UploadWorker.NotificationUpdateProgressListener +import fr.free.nrw.commons.wikidata.mwapi.MwException +import io.reactivex.Observable +import io.reactivex.disposables.CompositeDisposable +import okhttp3.MediaType +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.MultipartBody +import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import timber.log.Timber +import java.io.File +import java.io.IOException +import java.net.URLEncoder +import java.util.Date +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class UploadClient @Inject constructor( + private val uploadInterface: UploadInterface, + private val csrfTokenClient: CsrfTokenClient, + private val pageContentsCreator: PageContentsCreator, + private val fileUtilsWrapper: FileUtilsWrapper, + private val gson: Gson, private val timeProvider: TimeProvider +) { + private val CHUNK_SIZE = 512 * 1024 // 512 KB + + //This is maximum duration for which a stash is persisted on MediaWiki + // https://www.mediawiki.org/wiki/Manual:$wgUploadStashMaxAge + private val MAX_CHUNK_AGE = 6 * 3600 * 1000 // 6 hours + private val compositeDisposable = CompositeDisposable() + + /** + * Upload file to stash in chunks of specified size. Uploading files in chunks will make + * handling of large files easier. Also, it will be useful in supporting pause/resume of + * uploads + */ + @Throws(IOException::class) + fun uploadFileToStash( + filename: String, contribution: Contribution, + notificationUpdater: NotificationUpdateProgressListener + ): Observable { + if (contribution.isCompleted()) { + return Observable.just( + StashUploadResult(StashUploadState.SUCCESS, contribution.fileKey) + ) + } + + contribution.unpause() + + val file = contribution.localUriPath + val fileChunks = fileUtilsWrapper.getFileChunks(file, CHUNK_SIZE) + val mediaType = fileUtilsWrapper.getMimeType(file).toMediaTypeOrNull() + + val chunkInfo = AtomicReference() + if (isStashValid(contribution)) { + chunkInfo.set(contribution.chunkInfo) + Timber.d( + "Chunk: Next Chunk: %s, Total Chunks: %s", + contribution.chunkInfo!!.indexOfNextChunkToUpload, + contribution.chunkInfo!!.totalChunks + ) + } + + val index = AtomicInteger() + val failures = AtomicBoolean() + compositeDisposable.add( + Observable.fromIterable(fileChunks).forEach { chunkFile: File -> + if (canProcess(contribution, failures)) { + processChunk( + filename, contribution, notificationUpdater, chunkFile, + failures, chunkInfo, index, mediaType!!, file!!, fileChunks.size + ) + } + } + ) + + return when { + contribution.isPaused() -> { + Timber.d("Upload stash paused %s", contribution.pageId) + Observable.just(StashUploadResult(StashUploadState.PAUSED, null)) + } + failures.get() -> { + Timber.d("Upload stash contains failures %s", contribution.pageId) + Observable.just(StashUploadResult(StashUploadState.FAILED, null)) + } + chunkInfo.get() != null -> { + Timber.d("Upload stash success %s", contribution.pageId) + Observable.just( + StashUploadResult( + StashUploadState.SUCCESS, + chunkInfo.get()!!.uploadResult!!.filekey + ) + ) + } + else -> { + Timber.d("Upload stash failed %s", contribution.pageId) + Observable.just(StashUploadResult(StashUploadState.FAILED, null)) + } + } + } + + private fun processChunk( + filename: String, contribution: Contribution, + notificationUpdater: NotificationUpdateProgressListener, chunkFile: File, + failures: AtomicBoolean, chunkInfo: AtomicReference, index: AtomicInteger, + mediaType: MediaType, file: File, totalChunks: Int + ) { + if (shouldSkip(chunkInfo, index)) { + index.incrementAndGet() + Timber.d("Chunk: Increment and return: %s", index.get()) + return + } + + index.getAndIncrement() + + val offset = if (chunkInfo.get() != null) chunkInfo.get()!!.uploadResult!!.offset else 0 + Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset) + + val filekey = chunkInfo.get()?.let { it.uploadResult!!.filekey } + val requestBody = chunkFile.asRequestBody(mediaType) + val listener = { transferred: Long, total: Long -> + notificationUpdater.onProgress(transferred, total) + } + val countingRequestBody = CountingRequestBody(requestBody, listener, offset.toLong(), file.length()) + + compositeDisposable.add( + uploadChunkToStash( + filename, file.length(), offset.toLong(), filekey, countingRequestBody + ).subscribe( + { uploadResult: UploadResult -> + Timber.d( + "Chunk: Received Chunk number: %s, offset: %s", + index.get(), + uploadResult.offset + ) + chunkInfo.set(ChunkInfo(uploadResult, index.get(), totalChunks)) + notificationUpdater.onChunkUploaded(contribution, chunkInfo.get()) + }, { throwable: Throwable? -> + Timber.e(throwable, "Received error in chunk upload") + failures.set(true) + } + ) + ) + } + + /** + * Stash is valid for 6 hours. This function checks the validity of stash + * + * @param contribution + * @return + */ + private fun isStashValid(contribution: Contribution): Boolean { + return contribution.chunkInfo != null && + contribution.dateModified!!.after(Date( + timeProvider.currentTimeMillis() - MAX_CHUNK_AGE)) + } + + /** + * Uploads a file chunk to stash + * + * @param filename The name of the file being uploaded + * @param fileSize The total size of the file + * @param offset The offset returned by the previous chunk upload + * @param fileKey The filekey returned by the previous chunk upload + * @param countingRequestBody Request body with chunk file + * @return + */ + fun uploadChunkToStash( + filename: String?, + fileSize: Long, + offset: Long, + fileKey: String?, + countingRequestBody: CountingRequestBody + ): Observable { + val filePart: MultipartBody.Part + return try { + filePart = MultipartBody.Part.createFormData( + "chunk", + URLEncoder.encode(filename, "utf-8"), + countingRequestBody + ) + uploadInterface.uploadFileToStash( + toRequestBody(filename), + toRequestBody(fileSize.toString()), + toRequestBody(offset.toString()), + toRequestBody(fileKey), + toRequestBody(csrfTokenClient.getTokenBlocking()), + filePart + ).map(UploadResponse::upload) + } catch (throwable: Throwable) { + Timber.e(throwable, "Failed to upload chunk to stash") + Observable.error(throwable) + } + } + + /** + * Converts string value to request body + */ + private fun toRequestBody(value: String?): RequestBody? { + return value?.toRequestBody(MultipartBody.FORM) + } + + fun uploadFileFromStash( + contribution: Contribution?, + uniqueFileName: String?, + fileKey: String? + ): Observable { + return try { + uploadInterface.uploadFileFromStash( + csrfTokenClient.getTokenBlocking(), + pageContentsCreator.createFrom(contribution), + CommonsApplication.DEFAULT_EDIT_SUMMARY, + uniqueFileName!!, + fileKey!! + ).map { uploadResponse: JsonObject? -> + val uploadResult = gson.fromJson(uploadResponse, UploadResponse::class.java) + if (uploadResult.upload == null) { + val exception = gson.fromJson(uploadResponse, MwException::class.java) + Timber.e(exception, "Error in uploading file from stash") + throw Exception(exception.getErrorCode()) + } + uploadResult.upload + } + } catch (throwable: Throwable) { + Timber.e(throwable, "Exception occurred in uploading file from stash") + Observable.error(throwable) + } + } + + fun interface TimeProvider { + fun currentTimeMillis(): Long + } +} + +private fun canProcess(contribution: Contribution, failures: AtomicBoolean): Boolean { + // As long as the contribution hasn't been paused and there are no errors, + // we can process the current chunk. + return !(contribution.isPaused() || failures.get()) +} + +private fun shouldSkip( + chunkInfo: AtomicReference, + index: AtomicInteger +): Boolean { + return chunkInfo.get() != null && index.get() < chunkInfo.get()!!.indexOfNextChunkToUpload +} diff --git a/app/src/main/java/fr/free/nrw/commons/upload/UploadModule.java b/app/src/main/java/fr/free/nrw/commons/upload/UploadModule.java index 0094fda22..602d75542 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/UploadModule.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/UploadModule.java @@ -1,7 +1,11 @@ package fr.free.nrw.commons.upload; +import com.google.gson.Gson; import dagger.Binds; import dagger.Module; +import dagger.Provides; +import fr.free.nrw.commons.auth.csrf.CsrfTokenClient; +import fr.free.nrw.commons.di.NetworkingModule; import fr.free.nrw.commons.upload.categories.CategoriesContract; import fr.free.nrw.commons.upload.categories.CategoriesPresenter; import fr.free.nrw.commons.upload.depicts.DepictsContract; @@ -10,6 +14,7 @@ import fr.free.nrw.commons.upload.license.MediaLicenseContract; import fr.free.nrw.commons.upload.license.MediaLicensePresenter; import fr.free.nrw.commons.upload.mediaDetails.UploadMediaDetailsContract; import fr.free.nrw.commons.upload.mediaDetails.UploadMediaPresenter; +import javax.inject.Named; /** * The Dagger Module for upload related presenters and (some other objects maybe in future) @@ -40,4 +45,13 @@ public abstract class UploadModule { DepictsPresenter presenter ); + + @Provides + public static UploadClient provideUploadClient(final UploadInterface uploadInterface, + @Named(NetworkingModule.NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient, + final PageContentsCreator pageContentsCreator, final FileUtilsWrapper fileUtilsWrapper, + final Gson gson) { + return new UploadClient(uploadInterface, csrfTokenClient, pageContentsCreator, + fileUtilsWrapper, gson, System::currentTimeMillis); + } } diff --git a/app/src/main/java/fr/free/nrw/commons/upload/worker/UploadWorker.kt b/app/src/main/java/fr/free/nrw/commons/upload/worker/UploadWorker.kt index ee03ac845..864e3149a 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/worker/UploadWorker.kt +++ b/app/src/main/java/fr/free/nrw/commons/upload/worker/UploadWorker.kt @@ -43,7 +43,6 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import timber.log.Timber -import java.net.SocketTimeoutException import java.util.* import java.util.regex.Pattern import javax.inject.Inject @@ -337,7 +336,7 @@ class UploadWorker(var appContext: Context, workerParams: WorkerParameters) : try { //Upload the file to stash val stashUploadResult = uploadClient.uploadFileToStash( - appContext, filename, contribution, notificationProgressUpdater + filename!!, contribution, notificationProgressUpdater ).onErrorReturn{ return@onErrorReturn StashUploadResult(StashUploadState.FAILED,fileKey = null) }.blockingSingle() diff --git a/app/src/test/kotlin/fr/free/nrw/commons/upload/UploadClientTest.kt b/app/src/test/kotlin/fr/free/nrw/commons/upload/UploadClientTest.kt new file mode 100644 index 000000000..aa2265a47 --- /dev/null +++ b/app/src/test/kotlin/fr/free/nrw/commons/upload/UploadClientTest.kt @@ -0,0 +1,241 @@ +package fr.free.nrw.commons.upload + +import com.google.gson.Gson +import com.google.gson.JsonObject +import com.nhaarman.mockitokotlin2.KArgumentCaptor +import com.nhaarman.mockitokotlin2.any +import com.nhaarman.mockitokotlin2.anyOrNull +import com.nhaarman.mockitokotlin2.argumentCaptor +import com.nhaarman.mockitokotlin2.eq +import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.times +import com.nhaarman.mockitokotlin2.verify +import com.nhaarman.mockitokotlin2.whenever +import fr.free.nrw.commons.CommonsApplication.DEFAULT_EDIT_SUMMARY +import fr.free.nrw.commons.auth.csrf.CsrfTokenClient +import fr.free.nrw.commons.contributions.ChunkInfo +import fr.free.nrw.commons.contributions.Contribution +import fr.free.nrw.commons.upload.UploadClient.TimeProvider +import fr.free.nrw.commons.wikidata.mwapi.MwException +import fr.free.nrw.commons.wikidata.mwapi.MwServiceError +import io.reactivex.Observable +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertSame +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.MultipartBody +import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import okio.Buffer +import org.junit.Before +import org.junit.Test +import java.io.File +import java.util.Date + + +class UploadClientTest { + + private val contribution = mock() + private val uploadResult = mock() + private val uploadInterface = mock() + private val csrfTokenClient = mock() + private val pageContentsCreator = mock() + private val fileUtilsWrapper = mock() + private val gson = mock() + private val timeProvider = mock() + private val uploadClient = UploadClient(uploadInterface, csrfTokenClient, pageContentsCreator, fileUtilsWrapper, gson, timeProvider) + + private val expectedChunkSize = 512 * 1024 + private val testToken = "test-token" + private val createdContent = "content" + private val filename = "test.jpg" + private val filekey = "the-key" + private val errorCode = "the-code" + private val uploadJson = Gson().fromJson("{\"foo\" = 1}", JsonObject::class.java) + + private val uploadResponse = UploadResponse(uploadResult) + private val errorResponse = UploadResponse(null) + + @Before + fun setUp() { + whenever(csrfTokenClient.getTokenBlocking()).thenReturn(testToken) + whenever(pageContentsCreator.createFrom(contribution)).thenReturn(createdContent) + } + + @Test + fun testUploadFileFromStash_NoErrors() { + whenever(gson.fromJson(uploadJson, UploadResponse::class.java)).thenReturn(uploadResponse) + whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey)).thenReturn(Observable.just(uploadJson)) + + val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test() + + result.assertNoErrors() + assertSame(uploadResult, result.values()[0]) + } + + @Test + fun testUploadFileFromStash_WithError() { + val error = mock() + whenever(error.code).thenReturn(errorCode) + val uploadException = MwException(error, null) + + whenever(gson.fromJson(uploadJson, UploadResponse::class.java)).thenReturn(errorResponse) + whenever(gson.fromJson(uploadJson, MwException::class.java)).thenReturn(uploadException) + whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey)).thenReturn(Observable.just(uploadJson)) + + val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test() + + result.assertNoValues() + assertEquals(errorCode, result.errors()[0].message) + } + + @Test + fun testUploadFileFromStash_Failure() { + val exception = Exception("test") + whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey)) + .thenReturn(Observable.error(exception)) + + val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test() + + result.assertNoValues() + assertEquals(exception, result.errors()[0]) + } + + @Test + fun testUploadChunkToStash_Success() { + val fileContent = "content" + val requestBody: RequestBody = fileContent.toRequestBody("text/plain".toMediaType()) + val countingRequestBody = CountingRequestBody(requestBody, mock(), 0, fileContent.length.toLong()) + + val filenameCaptor: KArgumentCaptor = argumentCaptor() + val totalFileSizeCaptor = argumentCaptor() + val offsetCaptor = argumentCaptor() + val fileKeyCaptor = argumentCaptor() + val tokenCaptor = argumentCaptor() + val fileCaptor = argumentCaptor() + + whenever(uploadInterface.uploadFileToStash( + filenameCaptor.capture(), totalFileSizeCaptor.capture(), offsetCaptor.capture(), + fileKeyCaptor.capture(), tokenCaptor.capture(), fileCaptor.capture() + )).thenReturn(Observable.just(uploadResponse)) + + val result = uploadClient.uploadChunkToStash(filename, 100, 10, filekey, countingRequestBody).test() + + result.assertNoErrors() + assertSame(uploadResult, result.values()[0]) + + assertEquals(filename, filenameCaptor.asString()) + assertEquals("100", totalFileSizeCaptor.asString()) + assertEquals("10", offsetCaptor.asString()) + assertEquals(filekey, fileKeyCaptor.asString()) + assertEquals(testToken, tokenCaptor.asString()) + assertEquals(fileContent, fileCaptor.firstValue.body.asString()) + } + + @Test + fun testUploadChunkToStash_Failure() { + val exception = Exception("expected") + whenever(uploadInterface.uploadFileToStash(any(), any(), any(), any(), any(), any())) + .thenReturn(Observable.error(exception)) + + val result = uploadClient.uploadChunkToStash(filename, 100, 10, filekey, mock()).test() + + result.assertNoValues() + assertSame(exception, result.errors()[0]) + } + + @Test + fun uploadFileToStash_completedContribution() { + whenever(contribution.isCompleted()).thenReturn(true) + whenever(contribution.fileKey).thenReturn(filekey) + + val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test() + + result.assertNoErrors() + val stashResult = result.values()[0] + assertEquals(filekey, stashResult.fileKey) + assertEquals(StashUploadState.SUCCESS, stashResult.state) + } + + @Test + fun uploadFileToStash_contributionIsUnpaused() { + whenever(contribution.isCompleted()).thenReturn(false) + whenever(contribution.fileKey).thenReturn(filekey) + whenever(fileUtilsWrapper.getMimeType(anyOrNull())).thenReturn("image/png") + whenever(fileUtilsWrapper.getFileChunks(anyOrNull(), eq(expectedChunkSize))).thenReturn(emptyList()) + + val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test() + + result.assertNoErrors() + verify(contribution, times(1)).unpause() + } + + @Test + fun uploadFileToStash_returnsFailureIfNothingToUpload() { + whenever(contribution.isCompleted()).thenReturn(false) + whenever(contribution.fileKey).thenReturn(filekey) + whenever(fileUtilsWrapper.getMimeType(anyOrNull())).thenReturn("image/png") + whenever(fileUtilsWrapper.getFileChunks(anyOrNull(), eq(expectedChunkSize))).thenReturn(emptyList()) + + val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test() + + result.assertNoErrors() + assertEquals(StashUploadState.FAILED, result.values()[0].state) + } + + @Test + fun uploadFileToStash_returnsFailureIfAnyChunkFails() { + val mockFile = mock() + whenever(mockFile.length()).thenReturn(1) + whenever(contribution.localUriPath).thenReturn(mockFile) + whenever(contribution.isCompleted()).thenReturn(false) + whenever(contribution.fileKey).thenReturn(filekey) + whenever(fileUtilsWrapper.getMimeType(anyOrNull())).thenReturn("image/png") + whenever(fileUtilsWrapper.getFileChunks(anyOrNull(), eq(expectedChunkSize))).thenReturn(listOf(mockFile)) + whenever(uploadInterface.uploadFileToStash(any(), any(), any(), any(), any(), any())).thenReturn(Observable.just(uploadResponse)) + + val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test() + + result.assertNoErrors() + assertEquals(StashUploadState.FAILED, result.values()[0].state) + } + + @Test + fun uploadFileToStash_successWithOneChunk() { + val mockFile = mock() + val chunkInfo = mock() + whenever(mockFile.length()).thenReturn(10) + whenever(chunkInfo.uploadResult).thenReturn(uploadResult) + + whenever(uploadResult.offset).thenReturn(1) + whenever(uploadResult.filekey).thenReturn(filekey) + + whenever(contribution.localUriPath).thenReturn(mockFile) + whenever(contribution.chunkInfo).thenReturn(chunkInfo) + whenever(contribution.isCompleted()).thenReturn(false) + whenever(contribution.dateModified).thenReturn(Date(100)) + whenever(timeProvider.currentTimeMillis()).thenReturn(200) + whenever(contribution.fileKey).thenReturn(filekey) + + whenever(fileUtilsWrapper.getMimeType(anyOrNull())).thenReturn("image/png") + whenever(fileUtilsWrapper.getFileChunks(anyOrNull(), eq(expectedChunkSize))).thenReturn(listOf(mockFile)) + + whenever(uploadInterface.uploadFileToStash(anyOrNull(), anyOrNull(), anyOrNull(), + anyOrNull(), anyOrNull(), anyOrNull())).thenReturn(Observable.just(uploadResponse)) + + val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test() + + result.assertNoErrors() + assertEquals(StashUploadState.SUCCESS, result.values()[0].state) + assertEquals(filekey, result.values()[0].fileKey) + } + + + private fun KArgumentCaptor.asString(): String = + firstValue.asString() + + private fun RequestBody.asString(): String { + val b = Buffer() + writeTo(b) + return b.readUtf8() + } +} \ No newline at end of file diff --git a/app/src/test/kotlin/fr/free/nrw/commons/utils/FileUtilsTest.kt b/app/src/test/kotlin/fr/free/nrw/commons/utils/FileUtilsTest.kt index 4980ec61a..8e4438c64 100644 --- a/app/src/test/kotlin/fr/free/nrw/commons/utils/FileUtilsTest.kt +++ b/app/src/test/kotlin/fr/free/nrw/commons/utils/FileUtilsTest.kt @@ -1,5 +1,6 @@ package fr.free.nrw.commons.utils +import com.nhaarman.mockitokotlin2.mock import fr.free.nrw.commons.upload.FileUtils import fr.free.nrw.commons.upload.FileUtilsWrapper import org.junit.Assert.assertEquals @@ -19,7 +20,7 @@ class FileUtilsTest { @Test fun testSHA1() { - val fileUtilsWrapper = FileUtilsWrapper() + val fileUtilsWrapper = FileUtilsWrapper(mock()) assertEquals( "907d14fb3af2b0d4f18c2d46abe8aedce17367bd",