mirror of
https://github.com/commons-app/apps-android-commons.git
synced 2025-10-26 12:23:58 +01:00
Convert upload client to kotlin (#5568)
* Write unit tests for the UploadClient (with gentle refactoring to make it testable) * Convert Upload client to kotlin
This commit is contained in:
Paul Hawke
GitHub
parent
728712c4e1
commit
f0a1d036a5
10 changed files with 552 additions and 247 deletions
|
|
@ -5,6 +5,7 @@ import android.os.Parcelable
|
|||
import androidx.room.Embedded
|
||||
import androidx.room.Entity
|
||||
import androidx.room.PrimaryKey
|
||||
import fr.free.nrw.commons.CommonsApplication
|
||||
import fr.free.nrw.commons.Media
|
||||
import fr.free.nrw.commons.auth.SessionManager
|
||||
import fr.free.nrw.commons.upload.UploadItem
|
||||
|
|
@ -13,7 +14,8 @@ import fr.free.nrw.commons.upload.WikidataPlace
|
|||
import fr.free.nrw.commons.upload.WikidataPlace.Companion.from
|
||||
import fr.free.nrw.commons.upload.structure.depictions.DepictedItem
|
||||
import kotlinx.parcelize.Parcelize
|
||||
import java.util.*
|
||||
import java.io.File
|
||||
import java.util.Date
|
||||
|
||||
@Entity(tableName = "contribution")
|
||||
@Parcelize
|
||||
|
|
@ -117,4 +119,19 @@ data class Contribution constructor(
|
|||
descriptions.filter { it.descriptionText.isNotEmpty() }
|
||||
.joinToString(separator = "") { "{{${it.languageCode}|1=${it.descriptionText}}}" }
|
||||
}
|
||||
|
||||
val fileKey : String? get() = chunkInfo?.uploadResult?.filekey
|
||||
val localUriPath: File? get() = localUri?.path?.let { File(it) }
|
||||
|
||||
fun isCompleted(): Boolean {
|
||||
return chunkInfo != null && chunkInfo!!.totalChunks == chunkInfo!!.indexOfNextChunkToUpload
|
||||
}
|
||||
|
||||
fun isPaused(): Boolean {
|
||||
return CommonsApplication.pauseUploads[pageId] ?: false
|
||||
}
|
||||
|
||||
fun unpause() {
|
||||
CommonsApplication.pauseUploads[pageId] = false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class CountingRequestBody(
|
|||
}
|
||||
}
|
||||
|
||||
interface Listener {
|
||||
fun interface Listener {
|
||||
/**
|
||||
* Will be triggered when write progresses
|
||||
* @param bytesWritten
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package fr.free.nrw.commons.upload;
|
||||
|
||||
import android.content.Context;
|
||||
import android.net.Uri;
|
||||
import fr.free.nrw.commons.location.LatLng;
|
||||
import io.reactivex.Observable;
|
||||
import java.io.BufferedInputStream;
|
||||
|
|
@ -21,9 +22,11 @@ import timber.log.Timber;
|
|||
@Singleton
|
||||
public class FileUtilsWrapper {
|
||||
|
||||
@Inject
|
||||
public FileUtilsWrapper() {
|
||||
private final Context context;
|
||||
|
||||
@Inject
|
||||
public FileUtilsWrapper(final Context context) {
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
public String getFileExt(String fileName) {
|
||||
|
|
@ -42,11 +45,18 @@ public class FileUtilsWrapper {
|
|||
return FileUtils.getGeolocationOfFile(filePath, inAppPictureLocation);
|
||||
}
|
||||
|
||||
public String getMimeType(File file) {
|
||||
return getMimeType(Uri.parse(file.getPath()));
|
||||
}
|
||||
|
||||
public String getMimeType(Uri uri) {
|
||||
return FileUtils.getMimeType(context, uri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a file as input and returns an Observable of files with the specified chunk size
|
||||
*/
|
||||
public List<File> getFileChunks(Context context, File file, final int chunkSize)
|
||||
public List<File> getFileChunks(File file, final int chunkSize)
|
||||
throws IOException {
|
||||
final byte[] buffer = new byte[chunkSize];
|
||||
|
||||
|
|
@ -56,7 +66,7 @@ public class FileUtilsWrapper {
|
|||
final List<File> buffers = new ArrayList<>();
|
||||
int size;
|
||||
while ((size = bis.read(buffer)) > 0) {
|
||||
buffers.add(writeToFile(context, Arrays.copyOf(buffer, size), file.getName(),
|
||||
buffers.add(writeToFile(Arrays.copyOf(buffer, size), file.getName(),
|
||||
getFileExt(file.getName())));
|
||||
}
|
||||
return buffers;
|
||||
|
|
@ -66,7 +76,7 @@ public class FileUtilsWrapper {
|
|||
/**
|
||||
* Create a temp file containing the passed byte data.
|
||||
*/
|
||||
private File writeToFile(Context context, final byte[] data, final String fileName,
|
||||
private File writeToFile(final byte[] data, final String fileName,
|
||||
String fileExtension)
|
||||
throws IOException {
|
||||
final File file = File.createTempFile(fileName, fileExtension, context.getCacheDir());
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import javax.inject.Inject;
|
|||
import org.apache.commons.lang3.StringUtils;
|
||||
import timber.log.Timber;
|
||||
|
||||
class PageContentsCreator {
|
||||
public class PageContentsCreator {
|
||||
|
||||
//{{According to Exif data|2009-01-09}}
|
||||
private static final String TEMPLATE_DATE_ACC_TO_EXIF = "{{According to Exif data|%s}}";
|
||||
|
|
|
|||
|
|
@ -1,236 +0,0 @@
|
|||
package fr.free.nrw.commons.upload;
|
||||
|
||||
import static fr.free.nrw.commons.di.NetworkingModule.NAMED_COMMONS_CSRF;
|
||||
|
||||
import android.content.Context;
|
||||
import android.net.Uri;
|
||||
import androidx.annotation.Nullable;
|
||||
import com.google.gson.Gson;
|
||||
import fr.free.nrw.commons.CommonsApplication;
|
||||
import fr.free.nrw.commons.contributions.ChunkInfo;
|
||||
import fr.free.nrw.commons.contributions.Contribution;
|
||||
import fr.free.nrw.commons.upload.worker.UploadWorker.NotificationUpdateProgressListener;
|
||||
import io.reactivex.Observable;
|
||||
import io.reactivex.disposables.CompositeDisposable;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import javax.inject.Singleton;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.MultipartBody;
|
||||
import okhttp3.RequestBody;
|
||||
import fr.free.nrw.commons.auth.csrf.CsrfTokenClient;
|
||||
import fr.free.nrw.commons.wikidata.mwapi.MwException;
|
||||
import timber.log.Timber;
|
||||
|
||||
@Singleton
|
||||
public class UploadClient {
|
||||
|
||||
private final int CHUNK_SIZE = 512 * 1024; // 512 KB
|
||||
|
||||
//This is maximum duration for which a stash is persisted on MediaWiki
|
||||
// https://www.mediawiki.org/wiki/Manual:$wgUploadStashMaxAge
|
||||
private final int MAX_CHUNK_AGE = 6 * 3600 * 1000; // 6 hours
|
||||
|
||||
private final UploadInterface uploadInterface;
|
||||
private final CsrfTokenClient csrfTokenClient;
|
||||
private final PageContentsCreator pageContentsCreator;
|
||||
private final FileUtilsWrapper fileUtilsWrapper;
|
||||
private final Gson gson;
|
||||
|
||||
private final CompositeDisposable compositeDisposable = new CompositeDisposable();
|
||||
|
||||
@Inject
|
||||
public UploadClient(final UploadInterface uploadInterface,
|
||||
@Named(NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient,
|
||||
final PageContentsCreator pageContentsCreator,
|
||||
final FileUtilsWrapper fileUtilsWrapper, final Gson gson) {
|
||||
this.uploadInterface = uploadInterface;
|
||||
this.csrfTokenClient = csrfTokenClient;
|
||||
this.pageContentsCreator = pageContentsCreator;
|
||||
this.fileUtilsWrapper = fileUtilsWrapper;
|
||||
this.gson = gson;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload file to stash in chunks of specified size. Uploading files in chunks will make
|
||||
* handling of large files easier. Also, it will be useful in supporting pause/resume of
|
||||
* uploads
|
||||
*/
|
||||
public Observable<StashUploadResult> uploadFileToStash(
|
||||
final Context context, final String filename, final Contribution contribution,
|
||||
final NotificationUpdateProgressListener notificationUpdater) throws IOException {
|
||||
if (contribution.getChunkInfo() != null
|
||||
&& contribution.getChunkInfo().getTotalChunks() == contribution.getChunkInfo()
|
||||
.getIndexOfNextChunkToUpload()) {
|
||||
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
|
||||
contribution.getChunkInfo().getUploadResult().getFilekey()));
|
||||
}
|
||||
|
||||
CommonsApplication.pauseUploads.put(contribution.getPageId(), false);
|
||||
|
||||
final File file = new File(contribution.getLocalUri().getPath());
|
||||
final List<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
|
||||
|
||||
final int totalChunks = fileChunks.size();
|
||||
|
||||
final MediaType mediaType = MediaType
|
||||
.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath())));
|
||||
|
||||
final AtomicReference<ChunkInfo> chunkInfo = new AtomicReference<>();
|
||||
if (isStashValid(contribution)) {
|
||||
chunkInfo.set(contribution.getChunkInfo());
|
||||
|
||||
Timber.d("Chunk: Next Chunk: %s, Total Chunks: %s",
|
||||
contribution.getChunkInfo().getIndexOfNextChunkToUpload(),
|
||||
contribution.getChunkInfo().getTotalChunks());
|
||||
}
|
||||
|
||||
final AtomicInteger index = new AtomicInteger();
|
||||
final AtomicBoolean failures = new AtomicBoolean();
|
||||
|
||||
compositeDisposable.add(Observable.fromIterable(fileChunks).forEach(chunkFile -> {
|
||||
if (CommonsApplication.pauseUploads.get(contribution.getPageId()) || failures.get()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (chunkInfo.get() != null && index.get() < chunkInfo.get()
|
||||
.getIndexOfNextChunkToUpload()) {
|
||||
index.incrementAndGet();
|
||||
Timber.d("Chunk: Increment and return: %s", index.get());
|
||||
return;
|
||||
}
|
||||
index.getAndIncrement();
|
||||
final int offset =
|
||||
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getOffset() : 0;
|
||||
|
||||
Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset);
|
||||
final String filekey =
|
||||
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getFilekey() : null;
|
||||
|
||||
final RequestBody requestBody = RequestBody
|
||||
.create(mediaType, chunkFile);
|
||||
final CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
|
||||
notificationUpdater::onProgress, offset,
|
||||
file.length());
|
||||
|
||||
compositeDisposable.add(uploadChunkToStash(filename,
|
||||
file.length(),
|
||||
offset,
|
||||
filekey,
|
||||
countingRequestBody).subscribe(uploadResult -> {
|
||||
Timber.d("Chunk: Received Chunk number: %s, offset: %s", index.get(),
|
||||
uploadResult.getOffset());
|
||||
chunkInfo.set(
|
||||
new ChunkInfo(uploadResult, index.get(), totalChunks));
|
||||
notificationUpdater.onChunkUploaded(contribution, chunkInfo.get());
|
||||
}, throwable -> {
|
||||
Timber.e(throwable, "Received error in chunk upload");
|
||||
failures.set(true);
|
||||
}));
|
||||
}));
|
||||
|
||||
if (CommonsApplication.pauseUploads.get(contribution.getPageId())) {
|
||||
Timber.d("Upload stash paused %s", contribution.getPageId());
|
||||
return Observable.just(new StashUploadResult(StashUploadState.PAUSED, null));
|
||||
} else if (failures.get()) {
|
||||
Timber.d("Upload stash contains failures %s", contribution.getPageId());
|
||||
return Observable.just(new StashUploadResult(StashUploadState.FAILED, null));
|
||||
} else if (chunkInfo.get() != null) {
|
||||
Timber.d("Upload stash success %s", contribution.getPageId());
|
||||
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
|
||||
chunkInfo.get().getUploadResult().getFilekey()));
|
||||
} else {
|
||||
Timber.d("Upload stash failed %s", contribution.getPageId());
|
||||
return Observable.just(new StashUploadResult(StashUploadState.FAILED, null));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stash is valid for 6 hours. This function checks the validity of stash
|
||||
*
|
||||
* @param contribution
|
||||
* @return
|
||||
*/
|
||||
private boolean isStashValid(Contribution contribution) {
|
||||
return contribution.getChunkInfo() != null &&
|
||||
contribution.getDateModified()
|
||||
.after(new Date(System.currentTimeMillis() - MAX_CHUNK_AGE));
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads a file chunk to stash
|
||||
*
|
||||
* @param filename The name of the file being uploaded
|
||||
* @param fileSize The total size of the file
|
||||
* @param offset The offset returned by the previous chunk upload
|
||||
* @param fileKey The filekey returned by the previous chunk upload
|
||||
* @param countingRequestBody Request body with chunk file
|
||||
* @return
|
||||
*/
|
||||
Observable<UploadResult> uploadChunkToStash(final String filename,
|
||||
final long fileSize,
|
||||
final long offset,
|
||||
final String fileKey,
|
||||
final CountingRequestBody countingRequestBody) {
|
||||
final MultipartBody.Part filePart;
|
||||
try {
|
||||
filePart = MultipartBody.Part
|
||||
.createFormData("chunk", URLEncoder.encode(filename, "utf-8"), countingRequestBody);
|
||||
|
||||
return uploadInterface.uploadFileToStash(toRequestBody(filename),
|
||||
toRequestBody(String.valueOf(fileSize)),
|
||||
toRequestBody(String.valueOf(offset)),
|
||||
toRequestBody(fileKey),
|
||||
toRequestBody(csrfTokenClient.getTokenBlocking()),
|
||||
filePart)
|
||||
.map(UploadResponse::getUpload);
|
||||
} catch (final Throwable throwable) {
|
||||
Timber.e(throwable, "Failed to upload chunk to stash");
|
||||
return Observable.error(throwable);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts string value to request body
|
||||
*/
|
||||
@Nullable
|
||||
private RequestBody toRequestBody(@Nullable final String value) {
|
||||
return value == null ? null : RequestBody.create(okhttp3.MultipartBody.FORM, value);
|
||||
}
|
||||
|
||||
|
||||
public Observable<UploadResult> uploadFileFromStash(
|
||||
final Contribution contribution,
|
||||
final String uniqueFileName,
|
||||
final String fileKey) {
|
||||
try {
|
||||
return uploadInterface
|
||||
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
|
||||
pageContentsCreator.createFrom(contribution),
|
||||
CommonsApplication.DEFAULT_EDIT_SUMMARY,
|
||||
uniqueFileName,
|
||||
fileKey).map(uploadResponse -> {
|
||||
final UploadResponse uploadResult = gson
|
||||
.fromJson(uploadResponse, UploadResponse.class);
|
||||
if (uploadResult.getUpload() == null) {
|
||||
final MwException exception = gson
|
||||
.fromJson(uploadResponse, MwException.class);
|
||||
Timber.e(exception, "Error in uploading file from stash");
|
||||
throw new Exception(exception.getErrorCode());
|
||||
}
|
||||
return uploadResult.getUpload();
|
||||
});
|
||||
} catch (final Throwable throwable) {
|
||||
Timber.e(throwable, "Exception occurred in uploading file from stash");
|
||||
return Observable.error(throwable);
|
||||
}
|
||||
}
|
||||
}
|
||||
259
app/src/main/java/fr/free/nrw/commons/upload/UploadClient.kt
Normal file
259
app/src/main/java/fr/free/nrw/commons/upload/UploadClient.kt
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
package fr.free.nrw.commons.upload
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import fr.free.nrw.commons.CommonsApplication
|
||||
import fr.free.nrw.commons.auth.csrf.CsrfTokenClient
|
||||
import fr.free.nrw.commons.contributions.ChunkInfo
|
||||
import fr.free.nrw.commons.contributions.Contribution
|
||||
import fr.free.nrw.commons.upload.worker.UploadWorker.NotificationUpdateProgressListener
|
||||
import fr.free.nrw.commons.wikidata.mwapi.MwException
|
||||
import io.reactivex.Observable
|
||||
import io.reactivex.disposables.CompositeDisposable
|
||||
import okhttp3.MediaType
|
||||
import okhttp3.MediaType.Companion.toMediaTypeOrNull
|
||||
import okhttp3.MultipartBody
|
||||
import okhttp3.RequestBody
|
||||
import okhttp3.RequestBody.Companion.asRequestBody
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.net.URLEncoder
|
||||
import java.util.Date
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class UploadClient @Inject constructor(
|
||||
private val uploadInterface: UploadInterface,
|
||||
private val csrfTokenClient: CsrfTokenClient,
|
||||
private val pageContentsCreator: PageContentsCreator,
|
||||
private val fileUtilsWrapper: FileUtilsWrapper,
|
||||
private val gson: Gson, private val timeProvider: TimeProvider
|
||||
) {
|
||||
private val CHUNK_SIZE = 512 * 1024 // 512 KB
|
||||
|
||||
//This is maximum duration for which a stash is persisted on MediaWiki
|
||||
// https://www.mediawiki.org/wiki/Manual:$wgUploadStashMaxAge
|
||||
private val MAX_CHUNK_AGE = 6 * 3600 * 1000 // 6 hours
|
||||
private val compositeDisposable = CompositeDisposable()
|
||||
|
||||
/**
|
||||
* Upload file to stash in chunks of specified size. Uploading files in chunks will make
|
||||
* handling of large files easier. Also, it will be useful in supporting pause/resume of
|
||||
* uploads
|
||||
*/
|
||||
@Throws(IOException::class)
|
||||
fun uploadFileToStash(
|
||||
filename: String, contribution: Contribution,
|
||||
notificationUpdater: NotificationUpdateProgressListener
|
||||
): Observable<StashUploadResult> {
|
||||
if (contribution.isCompleted()) {
|
||||
return Observable.just(
|
||||
StashUploadResult(StashUploadState.SUCCESS, contribution.fileKey)
|
||||
)
|
||||
}
|
||||
|
||||
contribution.unpause()
|
||||
|
||||
val file = contribution.localUriPath
|
||||
val fileChunks = fileUtilsWrapper.getFileChunks(file, CHUNK_SIZE)
|
||||
val mediaType = fileUtilsWrapper.getMimeType(file).toMediaTypeOrNull()
|
||||
|
||||
val chunkInfo = AtomicReference<ChunkInfo?>()
|
||||
if (isStashValid(contribution)) {
|
||||
chunkInfo.set(contribution.chunkInfo)
|
||||
Timber.d(
|
||||
"Chunk: Next Chunk: %s, Total Chunks: %s",
|
||||
contribution.chunkInfo!!.indexOfNextChunkToUpload,
|
||||
contribution.chunkInfo!!.totalChunks
|
||||
)
|
||||
}
|
||||
|
||||
val index = AtomicInteger()
|
||||
val failures = AtomicBoolean()
|
||||
compositeDisposable.add(
|
||||
Observable.fromIterable(fileChunks).forEach { chunkFile: File ->
|
||||
if (canProcess(contribution, failures)) {
|
||||
processChunk(
|
||||
filename, contribution, notificationUpdater, chunkFile,
|
||||
failures, chunkInfo, index, mediaType!!, file!!, fileChunks.size
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return when {
|
||||
contribution.isPaused() -> {
|
||||
Timber.d("Upload stash paused %s", contribution.pageId)
|
||||
Observable.just(StashUploadResult(StashUploadState.PAUSED, null))
|
||||
}
|
||||
failures.get() -> {
|
||||
Timber.d("Upload stash contains failures %s", contribution.pageId)
|
||||
Observable.just(StashUploadResult(StashUploadState.FAILED, null))
|
||||
}
|
||||
chunkInfo.get() != null -> {
|
||||
Timber.d("Upload stash success %s", contribution.pageId)
|
||||
Observable.just(
|
||||
StashUploadResult(
|
||||
StashUploadState.SUCCESS,
|
||||
chunkInfo.get()!!.uploadResult!!.filekey
|
||||
)
|
||||
)
|
||||
}
|
||||
else -> {
|
||||
Timber.d("Upload stash failed %s", contribution.pageId)
|
||||
Observable.just(StashUploadResult(StashUploadState.FAILED, null))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun processChunk(
|
||||
filename: String, contribution: Contribution,
|
||||
notificationUpdater: NotificationUpdateProgressListener, chunkFile: File,
|
||||
failures: AtomicBoolean, chunkInfo: AtomicReference<ChunkInfo?>, index: AtomicInteger,
|
||||
mediaType: MediaType, file: File, totalChunks: Int
|
||||
) {
|
||||
if (shouldSkip(chunkInfo, index)) {
|
||||
index.incrementAndGet()
|
||||
Timber.d("Chunk: Increment and return: %s", index.get())
|
||||
return
|
||||
}
|
||||
|
||||
index.getAndIncrement()
|
||||
|
||||
val offset = if (chunkInfo.get() != null) chunkInfo.get()!!.uploadResult!!.offset else 0
|
||||
Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset)
|
||||
|
||||
val filekey = chunkInfo.get()?.let { it.uploadResult!!.filekey }
|
||||
val requestBody = chunkFile.asRequestBody(mediaType)
|
||||
val listener = { transferred: Long, total: Long ->
|
||||
notificationUpdater.onProgress(transferred, total)
|
||||
}
|
||||
val countingRequestBody = CountingRequestBody(requestBody, listener, offset.toLong(), file.length())
|
||||
|
||||
compositeDisposable.add(
|
||||
uploadChunkToStash(
|
||||
filename, file.length(), offset.toLong(), filekey, countingRequestBody
|
||||
).subscribe(
|
||||
{ uploadResult: UploadResult ->
|
||||
Timber.d(
|
||||
"Chunk: Received Chunk number: %s, offset: %s",
|
||||
index.get(),
|
||||
uploadResult.offset
|
||||
)
|
||||
chunkInfo.set(ChunkInfo(uploadResult, index.get(), totalChunks))
|
||||
notificationUpdater.onChunkUploaded(contribution, chunkInfo.get())
|
||||
}, { throwable: Throwable? ->
|
||||
Timber.e(throwable, "Received error in chunk upload")
|
||||
failures.set(true)
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stash is valid for 6 hours. This function checks the validity of stash
|
||||
*
|
||||
* @param contribution
|
||||
* @return
|
||||
*/
|
||||
private fun isStashValid(contribution: Contribution): Boolean {
|
||||
return contribution.chunkInfo != null &&
|
||||
contribution.dateModified!!.after(Date(
|
||||
timeProvider.currentTimeMillis() - MAX_CHUNK_AGE))
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads a file chunk to stash
|
||||
*
|
||||
* @param filename The name of the file being uploaded
|
||||
* @param fileSize The total size of the file
|
||||
* @param offset The offset returned by the previous chunk upload
|
||||
* @param fileKey The filekey returned by the previous chunk upload
|
||||
* @param countingRequestBody Request body with chunk file
|
||||
* @return
|
||||
*/
|
||||
fun uploadChunkToStash(
|
||||
filename: String?,
|
||||
fileSize: Long,
|
||||
offset: Long,
|
||||
fileKey: String?,
|
||||
countingRequestBody: CountingRequestBody
|
||||
): Observable<UploadResult> {
|
||||
val filePart: MultipartBody.Part
|
||||
return try {
|
||||
filePart = MultipartBody.Part.createFormData(
|
||||
"chunk",
|
||||
URLEncoder.encode(filename, "utf-8"),
|
||||
countingRequestBody
|
||||
)
|
||||
uploadInterface.uploadFileToStash(
|
||||
toRequestBody(filename),
|
||||
toRequestBody(fileSize.toString()),
|
||||
toRequestBody(offset.toString()),
|
||||
toRequestBody(fileKey),
|
||||
toRequestBody(csrfTokenClient.getTokenBlocking()),
|
||||
filePart
|
||||
).map(UploadResponse::upload)
|
||||
} catch (throwable: Throwable) {
|
||||
Timber.e(throwable, "Failed to upload chunk to stash")
|
||||
Observable.error(throwable)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts string value to request body
|
||||
*/
|
||||
private fun toRequestBody(value: String?): RequestBody? {
|
||||
return value?.toRequestBody(MultipartBody.FORM)
|
||||
}
|
||||
|
||||
fun uploadFileFromStash(
|
||||
contribution: Contribution?,
|
||||
uniqueFileName: String?,
|
||||
fileKey: String?
|
||||
): Observable<UploadResult?> {
|
||||
return try {
|
||||
uploadInterface.uploadFileFromStash(
|
||||
csrfTokenClient.getTokenBlocking(),
|
||||
pageContentsCreator.createFrom(contribution),
|
||||
CommonsApplication.DEFAULT_EDIT_SUMMARY,
|
||||
uniqueFileName!!,
|
||||
fileKey!!
|
||||
).map { uploadResponse: JsonObject? ->
|
||||
val uploadResult = gson.fromJson(uploadResponse, UploadResponse::class.java)
|
||||
if (uploadResult.upload == null) {
|
||||
val exception = gson.fromJson(uploadResponse, MwException::class.java)
|
||||
Timber.e(exception, "Error in uploading file from stash")
|
||||
throw Exception(exception.getErrorCode())
|
||||
}
|
||||
uploadResult.upload
|
||||
}
|
||||
} catch (throwable: Throwable) {
|
||||
Timber.e(throwable, "Exception occurred in uploading file from stash")
|
||||
Observable.error(throwable)
|
||||
}
|
||||
}
|
||||
|
||||
fun interface TimeProvider {
|
||||
fun currentTimeMillis(): Long
|
||||
}
|
||||
}
|
||||
|
||||
private fun canProcess(contribution: Contribution, failures: AtomicBoolean): Boolean {
|
||||
// As long as the contribution hasn't been paused and there are no errors,
|
||||
// we can process the current chunk.
|
||||
return !(contribution.isPaused() || failures.get())
|
||||
}
|
||||
|
||||
private fun shouldSkip(
|
||||
chunkInfo: AtomicReference<ChunkInfo?>,
|
||||
index: AtomicInteger
|
||||
): Boolean {
|
||||
return chunkInfo.get() != null && index.get() < chunkInfo.get()!!.indexOfNextChunkToUpload
|
||||
}
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
package fr.free.nrw.commons.upload;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import dagger.Binds;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
import fr.free.nrw.commons.auth.csrf.CsrfTokenClient;
|
||||
import fr.free.nrw.commons.di.NetworkingModule;
|
||||
import fr.free.nrw.commons.upload.categories.CategoriesContract;
|
||||
import fr.free.nrw.commons.upload.categories.CategoriesPresenter;
|
||||
import fr.free.nrw.commons.upload.depicts.DepictsContract;
|
||||
|
|
@ -10,6 +14,7 @@ import fr.free.nrw.commons.upload.license.MediaLicenseContract;
|
|||
import fr.free.nrw.commons.upload.license.MediaLicensePresenter;
|
||||
import fr.free.nrw.commons.upload.mediaDetails.UploadMediaDetailsContract;
|
||||
import fr.free.nrw.commons.upload.mediaDetails.UploadMediaPresenter;
|
||||
import javax.inject.Named;
|
||||
|
||||
/**
|
||||
* The Dagger Module for upload related presenters and (some other objects maybe in future)
|
||||
|
|
@ -40,4 +45,13 @@ public abstract class UploadModule {
|
|||
DepictsPresenter
|
||||
presenter
|
||||
);
|
||||
|
||||
@Provides
|
||||
public static UploadClient provideUploadClient(final UploadInterface uploadInterface,
|
||||
@Named(NetworkingModule.NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient,
|
||||
final PageContentsCreator pageContentsCreator, final FileUtilsWrapper fileUtilsWrapper,
|
||||
final Gson gson) {
|
||||
return new UploadClient(uploadInterface, csrfTokenClient, pageContentsCreator,
|
||||
fileUtilsWrapper, gson, System::currentTimeMillis);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import kotlinx.coroutines.flow.map
|
|||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import timber.log.Timber
|
||||
import java.net.SocketTimeoutException
|
||||
import java.util.*
|
||||
import java.util.regex.Pattern
|
||||
import javax.inject.Inject
|
||||
|
|
@ -337,7 +336,7 @@ class UploadWorker(var appContext: Context, workerParams: WorkerParameters) :
|
|||
try {
|
||||
//Upload the file to stash
|
||||
val stashUploadResult = uploadClient.uploadFileToStash(
|
||||
appContext, filename, contribution, notificationProgressUpdater
|
||||
filename!!, contribution, notificationProgressUpdater
|
||||
).onErrorReturn{
|
||||
return@onErrorReturn StashUploadResult(StashUploadState.FAILED,fileKey = null)
|
||||
}.blockingSingle()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,241 @@
|
|||
package fr.free.nrw.commons.upload
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import com.nhaarman.mockitokotlin2.KArgumentCaptor
|
||||
import com.nhaarman.mockitokotlin2.any
|
||||
import com.nhaarman.mockitokotlin2.anyOrNull
|
||||
import com.nhaarman.mockitokotlin2.argumentCaptor
|
||||
import com.nhaarman.mockitokotlin2.eq
|
||||
import com.nhaarman.mockitokotlin2.mock
|
||||
import com.nhaarman.mockitokotlin2.times
|
||||
import com.nhaarman.mockitokotlin2.verify
|
||||
import com.nhaarman.mockitokotlin2.whenever
|
||||
import fr.free.nrw.commons.CommonsApplication.DEFAULT_EDIT_SUMMARY
|
||||
import fr.free.nrw.commons.auth.csrf.CsrfTokenClient
|
||||
import fr.free.nrw.commons.contributions.ChunkInfo
|
||||
import fr.free.nrw.commons.contributions.Contribution
|
||||
import fr.free.nrw.commons.upload.UploadClient.TimeProvider
|
||||
import fr.free.nrw.commons.wikidata.mwapi.MwException
|
||||
import fr.free.nrw.commons.wikidata.mwapi.MwServiceError
|
||||
import io.reactivex.Observable
|
||||
import junit.framework.TestCase.assertEquals
|
||||
import junit.framework.TestCase.assertSame
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.MultipartBody
|
||||
import okhttp3.RequestBody
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import okio.Buffer
|
||||
import org.junit.Before
|
||||
import org.junit.Test
|
||||
import java.io.File
|
||||
import java.util.Date
|
||||
|
||||
|
||||
class UploadClientTest {
|
||||
|
||||
private val contribution = mock<Contribution>()
|
||||
private val uploadResult = mock<UploadResult>()
|
||||
private val uploadInterface = mock<UploadInterface>()
|
||||
private val csrfTokenClient = mock<CsrfTokenClient>()
|
||||
private val pageContentsCreator = mock<PageContentsCreator>()
|
||||
private val fileUtilsWrapper = mock<FileUtilsWrapper>()
|
||||
private val gson = mock<Gson>()
|
||||
private val timeProvider = mock<TimeProvider>()
|
||||
private val uploadClient = UploadClient(uploadInterface, csrfTokenClient, pageContentsCreator, fileUtilsWrapper, gson, timeProvider)
|
||||
|
||||
private val expectedChunkSize = 512 * 1024
|
||||
private val testToken = "test-token"
|
||||
private val createdContent = "content"
|
||||
private val filename = "test.jpg"
|
||||
private val filekey = "the-key"
|
||||
private val errorCode = "the-code"
|
||||
private val uploadJson = Gson().fromJson("{\"foo\" = 1}", JsonObject::class.java)
|
||||
|
||||
private val uploadResponse = UploadResponse(uploadResult)
|
||||
private val errorResponse = UploadResponse(null)
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
whenever(csrfTokenClient.getTokenBlocking()).thenReturn(testToken)
|
||||
whenever(pageContentsCreator.createFrom(contribution)).thenReturn(createdContent)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testUploadFileFromStash_NoErrors() {
|
||||
whenever(gson.fromJson(uploadJson, UploadResponse::class.java)).thenReturn(uploadResponse)
|
||||
whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey)).thenReturn(Observable.just(uploadJson))
|
||||
|
||||
val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
assertSame(uploadResult, result.values()[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testUploadFileFromStash_WithError() {
|
||||
val error = mock<MwServiceError>()
|
||||
whenever(error.code).thenReturn(errorCode)
|
||||
val uploadException = MwException(error, null)
|
||||
|
||||
whenever(gson.fromJson(uploadJson, UploadResponse::class.java)).thenReturn(errorResponse)
|
||||
whenever(gson.fromJson(uploadJson, MwException::class.java)).thenReturn(uploadException)
|
||||
whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey)).thenReturn(Observable.just(uploadJson))
|
||||
|
||||
val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test()
|
||||
|
||||
result.assertNoValues()
|
||||
assertEquals(errorCode, result.errors()[0].message)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testUploadFileFromStash_Failure() {
|
||||
val exception = Exception("test")
|
||||
whenever(uploadInterface.uploadFileFromStash(testToken, createdContent, DEFAULT_EDIT_SUMMARY, filename, filekey))
|
||||
.thenReturn(Observable.error(exception))
|
||||
|
||||
val result = uploadClient.uploadFileFromStash(contribution, filename, filekey).test()
|
||||
|
||||
result.assertNoValues()
|
||||
assertEquals(exception, result.errors()[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testUploadChunkToStash_Success() {
|
||||
val fileContent = "content"
|
||||
val requestBody: RequestBody = fileContent.toRequestBody("text/plain".toMediaType())
|
||||
val countingRequestBody = CountingRequestBody(requestBody, mock(), 0, fileContent.length.toLong())
|
||||
|
||||
val filenameCaptor: KArgumentCaptor<RequestBody> = argumentCaptor<RequestBody>()
|
||||
val totalFileSizeCaptor = argumentCaptor<RequestBody>()
|
||||
val offsetCaptor = argumentCaptor<RequestBody>()
|
||||
val fileKeyCaptor = argumentCaptor<RequestBody>()
|
||||
val tokenCaptor = argumentCaptor<RequestBody>()
|
||||
val fileCaptor = argumentCaptor<MultipartBody.Part>()
|
||||
|
||||
whenever(uploadInterface.uploadFileToStash(
|
||||
filenameCaptor.capture(), totalFileSizeCaptor.capture(), offsetCaptor.capture(),
|
||||
fileKeyCaptor.capture(), tokenCaptor.capture(), fileCaptor.capture()
|
||||
)).thenReturn(Observable.just(uploadResponse))
|
||||
|
||||
val result = uploadClient.uploadChunkToStash(filename, 100, 10, filekey, countingRequestBody).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
assertSame(uploadResult, result.values()[0])
|
||||
|
||||
assertEquals(filename, filenameCaptor.asString())
|
||||
assertEquals("100", totalFileSizeCaptor.asString())
|
||||
assertEquals("10", offsetCaptor.asString())
|
||||
assertEquals(filekey, fileKeyCaptor.asString())
|
||||
assertEquals(testToken, tokenCaptor.asString())
|
||||
assertEquals(fileContent, fileCaptor.firstValue.body.asString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testUploadChunkToStash_Failure() {
|
||||
val exception = Exception("expected")
|
||||
whenever(uploadInterface.uploadFileToStash(any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(Observable.error(exception))
|
||||
|
||||
val result = uploadClient.uploadChunkToStash(filename, 100, 10, filekey, mock()).test()
|
||||
|
||||
result.assertNoValues()
|
||||
assertSame(exception, result.errors()[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadFileToStash_completedContribution() {
|
||||
whenever(contribution.isCompleted()).thenReturn(true)
|
||||
whenever(contribution.fileKey).thenReturn(filekey)
|
||||
|
||||
val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
val stashResult = result.values()[0]
|
||||
assertEquals(filekey, stashResult.fileKey)
|
||||
assertEquals(StashUploadState.SUCCESS, stashResult.state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadFileToStash_contributionIsUnpaused() {
|
||||
whenever(contribution.isCompleted()).thenReturn(false)
|
||||
whenever(contribution.fileKey).thenReturn(filekey)
|
||||
whenever(fileUtilsWrapper.getMimeType(anyOrNull<File>())).thenReturn("image/png")
|
||||
whenever(fileUtilsWrapper.getFileChunks(anyOrNull<File>(), eq(expectedChunkSize))).thenReturn(emptyList())
|
||||
|
||||
val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
verify(contribution, times(1)).unpause()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadFileToStash_returnsFailureIfNothingToUpload() {
|
||||
whenever(contribution.isCompleted()).thenReturn(false)
|
||||
whenever(contribution.fileKey).thenReturn(filekey)
|
||||
whenever(fileUtilsWrapper.getMimeType(anyOrNull<File>())).thenReturn("image/png")
|
||||
whenever(fileUtilsWrapper.getFileChunks(anyOrNull<File>(), eq(expectedChunkSize))).thenReturn(emptyList())
|
||||
|
||||
val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
assertEquals(StashUploadState.FAILED, result.values()[0].state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadFileToStash_returnsFailureIfAnyChunkFails() {
|
||||
val mockFile = mock<File>()
|
||||
whenever(mockFile.length()).thenReturn(1)
|
||||
whenever(contribution.localUriPath).thenReturn(mockFile)
|
||||
whenever(contribution.isCompleted()).thenReturn(false)
|
||||
whenever(contribution.fileKey).thenReturn(filekey)
|
||||
whenever(fileUtilsWrapper.getMimeType(anyOrNull<File>())).thenReturn("image/png")
|
||||
whenever(fileUtilsWrapper.getFileChunks(anyOrNull<File>(), eq(expectedChunkSize))).thenReturn(listOf(mockFile))
|
||||
whenever(uploadInterface.uploadFileToStash(any(), any(), any(), any(), any(), any())).thenReturn(Observable.just(uploadResponse))
|
||||
|
||||
val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
assertEquals(StashUploadState.FAILED, result.values()[0].state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadFileToStash_successWithOneChunk() {
|
||||
val mockFile = mock<File>()
|
||||
val chunkInfo = mock<ChunkInfo>()
|
||||
whenever(mockFile.length()).thenReturn(10)
|
||||
whenever(chunkInfo.uploadResult).thenReturn(uploadResult)
|
||||
|
||||
whenever(uploadResult.offset).thenReturn(1)
|
||||
whenever(uploadResult.filekey).thenReturn(filekey)
|
||||
|
||||
whenever(contribution.localUriPath).thenReturn(mockFile)
|
||||
whenever(contribution.chunkInfo).thenReturn(chunkInfo)
|
||||
whenever(contribution.isCompleted()).thenReturn(false)
|
||||
whenever(contribution.dateModified).thenReturn(Date(100))
|
||||
whenever(timeProvider.currentTimeMillis()).thenReturn(200)
|
||||
whenever(contribution.fileKey).thenReturn(filekey)
|
||||
|
||||
whenever(fileUtilsWrapper.getMimeType(anyOrNull<File>())).thenReturn("image/png")
|
||||
whenever(fileUtilsWrapper.getFileChunks(anyOrNull<File>(), eq(expectedChunkSize))).thenReturn(listOf(mockFile))
|
||||
|
||||
whenever(uploadInterface.uploadFileToStash(anyOrNull(), anyOrNull(), anyOrNull(),
|
||||
anyOrNull(), anyOrNull(), anyOrNull())).thenReturn(Observable.just(uploadResponse))
|
||||
|
||||
val result = uploadClient.uploadFileToStash(filename, contribution, mock()).test()
|
||||
|
||||
result.assertNoErrors()
|
||||
assertEquals(StashUploadState.SUCCESS, result.values()[0].state)
|
||||
assertEquals(filekey, result.values()[0].fileKey)
|
||||
}
|
||||
|
||||
|
||||
private fun KArgumentCaptor<RequestBody>.asString(): String =
|
||||
firstValue.asString()
|
||||
|
||||
private fun RequestBody.asString(): String {
|
||||
val b = Buffer()
|
||||
writeTo(b)
|
||||
return b.readUtf8()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
package fr.free.nrw.commons.utils
|
||||
|
||||
import com.nhaarman.mockitokotlin2.mock
|
||||
import fr.free.nrw.commons.upload.FileUtils
|
||||
import fr.free.nrw.commons.upload.FileUtilsWrapper
|
||||
import org.junit.Assert.assertEquals
|
||||
|
|
@ -19,7 +20,7 @@ class FileUtilsTest {
|
|||
|
||||
@Test
|
||||
fun testSHA1() {
|
||||
val fileUtilsWrapper = FileUtilsWrapper()
|
||||
val fileUtilsWrapper = FileUtilsWrapper(mock())
|
||||
|
||||
assertEquals(
|
||||
"907d14fb3af2b0d4f18c2d46abe8aedce17367bd",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue