Stash upload (#2505)

* Introduce Single<UploadResult>

* Two stage upload process

- split upload process and use stash
- resolve filename conflict after upload not before
- use NotificationManagerCompat; add notification tag; assign temporaty stash file name

app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java

@@ -853,35 +853,65 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
 
     @Override
     @NonNull
-    public UploadResult uploadFile(String filename,
+    public Single<UploadStash> uploadFile(
-                                   @NonNull InputStream file,
+            String filename,
-                                   long dataLength,
+            @NonNull InputStream file,
-                                   String pageContents,
+            long dataLength,
-                                   String editSummary,
+            Uri fileUri,
-                                   Uri fileUri,
+            Uri contentProviderUri,
-                                   Uri contentProviderUri,
+            ProgressListener progressListener) throws IOException {
-                                   final ProgressListener progressListener) throws IOException {
+        return Single.fromCallable(() -> {
+            CustomApiResult result = api.uploadToStash(filename, file, dataLength, getEditToken(), progressListener::onProgress);
 
-        CustomApiResult result = api.upload(filename, file, dataLength, pageContents, editSummary, getEditToken(), progressListener::onProgress);
+            Timber.wtf("Result: " + result.toString());
 
-        Timber.d("Result: %s", result.toString());
+            String resultStatus = result.getString("/api/upload/@result");
-
+            if (!resultStatus.equals("Success")) {
-        String resultStatus = result.getString("/api/upload/@result");
+                String errorCode = result.getString("/api/error/@code");
-
+                Timber.e(errorCode);
-        if (!resultStatus.equals("Success")) {
+                
-            String errorCode = result.getString("/api/error/@code");
+                if (errorCode.equals(ERROR_CODE_BAD_TOKEN)) {
-            Timber.e(errorCode);
+                    ViewUtil.showLongToast(context, R.string.bad_token_error_proposed_solution);
-
+                }
-            if (errorCode.equals(ERROR_CODE_BAD_TOKEN)) {
+                return new UploadStash(errorCode, resultStatus, filename, "");
-                ViewUtil.showLongToast(context, R.string.bad_token_error_proposed_solution);
+            } else {
+                String filekey = result.getString("/api/upload/@filekey");
+                return new UploadStash("", resultStatus, filename, filekey);
             }
-            return new UploadResult(resultStatus, errorCode);
+        });
-        } else {
+    }
-            Date dateUploaded = parseMWDate(result.getString("/api/upload/imageinfo/@timestamp"));
+
-            String canonicalFilename = "File:" + result.getString("/api/upload/@filename").replace("_", " "); // Title vs Filename
+
-            String imageUrl = result.getString("/api/upload/imageinfo/@url");
+    @Override
-            return new UploadResult(resultStatus, dateUploaded, canonicalFilename, imageUrl);
+    @NonNull
-        }
+    public Single<UploadResult> uploadFileFinalize(
+            String filename,
+            String filekey,
+            String pageContents,
+            String editSummary) throws IOException {
+        return Single.fromCallable(() -> {
+            CustomApiResult result = api.uploadFromStash(
+                    filename, filekey, pageContents, editSummary,
+                    getEditToken());
+
+            Timber.d("Result: %s", result.toString());
+
+            String resultStatus = result.getString("/api/upload/@result");
+            if (!resultStatus.equals("Success")) {
+                String errorCode = result.getString("/api/error/@code");
+                Timber.e(errorCode);
+
+                if (errorCode.equals(ERROR_CODE_BAD_TOKEN)) {
+                    ViewUtil.showLongToast(context, R.string.bad_token_error_proposed_solution);
+                }
+                return new UploadResult(resultStatus, errorCode);
+            } else {
+                Date dateUploaded = parseMWDate(result.getString("/api/upload/imageinfo/@timestamp"));
+                String canonicalFilename = "File:" + result.getString("/api/upload/@filename").replace("_", " "); // Title vs Filename
+                String imageUrl = result.getString("/api/upload/imageinfo/@url");
+                return new UploadResult(resultStatus, dateUploaded, canonicalFilename, imageUrl);
+            }
+        });
     }
 
     /**

app/src/main/java/fr/free/nrw/commons/mwapi/CustomMwApi.java

@@ -131,22 +131,13 @@ public class CustomMwApi {
         }
     }
 
-    public CustomApiResult upload(String filename, InputStream file, long length, String text, String comment, String token) throws IOException {
+    public CustomApiResult uploadToStash(String filename, InputStream file, long length, String token, ProgressListener uploadProgressListener) throws IOException {
-        return this.upload(filename, file, length, text, comment, token, null);
+            Timber.d("Initiating upload for file %s", filename);
-    }
+            Http.HttpRequestBuilder builder = Http.multipart(apiURL)
-
-    public CustomApiResult upload(String filename, InputStream file, String text, String comment, String token) throws IOException {
-        return this.upload(filename, file, -1, text, comment, token, null);
-    }
-
-    public CustomApiResult upload(String filename, InputStream file, long length, String text, String comment, String token, ProgressListener uploadProgressListener) throws IOException {
-        Timber.d("Initiating upload for file %s", filename);
-        Http.HttpRequestBuilder builder = Http.multipart(apiURL)
                 .data("action", "upload")
+                .data("stash", "1")
                 .data("token", token)
-                .data("text", text)
                 .data("ignorewarnings", "1")
-                .data("comment", comment)
                 .data("filename", filename)
                 .sendProgressListener(uploadProgressListener);
         if (length != -1) {
@@ -155,7 +146,20 @@ public class CustomMwApi {
             builder.file("file", filename, file);
         }
 
-        return CustomApiResult.fromRequestBuilder("uploadFile", builder, client);
+        return CustomApiResult.fromRequestBuilder("uploadToStash", builder, client);
+    }
+
+    public CustomApiResult uploadFromStash(String filename, String filekey, String text, String comment, String token) throws IOException {
+        Http.HttpRequestBuilder builder = Http.multipart(apiURL)
+                .data("action", "upload")
+                .data("token", token)
+                .data("ignorewarnings", "1")
+                .data("text", text)
+                .data("comment", comment)
+                .data("filename", filename)
+                .data("filekey", filekey);
+
+        return CustomApiResult.fromRequestBuilder("uploadFromStash", builder, client);
     }
 
     public void logout() throws IOException {

app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java

@@ -11,6 +11,7 @@ import java.util.List;
 import fr.free.nrw.commons.Media;
 import fr.free.nrw.commons.notification.Notification;
 import io.reactivex.Observable;
+import io.reactivex.Single;
 
 public interface MediaWikiApi {
     String getUserAgent();
@@ -49,8 +50,13 @@ public interface MediaWikiApi {
     List<String> searchCategory(String title, int offset);
 
     @NonNull
-    UploadResult uploadFile(String filename, InputStream file, long dataLength, String pageContents, String editSummary, Uri fileUri, Uri contentProviderUri, ProgressListener progressListener) throws IOException;
+    Single<UploadStash> uploadFile(String filename, InputStream file,
+                                   long dataLength, Uri fileUri, Uri contentProviderUri,
+                                   final ProgressListener progressListener) throws IOException;
 
+    @NonNull
+    Single<UploadResult> uploadFileFinalize(String filename, String filekey,
+                                            String pageContents, String editSummary) throws IOException;
     @Nullable
     String edit(String editToken, String processedPageContent, String filename, String summary) throws IOException;
 

app/src/main/java/fr/free/nrw/commons/mwapi/UploadStash.java

@@ -0,0 +1,70 @@
+package fr.free.nrw.commons.mwapi;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+
+public class UploadStash {
+    @NonNull
+    private String errorCode;
+    @NonNull
+    private String resultStatus;
+    @NonNull
+    private String filename;
+    @NonNull
+    private String filekey;
+
+    @NonNull
+    public final String getErrorCode() {
+        return this.errorCode;
+    }
+
+    @NonNull
+    public final String getResultStatus() {
+        return this.resultStatus;
+    }
+
+    @NonNull
+    public final String getFilename() {
+        return this.filename;
+    }
+
+    @NonNull
+    public final String getFilekey() {
+        return this.filekey;
+    }
+
+    public UploadStash(@NonNull String errorCode, @NonNull String resultStatus, @NonNull String filename, @NonNull String filekey) {
+        this.errorCode = errorCode;
+        this.resultStatus = resultStatus;
+        this.filename = filename;
+        this.filekey = filekey;
+    }
+
+    public String toString() {
+        return "UploadStash(errorCode=" + this.errorCode + ", resultStatus=" + this.resultStatus + ", filename=" + this.filename + ", filekey=" + this.filekey + ")";
+    }
+
+    public int hashCode() {
+        return ((this.errorCode.hashCode() * 31 + this.resultStatus.hashCode()
+        ) * 31 + this.filename.hashCode()
+        ) * 31 + this.filekey.hashCode();
+    }
+
+    public boolean equals(@Nullable Object obj) {
+        if (this != obj) {
+            if (obj instanceof UploadStash) {
+                UploadStash that = (UploadStash)obj;
+                if (this.errorCode.equals(that.errorCode)
+                    && this.resultStatus.equals(that.resultStatus)
+                    && this.filename.equals(that.filename)
+                        && this.filekey.equals(that.filekey)) {
+                    return true;
+                }
+            }
+
+            return false;
+        } else {
+            return true;
+        }
+    }
+}

app/src/main/java/fr/free/nrw/commons/upload/UploadService.java

@@ -1,7 +1,6 @@
 package fr.free.nrw.commons.upload;
 
 import android.annotation.SuppressLint;
-import android.app.NotificationManager;
 import android.app.PendingIntent;
 import android.content.ContentResolver;
 import android.content.ContentValues;
@@ -10,6 +9,7 @@ import android.graphics.BitmapFactory;
 import android.net.Uri;
 import android.os.Bundle;
 import androidx.core.app.NotificationCompat;
+import androidx.core.app.NotificationManagerCompat;
 import android.widget.Toast;
 
 import java.io.File;
@@ -34,8 +34,9 @@ import fr.free.nrw.commons.contributions.ContributionDao;
 import fr.free.nrw.commons.contributions.ContributionsContentProvider;
 import fr.free.nrw.commons.contributions.MainActivity;
 import fr.free.nrw.commons.mwapi.MediaWikiApi;
-import fr.free.nrw.commons.mwapi.UploadResult;
 import fr.free.nrw.commons.wikidata.WikidataEditService;
+import io.reactivex.Single;
+import io.reactivex.schedulers.Schedulers;
 import timber.log.Timber;
 
 public class UploadService extends HandlerService<Contribution> {
@@ -54,7 +55,7 @@ public class UploadService extends HandlerService<Contribution> {
     @Inject SessionManager sessionManager;
     @Inject ContributionDao contributionDao;
 
-    private NotificationManager notificationManager;
+    private NotificationManagerCompat notificationManager;
     private NotificationCompat.Builder curNotification;
     private int toUpload;
 
@@ -106,7 +107,7 @@ public class UploadService extends HandlerService<Contribution> {
             } else {
                 curNotification.setProgress(100, (int) (((double) transferred / (double) total) * 100), false);
             }
-            notificationManager.notify(NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
+            notificationManager.notify(notificationTag, NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
 
             contribution.setTransferred(transferred);
             contributionDao.save(contribution);
@@ -124,7 +125,7 @@ public class UploadService extends HandlerService<Contribution> {
     public void onCreate() {
         super.onCreate();
         CommonsApplication.createNotificationChannel(getApplicationContext());
-        notificationManager = (NotificationManager) getSystemService(NOTIFICATION_SERVICE);
+        notificationManager = NotificationManagerCompat.from(this);
         curNotification = getNotificationBuilder(CommonsApplication.NOTIFICATION_CHANNEL_ID_ALL);
     }
 
@@ -152,7 +153,7 @@ public class UploadService extends HandlerService<Contribution> {
                 if (curNotification != null && toUpload != 1) {
                     curNotification.setContentText(getResources().getQuantityString(R.plurals.uploads_pending_notification_indicator, toUpload, toUpload));
                     Timber.d("%d uploads left", toUpload);
-                    notificationManager.notify(NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
+                    notificationManager.notify(contribution.getLocalUri().toString(), NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
                 }
 
                 super.queue(what, contribution);
@@ -217,15 +218,12 @@ public class UploadService extends HandlerService<Contribution> {
         curNotification.setContentTitle(getString(R.string.upload_progress_notification_title_start, contribution.getDisplayTitle()))
                 .setContentText(getResources().getQuantityString(R.plurals.uploads_pending_notification_indicator, toUpload, toUpload))
                 .setTicker(getString(R.string.upload_progress_notification_title_in_progress, contribution.getDisplayTitle()));
-        startForeground(NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
+        notificationManager.notify(notificationTag, NOTIFICATION_UPLOAD_IN_PROGRESS, curNotification.build());
 
         String filename = contribution.getFilename();
+
         try {
-            synchronized (unfinishedUploads) {
+
-                Timber.d("making sure of uniqueness of name: %s", filename);
-                filename = findUniqueFilename(filename);
-                unfinishedUploads.add(filename);
-            }
             if (!mwApi.validateLogin()) {
                 // Need to revalidate!
                 if (sessionManager.revalidateAuthToken()) {
@@ -244,27 +242,60 @@ public class UploadService extends HandlerService<Contribution> {
                     getString(R.string.upload_progress_notification_title_finishing, contribution.getDisplayTitle()),
                     contribution
             );
-            UploadResult uploadResult = mwApi.uploadFile(filename, fileInputStream, contribution.getDataLength(),
+            String stashFilename = "Temp_" + contribution.hashCode() + filename;
-                    contribution.getPageContents(getApplicationContext()), contribution.getEditSummary(), localUri, contribution.getContentProviderUri(), notificationUpdater);
+            mwApi.uploadFile(
+                    stashFilename, fileInputStream, contribution.getDataLength(),
+                    localUri, contribution.getContentProviderUri(), notificationUpdater)
+                    .subscribeOn(Schedulers.io())
+                    .observeOn(Schedulers.io())
+                    .flatMap(uploadStash -> {
+                        notificationManager.cancel(NOTIFICATION_UPLOAD_IN_PROGRESS);
 
-            notificationManager.cancel(NOTIFICATION_UPLOAD_IN_PROGRESS);
+                        Timber.d("Stash upload response 1 is %s", uploadStash.toString());
-            Timber.d("Response is %s", uploadResult.toString());
 
-            String resultStatus = uploadResult.getResultStatus();
+                        String resultStatus = uploadStash.getResultStatus();
-            if (!resultStatus.equals("Success")) {
+                        if (!resultStatus.equals("Success")) {
-                Timber.d("Contribution upload failed. Wikidata entity won't be edited");
+                            Timber.d("Contribution upload failed. Wikidata entity won't be edited");
-                showFailedNotification(contribution);
+                            showFailedNotification(contribution);
-            } else {
+                            return Single.never();
-                Timber.d("Contribution upload success. Initiating Wikidata edit for entity id %s", contribution.getWikiDataEntityId());
+                        } else {
-                wikidataEditService.createClaimWithLogging(contribution.getWikiDataEntityId(), filename);
+                            synchronized (unfinishedUploads) {
-                contribution.setFilename(uploadResult.getCanonicalFilename());
+                                Timber.d("making sure of uniqueness of name: %s", filename);
-                contribution.setImageUrl(uploadResult.getImageUrl());
+                                String uniqueFilename = findUniqueFilename(filename);
-                contribution.setState(Contribution.STATE_COMPLETED);
+                                unfinishedUploads.add(uniqueFilename);
-                contribution.setDateUploaded(uploadResult.getDateUploaded());
+                                return mwApi.uploadFileFinalize(
-                contributionDao.save(contribution);
+                                        uniqueFilename,
-            }
+                                        uploadStash.getFilekey(),
+                                        contribution.getPageContents(getApplicationContext()),
+                                        contribution.getEditSummary());
+                            }
+                        }
+                    })
+                    .subscribe(uploadResult -> {
+                        Timber.d("Stash upload response 2 is %s", uploadResult.toString());
+
+                        notificationManager.cancel(notificationTag, NOTIFICATION_UPLOAD_IN_PROGRESS);
+
+                        String resultStatus = uploadResult.getResultStatus();
+                        if (!resultStatus.equals("Success")) {
+                            Timber.d("Contribution upload failed. Wikidata entity won't be edited");
+                            showFailedNotification(contribution);
+                        } else {
+                            String canonicalFilename = uploadResult.getCanonicalFilename();
+                            Timber.d("Contribution upload success. Initiating Wikidata edit for entity id %s",
+                                    contribution.getWikiDataEntityId());
+                            wikidataEditService.createClaimWithLogging(contribution.getWikiDataEntityId(), canonicalFilename);
+                            contribution.setFilename(canonicalFilename);
+                            contribution.setImageUrl(uploadResult.getImageUrl());
+                            contribution.setState(Contribution.STATE_COMPLETED);
+                            contribution.setDateUploaded(uploadResult.getDateUploaded());
+                            contributionDao.save(contribution);
+                        }
+                    }, throwable -> {
+                        throw new RuntimeException(throwable);
+                    });
         } catch (IOException e) {
-            Timber.d("I have a network fuckup");
+            Timber.w(e,"IOException during upload");
             notificationManager.cancel(NOTIFICATION_UPLOAD_IN_PROGRESS);
             showFailedNotification(contribution);
         } finally {
@@ -287,7 +318,7 @@ public class UploadService extends HandlerService<Contribution> {
                 .setContentTitle(getString(R.string.upload_failed_notification_title, contribution.getDisplayTitle()))
                 .setContentText(getString(R.string.upload_failed_notification_subtitle))
                 .setProgress(0, 0, false);
-        notificationManager.notify(NOTIFICATION_UPLOAD_FAILED, curNotification.build());
+        notificationManager.notify(contribution.getLocalUri().toString(), NOTIFICATION_UPLOAD_FAILED, curNotification.build());
 
         contribution.setState(Contribution.STATE_FAILED);
         contributionDao.save(contribution);