5161: Fix repeating images in peer review (#5170)

* fix API call to fetch the latest changes

* add database table to keep a track of reviewed and skipped images

* fix repeating reviewed or skipped images

* add removed newline again

* add necessary comments

* change from timber.e to timber.i in case there is no exception

* reintroduce the parameter rctag in the API URL

* modify API URL to retrieve latest uploads

* remove unused imports and code

* modify ReviewHelperTest and add new unit tests

* modify tests in ReviewHelperTest.kt

* add comments about the value of gcmlimit

app/src/main/java/fr/free/nrw/commons/db/AppDatabase.kt

@@ -6,6 +6,8 @@ import androidx.room.TypeConverters
 import fr.free.nrw.commons.contributions.Contribution
 import fr.free.nrw.commons.contributions.ContributionDao
 import fr.free.nrw.commons.customselector.database.*
+import fr.free.nrw.commons.review.ReviewDao
+import fr.free.nrw.commons.review.ReviewEntity
 import fr.free.nrw.commons.upload.depicts.Depicts
 import fr.free.nrw.commons.upload.depicts.DepictsDao
 
@@ -13,11 +15,12 @@ import fr.free.nrw.commons.upload.depicts.DepictsDao
  * The database for accessing the respective DAOs
  *
  */
-@Database(entities = [Contribution::class, Depicts::class, UploadedStatus::class, NotForUploadStatus::class], version = 14, exportSchema = false)
+@Database(entities = [Contribution::class, Depicts::class, UploadedStatus::class, NotForUploadStatus::class, ReviewEntity::class], version = 15, exportSchema = false)
 @TypeConverters(Converters::class)
 abstract class AppDatabase : RoomDatabase() {
     abstract fun contributionDao(): ContributionDao
     abstract fun DepictsDao(): DepictsDao;
     abstract fun UploadedStatusDao(): UploadedStatusDao;
     abstract fun NotForUploadStatusDao(): NotForUploadStatusDao
+    abstract fun ReviewDao(): ReviewDao
 }

app/src/main/java/fr/free/nrw/commons/di/CommonsApplicationModule.java

@@ -24,6 +24,7 @@ import fr.free.nrw.commons.data.DBOpenHelper;
 import fr.free.nrw.commons.db.AppDatabase;
 import fr.free.nrw.commons.kvstore.JsonKvStore;
 import fr.free.nrw.commons.location.LocationServiceManager;
+import fr.free.nrw.commons.review.ReviewDao;
 import fr.free.nrw.commons.settings.Prefs;
 import fr.free.nrw.commons.upload.UploadController;
 import fr.free.nrw.commons.upload.depicts.DepictsDao;
@@ -299,6 +300,14 @@ public class CommonsApplicationModule {
         return appDatabase.NotForUploadStatusDao();
     }
 
+    /**
+     * Get the reference of ReviewDao class
+     */
+    @Provides
+    public ReviewDao providesReviewDao(AppDatabase appDatabase){
+        return appDatabase.ReviewDao();
+    }
+
     @Provides
     public ContentResolver providesContentResolver(Context context){
         return context.getContentResolver();

app/src/main/java/fr/free/nrw/commons/review/ReviewActivity.java

@@ -114,7 +114,7 @@ public class ReviewActivity extends BaseActivity {
         ButterKnife.bind(this);
         setSupportActionBar(toolbar);
         getSupportActionBar().setDisplayHomeAsUpEnabled(true);
-
+        
         reviewController = new ReviewController(deleteHelper, this);
 
         reviewPagerAdapter = new ReviewPagerAdapter(getSupportFragmentManager());
@@ -209,6 +209,7 @@ public class ReviewActivity extends BaseActivity {
 
     @SuppressLint("CheckResult")
     private void updateImage(Media media) {
+        reviewHelper.addViewedImagesToDB(media.getPageId());
         this.media = media;
         String fileName = media.getFilename();
         if (fileName.length() == 0) {
@@ -225,7 +226,7 @@ public class ReviewActivity extends BaseActivity {
                 .subscribe(revision -> {
                     reviewController.firstRevision = revision;
                     reviewPagerAdapter.updateFileInformation();
-                    String caption = String.format(getString(R.string.review_is_uploaded_by), fileName, revision.getUser());
+                    @SuppressLint({"StringFormatInvalid", "LocalSuppress"}) String caption = String.format(getString(R.string.review_is_uploaded_by), fileName, revision.getUser());
                     imageCaption.setText(caption);
                     progressBar.setVisibility(View.GONE);
                     reviewImageFragment = getInstanceOfReviewImageFragment();

app/src/main/java/fr/free/nrw/commons/review/ReviewDao.java

@@ -0,0 +1,32 @@
+package fr.free.nrw.commons.review;
+
+import androidx.room.Dao;
+import androidx.room.Insert;
+import androidx.room.OnConflictStrategy;
+import androidx.room.Query;
+
+/**
+ * Dao interface for reviewed images database
+ */
+@Dao
+public interface ReviewDao {
+
+    /**
+     * Inserts reviewed/skipped image identifier into the database
+     *
+     * @param reviewEntity
+     */
+    @Insert(onConflict = OnConflictStrategy.IGNORE)
+    void insert(ReviewEntity reviewEntity);
+
+    /**
+     * Checks if the image has already been reviewed/skipped by the user
+     * Returns true if the identifier exists in the reviewed images table
+     *
+     * @param imageId
+     * @return
+     */
+    @Query( "SELECT EXISTS (SELECT * from `reviewed-images` where imageId = (:imageId))")
+    Boolean isReviewedAlready(String imageId);
+
+}

app/src/main/java/fr/free/nrw/commons/review/ReviewEntity.java

@@ -0,0 +1,19 @@
+package fr.free.nrw.commons.review;
+
+import androidx.annotation.NonNull;
+import androidx.room.Entity;
+import androidx.room.PrimaryKey;
+
+/**
+ * Entity to store reviewed/skipped images identifier
+ */
+@Entity(tableName = "reviewed-images")
+public class ReviewEntity {
+    @PrimaryKey
+    @NonNull
+    String imageId;
+
+    public ReviewEntity(String imageId) {
+        this.imageId = imageId;
+    }
+}

app/src/main/java/fr/free/nrw/commons/review/ReviewHelper.java

@@ -1,19 +1,20 @@
 package fr.free.nrw.commons.review;
 
 
+import androidx.annotation.VisibleForTesting;
 import fr.free.nrw.commons.Media;
 import fr.free.nrw.commons.media.MediaClient;
+import io.reactivex.Completable;
 import io.reactivex.Observable;
 import io.reactivex.Single;
+import io.reactivex.android.schedulers.AndroidSchedulers;
+import io.reactivex.schedulers.Schedulers;
 import java.util.Collections;
-import java.util.Date;
-import java.util.Random;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import org.apache.commons.lang3.StringUtils;
 import org.wikipedia.dataclient.mwapi.MwQueryPage;
-import org.wikipedia.dataclient.mwapi.RecentChange;
-import org.wikipedia.util.DateUtil;
+import timber.log.Timber;
 
 @Singleton
 public class ReviewHelper {
@@ -23,6 +24,9 @@ public class ReviewHelper {
     private final MediaClient mediaClient;
     private final ReviewInterface reviewInterface;
 
+    @Inject
+    ReviewDao dao;
+
     @Inject
     public ReviewHelper(MediaClient mediaClient, ReviewInterface reviewInterface) {
         this.mediaClient = mediaClient;
@@ -31,21 +35,14 @@ public class ReviewHelper {
 
     /**
      * Fetches recent changes from MediaWiki API
-     * Calls the API to get 10 changes in the last 1 hour
-     * Earlier we were getting changes for the last 30 days but as the API returns just 10 results
-     * its best to fetch for just last 1 hour.
+     * Calls the API to get the latest 50 changes
+     * When more results are available, the query gets continued beyond this range
      *
      * @return
      */
-    private Observable<RecentChange> getRecentChanges() {
-        final int RANDOM_SECONDS = 60 * 60;
-        Random r = new Random();
-        Date now = new Date();
-        Date startDate = new Date(now.getTime() - r.nextInt(RANDOM_SECONDS) * 1000L);
-
-        String rcStart = DateUtil.iso8601DateFormat(startDate);
-        return reviewInterface.getRecentChanges(rcStart)
-                .map(mwQueryResponse -> mwQueryResponse.query().getRecentChanges())
+    private Observable<MwQueryPage> getRecentChanges() {
+        return reviewInterface.getRecentChanges()
+                .map(mwQueryResponse -> mwQueryResponse.query().pages())
                 .map(recentChanges -> {
                     Collections.shuffle(recentChanges);
                     return recentChanges;
@@ -56,7 +53,6 @@ public class ReviewHelper {
 
     /**
      * Gets a random file change for review.
-     * - Picks the most recent changes in the last 30 day window
      * - Picks a random file from those changes
      * - Checks if the file is nominated for deletion
      * - Retries upto 5 times for getting a file which is not nominated for deletion
@@ -66,7 +62,9 @@ public class ReviewHelper {
     public Single<Media> getRandomMedia() {
         return getRecentChanges()
                 .flatMapSingle(change -> getRandomMediaFromRecentChange(change))
-                .filter(media -> !StringUtils.isBlank(media.getFilename()))
+                .filter(media -> !StringUtils.isBlank(media.getFilename())
+                   && !getReviewStatus(media.getPageId())    // Check if the image has already been shown to the user
+                )
                 .firstOrError();
     }
 
@@ -77,18 +75,34 @@ public class ReviewHelper {
      * @param recentChange
      * @return
      */
-    private Single<Media> getRandomMediaFromRecentChange(RecentChange recentChange) {
+    private Single<Media> getRandomMediaFromRecentChange(MwQueryPage recentChange) {
         return Single.just(recentChange)
-                .flatMap(change -> mediaClient.checkPageExistsUsingTitle("Commons:Deletion_requests/" + change.getTitle()))
+                .flatMap(change -> mediaClient.checkPageExistsUsingTitle("Commons:Deletion_requests/" + change.title()))
                 .flatMap(isDeleted -> {
                     if (isDeleted) {
-                        return Single.error(new Exception(recentChange.getTitle() + " is deleted"));
+                        return Single.error(new Exception(recentChange.title() + " is deleted"));
                     }
-                    return mediaClient.getMedia(recentChange.getTitle());
+                    return mediaClient.getMedia(recentChange.title());
                 });
 
     }
 
+    /**
+     * Checks if the image exists in the reviewed images entity
+     *
+     * @param image
+     * @return
+     */
+    @VisibleForTesting
+    Boolean getReviewStatus(String image){
+        if(dao == null){
+            return false;
+        }
+        return Observable.fromCallable(()-> dao.isReviewedAlready(image))
+                .subscribeOn(Schedulers.io())
+                .observeOn(AndroidSchedulers.mainThread()).blockingSingle();
+    }
+
     /**
      * Gets the first revision of the file from filename
      *
@@ -121,18 +135,32 @@ public class ReviewHelper {
      * @param recentChange
      * @return
      */
-    private boolean isChangeReviewable(RecentChange recentChange) {
-        if ((recentChange.getType().equals("log") && !(recentChange.getOldRevisionId() == 0))
-                || !recentChange.getType().equals("log")) {
-            return false;
-        }
-
+    private boolean isChangeReviewable(MwQueryPage recentChange) {
         for (String extension : imageExtensions) {
-            if (recentChange.getTitle().endsWith(extension)) {
+            if (recentChange.title().endsWith(extension)) {
                 return true;
             }
         }
 
         return false;
     }
+
+    /**
+     * Adds reviewed/skipped images to the database
+     *
+     * @param imageId
+     */
+    public void addViewedImagesToDB(String imageId) {
+        Completable.fromAction(() -> dao.insert(new ReviewEntity(imageId)))
+            .subscribeOn(Schedulers.io())
+            .observeOn(AndroidSchedulers.mainThread())
+            .subscribe(() -> {
+                // Inserted successfully
+                Timber.i("Image inserted successfully.");
+                },
+                throwable -> {
+                    Timber.e("Image not inserted into the reviewed images database");
+                }
+            );
+    }
 }

app/src/main/java/fr/free/nrw/commons/review/ReviewInterface.java

@@ -10,8 +10,18 @@ import retrofit2.http.Query;
  * Interface class for peer review calls
  */
 public interface ReviewInterface {
-    @GET("w/api.php?action=query&format=json&formatversion=2&list=recentchanges&rcprop=title|ids&rctype=new|log&rctoponly=1&rcnamespace=6&rctag=android%20app%20edit")
-    Observable<MwQueryResponse> getRecentChanges(@Query("rcstart") String rcStart);
+
+    /**
+     * Fetch recent changes from MediaWiki API
+     * Calls the API for the latest 50 changes (the default limit is 10)
+     * More data can be fetched beyond this limit as the API call includes a continuation field
+     * However, since it takes longer to check the review status from the database and display the images
+     * as they get repeated before more data is fetched in the background
+     * the limit is increased from 10 to 50 using gcmlimit
+     *
+     */
+    @GET("w/api.php?action=query&format=json&formatversion=2&generator=categorymembers&gcmtype=file&gcmsort=timestamp&gcmdir=desc&gcmtitle=Category:Uploaded_with_Mobile/Android&gcmlimit=50")
+    Observable<MwQueryResponse> getRecentChanges();
 
     @GET("w/api.php?action=query&format=json&formatversion=2&prop=revisions&rvprop=timestamp|ids|user&rvdir=newer&rvlimit=1")
     Observable<MwQueryResponse> getFirstRevisionOfFile(@Query("titles") String titles);