Use JSON APIs for explore (#2731)

* Use JSON APIs for explore

* With tests

* Use JSON APIs for explore

* With tests

* BugFix #2731 (#4)

* Increased sdk version to 23

* with more robust tests

* Fix crashes and other reported issues

* Add javadocs

* Use common method for search and categories

* Add javadocs

app/src/main/java/fr/free/nrw/commons/Media.java

@@ -3,20 +3,24 @@ package fr.free.nrw.commons;
 import android.net.Uri;
 import android.os.Parcel;
 import android.os.Parcelable;
-import androidx.annotation.Nullable;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import androidx.annotation.Nullable;
 import fr.free.nrw.commons.location.LatLng;
+import fr.free.nrw.commons.media.model.ExtMetadata;
 import fr.free.nrw.commons.media.model.ImageInfo;
 import fr.free.nrw.commons.media.model.MwQueryPage;
 import fr.free.nrw.commons.utils.DateUtils;
+import fr.free.nrw.commons.utils.MediaDataExtractorUtil;
 import fr.free.nrw.commons.utils.StringUtils;
 
 public class Media implements Parcelable {
@@ -93,6 +97,8 @@ public class Media implements Parcelable {
         this.dateCreated = dateCreated;
         this.dateUploaded = dateUploaded;
         this.creator = creator;
+        this.categories = new ArrayList<>();
+        this.descriptions = new HashMap<>();
     }
 
     @SuppressWarnings("unchecked")
@@ -450,23 +456,51 @@ public class Media implements Parcelable {
         return requestedDeletion;
     }
 
+    /**
+     * Creating Media object from MWQueryPage.
+     * Earlier only basic details were set for the media object but going forward,
+     * a full media object(with categories, descriptions, coordinates etc) can be constructed using this method
+     *
+     * @param page response from the API
+     * @return Media object
+     */
+    @Nullable
     public static Media from(MwQueryPage page) {
         ImageInfo imageInfo = page.imageInfo();
         if(imageInfo == null) {
             return null;
         }
+        ExtMetadata metadata = imageInfo.getMetadata();
+        if (metadata == null) {
+            return new Media(null, imageInfo.getOriginalUrl(),
+                    page.title(), "", 0, null, null, null);
+        }
+
         Media media = new Media(null,
                 imageInfo.getOriginalUrl(),
                 page.title(),
-                imageInfo.getMetadata().imageDescription().value(),
+                "",
                 0,
-                DateUtils.getDateFromString(imageInfo.getMetadata().dateTimeOriginal().value()),
+                DateUtils.getDateFromString(metadata.dateTimeOriginal().value()),
-                DateUtils.getDateFromString(imageInfo.getMetadata().dateTime().value()),
+                DateUtils.getDateFromString(metadata.dateTime().value()),
-                StringUtils.getParsedStringFromHtml(imageInfo.getMetadata().artist().value())
+                StringUtils.getParsedStringFromHtml(metadata.artist().value())
         );
 
-        media.setLicense(imageInfo.getMetadata().licenseShortName().value());
+        String language = Locale.getDefault().getLanguage();
+        if (StringUtils.isNullOrWhiteSpace(language)) {
+            language = "default";
+        }
+        media.setDescriptions(Collections.singletonMap(language, metadata.imageDescription().value()));
+        media.setCategories(MediaDataExtractorUtil.extractCategoriesFromList(metadata.categories().value()));
+        String latitude = metadata.gpsLatitude().value();
+        String longitude = metadata.gpsLongitude().value();
 
+        if(!StringUtils.isNullOrWhiteSpace(latitude) && !StringUtils.isNullOrWhiteSpace(longitude)) {
+            LatLng latLng = new LatLng(Double.parseDouble(latitude), Double.parseDouble(longitude), 0);
+            media.setCoordinates(latLng);
+        }
+
+        media.setLicense(metadata.licenseShortName().value());
         return media;
     }
 }

app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java

@@ -36,7 +36,7 @@ public class BookmarkPicturesController {
         ArrayList<Media> medias = new ArrayList<>();
         for (Bookmark bookmark : bookmarks) {
             List<Media> tmpMedias = okHttpJsonApiClient
-                    .searchImages(bookmark.getMediaName(), 0)
+                    .getMediaList("search", bookmark.getMediaName())
                     .blockingGet();
             for (Media m : tmpMedias) {
                 if (m.getCreator().trim().equals(bookmark.getMediaCreator().trim())) {

app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java

@@ -6,16 +6,17 @@ import javax.inject.Inject;
 import javax.inject.Singleton;
 
 import fr.free.nrw.commons.Media;
-import fr.free.nrw.commons.mwapi.MediaWikiApi;
+import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient;
+import io.reactivex.Single;
 
 @Singleton
 public class CategoryImageController {
 
-    private MediaWikiApi mediaWikiApi;
+    private OkHttpJsonApiClient okHttpJsonApiClient;
 
     @Inject
-    public CategoryImageController(MediaWikiApi mediaWikiApi) {
+    public CategoryImageController(OkHttpJsonApiClient okHttpJsonApiClient) {
-        this.mediaWikiApi = mediaWikiApi;
+        this.okHttpJsonApiClient = okHttpJsonApiClient;
     }
 
     /**
@@ -23,7 +24,7 @@ public class CategoryImageController {
      * @param categoryName
      * @return
      */
-    public List<Media> getCategoryImages(String categoryName) {
+    public Single<List<Media>> getCategoryImages(String categoryName) {
-        return mediaWikiApi.getCategoryImages(categoryName);
+        return okHttpJsonApiClient.getMediaList("category", categoryName);
     }
 }

app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java

@@ -1,45 +1,15 @@
 package fr.free.nrw.commons.category;
 
-import androidx.annotation.NonNull;
-
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
 import java.util.List;
 
-import javax.annotation.Nullable;
-
-import fr.free.nrw.commons.Media;
-import fr.free.nrw.commons.utils.StringUtils;
-import timber.log.Timber;
-
 public class CategoryImageUtils {
 
-    /**
-     * The method iterates over the child nodes to return a list of Media objects
-     * @param childNodes
-     * @return
-     */
-    public static List<Media> getMediaList(NodeList childNodes) {
-        List<Media> categoryImages = new ArrayList<>();
-
-        for (int i = 0; i < childNodes.getLength(); i++) {
-            Node node = childNodes.item(i);
-
-            if (getFileName(node).substring(0, 5).equals("File:")) {
-                categoryImages.add(getMediaFromPage(node));
-            }
-        }
-
-        return categoryImages;
-    }
-
     /**
      * The method iterates over the child nodes to return a list of Subcategory name
      * sorted alphabetically
@@ -56,27 +26,6 @@ public class CategoryImageUtils {
         return subCategories;
     }
 
-    /**
-     * Creates a new Media object from the XML response as received by the API
-     * @param node
-     * @return
-     */
-    public static Media getMediaFromPage(Node node) {
-        Media media = new Media(null,
-                getImageUrl(node),
-                getFileName(node),
-                getDescription(node),
-                getDataLength(node),
-                getDateCreated(node),
-                getDateCreated(node),
-                getCreator(node)
-        );
-
-        media.setLicense(getLicense(node));
-
-        return media;
-    }
-
     /**
      * Extracts the filename of the uploaded image
      * @param document
@@ -87,162 +36,4 @@ public class CategoryImageUtils {
         return element.getAttribute("title");
     }
 
-    /**
-     * Extracts the image description for that particular upload
-     * @param document
-     * @return
-     */
-    private static String getDescription(Node document) {
-        return getMetaDataValue(document, "ImageDescription");
-    }
-
-    /**
-     * Extracts license information from the image meta data
-     * @param document
-     * @return
-     */
-    private static String getLicense(Node document) {
-        return getMetaDataValue(document, "License");
-    }
-
-    /**
-     * Returns the parsed value of artist from the response
-     * The artist information is returned as a HTML string from the API. Using HTML parser to parse the HTML
-     * @param document
-     * @return
-     */
-    @NonNull
-    private static String getCreator(Node document) {
-        String artist = getMetaDataValue(document, "Artist");
-        if (StringUtils.isNullOrWhiteSpace(artist)) {
-            return "";
-        }
-        return StringUtils.getParsedStringFromHtml(artist);
-    }
-
-    /**
-     * Returns the parsed date of creation of the image
-     * @param document
-     * @return
-     */
-    private static Date getDateCreated(Node document) {
-        String dateTime = getMetaDataValue(document, "DateTime");
-        if (dateTime != null && !dateTime.equals("")) {
-            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-            try {
-                return format.parse(dateTime);
-            } catch (ParseException e) {
-                Timber.d("Error occurred while parsing date %s", dateTime);
-                return new Date();
-            }
-        }
-        return new Date();
-    }
-
-    /**
-     * @param document
-     * @return Returns the url attribute from the imageInfo node
-     */
-    private static String getImageUrl(Node document) {
-        Element element = (Element) getImageInfo(document);
-        if (element != null) {
-            return element.getAttribute("url");
-        }
-        return null;
-    }
-
-    /**
-     * Takes the node document and gives out the attribute length from the node document
-     * @param document
-     * @return
-     */
-    private static long getDataLength(Node document) {
-        Element element = (Element) document;
-        if (element != null) {
-            String length = element.getAttribute("length");
-            if (length != null && !length.equals("")) {
-                return Long.parseLong(length);
-            }
-        }
-        return 0L;
-    }
-
-    /**
-     * Generic method to get the value of any meta as returned by the getMetaData function
-     * @param document node document as returned by API
-     * @param metaName the name of meta node to be returned
-     * @return
-     */
-    private static String getMetaDataValue(Node document, String metaName) {
-        Element metaData = getMetaData(document, metaName);
-        if (metaData != null) {
-            return metaData.getAttribute("value");
-        }
-        return null;
-    }
-
-    /**
-     * Generic method to return an element taking the node document and metaName as input
-     * @param document node document as returned by API
-     * @param metaName the name of meta node to be returned
-     * @return
-     */
-    @Nullable
-    private static Element getMetaData(Node document, String metaName) {
-        Node extraMetaData = getExtraMetaData(document);
-        if (extraMetaData != null) {
-            Node node = getNode(extraMetaData, metaName);
-            if (node != null) {
-                return (Element) node;
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Extracts extmetadata from the response XML
-     * @param document
-     * @return
-     */
-    @Nullable
-    private static Node getExtraMetaData(Node document) {
-        Node imageInfo = getImageInfo(document);
-        if (imageInfo != null) {
-            return getNode(imageInfo, "extmetadata");
-        }
-        return null;
-    }
-
-    /**
-     * Extracts the ii node from the imageinfo node
-     * @param document
-     * @return
-     */
-    @Nullable
-    private static Node getImageInfo(Node document) {
-        Node imageInfo = getNode(document, "imageinfo");
-        if (imageInfo != null) {
-            return getNode(imageInfo, "ii");
-        }
-        return null;
-    }
-
-    /**
-     * Takes a parent node as input and returns a child node if present
-     * @param node parent node
-     * @param nodeName child node name
-     * @return
-     */
-    @Nullable
-    public static Node getNode(Node node, String nodeName) {
-        NodeList childNodes = node.getChildNodes();
-        for (int i = 0; i < childNodes.getLength(); i++) {
-            Node nodeItem = childNodes.item(i);
-            Element item = (Element) nodeItem;
-            if (item.getTagName().equals(nodeName)) {
-                return nodeItem;
-            }
-        }
-        return null;
-    }
 }

app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java

@@ -2,7 +2,6 @@ package fr.free.nrw.commons.category;
 
 import android.annotation.SuppressLint;
 import android.os.Bundle;
-import androidx.annotation.Nullable;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.view.ViewGroup;
@@ -20,6 +19,7 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.inject.Named;
 
+import androidx.annotation.Nullable;
 import butterknife.BindView;
 import butterknife.ButterKnife;
 import dagger.android.support.DaggerFragment;
@@ -29,7 +29,6 @@ import fr.free.nrw.commons.explore.categories.ExploreActivity;
 import fr.free.nrw.commons.kvstore.JsonKvStore;
 import fr.free.nrw.commons.utils.NetworkUtils;
 import fr.free.nrw.commons.utils.ViewUtil;
-import io.reactivex.Observable;
 import io.reactivex.android.schedulers.AndroidSchedulers;
 import io.reactivex.disposables.CompositeDisposable;
 import io.reactivex.schedulers.Schedulers;
@@ -102,7 +101,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
      * @param keyword
      */
     private void resetQueryContinueValues(String keyword) {
-        categoryKvStore.remove(keyword);
+        categoryKvStore.remove("query_continue_" + keyword);
     }
 
     /**
@@ -117,7 +116,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
 
         isLoading = true;
         progressBar.setVisibility(VISIBLE);
-        compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName))
+        compositeDisposable.add(controller.getCategoryImages(categoryName)
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)
@@ -223,7 +222,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
         }
 
         progressBar.setVisibility(VISIBLE);
-        compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName))
+        compositeDisposable.add(controller.getCategoryImages(categoryName)
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)

app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java

@@ -75,12 +75,14 @@ public class NetworkingModule {
     @Singleton
     public OkHttpJsonApiClient provideOkHttpJsonApiClient(OkHttpClient okHttpClient,
                                                           @Named("tools_force") HttpUrl toolsForgeUrl,
+                                                          @Named("default_preferences") JsonKvStore defaultKvStore,
                                                           Gson gson) {
         return new OkHttpJsonApiClient(okHttpClient,
                 toolsForgeUrl,
                 WIKIDATA_SPARQL_QUERY_URL,
                 WIKIMEDIA_CAMPAIGNS_BASE_URL,
                 BuildConfig.WIKIMEDIA_API_HOST,
+                defaultKvStore,
                 gson);
     }
 

app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java

@@ -4,9 +4,6 @@ package fr.free.nrw.commons.explore.images;
 import android.annotation.SuppressLint;
 import android.content.res.Configuration;
 import android.os.Bundle;
-import androidx.recyclerview.widget.GridLayoutManager;
-import androidx.recyclerview.widget.LinearLayoutManager;
-import androidx.recyclerview.widget.RecyclerView;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.view.ViewGroup;
@@ -23,6 +20,9 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 import javax.inject.Named;
 
+import androidx.recyclerview.widget.GridLayoutManager;
+import androidx.recyclerview.widget.LinearLayoutManager;
+import androidx.recyclerview.widget.RecyclerView;
 import butterknife.BindView;
 import butterknife.ButterKnife;
 import fr.free.nrw.commons.Media;
@@ -141,7 +141,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment {
         bottomProgressBar.setVisibility(GONE);
         queryList.clear();
         imagesAdapter.clear();
-        compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size())
+        compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query)
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)
@@ -158,7 +158,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment {
         this.query = query;
         bottomProgressBar.setVisibility(View.VISIBLE);
         progressBar.setVisibility(GONE);
-        compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size())
+        compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query)
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)

app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java

@@ -14,6 +14,8 @@ public class ExtMetadata {
     @SuppressWarnings("unused") @SerializedName("Categories") @Nullable private Values categories;
     @SuppressWarnings("unused") @SerializedName("Assessments") @Nullable private Values assessments;
     @SuppressWarnings("unused") @SerializedName("ImageDescription") @Nullable private Values imageDescription;
+    @SuppressWarnings("unused") @SerializedName("GPSLatitude") @Nullable private Values gpsLatitude;
+    @SuppressWarnings("unused") @SerializedName("GPSLongitude") @Nullable private Values gpsLongitude;
     @SuppressWarnings("unused") @SerializedName("DateTimeOriginal") @Nullable private Values dateTimeOriginal;
     @SuppressWarnings("unused") @SerializedName("Artist") @Nullable private Values artist;
     @SuppressWarnings("unused") @SerializedName("Credit") @Nullable private Values credit;
@@ -51,6 +53,21 @@ public class ExtMetadata {
         return imageDescription != null ? imageDescription : new Values();
     }
 
+    @NonNull
+    public Values categories() {
+        return categories != null ? categories : new Values();
+    }
+
+    @NonNull
+    public Values gpsLatitude() {
+        return gpsLatitude != null ? gpsLatitude : new Values();
+    }
+
+    @NonNull
+    public Values gpsLongitude() {
+        return gpsLongitude != null ? gpsLongitude : new Values();
+    }
+
     @NonNull public Values objectName() {
         return objectName != null ? objectName : new Values();
     }

app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java

@@ -38,7 +38,6 @@ import java.util.concurrent.Callable;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import fr.free.nrw.commons.BuildConfig;
-import fr.free.nrw.commons.Media;
 import fr.free.nrw.commons.R;
 import fr.free.nrw.commons.auth.AccountUtil;
 import fr.free.nrw.commons.category.CategoryImageUtils;
@@ -54,8 +53,6 @@ import io.reactivex.Observable;
 import io.reactivex.Single;
 import timber.log.Timber;
 
-import static fr.free.nrw.commons.utils.ContinueUtils.getQueryContinue;
-
 /**
  * @author Addshore
  */
@@ -714,63 +711,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
         return CategoryImageUtils.getSubCategoryList(childNodes);
     }
 
-
-    /**
-     * The method takes categoryName as input and returns a List of Media objects
-     * It uses the generator query API to get the images in a category, 10 at a time.
-     * Uses the query continue values for fetching paginated responses
-     * @param categoryName Category name as defined on commons
-     * @return
-     */
-    @Override
-    @NonNull
-    public List<Media> getCategoryImages(String categoryName) {
-        CustomApiResult apiResult = null;
-        try {
-            CustomMwApi.RequestBuilder requestBuilder = api.action("query")
-                    .param("generator", "categorymembers")
-                    .param("format", "xml")
-                    .param("gcmtype", "file")
-                    .param("gcmtitle", categoryName)
-                    .param("gcmsort", "timestamp")//property to sort by;timestamp
-                    .param("gcmdir", "desc")//in which direction to sort;descending
-                    .param("prop", "imageinfo")
-                    .param("gcmlimit", "10")
-                    .param("iiprop", "url|extmetadata");
-
-            QueryContinue queryContinueValues = getQueryContinueValues(categoryName);
-            if (queryContinueValues != null) {
-                requestBuilder.param("continue", queryContinueValues.getContinueParam());
-                requestBuilder.param("gcmcontinue", queryContinueValues.getGcmContinueParam());
-            }
-            apiResult = requestBuilder.get();
-        } catch (IOException e) {
-            Timber.e(e, "Failed to obtain searchCategories");
-        }
-
-        if (apiResult == null) {
-            return new ArrayList<>();
-        }
-
-        CustomApiResult categoryImagesNode = apiResult.getNode("/api/query/pages");
-        if (categoryImagesNode == null
-                || categoryImagesNode.getDocument() == null
-                || categoryImagesNode.getDocument().getChildNodes() == null
-                || categoryImagesNode.getDocument().getChildNodes().getLength() == 0) {
-            return new ArrayList<>();
-        }
-
-        if (apiResult.getNode("/api/continue").getDocument()==null){
-            setQueryContinueValues(categoryName, null);
-        }else {
-            QueryContinue queryContinue = getQueryContinue(apiResult.getNode("/api/continue").getDocument());
-            setQueryContinueValues(categoryName, queryContinue);
-        }
-
-        NodeList childNodes = categoryImagesNode.getDocument().getChildNodes();
-        return CategoryImageUtils.getMediaList(childNodes);
-    }
-
     /**
      * This method takes search keyword as input and returns a list of categories objects filtered using search query
      * It uses the generator query API to get the categories searched using a query, 25 at a time.

app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java

@@ -8,7 +8,6 @@ import java.util.List;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
-import fr.free.nrw.commons.Media;
 import fr.free.nrw.commons.notification.Notification;
 import io.reactivex.Observable;
 import io.reactivex.Single;
@@ -40,8 +39,6 @@ public interface MediaWikiApi {
 
     boolean logEvents(LogBuilder[] logBuilders);
 
-    List<Media> getCategoryImages(String categoryName);
-
     List<String> getSubCategoryList(String categoryName);
 
     List<String> getParentCategoryList(String categoryName);

app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java

@@ -1,12 +1,15 @@
 package fr.free.nrw.commons.mwapi;
 
 import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
 
 import java.io.IOException;
+import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Random;
 
 import javax.inject.Inject;
@@ -19,6 +22,7 @@ import fr.free.nrw.commons.PageTitle;
 import fr.free.nrw.commons.achievements.FeaturedImages;
 import fr.free.nrw.commons.achievements.FeedbackResponse;
 import fr.free.nrw.commons.campaigns.CampaignResponseDTO;
+import fr.free.nrw.commons.kvstore.JsonKvStore;
 import fr.free.nrw.commons.location.LatLng;
 import fr.free.nrw.commons.media.model.MwQueryPage;
 import fr.free.nrw.commons.mwapi.model.MwQueryResponse;
@@ -28,6 +32,7 @@ import fr.free.nrw.commons.nearby.model.NearbyResponse;
 import fr.free.nrw.commons.nearby.model.NearbyResultItem;
 import fr.free.nrw.commons.upload.FileUtils;
 import fr.free.nrw.commons.utils.DateUtils;
+import fr.free.nrw.commons.utils.StringUtils;
 import fr.free.nrw.commons.wikidata.model.GetWikidataEditCountResponse;
 import io.reactivex.Observable;
 import io.reactivex.Single;
@@ -39,11 +44,16 @@ import timber.log.Timber;
 
 @Singleton
 public class OkHttpJsonApiClient {
+
+    public static final Type mapType = new TypeToken<Map<String, String>>() {
+    }.getType();
+
     private final OkHttpClient okHttpClient;
     private final HttpUrl wikiMediaToolforgeUrl;
     private final String sparqlQueryUrl;
     private final String campaignsUrl;
     private final String commonsBaseUrl;
+    private final JsonKvStore defaultKvStore;
     private Gson gson;
 
 
@@ -53,12 +63,14 @@ public class OkHttpJsonApiClient {
                                String sparqlQueryUrl,
                                String campaignsUrl,
                                String commonsBaseUrl,
+                               JsonKvStore defaultKvStore,
                                Gson gson) {
         this.okHttpClient = okHttpClient;
         this.wikiMediaToolforgeUrl = wikiMediaToolforgeUrl;
         this.sparqlQueryUrl = sparqlQueryUrl;
         this.campaignsUrl = campaignsUrl;
         this.commonsBaseUrl = commonsBaseUrl;
+        this.defaultKvStore = defaultKvStore;
         this.gson = gson;
     }
 
@@ -75,7 +87,6 @@ public class OkHttpJsonApiClient {
         return Single.fromCallable(() -> {
             Response response = okHttpClient.newCall(request).execute();
             if (response != null && response.isSuccessful()) {
-
                 return Integer.parseInt(response.body().string().trim());
             }
             return 0;
@@ -224,11 +235,8 @@ public class OkHttpJsonApiClient {
 
         return Single.fromCallable(() -> {
             Response response = okHttpClient.newCall(request).execute();
-            if (response != null && response.body() != null && response.isSuccessful()) {
+            if (response.body() != null && response.isSuccessful()) {
                 String json = response.body().string();
-                if (json == null) {
-                    return null;
-                }
                 MwQueryResponse mwQueryPage = gson.fromJson(json, MwQueryResponse.class);
                 return Media.from(mwQueryPage.query().firstPage());
             }
@@ -237,49 +245,130 @@ public class OkHttpJsonApiClient {
     }
 
     /**
-     * This method takes search keyword as input and returns a list of  Media objects filtered using search query
+     * This method takes the keyword and queryType as input and returns a list of  Media objects filtered using image generator query
-     * It uses the generator query API to get the images searched using a query, 25 at a time.
+     * It uses the generator query API to get the images searched using a query, 10 at a time.
-     * @param query keyword to search images on commons
+     * @param queryType queryType can be "search" OR "category"
+     * @param keyword
      * @return
      */
     @Nullable
-    public Single<List<Media>> searchImages(String query, int offset) {
+    public Single<List<Media>> getMediaList(String queryType, String keyword) {
         HttpUrl.Builder urlBuilder = HttpUrl
                 .parse(commonsBaseUrl)
                 .newBuilder()
                 .addQueryParameter("action", "query")
-                .addQueryParameter("generator", "search")
+                .addQueryParameter("format", "json");
-                .addQueryParameter("format", "json")
+
-                .addQueryParameter("gsrwhat", "text")
+
-                .addQueryParameter("gsrnamespace", "6")
+        if (queryType.equals("search")) {
-                .addQueryParameter("gsrlimit", "25")
+            appendSearchParam(keyword, urlBuilder);
-                .addQueryParameter("gsroffset", String.valueOf(offset))
+        } else {
-                .addQueryParameter("gsrsearch", query)
+            appendCategoryParams(keyword, urlBuilder);
-                .addQueryParameter("prop", "imageinfo")
+        }
-                .addQueryParameter("iiprop", "url|extmetadata");
+
+        appendQueryContinueValues(keyword, urlBuilder);
 
         Request request = new Request.Builder()
-                .url(urlBuilder.build())
+                .url(appendMediaProperties(urlBuilder).build())
                 .build();
 
         return Single.fromCallable(() -> {
             Response response = okHttpClient.newCall(request).execute();
             List<Media> mediaList = new ArrayList<>();
-            if (response != null && response.body() != null && response.isSuccessful()) {
+            if (response.body() != null && response.isSuccessful()) {
                 String json = response.body().string();
-                if (json == null) {
+                MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class);
+                putContinueValues(keyword, mwQueryResponse.continuation());
+                if (mwQueryResponse.query() == null) {
                     return mediaList;
                 }
-                MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class);
                 List<MwQueryPage> pages = mwQueryResponse.query().pages();
                 for (MwQueryPage page : pages) {
-                    mediaList.add(Media.from(page));
+                    Media media = Media.from(page);
+                    if (media != null) {
+                        mediaList.add(media);
+                    }
                 }
             }
             return mediaList;
         });
     }
 
+    /**
+     * Whenever imageInfo is fetched, these common properties can be specified for the API call
+     * https://www.mediawiki.org/wiki/API:Imageinfo
+     * @param builder
+     * @return
+     */
+    private HttpUrl.Builder appendMediaProperties(HttpUrl.Builder builder) {
+        builder.addQueryParameter("prop", "imageinfo")
+                .addQueryParameter("iiprop", "url|extmetadata")
+                .addQueryParameter("iiextmetadatafilter", "DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName");
+
+        String language = Locale.getDefault().getLanguage();
+        if (!StringUtils.isNullOrWhiteSpace(language)) {
+            builder.addQueryParameter("iiextmetadatalanguage", language);
+        }
+
+        return builder;
+    }
+
+    /**
+     * Append params for search query.
+     * @param query
+     * @param urlBuilder
+     */
+    private void appendSearchParam(String query, HttpUrl.Builder urlBuilder) {
+        urlBuilder.addQueryParameter("generator", "search")
+                .addQueryParameter("gsrwhat", "text")
+                .addQueryParameter("gsrnamespace", "6")
+                .addQueryParameter("gsrlimit", "25")
+                .addQueryParameter("gsrsearch", query);
+    }
+
+    /**
+     * It takes a urlBuilder and appends all the continue values as query parameters
+     * @param query
+     * @param urlBuilder
+     */
+    private void appendQueryContinueValues(String query, HttpUrl.Builder urlBuilder) {
+        Map<String, String> continueValues = getContinueValues(query);
+        if (continueValues != null && continueValues.size() > 0) {
+            for (Map.Entry<String, String> entry : continueValues.entrySet()) {
+                urlBuilder.addQueryParameter(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+
+    private void appendCategoryParams(String categoryName, HttpUrl.Builder urlBuilder) {
+        urlBuilder.addQueryParameter("generator", "categorymembers")
+                .addQueryParameter("gcmtype", "file")
+                .addQueryParameter("gcmtitle", categoryName)
+                .addQueryParameter("gcmsort", "timestamp")//property to sort by;timestamp
+                .addQueryParameter("gcmdir", "desc")//in which direction to sort;descending
+                .addQueryParameter("gcmlimit", "10");
+    }
+
+    /**
+     * Stores the continue values for action=query
+     * These values are sent to the server in the subsequent call to fetch results after this point
+     * @param keyword
+     * @param values
+     */
+    private void putContinueValues(String keyword, Map<String, String> values) {
+        defaultKvStore.putJson("query_continue_" + keyword, values);
+    }
+
+    /**
+     * Retrieves a map of continue values from shared preferences.
+     * These values are appended to the next API call
+     * @param keyword
+     * @return
+     */
+    private Map<String, String> getContinueValues(String keyword) {
+        return defaultKvStore.getJson("query_continue_" + keyword, mapType);
+    }
+
     /**
      * Returns recent changes on commons
      * @return list of recent changes made

app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java

@@ -1,6 +1,7 @@
 package fr.free.nrw.commons.utils;
 
 import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -25,4 +26,24 @@ public class MediaDataExtractorUtil {
     }
 
 
+    /**
+     * Extracts a list of categories from | separated category string
+     *
+     * @param source
+     * @return
+     */
+    public static List<String> extractCategoriesFromList(String source) {
+        if (StringUtils.isNullOrWhiteSpace(source)) {
+            return new ArrayList<>();
+        }
+        String[] cats = source.split("\\|");
+        List<String> categories = new ArrayList<>();
+        for (String category : cats) {
+            if (!StringUtils.isNullOrWhiteSpace(category.trim())) {
+                categories.add(category);
+            }
+        }
+        return categories;
+    }
+
 }

app/src/test/kotlin/fr/free/nrw/commons/mwapi/OkHttpJsonApiClientTest.kt

@@ -0,0 +1,219 @@
+package fr.free.nrw.commons.mwapi
+
+import com.google.gson.Gson
+import fr.free.nrw.commons.BuildConfig
+import fr.free.nrw.commons.TestCommonsApplication
+import fr.free.nrw.commons.kvstore.JsonKvStore
+import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient.mapType
+import junit.framework.Assert.assertEquals
+import okhttp3.HttpUrl
+import okhttp3.OkHttpClient
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import okhttp3.mockwebserver.RecordedRequest
+import org.junit.After
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.Mockito
+import org.mockito.Mockito.`when`
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.annotation.Config
+import java.net.URLDecoder
+
+@RunWith(RobolectricTestRunner::class)
+@Config(constants = BuildConfig::class, sdk = [23], application = TestCommonsApplication::class)
+class OkHttpJsonApiClientTest {
+
+    private lateinit var testObject: OkHttpJsonApiClient
+    private lateinit var toolsForgeServer: MockWebServer
+    private lateinit var sparqlServer: MockWebServer
+    private lateinit var campaignsServer: MockWebServer
+    private lateinit var server: MockWebServer
+    private lateinit var sharedPreferences: JsonKvStore
+    private lateinit var okHttpClient: OkHttpClient
+
+    @Before
+    fun setUp() {
+        server = MockWebServer()
+        toolsForgeServer = MockWebServer()
+        sparqlServer = MockWebServer()
+        campaignsServer = MockWebServer()
+        okHttpClient = OkHttpClient.Builder().build()
+        sharedPreferences = Mockito.mock(JsonKvStore::class.java)
+        val toolsForgeUrl = "http://" + toolsForgeServer.hostName + ":" + toolsForgeServer.port + "/"
+        val sparqlUrl = "http://" + sparqlServer.hostName + ":" + sparqlServer.port + "/"
+        val campaignsUrl = "http://" + campaignsServer.hostName + ":" + campaignsServer.port + "/"
+        val serverUrl = "http://" + server.hostName + ":" + server.port + "/"
+        testObject = OkHttpJsonApiClient(okHttpClient, HttpUrl.get(toolsForgeUrl), sparqlUrl, campaignsUrl, serverUrl, sharedPreferences, Gson())
+    }
+
+    @After
+    fun teardown() {
+        server.shutdown()
+    }
+
+    @Test
+    fun getCategoryImages() {
+        server.enqueue(getFirstPageOfImages())
+        testFirstPageQuery()
+    }
+
+    @Test
+    fun getCategoryImagesWithContinue() {
+        server.enqueue(getFirstPageOfImages())
+        server.enqueue(getSecondPageOfImages())
+        testFirstPageQuery()
+
+        `when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
+                .thenReturn(hashMapOf(Pair("gcmcontinue", "testvalue"), Pair("continue", "gcmcontinue||")))
+
+
+        val categoryImagesContinued = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
+
+        assertBasicRequestParameters(server, "GET").let { request ->
+            parseQueryParams(request).let { body ->
+                Assert.assertEquals("json", body["format"])
+                Assert.assertEquals("query", body["action"])
+                Assert.assertEquals("categorymembers", body["generator"])
+                Assert.assertEquals("file", body["gcmtype"])
+                Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
+                Assert.assertEquals("timestamp", body["gcmsort"])
+                Assert.assertEquals("desc", body["gcmdir"])
+                Assert.assertEquals("testvalue", body["gcmcontinue"])
+                Assert.assertEquals("gcmcontinue||", body["continue"])
+                Assert.assertEquals("imageinfo", body["prop"])
+                Assert.assertEquals("url|extmetadata", body["iiprop"])
+                Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
+            }
+        }
+
+        assertEquals(categoryImagesContinued.size, 2)
+    }
+
+    @Test
+    fun getSearchImages() {
+        server.enqueue(getFirstPageOfImages())
+        testFirstPageSearchQuery()
+    }
+
+    @Test
+    fun getSearchImagesWithContinue() {
+        server.enqueue(getFirstPageOfSearchImages())
+        server.enqueue(getSecondPageOfSearchImages())
+        testFirstPageSearchQuery()
+
+        `when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
+                .thenReturn(hashMapOf(Pair("gsroffset", "25"), Pair("continue", "gsroffset||")))
+
+
+        val categoryImagesContinued = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
+
+        assertBasicRequestParameters(server, "GET").let { request ->
+            parseQueryParams(request).let { body ->
+                Assert.assertEquals("json", body["format"])
+                Assert.assertEquals("query", body["action"])
+                Assert.assertEquals("search", body["generator"])
+                Assert.assertEquals("text", body["gsrwhat"])
+                Assert.assertEquals("6", body["gsrnamespace"])
+                Assert.assertEquals("25", body["gsrlimit"])
+                Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
+                Assert.assertEquals("25", body["gsroffset"])
+                Assert.assertEquals("gsroffset||", body["continue"])
+                Assert.assertEquals("imageinfo", body["prop"])
+                Assert.assertEquals("url|extmetadata", body["iiprop"])
+                Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
+            }
+        }
+
+        assertEquals(categoryImagesContinued.size, 2)
+    }
+
+    private fun testFirstPageSearchQuery() {
+        val categoryImages = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
+
+        assertBasicRequestParameters(server, "GET").let { request ->
+            parseQueryParams(request).let { body ->
+                Assert.assertEquals("json", body["format"])
+                Assert.assertEquals("query", body["action"])
+                Assert.assertEquals("search", body["generator"])
+                Assert.assertEquals("text", body["gsrwhat"])
+                Assert.assertEquals("6", body["gsrnamespace"])
+                Assert.assertEquals("25", body["gsrlimit"])
+                Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
+                Assert.assertEquals("imageinfo", body["prop"])
+                Assert.assertEquals("url|extmetadata", body["iiprop"])
+                Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
+            }
+        }
+        assertEquals(categoryImages.size, 2)
+    }
+
+    private fun testFirstPageQuery() {
+        val categoryImages = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
+
+        assertBasicRequestParameters(server, "GET").let { request ->
+            parseQueryParams(request).let { body ->
+                Assert.assertEquals("json", body["format"])
+                Assert.assertEquals("query", body["action"])
+                Assert.assertEquals("categorymembers", body["generator"])
+                Assert.assertEquals("file", body["gcmtype"])
+                Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
+                Assert.assertEquals("timestamp", body["gcmsort"])
+                Assert.assertEquals("desc", body["gcmdir"])
+                Assert.assertEquals("imageinfo", body["prop"])
+                Assert.assertEquals("url|extmetadata", body["iiprop"])
+                Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
+            }
+        }
+        assertEquals(categoryImages.size, 2)
+    }
+
+    private fun getFirstPageOfImages(): MockResponse {
+        val mockResponse = MockResponse()
+        mockResponse.setResponseCode(200)
+        mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
+        return mockResponse
+    }
+
+    private fun getSecondPageOfImages(): MockResponse {
+        val mockResponse = MockResponse()
+        mockResponse.setResponseCode(200)
+        mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue2\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
+        return mockResponse
+    }
+
+    private fun getFirstPageOfSearchImages(): MockResponse {
+        val mockResponse = MockResponse()
+        mockResponse.setResponseCode(200)
+        mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"25\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
+        return mockResponse
+    }
+
+    private fun getSecondPageOfSearchImages(): MockResponse {
+        val mockResponse = MockResponse()
+        mockResponse.setResponseCode(200)
+        mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"50\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
+        return mockResponse
+    }
+
+    private fun assertBasicRequestParameters(server: MockWebServer, method: String): RecordedRequest = server.takeRequest().let {
+        Assert.assertEquals("/", it.requestUrl.encodedPath())
+        Assert.assertEquals(method, it.method)
+        return it
+    }
+
+    private fun parseQueryParams(request: RecordedRequest) = HashMap<String, String?>().apply {
+        request.requestUrl.let {
+            it.queryParameterNames().forEach { name -> put(name, it.queryParameter(name)) }
+        }
+    }
+
+    private fun parseBody(body: String): Map<String, String> = HashMap<String, String>().apply {
+        body.split("&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray().forEach { prop ->
+            val pair = prop.split("=".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
+            put(pair[0], URLDecoder.decode(pair[1], "utf-8"))
+        }
+    }
+}

app/src/test/kotlin/fr/free/nrw/commons/utils/MediaDataExtractorUtilTest.java

@@ -0,0 +1,40 @@
+package fr.free.nrw.commons.utils;
+
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class MediaDataExtractorUtilTest {
+
+    @Test
+    public void extractCategoriesFromList() {
+        List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018|Watercraft|2018");
+        assertEquals(strings.size(), 3);
+    }
+
+    @Test
+    public void extractCategoriesFromEmptyList() {
+        List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("");
+        assertEquals(strings.size(), 0);
+    }
+
+    @Test
+    public void extractCategoriesFromNullList() {
+        List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList(null);
+        assertEquals(strings.size(), 0);
+    }
+
+    @Test
+    public void extractCategoriesFromListWithEmptyValues() {
+        List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018||");
+        assertEquals(strings.size(), 1);
+    }
+
+    @Test
+    public void extractCategoriesFromListWithWhitespaces() {
+        List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018| | ||");
+        assertEquals(strings.size(), 1);
+    }
+}