diff --git a/app/src/main/java/fr/free/nrw/commons/Media.java b/app/src/main/java/fr/free/nrw/commons/Media.java index 0e90ba3ad..512937f6a 100644 --- a/app/src/main/java/fr/free/nrw/commons/Media.java +++ b/app/src/main/java/fr/free/nrw/commons/Media.java @@ -3,20 +3,24 @@ package fr.free.nrw.commons; import android.net.Uri; import android.os.Parcel; import android.os.Parcelable; -import androidx.annotation.Nullable; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import androidx.annotation.Nullable; import fr.free.nrw.commons.location.LatLng; +import fr.free.nrw.commons.media.model.ExtMetadata; import fr.free.nrw.commons.media.model.ImageInfo; import fr.free.nrw.commons.media.model.MwQueryPage; import fr.free.nrw.commons.utils.DateUtils; +import fr.free.nrw.commons.utils.MediaDataExtractorUtil; import fr.free.nrw.commons.utils.StringUtils; public class Media implements Parcelable { @@ -93,6 +97,8 @@ public class Media implements Parcelable { this.dateCreated = dateCreated; this.dateUploaded = dateUploaded; this.creator = creator; + this.categories = new ArrayList<>(); + this.descriptions = new HashMap<>(); } @SuppressWarnings("unchecked") @@ -450,23 +456,51 @@ public class Media implements Parcelable { return requestedDeletion; } + /** + * Creating Media object from MWQueryPage. + * Earlier only basic details were set for the media object but going forward, + * a full media object(with categories, descriptions, coordinates etc) can be constructed using this method + * + * @param page response from the API + * @return Media object + */ + @Nullable public static Media from(MwQueryPage page) { ImageInfo imageInfo = page.imageInfo(); if(imageInfo == null) { return null; } + ExtMetadata metadata = imageInfo.getMetadata(); + if (metadata == null) { + return new Media(null, imageInfo.getOriginalUrl(), + page.title(), "", 0, null, null, null); + } + Media media = new Media(null, imageInfo.getOriginalUrl(), page.title(), - imageInfo.getMetadata().imageDescription().value(), + "", 0, - DateUtils.getDateFromString(imageInfo.getMetadata().dateTimeOriginal().value()), - DateUtils.getDateFromString(imageInfo.getMetadata().dateTime().value()), - StringUtils.getParsedStringFromHtml(imageInfo.getMetadata().artist().value()) + DateUtils.getDateFromString(metadata.dateTimeOriginal().value()), + DateUtils.getDateFromString(metadata.dateTime().value()), + StringUtils.getParsedStringFromHtml(metadata.artist().value()) ); - media.setLicense(imageInfo.getMetadata().licenseShortName().value()); + String language = Locale.getDefault().getLanguage(); + if (StringUtils.isNullOrWhiteSpace(language)) { + language = "default"; + } + media.setDescriptions(Collections.singletonMap(language, metadata.imageDescription().value())); + media.setCategories(MediaDataExtractorUtil.extractCategoriesFromList(metadata.categories().value())); + String latitude = metadata.gpsLatitude().value(); + String longitude = metadata.gpsLongitude().value(); + if(!StringUtils.isNullOrWhiteSpace(latitude) && !StringUtils.isNullOrWhiteSpace(longitude)) { + LatLng latLng = new LatLng(Double.parseDouble(latitude), Double.parseDouble(longitude), 0); + media.setCoordinates(latLng); + } + + media.setLicense(metadata.licenseShortName().value()); return media; } } diff --git a/app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java b/app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java index c7e74e532..ef4eeca74 100644 --- a/app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java +++ b/app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java @@ -36,7 +36,7 @@ public class BookmarkPicturesController { ArrayList medias = new ArrayList<>(); for (Bookmark bookmark : bookmarks) { List tmpMedias = okHttpJsonApiClient - .searchImages(bookmark.getMediaName(), 0) + .getMediaList("search", bookmark.getMediaName()) .blockingGet(); for (Media m : tmpMedias) { if (m.getCreator().trim().equals(bookmark.getMediaCreator().trim())) { diff --git a/app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java b/app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java index ae45952e4..5ab5ea6b3 100644 --- a/app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java +++ b/app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java @@ -6,16 +6,17 @@ import javax.inject.Inject; import javax.inject.Singleton; import fr.free.nrw.commons.Media; -import fr.free.nrw.commons.mwapi.MediaWikiApi; +import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient; +import io.reactivex.Single; @Singleton public class CategoryImageController { - private MediaWikiApi mediaWikiApi; + private OkHttpJsonApiClient okHttpJsonApiClient; @Inject - public CategoryImageController(MediaWikiApi mediaWikiApi) { - this.mediaWikiApi = mediaWikiApi; + public CategoryImageController(OkHttpJsonApiClient okHttpJsonApiClient) { + this.okHttpJsonApiClient = okHttpJsonApiClient; } /** @@ -23,7 +24,7 @@ public class CategoryImageController { * @param categoryName * @return */ - public List getCategoryImages(String categoryName) { - return mediaWikiApi.getCategoryImages(categoryName); + public Single> getCategoryImages(String categoryName) { + return okHttpJsonApiClient.getMediaList("category", categoryName); } } diff --git a/app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java b/app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java index d651ae2b6..cd97511cd 100644 --- a/app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java +++ b/app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java @@ -1,45 +1,15 @@ package fr.free.nrw.commons.category; -import androidx.annotation.NonNull; - import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.List; -import javax.annotation.Nullable; - -import fr.free.nrw.commons.Media; -import fr.free.nrw.commons.utils.StringUtils; -import timber.log.Timber; - public class CategoryImageUtils { - /** - * The method iterates over the child nodes to return a list of Media objects - * @param childNodes - * @return - */ - public static List getMediaList(NodeList childNodes) { - List categoryImages = new ArrayList<>(); - - for (int i = 0; i < childNodes.getLength(); i++) { - Node node = childNodes.item(i); - - if (getFileName(node).substring(0, 5).equals("File:")) { - categoryImages.add(getMediaFromPage(node)); - } - } - - return categoryImages; - } - /** * The method iterates over the child nodes to return a list of Subcategory name * sorted alphabetically @@ -56,27 +26,6 @@ public class CategoryImageUtils { return subCategories; } - /** - * Creates a new Media object from the XML response as received by the API - * @param node - * @return - */ - public static Media getMediaFromPage(Node node) { - Media media = new Media(null, - getImageUrl(node), - getFileName(node), - getDescription(node), - getDataLength(node), - getDateCreated(node), - getDateCreated(node), - getCreator(node) - ); - - media.setLicense(getLicense(node)); - - return media; - } - /** * Extracts the filename of the uploaded image * @param document @@ -87,162 +36,4 @@ public class CategoryImageUtils { return element.getAttribute("title"); } - /** - * Extracts the image description for that particular upload - * @param document - * @return - */ - private static String getDescription(Node document) { - return getMetaDataValue(document, "ImageDescription"); - } - - /** - * Extracts license information from the image meta data - * @param document - * @return - */ - private static String getLicense(Node document) { - return getMetaDataValue(document, "License"); - } - - /** - * Returns the parsed value of artist from the response - * The artist information is returned as a HTML string from the API. Using HTML parser to parse the HTML - * @param document - * @return - */ - @NonNull - private static String getCreator(Node document) { - String artist = getMetaDataValue(document, "Artist"); - if (StringUtils.isNullOrWhiteSpace(artist)) { - return ""; - } - return StringUtils.getParsedStringFromHtml(artist); - } - - /** - * Returns the parsed date of creation of the image - * @param document - * @return - */ - private static Date getDateCreated(Node document) { - String dateTime = getMetaDataValue(document, "DateTime"); - if (dateTime != null && !dateTime.equals("")) { - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - try { - return format.parse(dateTime); - } catch (ParseException e) { - Timber.d("Error occurred while parsing date %s", dateTime); - return new Date(); - } - } - return new Date(); - } - - /** - * @param document - * @return Returns the url attribute from the imageInfo node - */ - private static String getImageUrl(Node document) { - Element element = (Element) getImageInfo(document); - if (element != null) { - return element.getAttribute("url"); - } - return null; - } - - /** - * Takes the node document and gives out the attribute length from the node document - * @param document - * @return - */ - private static long getDataLength(Node document) { - Element element = (Element) document; - if (element != null) { - String length = element.getAttribute("length"); - if (length != null && !length.equals("")) { - return Long.parseLong(length); - } - } - return 0L; - } - - /** - * Generic method to get the value of any meta as returned by the getMetaData function - * @param document node document as returned by API - * @param metaName the name of meta node to be returned - * @return - */ - private static String getMetaDataValue(Node document, String metaName) { - Element metaData = getMetaData(document, metaName); - if (metaData != null) { - return metaData.getAttribute("value"); - } - return null; - } - - /** - * Generic method to return an element taking the node document and metaName as input - * @param document node document as returned by API - * @param metaName the name of meta node to be returned - * @return - */ - @Nullable - private static Element getMetaData(Node document, String metaName) { - Node extraMetaData = getExtraMetaData(document); - if (extraMetaData != null) { - Node node = getNode(extraMetaData, metaName); - if (node != null) { - return (Element) node; - } - } - return null; - } - - /** - * Extracts extmetadata from the response XML - * @param document - * @return - */ - @Nullable - private static Node getExtraMetaData(Node document) { - Node imageInfo = getImageInfo(document); - if (imageInfo != null) { - return getNode(imageInfo, "extmetadata"); - } - return null; - } - - /** - * Extracts the ii node from the imageinfo node - * @param document - * @return - */ - @Nullable - private static Node getImageInfo(Node document) { - Node imageInfo = getNode(document, "imageinfo"); - if (imageInfo != null) { - return getNode(imageInfo, "ii"); - } - return null; - } - - /** - * Takes a parent node as input and returns a child node if present - * @param node parent node - * @param nodeName child node name - * @return - */ - @Nullable - public static Node getNode(Node node, String nodeName) { - NodeList childNodes = node.getChildNodes(); - for (int i = 0; i < childNodes.getLength(); i++) { - Node nodeItem = childNodes.item(i); - Element item = (Element) nodeItem; - if (item.getTagName().equals(nodeName)) { - return nodeItem; - } - } - return null; - } } diff --git a/app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java b/app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java index 022cbb45e..2a846413d 100644 --- a/app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java +++ b/app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java @@ -2,7 +2,6 @@ package fr.free.nrw.commons.category; import android.annotation.SuppressLint; import android.os.Bundle; -import androidx.annotation.Nullable; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; @@ -20,6 +19,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.inject.Named; +import androidx.annotation.Nullable; import butterknife.BindView; import butterknife.ButterKnife; import dagger.android.support.DaggerFragment; @@ -29,7 +29,6 @@ import fr.free.nrw.commons.explore.categories.ExploreActivity; import fr.free.nrw.commons.kvstore.JsonKvStore; import fr.free.nrw.commons.utils.NetworkUtils; import fr.free.nrw.commons.utils.ViewUtil; -import io.reactivex.Observable; import io.reactivex.android.schedulers.AndroidSchedulers; import io.reactivex.disposables.CompositeDisposable; import io.reactivex.schedulers.Schedulers; @@ -102,7 +101,7 @@ public class CategoryImagesListFragment extends DaggerFragment { * @param keyword */ private void resetQueryContinueValues(String keyword) { - categoryKvStore.remove(keyword); + categoryKvStore.remove("query_continue_" + keyword); } /** @@ -117,7 +116,7 @@ public class CategoryImagesListFragment extends DaggerFragment { isLoading = true; progressBar.setVisibility(VISIBLE); - compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName)) + compositeDisposable.add(controller.getCategoryImages(categoryName) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) @@ -223,7 +222,7 @@ public class CategoryImagesListFragment extends DaggerFragment { } progressBar.setVisibility(VISIBLE); - compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName)) + compositeDisposable.add(controller.getCategoryImages(categoryName) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) diff --git a/app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java b/app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java index 5e6e2794d..77b420737 100644 --- a/app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java +++ b/app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java @@ -75,12 +75,14 @@ public class NetworkingModule { @Singleton public OkHttpJsonApiClient provideOkHttpJsonApiClient(OkHttpClient okHttpClient, @Named("tools_force") HttpUrl toolsForgeUrl, + @Named("default_preferences") JsonKvStore defaultKvStore, Gson gson) { return new OkHttpJsonApiClient(okHttpClient, toolsForgeUrl, WIKIDATA_SPARQL_QUERY_URL, WIKIMEDIA_CAMPAIGNS_BASE_URL, BuildConfig.WIKIMEDIA_API_HOST, + defaultKvStore, gson); } diff --git a/app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java b/app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java index 245ff9f1c..1dbaf5316 100644 --- a/app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java +++ b/app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java @@ -4,9 +4,6 @@ package fr.free.nrw.commons.explore.images; import android.annotation.SuppressLint; import android.content.res.Configuration; import android.os.Bundle; -import androidx.recyclerview.widget.GridLayoutManager; -import androidx.recyclerview.widget.LinearLayoutManager; -import androidx.recyclerview.widget.RecyclerView; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; @@ -23,6 +20,9 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.inject.Named; +import androidx.recyclerview.widget.GridLayoutManager; +import androidx.recyclerview.widget.LinearLayoutManager; +import androidx.recyclerview.widget.RecyclerView; import butterknife.BindView; import butterknife.ButterKnife; import fr.free.nrw.commons.Media; @@ -141,7 +141,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment { bottomProgressBar.setVisibility(GONE); queryList.clear(); imagesAdapter.clear(); - compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size()) + compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) @@ -158,7 +158,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment { this.query = query; bottomProgressBar.setVisibility(View.VISIBLE); progressBar.setVisibility(GONE); - compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size()) + compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS) diff --git a/app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java b/app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java index c5eb2d90c..a7ae2785b 100644 --- a/app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java +++ b/app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java @@ -14,6 +14,8 @@ public class ExtMetadata { @SuppressWarnings("unused") @SerializedName("Categories") @Nullable private Values categories; @SuppressWarnings("unused") @SerializedName("Assessments") @Nullable private Values assessments; @SuppressWarnings("unused") @SerializedName("ImageDescription") @Nullable private Values imageDescription; + @SuppressWarnings("unused") @SerializedName("GPSLatitude") @Nullable private Values gpsLatitude; + @SuppressWarnings("unused") @SerializedName("GPSLongitude") @Nullable private Values gpsLongitude; @SuppressWarnings("unused") @SerializedName("DateTimeOriginal") @Nullable private Values dateTimeOriginal; @SuppressWarnings("unused") @SerializedName("Artist") @Nullable private Values artist; @SuppressWarnings("unused") @SerializedName("Credit") @Nullable private Values credit; @@ -51,6 +53,21 @@ public class ExtMetadata { return imageDescription != null ? imageDescription : new Values(); } + @NonNull + public Values categories() { + return categories != null ? categories : new Values(); + } + + @NonNull + public Values gpsLatitude() { + return gpsLatitude != null ? gpsLatitude : new Values(); + } + + @NonNull + public Values gpsLongitude() { + return gpsLongitude != null ? gpsLongitude : new Values(); + } + @NonNull public Values objectName() { return objectName != null ? objectName : new Values(); } diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java b/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java index 95117cb46..ca8006d06 100644 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java +++ b/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java @@ -38,7 +38,6 @@ import java.util.concurrent.Callable; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import fr.free.nrw.commons.BuildConfig; -import fr.free.nrw.commons.Media; import fr.free.nrw.commons.R; import fr.free.nrw.commons.auth.AccountUtil; import fr.free.nrw.commons.category.CategoryImageUtils; @@ -54,8 +53,6 @@ import io.reactivex.Observable; import io.reactivex.Single; import timber.log.Timber; -import static fr.free.nrw.commons.utils.ContinueUtils.getQueryContinue; - /** * @author Addshore */ @@ -714,63 +711,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi { return CategoryImageUtils.getSubCategoryList(childNodes); } - - /** - * The method takes categoryName as input and returns a List of Media objects - * It uses the generator query API to get the images in a category, 10 at a time. - * Uses the query continue values for fetching paginated responses - * @param categoryName Category name as defined on commons - * @return - */ - @Override - @NonNull - public List getCategoryImages(String categoryName) { - CustomApiResult apiResult = null; - try { - CustomMwApi.RequestBuilder requestBuilder = api.action("query") - .param("generator", "categorymembers") - .param("format", "xml") - .param("gcmtype", "file") - .param("gcmtitle", categoryName) - .param("gcmsort", "timestamp")//property to sort by;timestamp - .param("gcmdir", "desc")//in which direction to sort;descending - .param("prop", "imageinfo") - .param("gcmlimit", "10") - .param("iiprop", "url|extmetadata"); - - QueryContinue queryContinueValues = getQueryContinueValues(categoryName); - if (queryContinueValues != null) { - requestBuilder.param("continue", queryContinueValues.getContinueParam()); - requestBuilder.param("gcmcontinue", queryContinueValues.getGcmContinueParam()); - } - apiResult = requestBuilder.get(); - } catch (IOException e) { - Timber.e(e, "Failed to obtain searchCategories"); - } - - if (apiResult == null) { - return new ArrayList<>(); - } - - CustomApiResult categoryImagesNode = apiResult.getNode("/api/query/pages"); - if (categoryImagesNode == null - || categoryImagesNode.getDocument() == null - || categoryImagesNode.getDocument().getChildNodes() == null - || categoryImagesNode.getDocument().getChildNodes().getLength() == 0) { - return new ArrayList<>(); - } - - if (apiResult.getNode("/api/continue").getDocument()==null){ - setQueryContinueValues(categoryName, null); - }else { - QueryContinue queryContinue = getQueryContinue(apiResult.getNode("/api/continue").getDocument()); - setQueryContinueValues(categoryName, queryContinue); - } - - NodeList childNodes = categoryImagesNode.getDocument().getChildNodes(); - return CategoryImageUtils.getMediaList(childNodes); - } - /** * This method takes search keyword as input and returns a list of categories objects filtered using search query * It uses the generator query API to get the categories searched using a query, 25 at a time. diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java b/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java index 362113de4..b38840086 100644 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java +++ b/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java @@ -8,7 +8,6 @@ import java.util.List; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import fr.free.nrw.commons.Media; import fr.free.nrw.commons.notification.Notification; import io.reactivex.Observable; import io.reactivex.Single; @@ -40,8 +39,6 @@ public interface MediaWikiApi { boolean logEvents(LogBuilder[] logBuilders); - List getCategoryImages(String categoryName); - List getSubCategoryList(String categoryName); List getParentCategoryList(String categoryName); diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java b/app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java index 47d89b11d..657786142 100644 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java +++ b/app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java @@ -1,12 +1,15 @@ package fr.free.nrw.commons.mwapi; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import java.io.IOException; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Random; import javax.inject.Inject; @@ -19,6 +22,7 @@ import fr.free.nrw.commons.PageTitle; import fr.free.nrw.commons.achievements.FeaturedImages; import fr.free.nrw.commons.achievements.FeedbackResponse; import fr.free.nrw.commons.campaigns.CampaignResponseDTO; +import fr.free.nrw.commons.kvstore.JsonKvStore; import fr.free.nrw.commons.location.LatLng; import fr.free.nrw.commons.media.model.MwQueryPage; import fr.free.nrw.commons.mwapi.model.MwQueryResponse; @@ -28,6 +32,7 @@ import fr.free.nrw.commons.nearby.model.NearbyResponse; import fr.free.nrw.commons.nearby.model.NearbyResultItem; import fr.free.nrw.commons.upload.FileUtils; import fr.free.nrw.commons.utils.DateUtils; +import fr.free.nrw.commons.utils.StringUtils; import fr.free.nrw.commons.wikidata.model.GetWikidataEditCountResponse; import io.reactivex.Observable; import io.reactivex.Single; @@ -39,11 +44,16 @@ import timber.log.Timber; @Singleton public class OkHttpJsonApiClient { + + public static final Type mapType = new TypeToken>() { + }.getType(); + private final OkHttpClient okHttpClient; private final HttpUrl wikiMediaToolforgeUrl; private final String sparqlQueryUrl; private final String campaignsUrl; private final String commonsBaseUrl; + private final JsonKvStore defaultKvStore; private Gson gson; @@ -53,12 +63,14 @@ public class OkHttpJsonApiClient { String sparqlQueryUrl, String campaignsUrl, String commonsBaseUrl, + JsonKvStore defaultKvStore, Gson gson) { this.okHttpClient = okHttpClient; this.wikiMediaToolforgeUrl = wikiMediaToolforgeUrl; this.sparqlQueryUrl = sparqlQueryUrl; this.campaignsUrl = campaignsUrl; this.commonsBaseUrl = commonsBaseUrl; + this.defaultKvStore = defaultKvStore; this.gson = gson; } @@ -75,7 +87,6 @@ public class OkHttpJsonApiClient { return Single.fromCallable(() -> { Response response = okHttpClient.newCall(request).execute(); if (response != null && response.isSuccessful()) { - return Integer.parseInt(response.body().string().trim()); } return 0; @@ -224,11 +235,8 @@ public class OkHttpJsonApiClient { return Single.fromCallable(() -> { Response response = okHttpClient.newCall(request).execute(); - if (response != null && response.body() != null && response.isSuccessful()) { + if (response.body() != null && response.isSuccessful()) { String json = response.body().string(); - if (json == null) { - return null; - } MwQueryResponse mwQueryPage = gson.fromJson(json, MwQueryResponse.class); return Media.from(mwQueryPage.query().firstPage()); } @@ -237,49 +245,130 @@ public class OkHttpJsonApiClient { } /** - * This method takes search keyword as input and returns a list of Media objects filtered using search query - * It uses the generator query API to get the images searched using a query, 25 at a time. - * @param query keyword to search images on commons + * This method takes the keyword and queryType as input and returns a list of Media objects filtered using image generator query + * It uses the generator query API to get the images searched using a query, 10 at a time. + * @param queryType queryType can be "search" OR "category" + * @param keyword * @return */ @Nullable - public Single> searchImages(String query, int offset) { + public Single> getMediaList(String queryType, String keyword) { HttpUrl.Builder urlBuilder = HttpUrl .parse(commonsBaseUrl) .newBuilder() .addQueryParameter("action", "query") - .addQueryParameter("generator", "search") - .addQueryParameter("format", "json") - .addQueryParameter("gsrwhat", "text") - .addQueryParameter("gsrnamespace", "6") - .addQueryParameter("gsrlimit", "25") - .addQueryParameter("gsroffset", String.valueOf(offset)) - .addQueryParameter("gsrsearch", query) - .addQueryParameter("prop", "imageinfo") - .addQueryParameter("iiprop", "url|extmetadata"); + .addQueryParameter("format", "json"); + + + if (queryType.equals("search")) { + appendSearchParam(keyword, urlBuilder); + } else { + appendCategoryParams(keyword, urlBuilder); + } + + appendQueryContinueValues(keyword, urlBuilder); Request request = new Request.Builder() - .url(urlBuilder.build()) + .url(appendMediaProperties(urlBuilder).build()) .build(); return Single.fromCallable(() -> { Response response = okHttpClient.newCall(request).execute(); List mediaList = new ArrayList<>(); - if (response != null && response.body() != null && response.isSuccessful()) { + if (response.body() != null && response.isSuccessful()) { String json = response.body().string(); - if (json == null) { + MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class); + putContinueValues(keyword, mwQueryResponse.continuation()); + if (mwQueryResponse.query() == null) { return mediaList; } - MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class); List pages = mwQueryResponse.query().pages(); for (MwQueryPage page : pages) { - mediaList.add(Media.from(page)); + Media media = Media.from(page); + if (media != null) { + mediaList.add(media); + } } } return mediaList; }); } + /** + * Whenever imageInfo is fetched, these common properties can be specified for the API call + * https://www.mediawiki.org/wiki/API:Imageinfo + * @param builder + * @return + */ + private HttpUrl.Builder appendMediaProperties(HttpUrl.Builder builder) { + builder.addQueryParameter("prop", "imageinfo") + .addQueryParameter("iiprop", "url|extmetadata") + .addQueryParameter("iiextmetadatafilter", "DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName"); + + String language = Locale.getDefault().getLanguage(); + if (!StringUtils.isNullOrWhiteSpace(language)) { + builder.addQueryParameter("iiextmetadatalanguage", language); + } + + return builder; + } + + /** + * Append params for search query. + * @param query + * @param urlBuilder + */ + private void appendSearchParam(String query, HttpUrl.Builder urlBuilder) { + urlBuilder.addQueryParameter("generator", "search") + .addQueryParameter("gsrwhat", "text") + .addQueryParameter("gsrnamespace", "6") + .addQueryParameter("gsrlimit", "25") + .addQueryParameter("gsrsearch", query); + } + + /** + * It takes a urlBuilder and appends all the continue values as query parameters + * @param query + * @param urlBuilder + */ + private void appendQueryContinueValues(String query, HttpUrl.Builder urlBuilder) { + Map continueValues = getContinueValues(query); + if (continueValues != null && continueValues.size() > 0) { + for (Map.Entry entry : continueValues.entrySet()) { + urlBuilder.addQueryParameter(entry.getKey(), entry.getValue()); + } + } + } + + private void appendCategoryParams(String categoryName, HttpUrl.Builder urlBuilder) { + urlBuilder.addQueryParameter("generator", "categorymembers") + .addQueryParameter("gcmtype", "file") + .addQueryParameter("gcmtitle", categoryName) + .addQueryParameter("gcmsort", "timestamp")//property to sort by;timestamp + .addQueryParameter("gcmdir", "desc")//in which direction to sort;descending + .addQueryParameter("gcmlimit", "10"); + } + + /** + * Stores the continue values for action=query + * These values are sent to the server in the subsequent call to fetch results after this point + * @param keyword + * @param values + */ + private void putContinueValues(String keyword, Map values) { + defaultKvStore.putJson("query_continue_" + keyword, values); + } + + /** + * Retrieves a map of continue values from shared preferences. + * These values are appended to the next API call + * @param keyword + * @return + */ + private Map getContinueValues(String keyword) { + return defaultKvStore.getJson("query_continue_" + keyword, mapType); + } + /** * Returns recent changes on commons * @return list of recent changes made diff --git a/app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java b/app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java index 63421a8e4..9cf61721e 100644 --- a/app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java +++ b/app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java @@ -1,6 +1,7 @@ package fr.free.nrw.commons.utils; import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,4 +26,24 @@ public class MediaDataExtractorUtil { } + /** + * Extracts a list of categories from | separated category string + * + * @param source + * @return + */ + public static List extractCategoriesFromList(String source) { + if (StringUtils.isNullOrWhiteSpace(source)) { + return new ArrayList<>(); + } + String[] cats = source.split("\\|"); + List categories = new ArrayList<>(); + for (String category : cats) { + if (!StringUtils.isNullOrWhiteSpace(category.trim())) { + categories.add(category); + } + } + return categories; + } + } diff --git a/app/src/test/kotlin/fr/free/nrw/commons/mwapi/OkHttpJsonApiClientTest.kt b/app/src/test/kotlin/fr/free/nrw/commons/mwapi/OkHttpJsonApiClientTest.kt new file mode 100644 index 000000000..9d2a468f5 --- /dev/null +++ b/app/src/test/kotlin/fr/free/nrw/commons/mwapi/OkHttpJsonApiClientTest.kt @@ -0,0 +1,219 @@ +package fr.free.nrw.commons.mwapi + +import com.google.gson.Gson +import fr.free.nrw.commons.BuildConfig +import fr.free.nrw.commons.TestCommonsApplication +import fr.free.nrw.commons.kvstore.JsonKvStore +import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient.mapType +import junit.framework.Assert.assertEquals +import okhttp3.HttpUrl +import okhttp3.OkHttpClient +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.After +import org.junit.Assert +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito +import org.mockito.Mockito.`when` +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.net.URLDecoder + +@RunWith(RobolectricTestRunner::class) +@Config(constants = BuildConfig::class, sdk = [23], application = TestCommonsApplication::class) +class OkHttpJsonApiClientTest { + + private lateinit var testObject: OkHttpJsonApiClient + private lateinit var toolsForgeServer: MockWebServer + private lateinit var sparqlServer: MockWebServer + private lateinit var campaignsServer: MockWebServer + private lateinit var server: MockWebServer + private lateinit var sharedPreferences: JsonKvStore + private lateinit var okHttpClient: OkHttpClient + + @Before + fun setUp() { + server = MockWebServer() + toolsForgeServer = MockWebServer() + sparqlServer = MockWebServer() + campaignsServer = MockWebServer() + okHttpClient = OkHttpClient.Builder().build() + sharedPreferences = Mockito.mock(JsonKvStore::class.java) + val toolsForgeUrl = "http://" + toolsForgeServer.hostName + ":" + toolsForgeServer.port + "/" + val sparqlUrl = "http://" + sparqlServer.hostName + ":" + sparqlServer.port + "/" + val campaignsUrl = "http://" + campaignsServer.hostName + ":" + campaignsServer.port + "/" + val serverUrl = "http://" + server.hostName + ":" + server.port + "/" + testObject = OkHttpJsonApiClient(okHttpClient, HttpUrl.get(toolsForgeUrl), sparqlUrl, campaignsUrl, serverUrl, sharedPreferences, Gson()) + } + + @After + fun teardown() { + server.shutdown() + } + + @Test + fun getCategoryImages() { + server.enqueue(getFirstPageOfImages()) + testFirstPageQuery() + } + + @Test + fun getCategoryImagesWithContinue() { + server.enqueue(getFirstPageOfImages()) + server.enqueue(getSecondPageOfImages()) + testFirstPageQuery() + + `when`(sharedPreferences.getJson>("query_continue_Watercraft moored off shore", mapType)) + .thenReturn(hashMapOf(Pair("gcmcontinue", "testvalue"), Pair("continue", "gcmcontinue||"))) + + + val categoryImagesContinued = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet() + + assertBasicRequestParameters(server, "GET").let { request -> + parseQueryParams(request).let { body -> + Assert.assertEquals("json", body["format"]) + Assert.assertEquals("query", body["action"]) + Assert.assertEquals("categorymembers", body["generator"]) + Assert.assertEquals("file", body["gcmtype"]) + Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"]) + Assert.assertEquals("timestamp", body["gcmsort"]) + Assert.assertEquals("desc", body["gcmdir"]) + Assert.assertEquals("testvalue", body["gcmcontinue"]) + Assert.assertEquals("gcmcontinue||", body["continue"]) + Assert.assertEquals("imageinfo", body["prop"]) + Assert.assertEquals("url|extmetadata", body["iiprop"]) + Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"]) + } + } + + assertEquals(categoryImagesContinued.size, 2) + } + + @Test + fun getSearchImages() { + server.enqueue(getFirstPageOfImages()) + testFirstPageSearchQuery() + } + + @Test + fun getSearchImagesWithContinue() { + server.enqueue(getFirstPageOfSearchImages()) + server.enqueue(getSecondPageOfSearchImages()) + testFirstPageSearchQuery() + + `when`(sharedPreferences.getJson>("query_continue_Watercraft moored off shore", mapType)) + .thenReturn(hashMapOf(Pair("gsroffset", "25"), Pair("continue", "gsroffset||"))) + + + val categoryImagesContinued = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet() + + assertBasicRequestParameters(server, "GET").let { request -> + parseQueryParams(request).let { body -> + Assert.assertEquals("json", body["format"]) + Assert.assertEquals("query", body["action"]) + Assert.assertEquals("search", body["generator"]) + Assert.assertEquals("text", body["gsrwhat"]) + Assert.assertEquals("6", body["gsrnamespace"]) + Assert.assertEquals("25", body["gsrlimit"]) + Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"]) + Assert.assertEquals("25", body["gsroffset"]) + Assert.assertEquals("gsroffset||", body["continue"]) + Assert.assertEquals("imageinfo", body["prop"]) + Assert.assertEquals("url|extmetadata", body["iiprop"]) + Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"]) + } + } + + assertEquals(categoryImagesContinued.size, 2) + } + + private fun testFirstPageSearchQuery() { + val categoryImages = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet() + + assertBasicRequestParameters(server, "GET").let { request -> + parseQueryParams(request).let { body -> + Assert.assertEquals("json", body["format"]) + Assert.assertEquals("query", body["action"]) + Assert.assertEquals("search", body["generator"]) + Assert.assertEquals("text", body["gsrwhat"]) + Assert.assertEquals("6", body["gsrnamespace"]) + Assert.assertEquals("25", body["gsrlimit"]) + Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"]) + Assert.assertEquals("imageinfo", body["prop"]) + Assert.assertEquals("url|extmetadata", body["iiprop"]) + Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"]) + } + } + assertEquals(categoryImages.size, 2) + } + + private fun testFirstPageQuery() { + val categoryImages = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet() + + assertBasicRequestParameters(server, "GET").let { request -> + parseQueryParams(request).let { body -> + Assert.assertEquals("json", body["format"]) + Assert.assertEquals("query", body["action"]) + Assert.assertEquals("categorymembers", body["generator"]) + Assert.assertEquals("file", body["gcmtype"]) + Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"]) + Assert.assertEquals("timestamp", body["gcmsort"]) + Assert.assertEquals("desc", body["gcmdir"]) + Assert.assertEquals("imageinfo", body["prop"]) + Assert.assertEquals("url|extmetadata", body["iiprop"]) + Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"]) + } + } + assertEquals(categoryImages.size, 2) + } + + private fun getFirstPageOfImages(): MockResponse { + val mockResponse = MockResponse() + mockResponse.setResponseCode(200) + mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}") + return mockResponse + } + + private fun getSecondPageOfImages(): MockResponse { + val mockResponse = MockResponse() + mockResponse.setResponseCode(200) + mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue2\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}") + return mockResponse + } + + private fun getFirstPageOfSearchImages(): MockResponse { + val mockResponse = MockResponse() + mockResponse.setResponseCode(200) + mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"25\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}") + return mockResponse + } + + private fun getSecondPageOfSearchImages(): MockResponse { + val mockResponse = MockResponse() + mockResponse.setResponseCode(200) + mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"50\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"Raphael\\n\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511
date QS:P571,+1511-00-00T00:00:00Z/9
\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}") + return mockResponse + } + + private fun assertBasicRequestParameters(server: MockWebServer, method: String): RecordedRequest = server.takeRequest().let { + Assert.assertEquals("/", it.requestUrl.encodedPath()) + Assert.assertEquals(method, it.method) + return it + } + + private fun parseQueryParams(request: RecordedRequest) = HashMap().apply { + request.requestUrl.let { + it.queryParameterNames().forEach { name -> put(name, it.queryParameter(name)) } + } + } + + private fun parseBody(body: String): Map = HashMap().apply { + body.split("&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray().forEach { prop -> + val pair = prop.split("=".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + put(pair[0], URLDecoder.decode(pair[1], "utf-8")) + } + } +} \ No newline at end of file diff --git a/app/src/test/kotlin/fr/free/nrw/commons/utils/MediaDataExtractorUtilTest.java b/app/src/test/kotlin/fr/free/nrw/commons/utils/MediaDataExtractorUtilTest.java new file mode 100644 index 000000000..2f96ef19f --- /dev/null +++ b/app/src/test/kotlin/fr/free/nrw/commons/utils/MediaDataExtractorUtilTest.java @@ -0,0 +1,40 @@ +package fr.free.nrw.commons.utils; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +public class MediaDataExtractorUtilTest { + + @Test + public void extractCategoriesFromList() { + List strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018|Watercraft|2018"); + assertEquals(strings.size(), 3); + } + + @Test + public void extractCategoriesFromEmptyList() { + List strings = MediaDataExtractorUtil.extractCategoriesFromList(""); + assertEquals(strings.size(), 0); + } + + @Test + public void extractCategoriesFromNullList() { + List strings = MediaDataExtractorUtil.extractCategoriesFromList(null); + assertEquals(strings.size(), 0); + } + + @Test + public void extractCategoriesFromListWithEmptyValues() { + List strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018||"); + assertEquals(strings.size(), 1); + } + + @Test + public void extractCategoriesFromListWithWhitespaces() { + List strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018| | ||"); + assertEquals(strings.size(), 1); + } +} \ No newline at end of file