mnalis/apps-android-commons
1
0
Fork 0
mirror of https://github.com/commons-app/apps-android-commons.git synced 2025-10-26 20:33:53 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Use JSON APIs for explore (#2731)

Browse source 
* Use JSON APIs for explore

* With tests

* Use JSON APIs for explore

* With tests

* BugFix #2731 (#4)

* Increased sdk version to 23

* with more robust tests

* Fix crashes and other reported issues

* Add javadocs

* Use common method for search and categories

* Add javadocs
This commit is contained in:
Vivek Maskara 2019-03-26 18:02:58 +05:30 • committed by Ashish Kumar
parent c1a941eaae
commit c45b945526
14 changed files with 468 additions and 318 deletions
Download patch file Download diff file Expand all files Collapse all files

46
app/src/main/java/fr/free/nrw/commons/Media.java
View file

@ -3,20 +3,24 @@ package fr.free.nrw.commons;
import android.net.Uri;
import android.os.Parcel;
import android.os.Parcelable;
import androidx.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import androidx.annotation.Nullable;
import fr.free.nrw.commons.location.LatLng;
import fr.free.nrw.commons.media.model.ExtMetadata;
import fr.free.nrw.commons.media.model.ImageInfo;
import fr.free.nrw.commons.media.model.MwQueryPage;
import fr.free.nrw.commons.utils.DateUtils;
import fr.free.nrw.commons.utils.MediaDataExtractorUtil;
import fr.free.nrw.commons.utils.StringUtils;
public class Media implements Parcelable {
@ -93,6 +97,8 @@ public class Media implements Parcelable {
this.dateCreated = dateCreated;
this.dateUploaded = dateUploaded;
this.creator = creator;
this.categories = new ArrayList<>();
this.descriptions = new HashMap<>();
}
@SuppressWarnings("unchecked")
@ -450,23 +456,51 @@ public class Media implements Parcelable {
return requestedDeletion;
}
/**
* Creating Media object from MWQueryPage.
* Earlier only basic details were set for the media object but going forward,
* a full media object(with categories, descriptions, coordinates etc) can be constructed using this method
*
* @param page response from the API
* @return Media object
*/
@Nullable
public static Media from(MwQueryPage page) {
ImageInfo imageInfo = page.imageInfo();
if(imageInfo == null) {
return null;
}
ExtMetadata metadata = imageInfo.getMetadata();
if (metadata == null) {
return new Media(null, imageInfo.getOriginalUrl(),
page.title(), "", 0, null, null, null);
}
Media media = new Media(null,
imageInfo.getOriginalUrl(),
page.title(),
imageInfo.getMetadata().imageDescription().value(),
"",
0,
DateUtils.getDateFromString(imageInfo.getMetadata().dateTimeOriginal().value()),
DateUtils.getDateFromString(imageInfo.getMetadata().dateTime().value()),
StringUtils.getParsedStringFromHtml(imageInfo.getMetadata().artist().value())
DateUtils.getDateFromString(metadata.dateTimeOriginal().value()),
DateUtils.getDateFromString(metadata.dateTime().value()),
StringUtils.getParsedStringFromHtml(metadata.artist().value())
);
media.setLicense(imageInfo.getMetadata().licenseShortName().value());
String language = Locale.getDefault().getLanguage();
if (StringUtils.isNullOrWhiteSpace(language)) {
language = "default";
}
media.setDescriptions(Collections.singletonMap(language, metadata.imageDescription().value()));
media.setCategories(MediaDataExtractorUtil.extractCategoriesFromList(metadata.categories().value()));
String latitude = metadata.gpsLatitude().value();
String longitude = metadata.gpsLongitude().value();
if(!StringUtils.isNullOrWhiteSpace(latitude) && !StringUtils.isNullOrWhiteSpace(longitude)) {
LatLng latLng = new LatLng(Double.parseDouble(latitude), Double.parseDouble(longitude), 0);
media.setCoordinates(latLng);
}
media.setLicense(metadata.licenseShortName().value());
return media;
}
}

2
app/src/main/java/fr/free/nrw/commons/bookmarks/pictures/BookmarkPicturesController.java
View file

@ -36,7 +36,7 @@ public class BookmarkPicturesController {
ArrayList<Media> medias = new ArrayList<>();
for (Bookmark bookmark : bookmarks) {
List<Media> tmpMedias = okHttpJsonApiClient
.searchImages(bookmark.getMediaName(), 0)
.getMediaList("search", bookmark.getMediaName())
.blockingGet();
for (Media m : tmpMedias) {
if (m.getCreator().trim().equals(bookmark.getMediaCreator().trim())) {

13
app/src/main/java/fr/free/nrw/commons/category/CategoryImageController.java
View file

@ -6,16 +6,17 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.mwapi.MediaWikiApi;
import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient;
import io.reactivex.Single;
@Singleton
public class CategoryImageController {
private MediaWikiApi mediaWikiApi;
private OkHttpJsonApiClient okHttpJsonApiClient;
@Inject
public CategoryImageController(MediaWikiApi mediaWikiApi) {
this.mediaWikiApi = mediaWikiApi;
public CategoryImageController(OkHttpJsonApiClient okHttpJsonApiClient) {
this.okHttpJsonApiClient = okHttpJsonApiClient;
}
/**
@ -23,7 +24,7 @@ public class CategoryImageController {
* @param categoryName
* @return
*/
public List<Media> getCategoryImages(String categoryName) {
return mediaWikiApi.getCategoryImages(categoryName);
public Single<List<Media>> getCategoryImages(String categoryName) {
return okHttpJsonApiClient.getMediaList("category", categoryName);
}
}

209
app/src/main/java/fr/free/nrw/commons/category/CategoryImageUtils.java
View file

@ -1,45 +1,15 @@
package fr.free.nrw.commons.category;
import androidx.annotation.NonNull;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import javax.annotation.Nullable;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.utils.StringUtils;
import timber.log.Timber;
public class CategoryImageUtils {
/**
* The method iterates over the child nodes to return a list of Media objects
* @param childNodes
* @return
*/
public static List<Media> getMediaList(NodeList childNodes) {
List<Media> categoryImages = new ArrayList<>();
for (int i = 0; i < childNodes.getLength(); i++) {
Node node = childNodes.item(i);
if (getFileName(node).substring(0, 5).equals("File:")) {
categoryImages.add(getMediaFromPage(node));
}
}
return categoryImages;
}
/**
* The method iterates over the child nodes to return a list of Subcategory name
* sorted alphabetically
@ -56,27 +26,6 @@ public class CategoryImageUtils {
return subCategories;
}
/**
* Creates a new Media object from the XML response as received by the API
* @param node
* @return
*/
public static Media getMediaFromPage(Node node) {
Media media = new Media(null,
getImageUrl(node),
getFileName(node),
getDescription(node),
getDataLength(node),
getDateCreated(node),
getDateCreated(node),
getCreator(node)
);
media.setLicense(getLicense(node));
return media;
}
/**
* Extracts the filename of the uploaded image
* @param document
@ -87,162 +36,4 @@ public class CategoryImageUtils {
return element.getAttribute("title");
}
/**
* Extracts the image description for that particular upload
* @param document
* @return
*/
private static String getDescription(Node document) {
return getMetaDataValue(document, "ImageDescription");
}
/**
* Extracts license information from the image meta data
* @param document
* @return
*/
private static String getLicense(Node document) {
return getMetaDataValue(document, "License");
}
/**
* Returns the parsed value of artist from the response
* The artist information is returned as a HTML string from the API. Using HTML parser to parse the HTML
* @param document
* @return
*/
@NonNull
private static String getCreator(Node document) {
String artist = getMetaDataValue(document, "Artist");
if (StringUtils.isNullOrWhiteSpace(artist)) {
return "";
}
return StringUtils.getParsedStringFromHtml(artist);
}
/**
* Returns the parsed date of creation of the image
* @param document
* @return
*/
private static Date getDateCreated(Node document) {
String dateTime = getMetaDataValue(document, "DateTime");
if (dateTime != null && !dateTime.equals("")) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
return format.parse(dateTime);
} catch (ParseException e) {
Timber.d("Error occurred while parsing date %s", dateTime);
return new Date();
}
}
return new Date();
}
/**
* @param document
* @return Returns the url attribute from the imageInfo node
*/
private static String getImageUrl(Node document) {
Element element = (Element) getImageInfo(document);
if (element != null) {
return element.getAttribute("url");
}
return null;
}
/**
* Takes the node document and gives out the attribute length from the node document
* @param document
* @return
*/
private static long getDataLength(Node document) {
Element element = (Element) document;
if (element != null) {
String length = element.getAttribute("length");
if (length != null && !length.equals("")) {
return Long.parseLong(length);
}
}
return 0L;
}
/**
* Generic method to get the value of any meta as returned by the getMetaData function
* @param document node document as returned by API
* @param metaName the name of meta node to be returned
* @return
*/
private static String getMetaDataValue(Node document, String metaName) {
Element metaData = getMetaData(document, metaName);
if (metaData != null) {
return metaData.getAttribute("value");
}
return null;
}
/**
* Generic method to return an element taking the node document and metaName as input
* @param document node document as returned by API
* @param metaName the name of meta node to be returned
* @return
*/
@Nullable
private static Element getMetaData(Node document, String metaName) {
Node extraMetaData = getExtraMetaData(document);
if (extraMetaData != null) {
Node node = getNode(extraMetaData, metaName);
if (node != null) {
return (Element) node;
}
}
return null;
}
/**
* Extracts extmetadata from the response XML
* @param document
* @return
*/
@Nullable
private static Node getExtraMetaData(Node document) {
Node imageInfo = getImageInfo(document);
if (imageInfo != null) {
return getNode(imageInfo, "extmetadata");
}
return null;
}
/**
* Extracts the ii node from the imageinfo node
* @param document
* @return
*/
@Nullable
private static Node getImageInfo(Node document) {
Node imageInfo = getNode(document, "imageinfo");
if (imageInfo != null) {
return getNode(imageInfo, "ii");
}
return null;
}
/**
* Takes a parent node as input and returns a child node if present
* @param node parent node
* @param nodeName child node name
* @return
*/
@Nullable
public static Node getNode(Node node, String nodeName) {
NodeList childNodes = node.getChildNodes();
for (int i = 0; i < childNodes.getLength(); i++) {
Node nodeItem = childNodes.item(i);
Element item = (Element) nodeItem;
if (item.getTagName().equals(nodeName)) {
return nodeItem;
}
}
return null;
}
}

9
app/src/main/java/fr/free/nrw/commons/category/CategoryImagesListFragment.java
View file

@ -2,7 +2,6 @@ package fr.free.nrw.commons.category;
import android.annotation.SuppressLint;
import android.os.Bundle;
import androidx.annotation.Nullable;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
@ -20,6 +19,7 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Named;
import androidx.annotation.Nullable;
import butterknife.BindView;
import butterknife.ButterKnife;
import dagger.android.support.DaggerFragment;
@ -29,7 +29,6 @@ import fr.free.nrw.commons.explore.categories.ExploreActivity;
import fr.free.nrw.commons.kvstore.JsonKvStore;
import fr.free.nrw.commons.utils.NetworkUtils;
import fr.free.nrw.commons.utils.ViewUtil;
import io.reactivex.Observable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.disposables.CompositeDisposable;
import io.reactivex.schedulers.Schedulers;
@ -102,7 +101,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
* @param keyword
*/
private void resetQueryContinueValues(String keyword) {
categoryKvStore.remove(keyword);
categoryKvStore.remove("query_continue_" + keyword);
}
/**
@ -117,7 +116,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
isLoading = true;
progressBar.setVisibility(VISIBLE);
compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName))
compositeDisposable.add(controller.getCategoryImages(categoryName)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)
@ -223,7 +222,7 @@ public class CategoryImagesListFragment extends DaggerFragment {
}
progressBar.setVisibility(VISIBLE);
compositeDisposable.add(Observable.fromCallable(() -> controller.getCategoryImages(categoryName))
compositeDisposable.add(controller.getCategoryImages(categoryName)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)

2
app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java
View file

@ -75,12 +75,14 @@ public class NetworkingModule {
@Singleton
public OkHttpJsonApiClient provideOkHttpJsonApiClient(OkHttpClient okHttpClient,
@Named("tools_force") HttpUrl toolsForgeUrl,
@Named("default_preferences") JsonKvStore defaultKvStore,
Gson gson) {
return new OkHttpJsonApiClient(okHttpClient,
toolsForgeUrl,
WIKIDATA_SPARQL_QUERY_URL,
WIKIMEDIA_CAMPAIGNS_BASE_URL,
BuildConfig.WIKIMEDIA_API_HOST,
defaultKvStore,
gson);
}

10
app/src/main/java/fr/free/nrw/commons/explore/images/SearchImageFragment.java
View file

@ -4,9 +4,6 @@ package fr.free.nrw.commons.explore.images;
import android.annotation.SuppressLint;
import android.content.res.Configuration;
import android.os.Bundle;
import androidx.recyclerview.widget.GridLayoutManager;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
@ -23,6 +20,9 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Named;
import androidx.recyclerview.widget.GridLayoutManager;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import butterknife.BindView;
import butterknife.ButterKnife;
import fr.free.nrw.commons.Media;
@ -141,7 +141,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment {
bottomProgressBar.setVisibility(GONE);
queryList.clear();
imagesAdapter.clear();
compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size())
compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)
@ -158,7 +158,7 @@ public class SearchImageFragment extends CommonsDaggerSupportFragment {
this.query = query;
bottomProgressBar.setVisibility(View.VISIBLE);
progressBar.setVisibility(GONE);
compositeDisposable.add(okHttpJsonApiClient.searchImages(query, queryList.size())
compositeDisposable.add(okHttpJsonApiClient.getMediaList("search", query)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.timeout(TIMEOUT_SECONDS, TimeUnit.SECONDS)

17
app/src/main/java/fr/free/nrw/commons/media/model/ExtMetadata.java
View file

@ -14,6 +14,8 @@ public class ExtMetadata {
@SuppressWarnings("unused") @SerializedName("Categories") @Nullable private Values categories;
@SuppressWarnings("unused") @SerializedName("Assessments") @Nullable private Values assessments;
@SuppressWarnings("unused") @SerializedName("ImageDescription") @Nullable private Values imageDescription;
@SuppressWarnings("unused") @SerializedName("GPSLatitude") @Nullable private Values gpsLatitude;
@SuppressWarnings("unused") @SerializedName("GPSLongitude") @Nullable private Values gpsLongitude;
@SuppressWarnings("unused") @SerializedName("DateTimeOriginal") @Nullable private Values dateTimeOriginal;
@SuppressWarnings("unused") @SerializedName("Artist") @Nullable private Values artist;
@SuppressWarnings("unused") @SerializedName("Credit") @Nullable private Values credit;
@ -51,6 +53,21 @@ public class ExtMetadata {
return imageDescription != null ? imageDescription : new Values();
}
@NonNull
public Values categories() {
return categories != null ? categories : new Values();
}
@NonNull
public Values gpsLatitude() {
return gpsLatitude != null ? gpsLatitude : new Values();
}
@NonNull
public Values gpsLongitude() {
return gpsLongitude != null ? gpsLongitude : new Values();
}
@NonNull public Values objectName() {
return objectName != null ? objectName : new Values();
}

60
app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java
View file

@ -38,7 +38,6 @@ import java.util.concurrent.Callable;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import fr.free.nrw.commons.BuildConfig;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.R;
import fr.free.nrw.commons.auth.AccountUtil;
import fr.free.nrw.commons.category.CategoryImageUtils;
@ -54,8 +53,6 @@ import io.reactivex.Observable;
import io.reactivex.Single;
import timber.log.Timber;
import static fr.free.nrw.commons.utils.ContinueUtils.getQueryContinue;
/**
* @author Addshore
*/
@ -714,63 +711,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
return CategoryImageUtils.getSubCategoryList(childNodes);
}
/**
* The method takes categoryName as input and returns a List of Media objects
* It uses the generator query API to get the images in a category, 10 at a time.
* Uses the query continue values for fetching paginated responses
* @param categoryName Category name as defined on commons
* @return
*/
@Override
@NonNull
public List<Media> getCategoryImages(String categoryName) {
CustomApiResult apiResult = null;
try {
CustomMwApi.RequestBuilder requestBuilder = api.action("query")
.param("generator", "categorymembers")
.param("format", "xml")
.param("gcmtype", "file")
.param("gcmtitle", categoryName)
.param("gcmsort", "timestamp")//property to sort by;timestamp
.param("gcmdir", "desc")//in which direction to sort;descending
.param("prop", "imageinfo")
.param("gcmlimit", "10")
.param("iiprop", "url|extmetadata");
QueryContinue queryContinueValues = getQueryContinueValues(categoryName);
if (queryContinueValues != null) {
requestBuilder.param("continue", queryContinueValues.getContinueParam());
requestBuilder.param("gcmcontinue", queryContinueValues.getGcmContinueParam());
}
apiResult = requestBuilder.get();
} catch (IOException e) {
Timber.e(e, "Failed to obtain searchCategories");
}
if (apiResult == null) {
return new ArrayList<>();
}
CustomApiResult categoryImagesNode = apiResult.getNode("/api/query/pages");
if (categoryImagesNode == null
|| categoryImagesNode.getDocument() == null
|| categoryImagesNode.getDocument().getChildNodes() == null
|| categoryImagesNode.getDocument().getChildNodes().getLength() == 0) {
return new ArrayList<>();
}
if (apiResult.getNode("/api/continue").getDocument()==null){
setQueryContinueValues(categoryName, null);
}else {
QueryContinue queryContinue = getQueryContinue(apiResult.getNode("/api/continue").getDocument());
setQueryContinueValues(categoryName, queryContinue);
}
NodeList childNodes = categoryImagesNode.getDocument().getChildNodes();
return CategoryImageUtils.getMediaList(childNodes);
}
/**
* This method takes search keyword as input and returns a list of categories objects filtered using search query
* It uses the generator query API to get the categories searched using a query, 25 at a time.

3
app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java
View file

@ -8,7 +8,6 @@ import java.util.List;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.notification.Notification;
import io.reactivex.Observable;
import io.reactivex.Single;
@ -40,8 +39,6 @@ public interface MediaWikiApi {
boolean logEvents(LogBuilder[] logBuilders);
List<Media> getCategoryImages(String categoryName);
List<String> getSubCategoryList(String categoryName);
List<String> getParentCategoryList(String categoryName);

135
app/src/main/java/fr/free/nrw/commons/mwapi/OkHttpJsonApiClient.java
View file

@ -1,12 +1,15 @@
package fr.free.nrw.commons.mwapi;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import javax.inject.Inject;
@ -19,6 +22,7 @@ import fr.free.nrw.commons.PageTitle;
import fr.free.nrw.commons.achievements.FeaturedImages;
import fr.free.nrw.commons.achievements.FeedbackResponse;
import fr.free.nrw.commons.campaigns.CampaignResponseDTO;
import fr.free.nrw.commons.kvstore.JsonKvStore;
import fr.free.nrw.commons.location.LatLng;
import fr.free.nrw.commons.media.model.MwQueryPage;
import fr.free.nrw.commons.mwapi.model.MwQueryResponse;
@ -28,6 +32,7 @@ import fr.free.nrw.commons.nearby.model.NearbyResponse;
import fr.free.nrw.commons.nearby.model.NearbyResultItem;
import fr.free.nrw.commons.upload.FileUtils;
import fr.free.nrw.commons.utils.DateUtils;
import fr.free.nrw.commons.utils.StringUtils;
import fr.free.nrw.commons.wikidata.model.GetWikidataEditCountResponse;
import io.reactivex.Observable;
import io.reactivex.Single;
@ -39,11 +44,16 @@ import timber.log.Timber;
@Singleton
public class OkHttpJsonApiClient {
public static final Type mapType = new TypeToken<Map<String, String>>() {
}.getType();
private final OkHttpClient okHttpClient;
private final HttpUrl wikiMediaToolforgeUrl;
private final String sparqlQueryUrl;
private final String campaignsUrl;
private final String commonsBaseUrl;
private final JsonKvStore defaultKvStore;
private Gson gson;
@ -53,12 +63,14 @@ public class OkHttpJsonApiClient {
String sparqlQueryUrl,
String campaignsUrl,
String commonsBaseUrl,
JsonKvStore defaultKvStore,
Gson gson) {
this.okHttpClient = okHttpClient;
this.wikiMediaToolforgeUrl = wikiMediaToolforgeUrl;
this.sparqlQueryUrl = sparqlQueryUrl;
this.campaignsUrl = campaignsUrl;
this.commonsBaseUrl = commonsBaseUrl;
this.defaultKvStore = defaultKvStore;
this.gson = gson;
}
@ -75,7 +87,6 @@ public class OkHttpJsonApiClient {
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.isSuccessful()) {
return Integer.parseInt(response.body().string().trim());
}
return 0;
@ -224,11 +235,8 @@ public class OkHttpJsonApiClient {
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
if (response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
MwQueryResponse mwQueryPage = gson.fromJson(json, MwQueryResponse.class);
return Media.from(mwQueryPage.query().firstPage());
}
@ -237,49 +245,130 @@ public class OkHttpJsonApiClient {
}
/**
* This method takes search keyword as input and returns a list of Media objects filtered using search query
* It uses the generator query API to get the images searched using a query, 25 at a time.
* @param query keyword to search images on commons
* This method takes the keyword and queryType as input and returns a list of Media objects filtered using image generator query
* It uses the generator query API to get the images searched using a query, 10 at a time.
* @param queryType queryType can be "search" OR "category"
* @param keyword
* @return
*/
@Nullable
public Single<List<Media>> searchImages(String query, int offset) {
public Single<List<Media>> getMediaList(String queryType, String keyword) {
HttpUrl.Builder urlBuilder = HttpUrl
.parse(commonsBaseUrl)
.newBuilder()
.addQueryParameter("action", "query")
.addQueryParameter("generator", "search")
.addQueryParameter("format", "json")
.addQueryParameter("gsrwhat", "text")
.addQueryParameter("gsrnamespace", "6")
.addQueryParameter("gsrlimit", "25")
.addQueryParameter("gsroffset", String.valueOf(offset))
.addQueryParameter("gsrsearch", query)
.addQueryParameter("prop", "imageinfo")
.addQueryParameter("iiprop", "url|extmetadata");
.addQueryParameter("format", "json");
if (queryType.equals("search")) {
appendSearchParam(keyword, urlBuilder);
} else {
appendCategoryParams(keyword, urlBuilder);
}
appendQueryContinueValues(keyword, urlBuilder);
Request request = new Request.Builder()
.url(urlBuilder.build())
.url(appendMediaProperties(urlBuilder).build())
.build();
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
List<Media> mediaList = new ArrayList<>();
if (response != null && response.body() != null && response.isSuccessful()) {
if (response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class);
putContinueValues(keyword, mwQueryResponse.continuation());
if (mwQueryResponse.query() == null) {
return mediaList;
}
MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class);
List<MwQueryPage> pages = mwQueryResponse.query().pages();
for (MwQueryPage page : pages) {
mediaList.add(Media.from(page));
Media media = Media.from(page);
if (media != null) {
mediaList.add(media);
}
}
}
return mediaList;
});
}
/**
* Whenever imageInfo is fetched, these common properties can be specified for the API call
* https://www.mediawiki.org/wiki/API:Imageinfo
* @param builder
* @return
*/
private HttpUrl.Builder appendMediaProperties(HttpUrl.Builder builder) {
builder.addQueryParameter("prop", "imageinfo")
.addQueryParameter("iiprop", "url|extmetadata")
.addQueryParameter("iiextmetadatafilter", "DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName");
String language = Locale.getDefault().getLanguage();
if (!StringUtils.isNullOrWhiteSpace(language)) {
builder.addQueryParameter("iiextmetadatalanguage", language);
}
return builder;
}
/**
* Append params for search query.
* @param query
* @param urlBuilder
*/
private void appendSearchParam(String query, HttpUrl.Builder urlBuilder) {
urlBuilder.addQueryParameter("generator", "search")
.addQueryParameter("gsrwhat", "text")
.addQueryParameter("gsrnamespace", "6")
.addQueryParameter("gsrlimit", "25")
.addQueryParameter("gsrsearch", query);
}
/**
* It takes a urlBuilder and appends all the continue values as query parameters
* @param query
* @param urlBuilder
*/
private void appendQueryContinueValues(String query, HttpUrl.Builder urlBuilder) {
Map<String, String> continueValues = getContinueValues(query);
if (continueValues != null && continueValues.size() > 0) {
for (Map.Entry<String, String> entry : continueValues.entrySet()) {
urlBuilder.addQueryParameter(entry.getKey(), entry.getValue());
}
}
}
private void appendCategoryParams(String categoryName, HttpUrl.Builder urlBuilder) {
urlBuilder.addQueryParameter("generator", "categorymembers")
.addQueryParameter("gcmtype", "file")
.addQueryParameter("gcmtitle", categoryName)
.addQueryParameter("gcmsort", "timestamp")//property to sort by;timestamp
.addQueryParameter("gcmdir", "desc")//in which direction to sort;descending
.addQueryParameter("gcmlimit", "10");
}
/**
* Stores the continue values for action=query
* These values are sent to the server in the subsequent call to fetch results after this point
* @param keyword
* @param values
*/
private void putContinueValues(String keyword, Map<String, String> values) {
defaultKvStore.putJson("query_continue_" + keyword, values);
}
/**
* Retrieves a map of continue values from shared preferences.
* These values are appended to the next API call
* @param keyword
* @return
*/
private Map<String, String> getContinueValues(String keyword) {
return defaultKvStore.getJson("query_continue_" + keyword, mapType);
}
/**
* Returns recent changes on commons
* @return list of recent changes made

21
app/src/main/java/fr/free/nrw/commons/utils/MediaDataExtractorUtil.java
View file

@ -1,6 +1,7 @@
package fr.free.nrw.commons.utils;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -25,4 +26,24 @@ public class MediaDataExtractorUtil {
}
/**
* Extracts a list of categories from | separated category string
*
* @param source
* @return
*/
public static List<String> extractCategoriesFromList(String source) {
if (StringUtils.isNullOrWhiteSpace(source)) {
return new ArrayList<>();
}
String[] cats = source.split("\\|");
List<String> categories = new ArrayList<>();
for (String category : cats) {
if (!StringUtils.isNullOrWhiteSpace(category.trim())) {
categories.add(category);
}
}
return categories;
}
}

219
app/src/test/kotlin/fr/free/nrw/commons/mwapi/OkHttpJsonApiClientTest.kt Normal file
View file

@ -0,0 +1,219 @@
package fr.free.nrw.commons.mwapi
import com.google.gson.Gson
import fr.free.nrw.commons.BuildConfig
import fr.free.nrw.commons.TestCommonsApplication
import fr.free.nrw.commons.kvstore.JsonKvStore
import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient.mapType
import junit.framework.Assert.assertEquals
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import okhttp3.mockwebserver.RecordedRequest
import org.junit.After
import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.mockito.Mockito
import org.mockito.Mockito.`when`
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import java.net.URLDecoder
@RunWith(RobolectricTestRunner::class)
@Config(constants = BuildConfig::class, sdk = [23], application = TestCommonsApplication::class)
class OkHttpJsonApiClientTest {
private lateinit var testObject: OkHttpJsonApiClient
private lateinit var toolsForgeServer: MockWebServer
private lateinit var sparqlServer: MockWebServer
private lateinit var campaignsServer: MockWebServer
private lateinit var server: MockWebServer
private lateinit var sharedPreferences: JsonKvStore
private lateinit var okHttpClient: OkHttpClient
@Before
fun setUp() {
server = MockWebServer()
toolsForgeServer = MockWebServer()
sparqlServer = MockWebServer()
campaignsServer = MockWebServer()
okHttpClient = OkHttpClient.Builder().build()
sharedPreferences = Mockito.mock(JsonKvStore::class.java)
val toolsForgeUrl = "http://" + toolsForgeServer.hostName + ":" + toolsForgeServer.port + "/"
val sparqlUrl = "http://" + sparqlServer.hostName + ":" + sparqlServer.port + "/"
val campaignsUrl = "http://" + campaignsServer.hostName + ":" + campaignsServer.port + "/"
val serverUrl = "http://" + server.hostName + ":" + server.port + "/"
testObject = OkHttpJsonApiClient(okHttpClient, HttpUrl.get(toolsForgeUrl), sparqlUrl, campaignsUrl, serverUrl, sharedPreferences, Gson())
}
@After
fun teardown() {
server.shutdown()
}
@Test
fun getCategoryImages() {
server.enqueue(getFirstPageOfImages())
testFirstPageQuery()
}
@Test
fun getCategoryImagesWithContinue() {
server.enqueue(getFirstPageOfImages())
server.enqueue(getSecondPageOfImages())
testFirstPageQuery()
`when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
.thenReturn(hashMapOf(Pair("gcmcontinue", "testvalue"), Pair("continue", "gcmcontinue||")))
val categoryImagesContinued = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("categorymembers", body["generator"])
Assert.assertEquals("file", body["gcmtype"])
Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
Assert.assertEquals("timestamp", body["gcmsort"])
Assert.assertEquals("desc", body["gcmdir"])
Assert.assertEquals("testvalue", body["gcmcontinue"])
Assert.assertEquals("gcmcontinue||", body["continue"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImagesContinued.size, 2)
}
@Test
fun getSearchImages() {
server.enqueue(getFirstPageOfImages())
testFirstPageSearchQuery()
}
@Test
fun getSearchImagesWithContinue() {
server.enqueue(getFirstPageOfSearchImages())
server.enqueue(getSecondPageOfSearchImages())
testFirstPageSearchQuery()
`when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
.thenReturn(hashMapOf(Pair("gsroffset", "25"), Pair("continue", "gsroffset||")))
val categoryImagesContinued = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("search", body["generator"])
Assert.assertEquals("text", body["gsrwhat"])
Assert.assertEquals("6", body["gsrnamespace"])
Assert.assertEquals("25", body["gsrlimit"])
Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
Assert.assertEquals("25", body["gsroffset"])
Assert.assertEquals("gsroffset||", body["continue"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImagesContinued.size, 2)
}
private fun testFirstPageSearchQuery() {
val categoryImages = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("search", body["generator"])
Assert.assertEquals("text", body["gsrwhat"])
Assert.assertEquals("6", body["gsrnamespace"])
Assert.assertEquals("25", body["gsrlimit"])
Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImages.size, 2)
}
private fun testFirstPageQuery() {
val categoryImages = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("categorymembers", body["generator"])
Assert.assertEquals("file", body["gcmtype"])
Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
Assert.assertEquals("timestamp", body["gcmsort"])
Assert.assertEquals("desc", body["gcmdir"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImages.size, 2)
}
private fun getFirstPageOfImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getSecondPageOfImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue2\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getFirstPageOfSearchImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"25\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getSecondPageOfSearchImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"50\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun assertBasicRequestParameters(server: MockWebServer, method: String): RecordedRequest = server.takeRequest().let {
Assert.assertEquals("/", it.requestUrl.encodedPath())
Assert.assertEquals(method, it.method)
return it
}
private fun parseQueryParams(request: RecordedRequest) = HashMap<String, String?>().apply {
request.requestUrl.let {
it.queryParameterNames().forEach { name -> put(name, it.queryParameter(name)) }
}
}
private fun parseBody(body: String): Map<String, String> = HashMap<String, String>().apply {
body.split("&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray().forEach { prop ->
val pair = prop.split("=".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
put(pair[0], URLDecoder.decode(pair[1], "utf-8"))
}
}
}

40
app/src/test/kotlin/fr/free/nrw/commons/utils/MediaDataExtractorUtilTest.java Normal file
View file

@ -0,0 +1,40 @@
package fr.free.nrw.commons.utils;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.*;
public class MediaDataExtractorUtilTest {
@Test
public void extractCategoriesFromList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018|Watercraft|2018");
assertEquals(strings.size(), 3);
}
@Test
public void extractCategoriesFromEmptyList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("");
assertEquals(strings.size(), 0);
}
@Test
public void extractCategoriesFromNullList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList(null);
assertEquals(strings.size(), 0);
}
@Test
public void extractCategoriesFromListWithEmptyValues() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018||");
assertEquals(strings.size(), 1);
}
@Test
public void extractCategoriesFromListWithWhitespaces() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018| | ||");
assertEquals(strings.size(), 1);
}
}