Use JSON SPARQL query for fetching nearby places (#2398)

* Use JSON response for nearby places

* Move okhttp calls to a different class

* wip

* Fetch picture of the day using JSON API

* Search images using JSON APIs

* tests

* Fix injection based on code review comments
This commit is contained in:
Vivek Maskara 2019-02-06 10:40:30 +05:30 committed by Ashish Kumar
parent 323527b3be
commit f12837650a
44 changed files with 1472 additions and 418 deletions

View file

@ -5,7 +5,6 @@ import android.net.Uri;
import android.os.Build;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.annotation.VisibleForTesting;
import android.text.TextUtils;
import com.google.gson.Gson;
@ -21,7 +20,6 @@ import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -41,12 +39,8 @@ import java.util.concurrent.Callable;
import fr.free.nrw.commons.BuildConfig;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.PageTitle;
import fr.free.nrw.commons.R;
import fr.free.nrw.commons.achievements.FeaturedImages;
import fr.free.nrw.commons.achievements.FeedbackResponse;
import fr.free.nrw.commons.auth.AccountUtil;
import fr.free.nrw.commons.campaigns.CampaignResponseDTO;
import fr.free.nrw.commons.category.CategoryImageUtils;
import fr.free.nrw.commons.category.QueryContinue;
import fr.free.nrw.commons.kvstore.BasicKvStore;
@ -59,10 +53,6 @@ import fr.free.nrw.commons.utils.ViewUtil;
import in.yuvi.http.fluent.Http;
import io.reactivex.Observable;
import io.reactivex.Single;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import timber.log.Timber;
import static fr.free.nrw.commons.utils.ContinueUtils.getQueryContinue;
@ -71,8 +61,6 @@ import static fr.free.nrw.commons.utils.ContinueUtils.getQueryContinue;
* @author Addshore
*/
public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
private String wikiMediaToolforgeUrl = "https://tools.wmflabs.org/";
private static final String THUMB_SIZE = "640";
private AbstractHttpClient httpClient;
private CustomMwApi api;
@ -81,9 +69,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
private BasicKvStore defaultKvStore;
private BasicKvStore categoryKvStore;
private Gson gson;
private final OkHttpClient okHttpClient;
private final String WIKIMEDIA_CAMPAIGNS_BASE_URL =
"https://raw.githubusercontent.com/commons-app/campaigns/master/campaigns.json";
private final String ERROR_CODE_BAD_TOKEN = "badtoken";
@ -92,10 +77,8 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
String wikidatApiURL,
BasicKvStore defaultKvStore,
BasicKvStore categoryKvStore,
Gson gson,
OkHttpClient okHttpClient) {
Gson gson) {
this.context = context;
this.okHttpClient = okHttpClient;
BasicHttpParams params = new BasicHttpParams();
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
@ -120,11 +103,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
return "Commons/" + ConfigUtils.getVersionNameWithSha(context) + " (https://mediawiki.org/wiki/Apps/Commons) Android/" + Build.VERSION.RELEASE;
}
@VisibleForTesting
public void setWikiMediaToolforgeUrl(String wikiMediaToolforgeUrl) {
this.wikiMediaToolforgeUrl = wikiMediaToolforgeUrl;
}
/**
* @param username String
* @param password String
@ -760,44 +738,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
return CategoryImageUtils.getMediaList(childNodes);
}
/**
* This method takes search keyword as input and returns a list of Media objects filtered using search query
* It uses the generator query API to get the images searched using a query, 25 at a time.
* @param query keyword to search images on commons
* @return
*/
@Override
@NonNull
public List<Media> searchImages(String query, int offset) {
CustomApiResult apiResult=null;
try {
apiResult= api.action("query")
.param("format", "xml")
.param("generator", "search")
.param("gsrwhat", "text")
.param("gsrnamespace", "6")
.param("gsrlimit", "25")
.param("gsroffset",offset)
.param("gsrsearch", query)
.param("prop", "imageinfo")
.param("iiprop", "url|extmetadata")
.get();
} catch (IOException e) {
Timber.e(e, "Failed to obtain searchImages");
}
CustomApiResult searchImagesNode = apiResult.getNode("/api/query/pages");
if (searchImagesNode == null
|| searchImagesNode.getDocument() == null
|| searchImagesNode.getDocument().getChildNodes() == null
|| searchImagesNode.getDocument().getChildNodes().getLength() == 0) {
return new ArrayList<>();
}
NodeList childNodes = searchImagesNode.getDocument().getChildNodes();
return CategoryImageUtils.getMediaList(childNodes);
}
/**
* This method takes search keyword as input and returns a list of categories objects filtered using search query
* It uses the generator query API to get the categories searched using a query, 25 at a time.
@ -924,25 +864,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
}
}
@Override
@NonNull
public Single<Integer> getUploadCount(String userName) {
final String uploadCountUrlTemplate =
wikiMediaToolforgeUrl + "urbanecmbot/commonsmisc/uploadsbyuser.py";
return Single.fromCallable(() -> {
String url = String.format(
Locale.ENGLISH,
uploadCountUrlTemplate,
new PageTitle(userName).getText());
HttpResponse response = Http.get(url).use(httpClient)
.data("user", userName)
.asResponse();
String uploadCount = EntityUtils.toString(response.getEntity()).trim();
return Integer.parseInt(uploadCount);
});
}
/**
* Checks to see if a user is currently blocked from Commons
@ -976,86 +897,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
return userBlocked;
}
/**
* This takes userName as input, which is then used to fetch the feedback/achievements
* statistics using OkHttp and JavaRx. This function return JSONObject
* @param userName MediaWiki user name
* @return
*/
@Override
public Single<FeedbackResponse> getAchievements(String userName) {
final String fetchAchievementUrlTemplate =
wikiMediaToolforgeUrl + "urbanecmbot/commonsmisc/feedback.py";
return Single.fromCallable(() -> {
String url = String.format(
Locale.ENGLISH,
fetchAchievementUrlTemplate,
new PageTitle(userName).getText());
HttpUrl.Builder urlBuilder = HttpUrl.parse(url).newBuilder();
urlBuilder.addQueryParameter("user", userName);
Timber.i("Url %s", urlBuilder.toString());
Request request = new Request.Builder()
.url(urlBuilder.toString())
.build();
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
Timber.d("Response for achievements is %s", json);
try {
return gson.fromJson(json, FeedbackResponse.class);
}
catch (Exception e){
return new FeedbackResponse("",0,0,0,new FeaturedImages(0,0),0,"",0);
}
}
return null;
});
}
/**
* The method returns the picture of the day
*
* @return Media object corresponding to the picture of the day
*/
@Override
@Nullable
public Single<Media> getPictureOfTheDay() {
return Single.fromCallable(() -> {
CustomApiResult apiResult = null;
try {
String template = "Template:Potd/" + DateUtils.getCurrentDate();
CustomMwApi.RequestBuilder requestBuilder = api.action("query")
.param("generator", "images")
.param("format", "xml")
.param("titles", template)
.param("prop", "imageinfo")
.param("iiprop", "url|extmetadata");
apiResult = requestBuilder.get();
} catch (IOException e) {
Timber.e(e, "Failed to obtain searchCategories");
}
if (apiResult == null) {
return null;
}
CustomApiResult imageNode = apiResult.getNode("/api/query/pages/page");
if (imageNode == null
|| imageNode.getDocument() == null) {
return null;
}
return CategoryImageUtils.getMediaFromPage(imageNode.getDocument());
});
}
private Date parseMWDate(String mwDate) {
SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ENGLISH); // Assuming MW always gives me UTC
isoFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@ -1076,19 +917,4 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi {
Timber.e(e, "Error occurred while logging out");
}
}
@Override public Single<CampaignResponseDTO> getCampaigns() {
return Single.fromCallable(() -> {
Request request = new Request.Builder().url(WIKIMEDIA_CAMPAIGNS_BASE_URL).build();
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
return gson.fromJson(json, CampaignResponseDTO.class);
}
return null;
});
}
}

View file

@ -11,6 +11,8 @@ import java.util.List;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.achievements.FeedbackResponse;
import fr.free.nrw.commons.location.LatLng;
import fr.free.nrw.commons.nearby.Place;
import fr.free.nrw.commons.notification.Notification;
import io.reactivex.Observable;
import io.reactivex.Single;
@ -48,9 +50,6 @@ public interface MediaWikiApi {
List<String> getParentCategoryList(String categoryName);
@NonNull
List<Media> searchImages(String title, int offset);
@NonNull
List<String> searchCategory(String title, int offset);
@ -98,19 +97,11 @@ public interface MediaWikiApi {
@NonNull
LogEventResult logEvents(String user, String lastModified, String queryContinue, int limit) throws IOException;
@NonNull
Single<Integer> getUploadCount(String userName);
boolean isUserBlockedFromCommons();
Single<FeedbackResponse> getAchievements(String userName);
Single<Media> getPictureOfTheDay();
void logout();
Single<CampaignResponseDTO> getCampaigns();
interface ProgressListener {
void onProgress(long transferred, long total);
}

View file

@ -0,0 +1,253 @@
package fr.free.nrw.commons.mwapi;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import com.google.gson.Gson;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import javax.inject.Inject;
import javax.inject.Singleton;
import fr.free.nrw.commons.Media;
import fr.free.nrw.commons.PageTitle;
import fr.free.nrw.commons.achievements.FeaturedImages;
import fr.free.nrw.commons.achievements.FeedbackResponse;
import fr.free.nrw.commons.campaigns.CampaignResponseDTO;
import fr.free.nrw.commons.location.LatLng;
import fr.free.nrw.commons.media.model.MwQueryPage;
import fr.free.nrw.commons.mwapi.model.MwQueryResponse;
import fr.free.nrw.commons.nearby.Place;
import fr.free.nrw.commons.nearby.model.NearbyResponse;
import fr.free.nrw.commons.nearby.model.NearbyResultItem;
import fr.free.nrw.commons.upload.FileUtils;
import fr.free.nrw.commons.utils.DateUtils;
import io.reactivex.Observable;
import io.reactivex.Single;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import timber.log.Timber;
@Singleton
public class OkHttpJsonApiClient {
private final OkHttpClient okHttpClient;
private final HttpUrl wikiMediaToolforgeUrl;
private final String sparqlQueryUrl;
private final String campaignsUrl;
private final String commonsBaseUrl;
private Gson gson;
@Inject
public OkHttpJsonApiClient(OkHttpClient okHttpClient,
HttpUrl wikiMediaToolforgeUrl,
String sparqlQueryUrl,
String campaignsUrl,
String commonsBaseUrl,
Gson gson) {
this.okHttpClient = okHttpClient;
this.wikiMediaToolforgeUrl = wikiMediaToolforgeUrl;
this.sparqlQueryUrl = sparqlQueryUrl;
this.campaignsUrl = campaignsUrl;
this.commonsBaseUrl = commonsBaseUrl;
this.gson = gson;
}
@NonNull
public Single<Integer> getUploadCount(String userName) {
HttpUrl.Builder urlBuilder = wikiMediaToolforgeUrl.newBuilder();
urlBuilder
.addPathSegments("urbanecmbot/commonsmisc/uploadsbyuser.py")
.addQueryParameter("user", userName);
Request request = new Request.Builder()
.url(urlBuilder.build())
.build();
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.isSuccessful()) {
return Integer.parseInt(response.body().string().trim());
}
return 0;
});
}
/**
* This takes userName as input, which is then used to fetch the feedback/achievements
* statistics using OkHttp and JavaRx. This function return JSONObject
*
* @param userName MediaWiki user name
* @return
*/
public Single<FeedbackResponse> getAchievements(String userName) {
final String fetchAchievementUrlTemplate =
wikiMediaToolforgeUrl + "urbanecmbot/commonsmisc/feedback.py";
return Single.fromCallable(() -> {
String url = String.format(
Locale.ENGLISH,
fetchAchievementUrlTemplate,
new PageTitle(userName).getText());
HttpUrl.Builder urlBuilder = HttpUrl.parse(url).newBuilder();
urlBuilder.addQueryParameter("user", userName);
Timber.i("Url %s", urlBuilder.toString());
Request request = new Request.Builder()
.url(urlBuilder.toString())
.build();
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
Timber.d("Response for achievements is %s", json);
try {
return gson.fromJson(json, FeedbackResponse.class);
} catch (Exception e) {
return new FeedbackResponse("", 0, 0, 0, new FeaturedImages(0, 0), 0, "", 0);
}
}
return null;
});
}
public Observable<List<Place>> getNearbyPlaces(LatLng cur, String lang, double radius) throws IOException {
String wikidataQuery = FileUtils.readFromResource("/queries/nearby_query.rq");
String query = wikidataQuery
.replace("${RAD}", String.format(Locale.ROOT, "%.2f", radius))
.replace("${LAT}", String.format(Locale.ROOT, "%.4f", cur.getLatitude()))
.replace("${LONG}", String.format(Locale.ROOT, "%.4f", cur.getLongitude()))
.replace("${LANG}", lang);
HttpUrl.Builder urlBuilder = HttpUrl
.parse(sparqlQueryUrl)
.newBuilder()
.addQueryParameter("query", query)
.addQueryParameter("format", "json");
Request request = new Request.Builder()
.url(urlBuilder.build())
.build();
return Observable.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return new ArrayList<>();
}
NearbyResponse nearbyResponse = gson.fromJson(json, NearbyResponse.class);
List<NearbyResultItem> bindings = nearbyResponse.getResults().getBindings();
List<Place> places = new ArrayList<>();
for (NearbyResultItem item : bindings) {
places.add(Place.from(item));
}
return places;
}
return new ArrayList<>();
});
}
public Single<CampaignResponseDTO> getCampaigns() {
return Single.fromCallable(() -> {
Request request = new Request.Builder().url(campaignsUrl)
.build();
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
return gson.fromJson(json, CampaignResponseDTO.class);
}
return null;
});
}
/**
* The method returns the picture of the day
*
* @return Media object corresponding to the picture of the day
*/
@Nullable
public Single<Media> getPictureOfTheDay() {
String template = "Template:Potd/" + DateUtils.getCurrentDate();
HttpUrl.Builder urlBuilder = HttpUrl
.parse(commonsBaseUrl)
.newBuilder()
.addQueryParameter("action", "query")
.addQueryParameter("generator", "images")
.addQueryParameter("format", "json")
.addQueryParameter("titles", template)
.addQueryParameter("prop", "imageinfo")
.addQueryParameter("iiprop", "url|extmetadata");
Request request = new Request.Builder()
.url(urlBuilder.build())
.build();
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return null;
}
MwQueryResponse mwQueryPage = gson.fromJson(json, MwQueryResponse.class);
return Media.from(mwQueryPage.query().firstPage());
}
return null;
});
}
/**
* This method takes search keyword as input and returns a list of Media objects filtered using search query
* It uses the generator query API to get the images searched using a query, 25 at a time.
* @param query keyword to search images on commons
* @return
*/
@Nullable
public Single<List<Media>> searchImages(String query, int offset) {
HttpUrl.Builder urlBuilder = HttpUrl
.parse(commonsBaseUrl)
.newBuilder()
.addQueryParameter("action", "query")
.addQueryParameter("generator", "search")
.addQueryParameter("format", "json")
.addQueryParameter("gsrwhat", "text")
.addQueryParameter("gsrnamespace", "6")
.addQueryParameter("gsrlimit", "25")
.addQueryParameter("gsroffset", String.valueOf(offset))
.addQueryParameter("gsrsearch", query)
.addQueryParameter("prop", "imageinfo")
.addQueryParameter("iiprop", "url|extmetadata");
Request request = new Request.Builder()
.url(urlBuilder.build())
.build();
return Single.fromCallable(() -> {
Response response = okHttpClient.newCall(request).execute();
List<Media> mediaList = new ArrayList<>();
if (response != null && response.body() != null && response.isSuccessful()) {
String json = response.body().string();
if (json == null) {
return mediaList;
}
MwQueryResponse mwQueryResponse = gson.fromJson(json, MwQueryResponse.class);
List<MwQueryPage> pages = mwQueryResponse.query().pages();
for (MwQueryPage page : pages) {
mediaList.add(Media.from(page));
}
}
return mediaList;
});
}
}

View file

@ -0,0 +1,30 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
public class MwException extends RuntimeException {
@SuppressWarnings("unused")
@NonNull
private final MwServiceError error;
public MwException(@NonNull MwServiceError error) {
this.error = error;
}
@NonNull
public MwServiceError getError() {
return error;
}
@Nullable
public String getTitle() {
return error.getTitle();
}
@Override
@Nullable
public String getMessage() {
return error.getDetails();
}
}

View file

@ -0,0 +1,39 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.Nullable;
import android.support.annotation.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
import java.util.Map;
public class MwQueryResponse extends MwResponse {
@SuppressWarnings("unused") @SerializedName("batchcomplete") private boolean batchComplete;
@SuppressWarnings("unused") @SerializedName("continue") @Nullable
private Map<String, String> continuation;
@Nullable private MwQueryResult query;
public boolean batchComplete() {
return batchComplete;
}
@Nullable public Map<String, String> continuation() {
return continuation;
}
@Nullable public MwQueryResult query() {
return query;
}
public boolean success() {
return query != null;
}
@VisibleForTesting
protected void setQuery(@Nullable MwQueryResult query) {
this.query = query;
}
}

View file

@ -0,0 +1,44 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import fr.free.nrw.commons.media.model.ImageInfo;
import fr.free.nrw.commons.media.model.MwQueryPage;
public class MwQueryResult {
@SuppressWarnings("unused")
@Nullable
private HashMap<String, MwQueryPage> pages;
@NonNull
public List<MwQueryPage> pages() {
if (pages == null) {
return new ArrayList<>();
}
return new ArrayList<>(pages.values());
}
@Nullable
public MwQueryPage firstPage() {
return pages().get(0);
}
@NonNull
public Map<String, ImageInfo> images() {
Map<String, ImageInfo> result = new HashMap<>();
if (pages != null) {
for (MwQueryPage page : pages()) {
if (page.imageInfo() != null) {
result.put(page.title(), page.imageInfo());
}
}
}
return result;
}
}

View file

@ -0,0 +1,38 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import com.google.gson.annotations.SerializedName;
import java.util.Map;
import fr.free.nrw.commons.json.PostProcessingTypeAdapter;
public abstract class MwResponse implements PostProcessingTypeAdapter.PostProcessable {
@SuppressWarnings("unused")
@Nullable
private MwServiceError error;
@SuppressWarnings("unused")
@Nullable
private Map<String, Warning> warnings;
@SuppressWarnings("unused,NullableProblems")
@SerializedName("servedby")
@NonNull
private String servedBy;
@Override
public void postProcess() {
if (error != null) {
throw new MwException(error);
}
}
private class Warning {
@SuppressWarnings("unused,NullableProblems")
@NonNull
private String warnings;
}
}

View file

@ -0,0 +1,85 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import fr.free.nrw.commons.utils.StringUtils;
/**
* Gson POJO for a MediaWiki API error.
*/
public class MwServiceError implements ServiceError {
@SuppressWarnings("unused")
@Nullable
private String code;
@SuppressWarnings("unused")
@Nullable
private String info;
@SuppressWarnings("unused")
@Nullable
private String docref;
@SuppressWarnings("unused")
@NonNull
private List<Message> messages = Collections.emptyList();
@Override
@NonNull
public String getTitle() {
return StringUtils.defaultString(code);
}
@Override
@NonNull
public String getDetails() {
return StringUtils.defaultString(info);
}
@Nullable
public String getDocRef() {
return docref;
}
public boolean badToken() {
return "badtoken".equals(code);
}
public boolean badLoginState() {
return "assertuserfailed".equals(code);
}
public boolean hasMessageName(@NonNull String messageName) {
for (Message msg : messages) {
if (messageName.equals(msg.name)) {
return true;
}
}
return false;
}
@Nullable
public String getMessageHtml(@NonNull String messageName) {
for (Message msg : messages) {
if (messageName.equals(msg.name)) {
return msg.html();
}
}
return null;
}
private static final class Message {
@SuppressWarnings("unused")
@Nullable
private String name;
@SuppressWarnings("unused")
@Nullable
private String html;
@NonNull
private String html() {
return StringUtils.defaultString(html);
}
}
}

View file

@ -0,0 +1,13 @@
package fr.free.nrw.commons.mwapi.model;
import android.support.annotation.NonNull;
/**
* The API reported an error in the payload.
*/
public interface ServiceError {
@NonNull
String getTitle();
@NonNull String getDetails();
}