diff --git a/app/src/main/java/fr/free/nrw/commons/MediaDataExtractor.java b/app/src/main/java/fr/free/nrw/commons/MediaDataExtractor.java index c2e301826..d07bd4bfb 100644 --- a/app/src/main/java/fr/free/nrw/commons/MediaDataExtractor.java +++ b/app/src/main/java/fr/free/nrw/commons/MediaDataExtractor.java @@ -66,8 +66,7 @@ public class MediaDataExtractor { * @return */ private Single getDiscussion(String filename) { - return mediaWikiApi.fetchMediaByFilename(filename.replace("File", "File talk")) - .flatMap(mediaResult -> mediaWikiApi.parseWikicode(mediaResult.getWikiSource())) + return mediaClient.getPageHtml(filename.replace("File", "File talk")) .map(discussion -> HtmlCompat.fromHtml(discussion, HtmlCompat.FROM_HTML_MODE_LEGACY).toString()) .onErrorReturn(throwable -> { Timber.e(throwable, "Error occurred while fetching discussion"); diff --git a/app/src/main/java/fr/free/nrw/commons/media/MediaClient.java b/app/src/main/java/fr/free/nrw/commons/media/MediaClient.java index 58e014d42..ca0d83270 100644 --- a/app/src/main/java/fr/free/nrw/commons/media/MediaClient.java +++ b/app/src/main/java/fr/free/nrw/commons/media/MediaClient.java @@ -161,4 +161,13 @@ public class MediaClient { .map(Media::from) .single(Media.EMPTY); } + + @NonNull + public Single getPageHtml(String title){ + return mediaInterface.getPageHtml(title) + .filter(MwParseResponse::success) + .map(MwParseResponse::parse) + .map(MwParseResult::text) + .first(""); + } } diff --git a/app/src/main/java/fr/free/nrw/commons/media/MediaInterface.java b/app/src/main/java/fr/free/nrw/commons/media/MediaInterface.java index c54e33e62..68d311d9c 100644 --- a/app/src/main/java/fr/free/nrw/commons/media/MediaInterface.java +++ b/app/src/main/java/fr/free/nrw/commons/media/MediaInterface.java @@ -14,6 +14,9 @@ import retrofit2.http.QueryMap; * Interface for interacting with Commons media related APIs */ public interface MediaInterface { + String MEDIA_PARAMS="&prop=imageinfo&iiprop=url|extmetadata&iiurlwidth=640" + + "&iiextmetadatafilter=DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal" + + "|Artist|LicenseShortName|LicenseUrl"; /** * Checks if a page exists or not. * @@ -42,9 +45,7 @@ public interface MediaInterface { */ @GET("w/api.php?action=query&format=json&formatversion=2" + //Basic parameters "&generator=categorymembers&gcmtype=file&gcmsort=timestamp&gcmdir=desc" + //Category parameters - "&prop=imageinfo&iiprop=url|extmetadata&iiurlwidth=640" + //Media property parameters - "&iiextmetadatafilter=DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal" + - "|Artist|LicenseShortName|LicenseUrl") + MEDIA_PARAMS) Observable getMediaListFromCategory(@Query("gcmtitle") String category, @Query("gcmlimit") int itemLimit, @QueryMap Map continuation); /** @@ -57,9 +58,7 @@ public interface MediaInterface { */ @GET("w/api.php?action=query&format=json&formatversion=2" + //Basic parameters "&generator=search&gsrwhat=text&gsrnamespace=6" + //Search parameters - "&prop=imageinfo&iiprop=url|extmetadata&iiurlwidth=640" + //Media property parameters - "&iiextmetadatafilter=DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal" + - "|Artist|LicenseShortName|LicenseUrl") + MEDIA_PARAMS) Observable getMediaListFromSearch(@Query("gsrsearch") String keyword, @Query("gsrlimit") int itemLimit, @QueryMap Map continuation); /** @@ -69,9 +68,7 @@ public interface MediaInterface { * @return */ @GET("w/api.php?action=query&format=json&formatversion=2" + - "&prop=imageinfo&iiprop=url|extmetadata&iiurlwidth=640" + - "&iiextmetadatafilter=DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal" + - "|Artist|LicenseShortName|LicenseUrl") + MEDIA_PARAMS) Observable getMedia(@Query("titles") String title); /** @@ -82,8 +79,9 @@ public interface MediaInterface { * @return */ @GET("w/api.php?action=query&format=json&formatversion=2&generator=images" + - "&prop=imageinfo&iiprop=url|extmetadata&iiurlwidth=640" + - "&iiextmetadatafilter=DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal" + - "|Artist|LicenseShortName|LicenseUrl") + MEDIA_PARAMS) Observable getMediaWithGenerator(@Query("titles") String title); + + @GET("w/api.php?format=json&action=parse&prop=text") + Observable getPageHtml(@Query("page") String title); } diff --git a/app/src/main/java/fr/free/nrw/commons/media/MwParseResponse.java b/app/src/main/java/fr/free/nrw/commons/media/MwParseResponse.java new file mode 100644 index 000000000..c65874532 --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/media/MwParseResponse.java @@ -0,0 +1,25 @@ +package fr.free.nrw.commons.media; + +import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import org.wikipedia.dataclient.mwapi.MwResponse; + +public class MwParseResponse extends MwResponse { + @Nullable + private MwParseResult parse; + + @Nullable + public MwParseResult parse() { + return parse; + } + + public boolean success() { + return parse != null; + } + + @VisibleForTesting + protected void setParse(@Nullable MwParseResult parse) { + this.parse = parse; + } +} diff --git a/app/src/main/java/fr/free/nrw/commons/media/MwParseResult.java b/app/src/main/java/fr/free/nrw/commons/media/MwParseResult.java new file mode 100644 index 000000000..961936935 --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/media/MwParseResult.java @@ -0,0 +1,20 @@ +package fr.free.nrw.commons.media; + +import androidx.annotation.Nullable; + +import com.google.gson.annotations.SerializedName; + +public class MwParseResult { + @SuppressWarnings("unused") private int pageid; + @SuppressWarnings("unused") private int index; + private MwParseText text; + + public String text() { + return text.text; + } + + + public class MwParseText{ + @SerializedName("*") private String text; + } +} diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java b/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java index ea92214cc..d7cd7de46 100644 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java +++ b/app/src/main/java/fr/free/nrw/commons/mwapi/ApacheHttpClientMediaWikiApi.java @@ -4,7 +4,6 @@ import android.text.TextUtils; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import com.google.gson.Gson; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.scheme.PlainSocketFactory; @@ -26,10 +25,6 @@ import java.util.Date; import fr.free.nrw.commons.BuildConfig; import fr.free.nrw.commons.CommonsApplication; -import fr.free.nrw.commons.BuildConfig; -import fr.free.nrw.commons.CommonsApplication; - -import io.reactivex.Single; import timber.log.Timber; /** @@ -54,35 +49,6 @@ public class ApacheHttpClientMediaWikiApi implements MediaWikiApi { api = new CustomMwApi(apiURL, httpClient); } - @Override - public Single parseWikicode(String source) { - return Single.fromCallable(() -> api.action("flow-parsoid-utils") - .param("from", "wikitext") - .param("to", "html") - .param("content", source) - .param("title", "Main_page") - .get() - .getString("/api/flow-parsoid-utils/@content")); - } - - @Override - @NonNull - public Single fetchMediaByFilename(String filename) { - return Single.fromCallable(() -> { - CustomApiResult apiResult = api.action("query") - .param("prop", "revisions") - .param("titles", filename) - .param("rvprop", "content") - .param("rvlimit", 1) - .param("rvgeneratexml", 1) - .get(); - - return new MediaResult( - apiResult.getString("/api/query/pages/page/revisions/rev"), - apiResult.getString("/api/query/pages/page/revisions/rev/@parsetree")); - }); - } - /** * Checks to see if a user is currently blocked from Commons * diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaResult.java b/app/src/main/java/fr/free/nrw/commons/mwapi/MediaResult.java deleted file mode 100644 index f2f34ce6d..000000000 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaResult.java +++ /dev/null @@ -1,33 +0,0 @@ -package fr.free.nrw.commons.mwapi; - -public class MediaResult { - private final String wikiSource; - private final String parseTreeXmlSource; - - /** - * Full-fledged constructor of MediaResult - * - * @param wikiSource Media wiki source - * @param parseTreeXmlSource Media tree parsed in XML - */ - MediaResult(String wikiSource, String parseTreeXmlSource) { - this.wikiSource = wikiSource; - this.parseTreeXmlSource = parseTreeXmlSource; - } - - /** - * Gets wiki source - * @return Wiki source - */ - public String getWikiSource() { - return wikiSource; - } - - /** - * Gets tree parsed in XML - * @return XML parsed tree - */ - public String getParseTreeXmlSource() { - return parseTreeXmlSource; - } -} diff --git a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java b/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java index 0603581c2..2d2960a0a 100644 --- a/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java +++ b/app/src/main/java/fr/free/nrw/commons/mwapi/MediaWikiApi.java @@ -9,11 +9,6 @@ import io.reactivex.Single; public interface MediaWikiApi { - Single parseWikicode(String source); - - @NonNull - Single fetchMediaByFilename(String filename); - boolean isUserBlockedFromCommons(); void logout(); diff --git a/app/src/main/java/fr/free/nrw/commons/review/ReviewHelper.java b/app/src/main/java/fr/free/nrw/commons/review/ReviewHelper.java index 45880a160..2b44fbdcf 100644 --- a/app/src/main/java/fr/free/nrw/commons/review/ReviewHelper.java +++ b/app/src/main/java/fr/free/nrw/commons/review/ReviewHelper.java @@ -35,7 +35,7 @@ public class ReviewHelper { } /** - * Fetches recent changes from MediaWiki AP + * Fetches recent changes from MediaWiki API * Calls the API to get 10 changes in the last 1 hour * Earlier we were getting changes for the last 30 days but as the API returns just 10 results * its best to fetch for just last 1 hour. diff --git a/app/src/test/kotlin/fr/free/nrw/commons/MediaDataExtractorTest.kt b/app/src/test/kotlin/fr/free/nrw/commons/MediaDataExtractorTest.kt index d66dc4e33..c40ce6f11 100644 --- a/app/src/test/kotlin/fr/free/nrw/commons/MediaDataExtractorTest.kt +++ b/app/src/test/kotlin/fr/free/nrw/commons/MediaDataExtractorTest.kt @@ -1,9 +1,7 @@ package fr.free.nrw.commons import fr.free.nrw.commons.media.MediaClient -import fr.free.nrw.commons.mwapi.MediaResult import fr.free.nrw.commons.mwapi.MediaWikiApi -import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient import io.reactivex.Single import org.junit.Assert.assertTrue import org.junit.Before @@ -49,13 +47,8 @@ class MediaDataExtractorTest { `when`(mediaClient?.checkPageExistsUsingTitle(ArgumentMatchers.anyString())) .thenReturn(Single.just(true)) - val mediaResult = mock(MediaResult::class.java) - `when`(mediaResult.wikiSource).thenReturn("some wiki source") - `when`(mwApi?.fetchMediaByFilename(ArgumentMatchers.anyString())) - .thenReturn(Single.just(mediaResult)) - - `when`(mwApi?.parseWikicode(ArgumentMatchers.anyString())) - .thenReturn(Single.just("discussion text")) + `when`(mediaClient?.getPageHtml(ArgumentMatchers.anyString())) + .thenReturn(Single.just("Test")) val fetchMediaDetails = mediaDataExtractor?.fetchMediaDetails("test.jpg")?.blockingGet() diff --git a/app/src/test/kotlin/fr/free/nrw/commons/media/MediaClientTest.kt b/app/src/test/kotlin/fr/free/nrw/commons/media/MediaClientTest.kt index 7403ae99a..5579ce72f 100644 --- a/app/src/test/kotlin/fr/free/nrw/commons/media/MediaClientTest.kt +++ b/app/src/test/kotlin/fr/free/nrw/commons/media/MediaClientTest.kt @@ -196,4 +196,30 @@ class MediaClientTest { assertEquals(media1.filename, "Test") assertEquals(media2.filename, "Test") } + + @Test + fun getPageHtmlTest() { + val mwParseResult = mock(MwParseResult::class.java) + + `when`(mwParseResult.text()).thenReturn("Test") + + val mockResponse = MwParseResponse() + mockResponse.setParse(mwParseResult) + + `when`(mediaInterface!!.getPageHtml(ArgumentMatchers.anyString())) + .thenReturn(Observable.just(mockResponse)) + + assertEquals("Test", mediaClient!!.getPageHtml("abcde").blockingGet()) + } + + @Test + fun getPageHtmlTestNull() { + val mockResponse = MwParseResponse() + mockResponse.setParse(null) + + `when`(mediaInterface!!.getPageHtml(ArgumentMatchers.anyString())) + .thenReturn(Observable.just(mockResponse)) + + assertEquals("", mediaClient!!.getPageHtml("abcde").blockingGet()) + } } \ No newline at end of file