Use JSON APIs for explore (#2731)

* Use JSON APIs for explore

* With tests

* Use JSON APIs for explore

* With tests

* BugFix #2731 (#4)

* Increased sdk version to 23

* with more robust tests

* Fix crashes and other reported issues

* Add javadocs

* Use common method for search and categories

* Add javadocs
This commit is contained in:
Vivek Maskara 2019-03-26 18:02:58 +05:30 committed by Ashish Kumar
parent c1a941eaae
commit c45b945526
14 changed files with 468 additions and 318 deletions

View file

@ -0,0 +1,219 @@
package fr.free.nrw.commons.mwapi
import com.google.gson.Gson
import fr.free.nrw.commons.BuildConfig
import fr.free.nrw.commons.TestCommonsApplication
import fr.free.nrw.commons.kvstore.JsonKvStore
import fr.free.nrw.commons.mwapi.OkHttpJsonApiClient.mapType
import junit.framework.Assert.assertEquals
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import okhttp3.mockwebserver.RecordedRequest
import org.junit.After
import org.junit.Assert
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.mockito.Mockito
import org.mockito.Mockito.`when`
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import java.net.URLDecoder
@RunWith(RobolectricTestRunner::class)
@Config(constants = BuildConfig::class, sdk = [23], application = TestCommonsApplication::class)
class OkHttpJsonApiClientTest {
private lateinit var testObject: OkHttpJsonApiClient
private lateinit var toolsForgeServer: MockWebServer
private lateinit var sparqlServer: MockWebServer
private lateinit var campaignsServer: MockWebServer
private lateinit var server: MockWebServer
private lateinit var sharedPreferences: JsonKvStore
private lateinit var okHttpClient: OkHttpClient
@Before
fun setUp() {
server = MockWebServer()
toolsForgeServer = MockWebServer()
sparqlServer = MockWebServer()
campaignsServer = MockWebServer()
okHttpClient = OkHttpClient.Builder().build()
sharedPreferences = Mockito.mock(JsonKvStore::class.java)
val toolsForgeUrl = "http://" + toolsForgeServer.hostName + ":" + toolsForgeServer.port + "/"
val sparqlUrl = "http://" + sparqlServer.hostName + ":" + sparqlServer.port + "/"
val campaignsUrl = "http://" + campaignsServer.hostName + ":" + campaignsServer.port + "/"
val serverUrl = "http://" + server.hostName + ":" + server.port + "/"
testObject = OkHttpJsonApiClient(okHttpClient, HttpUrl.get(toolsForgeUrl), sparqlUrl, campaignsUrl, serverUrl, sharedPreferences, Gson())
}
@After
fun teardown() {
server.shutdown()
}
@Test
fun getCategoryImages() {
server.enqueue(getFirstPageOfImages())
testFirstPageQuery()
}
@Test
fun getCategoryImagesWithContinue() {
server.enqueue(getFirstPageOfImages())
server.enqueue(getSecondPageOfImages())
testFirstPageQuery()
`when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
.thenReturn(hashMapOf(Pair("gcmcontinue", "testvalue"), Pair("continue", "gcmcontinue||")))
val categoryImagesContinued = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("categorymembers", body["generator"])
Assert.assertEquals("file", body["gcmtype"])
Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
Assert.assertEquals("timestamp", body["gcmsort"])
Assert.assertEquals("desc", body["gcmdir"])
Assert.assertEquals("testvalue", body["gcmcontinue"])
Assert.assertEquals("gcmcontinue||", body["continue"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImagesContinued.size, 2)
}
@Test
fun getSearchImages() {
server.enqueue(getFirstPageOfImages())
testFirstPageSearchQuery()
}
@Test
fun getSearchImagesWithContinue() {
server.enqueue(getFirstPageOfSearchImages())
server.enqueue(getSecondPageOfSearchImages())
testFirstPageSearchQuery()
`when`(sharedPreferences.getJson<HashMap<String, String>>("query_continue_Watercraft moored off shore", mapType))
.thenReturn(hashMapOf(Pair("gsroffset", "25"), Pair("continue", "gsroffset||")))
val categoryImagesContinued = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("search", body["generator"])
Assert.assertEquals("text", body["gsrwhat"])
Assert.assertEquals("6", body["gsrnamespace"])
Assert.assertEquals("25", body["gsrlimit"])
Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
Assert.assertEquals("25", body["gsroffset"])
Assert.assertEquals("gsroffset||", body["continue"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImagesContinued.size, 2)
}
private fun testFirstPageSearchQuery() {
val categoryImages = testObject.getMediaList("search", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("search", body["generator"])
Assert.assertEquals("text", body["gsrwhat"])
Assert.assertEquals("6", body["gsrnamespace"])
Assert.assertEquals("25", body["gsrlimit"])
Assert.assertEquals("Watercraft moored off shore", body["gsrsearch"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImages.size, 2)
}
private fun testFirstPageQuery() {
val categoryImages = testObject.getMediaList("category", "Watercraft moored off shore")!!.blockingGet()
assertBasicRequestParameters(server, "GET").let { request ->
parseQueryParams(request).let { body ->
Assert.assertEquals("json", body["format"])
Assert.assertEquals("query", body["action"])
Assert.assertEquals("categorymembers", body["generator"])
Assert.assertEquals("file", body["gcmtype"])
Assert.assertEquals("Watercraft moored off shore", body["gcmtitle"])
Assert.assertEquals("timestamp", body["gcmsort"])
Assert.assertEquals("desc", body["gcmdir"])
Assert.assertEquals("imageinfo", body["prop"])
Assert.assertEquals("url|extmetadata", body["iiprop"])
Assert.assertEquals("DateTime|Categories|GPSLatitude|GPSLongitude|ImageDescription|DateTimeOriginal|Artist|LicenseShortName", body["iiextmetadatafilter"])
}
}
assertEquals(categoryImages.size, 2)
}
private fun getFirstPageOfImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getSecondPageOfImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"gcmcontinue\":\"testvalue2\",\"continue\":\"gcmcontinue||\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getFirstPageOfSearchImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"25\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test1.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test1.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test1.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat1|cat2\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test2.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test2.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test2.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat3|cat4\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun getSecondPageOfSearchImages(): MockResponse {
val mockResponse = MockResponse()
mockResponse.setResponseCode(200)
mockResponse.setBody("{\"batchcomplete\":\"\",\"continue\":{\"continue\":\"gsroffset||\",\"gsroffset\":\"50\"},\"query\":{\"pages\":{\"4406048\":{\"pageid\":4406048,\"ns\":6,\"title\":\"File:test3.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test3.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test3.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat5|cat6\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]},\"24259710\":{\"pageid\":24259710,\"ns\":6,\"title\":\"File:test4.jpg\",\"imagerepository\":\"local\",\"imageinfo\":[{\"url\":\"https://upload.wikimedia.org/test4.jpg\",\"descriptionurl\":\"https://commons.wikimedia.org/wiki/File:test4.jpg\",\"descriptionshorturl\":\"https://commons.wikimedia.org/w/index.php?curid=4406048\",\"extmetadata\":{\"DateTime\":{\"value\":\"2013-04-13 15:12:11\",\"source\":\"mediawiki-metadata\",\"hidden\":\"\"},\"Categories\":{\"value\":\"cat7\",\"source\":\"commons-categories\",\"hidden\":\"\"},\"Artist\":{\"value\":\"<bdi><a href=\\\"https://en.wikipedia.org/wiki/en:Raphael\\\" class=\\\"extiw\\\" title=\\\"w:en:Raphael\\\">Raphael</a>\\n</bdi>\",\"source\":\"commons-desc-page\"},\"ImageDescription\":{\"value\":\"test desc\",\"source\":\"commons-desc-page\"},\"DateTimeOriginal\":{\"value\":\"1511<div style=\\\"display: none;\\\">date QS:P571,+1511-00-00T00:00:00Z/9</div>\",\"source\":\"commons-desc-page\"},\"LicenseShortName\":{\"value\":\"Public domain\",\"source\":\"commons-desc-page\",\"hidden\":\"\"}}}]}}}}")
return mockResponse
}
private fun assertBasicRequestParameters(server: MockWebServer, method: String): RecordedRequest = server.takeRequest().let {
Assert.assertEquals("/", it.requestUrl.encodedPath())
Assert.assertEquals(method, it.method)
return it
}
private fun parseQueryParams(request: RecordedRequest) = HashMap<String, String?>().apply {
request.requestUrl.let {
it.queryParameterNames().forEach { name -> put(name, it.queryParameter(name)) }
}
}
private fun parseBody(body: String): Map<String, String> = HashMap<String, String>().apply {
body.split("&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray().forEach { prop ->
val pair = prop.split("=".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
put(pair[0], URLDecoder.decode(pair[1], "utf-8"))
}
}
}

View file

@ -0,0 +1,40 @@
package fr.free.nrw.commons.utils;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.*;
public class MediaDataExtractorUtilTest {
@Test
public void extractCategoriesFromList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018|Watercraft|2018");
assertEquals(strings.size(), 3);
}
@Test
public void extractCategoriesFromEmptyList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("");
assertEquals(strings.size(), 0);
}
@Test
public void extractCategoriesFromNullList() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList(null);
assertEquals(strings.size(), 0);
}
@Test
public void extractCategoriesFromListWithEmptyValues() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018||");
assertEquals(strings.size(), 1);
}
@Test
public void extractCategoriesFromListWithWhitespaces() {
List<String> strings = MediaDataExtractorUtil.extractCategoriesFromList("Watercraft 2018| | ||");
assertEquals(strings.size(), 1);
}
}