From e50adf858a7cd20a35cc1205db115c03d310b122 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sun, 18 Feb 2018 15:01:42 +0530 Subject: [PATCH] Detecting pictures that are too dark (#926) * Made the following changes: ->Added OpenCV library to the project ->Added functionality to detect if an image being uploaded is too dark ->Added functionality to detect if an image being uploaded is blurred * Made corrections and changes based on gradle checkstyle requirements * Updated gitignore to remove binary files related to OpenCV from project * Image blurriness detection was undone. Images are checked only for being too dark now * Removed OpenCV documentation folder containing a lot of html files * Removed unnecessary buildScript usage in build.gradle file for opencv library and also added abi splits * Removed OpenCV library usages and references from project * Removed OpenCV library folder from project --- .gitignore | 7 + app/build.gradle | 22 +++ .../upload/DetectUnwantedPicturesAsync.java | 59 ++++++++ .../fr/free/nrw/commons/upload/FileUtils.java | 23 +-- .../fr/free/nrw/commons/utils/ImageUtils.java | 135 ++++++++++++++++++ 5 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 app/src/main/java/fr/free/nrw/commons/upload/DetectUnwantedPicturesAsync.java create mode 100644 app/src/main/java/fr/free/nrw/commons/utils/ImageUtils.java diff --git a/.gitignore b/.gitignore index 1ab05305e..5514c2b09 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,10 @@ app/gradle/wrapper/gradle-wrapper.jar app/gradlew app/gradlew.bat app/gradle/wrapper/gradle-wrapper.properties + +#related to OpenCV +/libraries/opencv/build +app/src/main/jniLibs +#Below removes all the HTML files related to OpenCV documentation. The documentation can be otherwise found at: +#https://docs.opencv.org/3.3.0/ +/libraries/opencv/javadoc/ diff --git a/app/build.gradle b/app/build.gradle index 98693126e..480916ee8 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -38,6 +38,26 @@ dependencies { implementation 'io.reactivex.rxjava2:rxandroid:2.0.1' // Because RxAndroid releases are few and far between, it is recommended you also // explicitly depend on RxJava's latest version for bug fixes and new features. + compile 'io.reactivex.rxjava2:rxjava:2.1.2' + compile 'com.jakewharton.rxbinding2:rxbinding:2.0.0' + compile 'com.jakewharton.rxbinding2:rxbinding-support-v4:2.0.0' + compile 'com.jakewharton.rxbinding2:rxbinding-appcompat-v7:2.0.0' + compile 'com.jakewharton.rxbinding2:rxbinding-design:2.0.0' + + compile 'com.facebook.fresco:fresco:1.3.0' + compile 'com.facebook.stetho:stetho:1.5.0' + + testCompile 'junit:junit:4.12' + testCompile 'org.robolectric:robolectric:3.4' + + testCompile 'com.squareup.okhttp3:mockwebserver:3.8.1' + androidTestCompile 'com.squareup.okhttp3:mockwebserver:3.8.1' + androidTestCompile "com.android.support:support-annotations:${project.SUPPORT_LIB_VERSION}" + androidTestCompile 'com.android.support.test.espresso:espresso-core:3.0.1' + + debugCompile 'com.squareup.leakcanary:leakcanary-android:1.5.1' + releaseCompile 'com.squareup.leakcanary:leakcanary-android-no-op:1.5.1' + testCompile 'com.squareup.leakcanary:leakcanary-android-no-op:1.5.1' implementation 'io.reactivex.rxjava2:rxjava:2.1.2' implementation 'com.jakewharton.rxbinding2:rxbinding:2.0.0' implementation 'com.jakewharton.rxbinding2:rxbinding-support-v4:2.0.0' @@ -91,6 +111,8 @@ android { targetSdkVersion project.targetSdkVersion testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" vectorDrawables.useSupportLibrary = true + + multiDexEnabled true } sourceSets { diff --git a/app/src/main/java/fr/free/nrw/commons/upload/DetectUnwantedPicturesAsync.java b/app/src/main/java/fr/free/nrw/commons/upload/DetectUnwantedPicturesAsync.java new file mode 100644 index 000000000..b383601ec --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/upload/DetectUnwantedPicturesAsync.java @@ -0,0 +1,59 @@ +package fr.free.nrw.commons.upload; + +import android.content.Context; +import android.graphics.BitmapRegionDecoder; +import android.net.Uri; +import android.os.AsyncTask; + +import java.io.IOException; + +import fr.free.nrw.commons.utils.ImageUtils; +import timber.log.Timber; + +/** + * Created by bluesir9 on 16/9/17. + * + *

Responsible for checking if the picture that the user is trying to upload is useful or not. Will attempt to filter + * away completely black,fuzzy/blurry pictures(for now). + * + *

todo: Detect selfies? + */ + +public class DetectUnwantedPicturesAsync extends AsyncTask { + + interface Callback { + void onResult(ImageUtils.Result result); + } + + private final Callback callback; + private final String imageMediaFilePath; + + DetectUnwantedPicturesAsync(String imageMediaFilePath, Callback callback) { + this.callback = callback; + this.imageMediaFilePath = imageMediaFilePath; + } + + @Override + protected ImageUtils.Result doInBackground(Void... voids) { + try { + Timber.d("FilePath: " + imageMediaFilePath); + if (imageMediaFilePath == null) { + return ImageUtils.Result.IMAGE_OK; + } + + BitmapRegionDecoder decoder = BitmapRegionDecoder.newInstance(imageMediaFilePath,false); + + return ImageUtils.checkIfImageIsTooDark(decoder); + } catch (IOException ioe) { + Timber.e(ioe, "IO Exception"); + return ImageUtils.Result.IMAGE_OK; + } + } + + @Override + protected void onPostExecute(ImageUtils.Result result) { + super.onPostExecute(result); + //callback to UI so that it can take necessary decision based on the result obtained + callback.onResult(result); + } +} diff --git a/app/src/main/java/fr/free/nrw/commons/upload/FileUtils.java b/app/src/main/java/fr/free/nrw/commons/upload/FileUtils.java index 2654a95cc..016ae8cc0 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/FileUtils.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/FileUtils.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.util.Date; -import fr.free.nrw.commons.CommonsApplication; import timber.log.Timber; public class FileUtils { @@ -42,6 +41,7 @@ public class FileUtils { @Nullable public static String getPath(Context context, Uri uri) { + String returnPath = null; final boolean isKitKat = Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT; // DocumentProvider @@ -53,7 +53,7 @@ public class FileUtils { final String type = split[0]; if ("primary".equalsIgnoreCase(type)) { - return Environment.getExternalStorageDirectory() + "/" + split[1]; + returnPath = Environment.getExternalStorageDirectory() + "/" + split[1]; } } else if (isDownloadsDocument(uri)) { // DownloadsProvider @@ -61,8 +61,9 @@ public class FileUtils { final Uri contentUri = ContentUris.withAppendedId( Uri.parse("content://downloads/public_downloads"), Long.valueOf(id)); - return getDataColumn(context, contentUri, null, null); + returnPath = getDataColumn(context, contentUri, null, null); } else if (isMediaDocument(uri)) { // MediaProvider + final String docId = DocumentsContract.getDocumentId(uri); final String[] split = docId.split(":"); final String type = split[0]; @@ -87,17 +88,19 @@ public class FileUtils { split[1] }; - return getDataColumn(context, contentUri, selection, selectionArgs); + returnPath = getDataColumn(context, contentUri, selection, selectionArgs); } } // MediaStore (and general) else if ("content".equalsIgnoreCase(uri.getScheme())) { - return getDataColumn(context, uri, null, null); + returnPath = getDataColumn(context, uri, null, null); } // File else if ("file".equalsIgnoreCase(uri.getScheme())) { - return uri.getPath(); - } else { + returnPath = uri.getPath(); + } + + if(returnPath == null) { //fetching path may fail depending on the source URI and all hope is lost //so we will create and use a copy of the file, which seems to work String copyPath = null; @@ -105,6 +108,7 @@ public class FileUtils { ParcelFileDescriptor descriptor = context.getContentResolver().openFileDescriptor(uri, "r"); if (descriptor != null) { + SharedPreferences sharedPref = PreferenceManager .getDefaultSharedPreferences(CommonsApplication.getInstance()); boolean useExtStorage = sharedPref.getBoolean("useExternalStorage", true); @@ -131,7 +135,10 @@ public class FileUtils { Timber.w(e, "Error in file " + copyPath); return null; } + } else { + return returnPath; } + return null; } @@ -150,7 +157,7 @@ public class FileUtils { String[] selectionArgs) { Cursor cursor = null; - final String column = "_data"; + final String column = MediaStore.Images.ImageColumns.DATA; final String[] projection = { column }; diff --git a/app/src/main/java/fr/free/nrw/commons/utils/ImageUtils.java b/app/src/main/java/fr/free/nrw/commons/utils/ImageUtils.java new file mode 100644 index 000000000..6627f2886 --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/utils/ImageUtils.java @@ -0,0 +1,135 @@ +package fr.free.nrw.commons.utils; + +import android.graphics.Bitmap; +import android.graphics.BitmapRegionDecoder; +import android.graphics.Color; +import android.graphics.Rect; + +import timber.log.Timber; + +/** + * Created by bluesir9 on 3/10/17. + */ + +public class ImageUtils { + //atleast 50% of the image in question should be considered dark for the entire image to be dark + private static final double MINIMUM_DARKNESS_FACTOR = 0.50; + //atleast 50% of the image in question should be considered blurry for the entire image to be blurry + private static final double MINIMUM_BLURRYNESS_FACTOR = 0.50; + private static final int LAPLACIAN_VARIANCE_THRESHOLD = 70; + + public enum Result { + IMAGE_DARK, + IMAGE_OK + } + + /** + * BitmapRegionDecoder allows us to process a large bitmap by breaking it down into smaller rectangles. The rectangles + * are obtained by setting an initial width, height and start position of the rectangle as a factor of the width and + * height of the original bitmap and then manipulating the width, height and position to loop over the entire original + * bitmap. Each individual rectangle is independently processed to check if its too dark. Based on + * the factor of "bright enough" individual rectangles amongst the total rectangles into which the image + * was divided, we will declare the image as wanted/unwanted + * + * @param bitmapRegionDecoder BitmapRegionDecoder for the image we wish to process + * @return Result.IMAGE_OK if image is neither dark nor blurry or if the input bitmapRegionDecoder provided is null + * Result.IMAGE_DARK if image is too dark + */ + public static Result checkIfImageIsTooDark(BitmapRegionDecoder bitmapRegionDecoder) { + if (bitmapRegionDecoder == null) { + Timber.e("Expected bitmapRegionDecoder was null"); + return Result.IMAGE_OK; + } + + int loadImageHeight = bitmapRegionDecoder.getHeight(); + int loadImageWidth = bitmapRegionDecoder.getWidth(); + + int checkImageTopPosition = 0; + int checkImageBottomPosition = loadImageHeight / 10; + int checkImageLeftPosition = 0; + int checkImageRightPosition = loadImageWidth / 10; + + int totalDividedRectangles = 0; + int numberOfDarkRectangles = 0; + + while ((checkImageRightPosition <= loadImageWidth) && (checkImageLeftPosition < checkImageRightPosition)) { + while ((checkImageBottomPosition <= loadImageHeight) && (checkImageTopPosition < checkImageBottomPosition)) { + Timber.d("left: " + checkImageLeftPosition + " right: " + checkImageRightPosition + " top: " + checkImageTopPosition + " bottom: " + checkImageBottomPosition); + + Rect rect = new Rect(checkImageLeftPosition,checkImageTopPosition,checkImageRightPosition,checkImageBottomPosition); + totalDividedRectangles++; + + Bitmap processBitmap = bitmapRegionDecoder.decodeRegion(rect,null); + + if (checkIfImageIsDark(processBitmap)) { + numberOfDarkRectangles++; + } + + checkImageTopPosition = checkImageBottomPosition; + checkImageBottomPosition += (checkImageBottomPosition < (loadImageHeight - checkImageBottomPosition)) ? checkImageBottomPosition : (loadImageHeight - checkImageBottomPosition); + } + + checkImageTopPosition = 0; //reset to start + checkImageBottomPosition = loadImageHeight / 10; //reset to start + checkImageLeftPosition = checkImageRightPosition; + checkImageRightPosition += (checkImageRightPosition < (loadImageWidth - checkImageRightPosition)) ? checkImageRightPosition : (loadImageWidth - checkImageRightPosition); + } + + Timber.d("dark rectangles count = " + numberOfDarkRectangles + ", total rectangles count = " + totalDividedRectangles); + + if (numberOfDarkRectangles > totalDividedRectangles * MINIMUM_DARKNESS_FACTOR) { + return Result.IMAGE_DARK; + } + + return Result.IMAGE_OK; + } + + /** + * Pulls the pixels into an array and then runs through it while checking the brightness of each pixel. + * The calculation of brightness of each pixel is done by extracting the RGB constituents of the pixel + * and then applying the formula to calculate its "Luminance". If this brightness value is less than + * 50 then the pixel is considered to be dark. Based on the MINIMUM_DARKNESS_FACTOR if enough pixels + * are dark then the entire bitmap is considered to be dark. + * + *

For more information on this brightness/darkness calculation technique refer the accepted answer + * on this -> https://stackoverflow.com/questions/35914461/how-to-detect-dark-photos-in-android/35914745 + * SO question and follow the trail. + * + * @param bitmap The bitmap that needs to be checked. + * @return true if bitmap is dark or null, false if bitmap is bright + */ + private static boolean checkIfImageIsDark(Bitmap bitmap) { + if (bitmap == null) { + Timber.e("Expected bitmap was null"); + return true; + } + + int bitmapWidth = bitmap.getWidth(); + int bitmapHeight = bitmap.getHeight(); + + int allPixelsCount = bitmapWidth * bitmapHeight; + int[] bitmapPixels = new int[allPixelsCount]; + + bitmap.getPixels(bitmapPixels,0,bitmapWidth,0,0,bitmapWidth,bitmapHeight); + boolean isImageDark = false; + int darkPixelsCount = 0; + + for (int pixel : bitmapPixels) { + int r = Color.red(pixel); + int g = Color.green(pixel); + int b = Color.blue(pixel); + + int brightness = (int) (0.2126 * r + 0.7152 * g + 0.0722 * b); + if (brightness < 50) { + //pixel is dark + darkPixelsCount++; + if (darkPixelsCount > allPixelsCount * MINIMUM_DARKNESS_FACTOR) { + isImageDark = true; + break; + } + } + } + + return isImageDark; + } +}