From a3dcf41786e7e97cb325f433c189ceffdc256425 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Thu, 25 Apr 2024 00:37:36 +0200 Subject: [PATCH] improved memory usage and check for video thumbnail decoding --- .../thibault/aves/decoder/SvgGlideModule.kt | 2 +- .../thibault/aves/decoder/TiffGlideModule.kt | 2 +- .../aves/decoder/VideoThumbnailGlideModule.kt | 85 +++++++++++++++++-- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/SvgGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/SvgGlideModule.kt index bf4e3b6d0..0a8cef472 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/SvgGlideModule.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/SvgGlideModule.kt @@ -61,7 +61,7 @@ internal class SvgFetcher(val model: SvgImage, val width: Int, val height: Int) val bitmapWidth: Int val bitmapHeight: Int - if (width / height > svgWidth / svgHeight) { + if (width / height.toFloat() > svgWidth / svgHeight) { bitmapWidth = ceil(svgWidth * height / svgHeight).toInt() bitmapHeight = height } else { diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/TiffGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/TiffGlideModule.kt index c96631bf9..0671237a8 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/TiffGlideModule.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/TiffGlideModule.kt @@ -63,7 +63,7 @@ internal class TiffFetcher(val model: TiffImage, val width: Int, val height: Int TiffBitmapFactory.decodeFileDescriptor(fd, options) val imageWidth = options.outWidth val imageHeight = options.outHeight - if (imageHeight > height || imageWidth > width) { + if (imageWidth > width || imageHeight > height) { while (imageHeight / (sampleSize * 2) >= height && imageWidth / (sampleSize * 2) >= width) { sampleSize *= 2 } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt index f13a85f8e..ddb1e7096 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt @@ -1,8 +1,11 @@ package deckers.thibault.aves.decoder import android.content.Context +import android.graphics.Bitmap import android.media.MediaMetadataRetriever import android.net.Uri +import android.os.Build +import androidx.annotation.RequiresApi import com.bumptech.glide.Glide import com.bumptech.glide.Priority import com.bumptech.glide.Registry @@ -16,7 +19,9 @@ import com.bumptech.glide.load.model.ModelLoaderFactory import com.bumptech.glide.load.model.MultiModelLoaderFactory import com.bumptech.glide.module.LibraryGlideModule import com.bumptech.glide.signature.ObjectKey +import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -24,6 +29,8 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.launch import java.io.ByteArrayInputStream import java.io.InputStream +import kotlin.math.ceil +import kotlin.math.roundToInt @GlideModule class VideoThumbnailGlideModule : LibraryGlideModule() { @@ -36,7 +43,7 @@ class VideoThumbnail(val context: Context, val uri: Uri) internal class VideoThumbnailLoader : ModelLoader { override fun buildLoadData(model: VideoThumbnail, width: Int, height: Int, options: Options): ModelLoader.LoadData { - return ModelLoader.LoadData(ObjectKey(model.uri), VideoThumbnailFetcher(model)) + return ModelLoader.LoadData(ObjectKey(model.uri), VideoThumbnailFetcher(model, width, height)) } override fun handles(model: VideoThumbnail): Boolean = true @@ -48,7 +55,7 @@ internal class VideoThumbnailLoader : ModelLoader { } } -internal class VideoThumbnailFetcher(private val model: VideoThumbnail) : DataFetcher { +internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val width: Int, val height: Int) : DataFetcher { private val ioScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) override fun loadData(priority: Priority, callback: DataCallback) { @@ -68,10 +75,62 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail) : DataFe if (durationMillis != null) { timeMillis = if (durationMillis < 15000) 0 else 15000 } - val frame = if (timeMillis != null) { - retriever.getFrameAtTime(timeMillis * 1000) + val timeMicros = if (timeMillis != null) timeMillis * 1000 else -1 + val option = MediaMetadataRetriever.OPTION_CLOSEST_SYNC + + var videoWidth = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toFloatOrNull() + var videoHeight = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toFloatOrNull() + if (videoWidth == null || videoHeight == null) { + throw Exception("failed to get video dimensions") + } + + var dstWidth = 0 + var dstHeight = 0 + if (width > 0 && height > 0) { + val rotationDegrees = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)?.toIntOrNull() + if (rotationDegrees != null) { + val isRotated = rotationDegrees % 180 == 90 + if (isRotated) { + val temp = videoWidth + videoWidth = videoHeight + videoHeight = temp + } + + // cover fit + val videoAspectRatio = videoWidth / videoHeight + if (videoWidth > width || videoHeight > height) { + if (width / height.toFloat() > videoAspectRatio) { + dstHeight = ceil(videoHeight * width / videoWidth).toInt() + dstWidth = (dstHeight * videoAspectRatio).roundToInt() + } else { + dstWidth = ceil(videoWidth * height / videoHeight).toInt() + dstHeight = (dstWidth / videoAspectRatio).roundToInt() + } + } + } + } + + // the returned frame is already rotated according to the video metadata + val frame = if (dstWidth > 0 && dstHeight > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + val targetBitmapSizeBytes: Long = FORMAT_BYTE_SIZE.toLong() * dstWidth * dstHeight + if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { + throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the scaled frame at $dstWidth x $dstHeight") + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight, getBitmapParams()) + } else { + retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight) + } } else { - retriever.frameAtTime + val targetBitmapSizeBytes: Long = (FORMAT_BYTE_SIZE.toLong() * videoWidth * videoHeight).toLong() + if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { + throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the full frame at $videoWidth x $videoHeight") + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + retriever.getFrameAtTime(timeMicros, option, getBitmapParams()) + } else { + retriever.getFrameAtTime(timeMicros, option) + } } bytes = frame?.getBytes(canHaveAlpha = false, recycle = false) } @@ -91,6 +150,17 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail) : DataFe } } + @RequiresApi(Build.VERSION_CODES.P) + private fun getBitmapParams() = MediaMetadataRetriever.BitmapParams().apply { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + // improved precision with the same memory cost as `ARGB_8888` (4 bytes per pixel) + // for wide-gamut and HDR content which does not require alpha blending + setPreferredConfig(Bitmap.Config.RGBA_1010102) + } else { + setPreferredConfig(Bitmap.Config.ARGB_8888) + } + } + // already cleaned up in loadData and ByteArrayInputStream will be GC'd override fun cleanup() {} @@ -100,4 +170,9 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail) : DataFe override fun getDataClass(): Class = InputStream::class.java override fun getDataSource(): DataSource = DataSource.LOCAL + + companion object { + // same for either `ARGB_8888` or `RGBA_1010102` + private const val FORMAT_BYTE_SIZE = BitmapUtils.ARGB_8888_BYTE_SIZE + } } \ No newline at end of file