Fabio/aves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

region decoding: use raw image descriptor in Flutter on decoded bytes from Android

Browse source
This commit is contained in:
Thibault Deckers 2025-03-02 13:06:58 +01:00
parent 5805bb2b5b
commit f850178afd
13 changed files with 205 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt
View file

@ -38,7 +38,7 @@ import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
import deckers.thibault.aves.model.FieldMap import deckers.thibault.aves.model.FieldMap
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.LogUtils
import deckers.thibault.aves.utils.anyCauseIs import deckers.thibault.aves.utils.anyCauseIs
import deckers.thibault.aves.utils.getApplicationInfoCompat import deckers.thibault.aves.utils.getApplicationInfoCompat
@ -175,7 +175,7 @@ class AppAdapterHandler(private val context: Context) : MethodCallHandler {
try { try {
val bitmap = withContext(Dispatchers.IO) { target.get() } val bitmap = withContext(Dispatchers.IO) { target.get() }
data = bitmap?.getBytes(canHaveAlpha = true, recycle = false) data = bitmap?.getEncodedBytes(canHaveAlpha = true, recycle = false)
} catch (e: Exception) { } catch (e: Exception) {
Log.w(LOG_TAG, "failed to decode app icon for packageName=$packageName", e) Log.w(LOG_TAG, "failed to decode app icon for packageName=$packageName", e)
} }

4
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt
View file

@ -23,7 +23,7 @@ import deckers.thibault.aves.model.FieldMap
import deckers.thibault.aves.model.provider.ImageProvider import deckers.thibault.aves.model.provider.ImageProvider
import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
import deckers.thibault.aves.utils.FileUtils.transferFrom import deckers.thibault.aves.utils.FileUtils.transferFrom
import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.LogUtils
import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.MimeTypes
@ -75,7 +75,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
val orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) val orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
exif.thumbnailBitmap?.let { bitmap -> exif.thumbnailBitmap?.let { bitmap ->
TransformationUtils.rotateImageExif(BitmapUtils.getBitmapPool(context), bitmap, orientation)?.let { TransformationUtils.rotateImageExif(BitmapUtils.getBitmapPool(context), bitmap, orientation)?.let {
it.getBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) } it.getEncodedBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) }
} }
} }
} }

7
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt
View file

@ -3,6 +3,7 @@ package deckers.thibault.aves.channel.calls
import android.content.Context import android.content.Context
import android.graphics.Rect import android.graphics.Rect
import androidx.core.net.toUri import androidx.core.net.toUri
import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher
import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher
@ -29,7 +30,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
when (call.method) { when (call.method) {
"getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) } "getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) }
"getRegion" -> ioScope.launch { safeSuspend(call, result, ::getRegion) } "getRegion" -> ioScope.launch { safe(call, result, ::getRegion) }
else -> result.notImplemented() else -> result.notImplemented()
} }
} }
@ -68,7 +69,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
).fetch() ).fetch()
} }
private suspend fun getRegion(call: MethodCall, result: MethodChannel.Result) { private fun getRegion(call: MethodCall, result: MethodChannel.Result) {
val uri = call.argument<String>("uri")?.toUri() val uri = call.argument<String>("uri")?.toUri()
val mimeType = call.argument<String>("mimeType") val mimeType = call.argument<String>("mimeType")
val pageId = call.argument<Int>("pageId") val pageId = call.argument<Int>("pageId")
@ -97,6 +98,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
imageHeight = imageHeight, imageHeight = imageHeight,
result = result, result = result,
) )
MimeTypes.TIFF -> TiffRegionFetcher(context).fetch( MimeTypes.TIFF -> TiffRegionFetcher(context).fetch(
uri = uri, uri = uri,
page = pageId ?: 0, page = pageId ?: 0,
@ -104,6 +106,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
regionRect = regionRect, regionRect = regionRect,
result = result, result = result,
) )
else -> regionFetcher.fetch( else -> regionFetcher.fetch(
uri = uri, uri = uri,
mimeType = mimeType, mimeType = mimeType,

48
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt
View file

@ -4,16 +4,17 @@ import android.content.Context
import android.graphics.Bitmap import android.graphics.Bitmap
import android.graphics.BitmapFactory import android.graphics.BitmapFactory
import android.graphics.BitmapRegionDecoder import android.graphics.BitmapRegionDecoder
import android.graphics.ColorSpace
import android.graphics.Rect import android.graphics.Rect
import android.net.Uri import android.net.Uri
import android.os.Build
import android.util.Log import android.util.Log
import com.bumptech.glide.Glide import com.bumptech.glide.Glide
import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.decoder.AvesAppGlideModule
import deckers.thibault.aves.decoder.MultiPageImage import deckers.thibault.aves.decoder.MultiPageImage
import deckers.thibault.aves.utils.BitmapRegionDecoderCompat import deckers.thibault.aves.utils.BitmapRegionDecoderCompat
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
import deckers.thibault.aves.utils.BitmapUtils.getBytes
import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.LogUtils
import deckers.thibault.aves.utils.MathUtils import deckers.thibault.aves.utils.MathUtils
import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MemoryUtils
@ -33,7 +34,10 @@ class RegionFetcher internal constructor(
private val exportUris = HashMap<Pair<Uri, Int?>, Uri>() private val exportUris = HashMap<Pair<Uri, Int?>, Uri>()
suspend fun fetch( // return decoded bytes in ARGB_8888, with trailer bytes:
// - width (int32)
// - height (int32)
fun fetch(
uri: Uri, uri: Uri,
mimeType: String, mimeType: String,
pageId: Int?, pageId: Int?,
@ -99,26 +103,37 @@ class RegionFetcher internal constructor(
} }
} }
// use `Long` as rect size could be unexpectedly large and go beyond `Int` max val options = BitmapFactory.Options().apply {
val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * effectiveRect.width() * effectiveRect.height() / effectiveSampleSize inSampleSize = effectiveSampleSize
// Specifying preferred config and color space avoids the need for conversion afterwards,
// but may prevent decoding (e.g. from RGBA_1010102 to ARGB_8888 on some devices).
inPreferredConfig = PREFERRED_CONFIG
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB)
}
}
val pixelCount = effectiveRect.width() * effectiveRect.height() / effectiveSampleSize
val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), options.inPreferredConfig)
if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
// decoding a region that large would yield an OOM when creating the bitmap // decoding a region that large would yield an OOM when creating the bitmap
result.error("fetch-large-region", "Region too large for uri=$uri regionRect=$regionRect", null) result.error("fetch-large-region", "Region too large for uri=$uri regionRect=$regionRect", null)
return return
} }
val options = BitmapFactory.Options().apply { var bitmap = decoder.decodeRegion(effectiveRect, options)
inSampleSize = effectiveSampleSize if (bitmap == null) {
} // retry without specifying config or color space,
val bitmap = decoder.decodeRegion(effectiveRect, options) // falling back to custom byte conversion afterwards
if (bitmap != null) { options.inPreferredConfig = null
val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && options.inPreferredColorSpace != null) {
val recycle = false options.inPreferredColorSpace = null
var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle)
if (bytes != null && bytes.isEmpty()) {
bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle)
} }
bitmap.recycle() bitmap = decoder.decodeRegion(effectiveRect, options)
}
val bytes = bitmap?.getDecodedBytes(recycle = true)
if (bytes != null) {
result.success(bytes) result.success(bytes)
} else { } else {
result.error("fetch-null", "failed to decode region for uri=$uri regionRect=$regionRect", null) result.error("fetch-null", "failed to decode region for uri=$uri regionRect=$regionRect", null)
@ -173,5 +188,6 @@ class RegionFetcher internal constructor(
companion object { companion object {
private val LOG_TAG = LogUtils.createTag<RegionFetcher>() private val LOG_TAG = LogUtils.createTag<RegionFetcher>()
private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888
} }
} }

26
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt
View file

@ -13,8 +13,8 @@ import com.caverock.androidsvg.SVG
import com.caverock.androidsvg.SVGParseException import com.caverock.androidsvg.SVGParseException
import deckers.thibault.aves.metadata.SVGParserBufferedInputStream import deckers.thibault.aves.metadata.SVGParserBufferedInputStream
import deckers.thibault.aves.metadata.SvgHelper.normalizeSize import deckers.thibault.aves.metadata.SvgHelper.normalizeSize
import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MemoryUtils
import deckers.thibault.aves.utils.StorageUtils import deckers.thibault.aves.utils.StorageUtils
import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel
@ -25,7 +25,7 @@ class SvgRegionFetcher internal constructor(
) { ) {
private var lastSvgRef: LastSvgRef? = null private var lastSvgRef: LastSvgRef? = null
suspend fun fetch( fun fetch(
uri: Uri, uri: Uri,
sizeBytes: Long?, sizeBytes: Long?,
scale: Int, scale: Int,
@ -92,25 +92,25 @@ class SvgRegionFetcher internal constructor(
val targetBitmapWidth = regionRect.width() val targetBitmapWidth = regionRect.width()
val targetBitmapHeight = regionRect.height() val targetBitmapHeight = regionRect.height()
val canvasWidth = targetBitmapWidth + bleedX * 2
val canvasHeight = targetBitmapHeight + bleedY * 2
// use `Long` as rect size could be unexpectedly large and go beyond `Int` max val config = PREFERRED_CONFIG
val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * targetBitmapWidth * targetBitmapHeight val pixelCount = canvasWidth * canvasHeight
val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), config)
if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
// decoding a region that large would yield an OOM when creating the bitmap // decoding a region that large would yield an OOM when creating the bitmap
result.error("fetch-read-large-region", "SVG region too large for uri=$uri regionRect=$regionRect", null) result.error("fetch-read-large-region", "SVG region too large for uri=$uri regionRect=$regionRect", null)
return return
} }
var bitmap = createBitmap( var bitmap = createBitmap(canvasWidth, canvasHeight, config)
targetBitmapWidth + bleedX * 2,
targetBitmapHeight + bleedY * 2,
Bitmap.Config.ARGB_8888
)
val canvas = Canvas(bitmap) val canvas = Canvas(bitmap)
svg.renderToCanvas(canvas, renderOptions) svg.renderToCanvas(canvas, renderOptions)
bitmap = Bitmap.createBitmap(bitmap, bleedX, bleedY, targetBitmapWidth, targetBitmapHeight) bitmap = Bitmap.createBitmap(bitmap, bleedX, bleedY, targetBitmapWidth, targetBitmapHeight)
result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true)) val bytes = bitmap.getDecodedBytes(recycle = true)
result.success(bytes)
} catch (e: Exception) { } catch (e: Exception) {
result.error("fetch-read-exception", "failed to initialize region decoder for uri=$uri regionRect=$regionRect", e.message) result.error("fetch-read-exception", "failed to initialize region decoder for uri=$uri regionRect=$regionRect", e.message)
} }
@ -120,4 +120,8 @@ class SvgRegionFetcher internal constructor(
val uri: Uri, val uri: Uri,
val svg: SVG, val svg: SVG,
) )
companion object {
private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888
}
} }

6
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt
View file

@ -17,7 +17,7 @@ import deckers.thibault.aves.decoder.AvesAppGlideModule
import deckers.thibault.aves.decoder.MultiPageImage import deckers.thibault.aves.decoder.MultiPageImage
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.MimeTypes
import deckers.thibault.aves.utils.MimeTypes.SVG import deckers.thibault.aves.utils.MimeTypes.SVG
import deckers.thibault.aves.utils.MimeTypes.isVideo import deckers.thibault.aves.utils.MimeTypes.isVideo
@ -81,9 +81,9 @@ class ThumbnailFetcher internal constructor(
if (bitmap != null) { if (bitmap != null) {
val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
val recycle = false val recycle = false
var bytes = bitmap.getBytes(canHaveAlpha, quality, recycle) var bytes = bitmap.getEncodedBytes(canHaveAlpha, quality, recycle)
if (bytes != null && bytes.isEmpty()) { if (bytes != null && bytes.isEmpty()) {
bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, quality, recycle) bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, quality, recycle)
} }
result.success(bytes) result.success(bytes)
} else { } else {

9
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt
View file

@ -3,7 +3,7 @@ package deckers.thibault.aves.channel.calls.fetchers
import android.content.Context import android.content.Context
import android.graphics.Rect import android.graphics.Rect
import android.net.Uri import android.net.Uri
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel
import org.beyka.tiffbitmapfactory.DecodeArea import org.beyka.tiffbitmapfactory.DecodeArea
import org.beyka.tiffbitmapfactory.TiffBitmapFactory import org.beyka.tiffbitmapfactory.TiffBitmapFactory
@ -11,7 +11,7 @@ import org.beyka.tiffbitmapfactory.TiffBitmapFactory
class TiffRegionFetcher internal constructor( class TiffRegionFetcher internal constructor(
private val context: Context, private val context: Context,
) { ) {
suspend fun fetch( fun fetch(
uri: Uri, uri: Uri,
page: Int, page: Int,
sampleSize: Int, sampleSize: Int,
@ -32,8 +32,9 @@ class TiffRegionFetcher internal constructor(
inDecodeArea = DecodeArea(regionRect.left, regionRect.top, regionRect.width(), regionRect.height()) inDecodeArea = DecodeArea(regionRect.left, regionRect.top, regionRect.width(), regionRect.height())
} }
val bitmap = TiffBitmapFactory.decodeFileDescriptor(fd, options) val bitmap = TiffBitmapFactory.decodeFileDescriptor(fd, options)
if (bitmap != null) { val bytes = bitmap?.getDecodedBytes(recycle = true)
result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true)) if (bytes != null) {
result.success(bytes)
} else { } else {
result.error("getRegion-tiff-null", "failed to decode region for uri=$uri page=$page regionRect=$regionRect", null) result.error("getRegion-tiff-null", "failed to decode region for uri=$uri page=$page regionRect=$regionRect", null)
} }

8
android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt
View file

@ -10,7 +10,7 @@ import com.bumptech.glide.Glide
import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.decoder.AvesAppGlideModule
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.LogUtils
import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MemoryUtils
import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.MimeTypes
@ -140,9 +140,9 @@ class ImageByteStreamHandler(private val context: Context, private val arguments
if (bitmap != null) { if (bitmap != null) {
val recycle = false val recycle = false
val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle) var bytes = bitmap.getEncodedBytes(canHaveAlpha, recycle = recycle)
if (bytes != null && bytes.isEmpty()) { if (bytes != null && bytes.isEmpty()) {
bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle) bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, recycle = recycle)
} }
if (MemoryUtils.canAllocate(sizeBytes)) { if (MemoryUtils.canAllocate(sizeBytes)) {
success(bytes) success(bytes)
@ -168,7 +168,7 @@ class ImageByteStreamHandler(private val context: Context, private val arguments
try { try {
val bitmap = withContext(Dispatchers.IO) { target.get() } val bitmap = withContext(Dispatchers.IO) { target.get() }
if (bitmap != null) { if (bitmap != null) {
val bytes = bitmap.getBytes(canHaveAlpha = false, recycle = false) val bytes = bitmap.getEncodedBytes(canHaveAlpha = false, recycle = false)
if (MemoryUtils.canAllocate(sizeBytes)) { if (MemoryUtils.canAllocate(sizeBytes)) {
success(bytes) success(bytes)
} else { } else {

25
android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt
View file

@ -20,7 +20,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory
import com.bumptech.glide.module.LibraryGlideModule import com.bumptech.glide.module.LibraryGlideModule
import com.bumptech.glide.signature.ObjectKey import com.bumptech.glide.signature.ObjectKey
import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils
import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MemoryUtils
import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
@ -112,7 +112,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
// the returned frame is already rotated according to the video metadata // the returned frame is already rotated according to the video metadata
val frame = if (dstWidth > 0 && dstHeight > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { val frame = if (dstWidth > 0 && dstHeight > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
val targetBitmapSizeBytes: Long = FORMAT_BYTE_SIZE.toLong() * dstWidth * dstHeight val pixelCount = dstWidth * dstHeight
val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig())
if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the scaled frame at $dstWidth x $dstHeight") throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the scaled frame at $dstWidth x $dstHeight")
} }
@ -122,7 +123,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight) retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight)
} }
} else { } else {
val targetBitmapSizeBytes: Long = (FORMAT_BYTE_SIZE.toLong() * videoWidth * videoHeight).toLong() val pixelCount = videoWidth * videoHeight
val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig())
if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) {
throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the full frame at $videoWidth x $videoHeight") throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the full frame at $videoWidth x $videoHeight")
} }
@ -132,7 +134,7 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
retriever.getFrameAtTime(timeMicros, option) retriever.getFrameAtTime(timeMicros, option)
} }
} }
bytes = frame?.getBytes(canHaveAlpha = false, recycle = false) bytes = frame?.getEncodedBytes(canHaveAlpha = false, recycle = false)
} }
if (bytes != null) { if (bytes != null) {
@ -151,8 +153,14 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
} }
@RequiresApi(Build.VERSION_CODES.P) @RequiresApi(Build.VERSION_CODES.P)
private fun getBitmapParams() = MediaMetadataRetriever.BitmapParams().apply { private fun getBitmapParams(): MediaMetadataRetriever.BitmapParams {
preferredConfig = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { val params = MediaMetadataRetriever.BitmapParams()
params.preferredConfig = this.getPreferredConfig()
return params
}
private fun getPreferredConfig(): Bitmap.Config {
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
// improved precision with the same memory cost as `ARGB_8888` (4 bytes per pixel) // improved precision with the same memory cost as `ARGB_8888` (4 bytes per pixel)
// for wide-gamut and HDR content which does not require alpha blending // for wide-gamut and HDR content which does not require alpha blending
Bitmap.Config.RGBA_1010102 Bitmap.Config.RGBA_1010102
@ -170,9 +178,4 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt
override fun getDataClass(): Class<InputStream> = InputStream::class.java override fun getDataClass(): Class<InputStream> = InputStream::class.java
override fun getDataSource(): DataSource = DataSource.LOCAL override fun getDataSource(): DataSource = DataSource.LOCAL
companion object {
// same for either `ARGB_8888` or `RGBA_1010102`
private const val FORMAT_BYTE_SIZE = BitmapUtils.ARGB_8888_BYTE_SIZE
}
} }

119
android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt
View file

@ -25,10 +25,81 @@ object BitmapUtils {
private val freeBaos = ArrayList<ByteArrayOutputStream>() private val freeBaos = ArrayList<ByteArrayOutputStream>()
private val mutex = Mutex() private val mutex = Mutex()
const val ARGB_8888_BYTE_SIZE = 4 private const val INT_BYTE_SIZE = 4
private const val RGBA_1010102_BYTE_SIZE = 4 private const val MAX_2_BITS_FLOAT = 0x3.toFloat()
private const val MAX_8_BITS_FLOAT = 0xff.toFloat()
private const val MAX_10_BITS_FLOAT = 0x3ff.toFloat()
suspend fun Bitmap.getBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean): ByteArray? { // bytes per pixel with different bitmap config
private const val BPP_ALPHA_8 = 1
private const val BPP_RGB_565 = 2
private const val BPP_ARGB_8888 = 4
private const val BPP_RGBA_1010102 = 4
private const val BPP_RGBA_F16 = 8
private fun getBytePerPixel(config: Bitmap.Config?): Int {
return when (config) {
Bitmap.Config.ALPHA_8 -> BPP_ALPHA_8
Bitmap.Config.RGB_565 -> BPP_RGB_565
Bitmap.Config.ARGB_8888 -> BPP_ARGB_8888
else -> {
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && config == Bitmap.Config.RGBA_F16) {
BPP_RGBA_F16
} else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && config == Bitmap.Config.RGBA_1010102) {
BPP_RGBA_1010102
} else {
// default
BPP_ARGB_8888
}
}
}
}
fun getExpectedImageSize(pixelCount: Long, config: Bitmap.Config?): Long {
return pixelCount * getBytePerPixel(config)
}
fun Bitmap.getDecodedBytes(recycle: Boolean): ByteArray? {
if (!MemoryUtils.canAllocate(byteCount)) {
throw Exception("bitmap buffer is $byteCount bytes, which cannot be allocated to a new byte array")
}
try {
val bytes = ByteBuffer.allocate(byteCount + INT_BYTE_SIZE * 2).apply {
copyPixelsToBuffer(this)
// append bitmap size for use by the caller
putInt(width)
putInt(height)
rewind()
}.array()
// convert pixel format and color space, if necessary
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
colorSpace?.let { srcColorSpace ->
val dstColorSpace = ColorSpace.get(ColorSpace.Named.SRGB)
val connector = ColorSpace.connect(srcColorSpace, dstColorSpace)
if (config == Bitmap.Config.ARGB_8888) {
if (srcColorSpace != dstColorSpace) {
argb8888toArgb8888(bytes, connector, end = byteCount)
}
} else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && config == Bitmap.Config.RGBA_1010102) {
rgba1010102toArgb8888(bytes, connector, end = byteCount)
}
}
}
// should not be called before accessing color space or other properties
if (recycle) this.recycle()
return bytes
} catch (e: Exception) {
Log.e(LOG_TAG, "failed to get bytes from bitmap", e)
}
return null
}
suspend fun Bitmap.getEncodedBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean): ByteArray? {
val stream: ByteArrayOutputStream val stream: ByteArrayOutputStream
mutex.withLock { mutex.withLock {
// this method is called a lot, so we try and reuse output streams // this method is called a lot, so we try and reuse output streams
@ -101,37 +172,59 @@ object BitmapUtils {
return null return null
} }
@RequiresApi(Build.VERSION_CODES.O)
private fun argb8888toArgb8888(bytes: ByteArray, connector: ColorSpace.Connector, start: Int = 0, end: Int = bytes.size) {
// unpacking from ARGB_8888 and packing to ARGB_8888
// stored as [3,2,1,0] -> [AAAAAAAA BBBBBBBB GGGGGGGG RRRRRRRR]
for (i in start..<end step BPP_ARGB_8888) {
// mask with `0xff` to yield values in [0, 255], instead of [-128, 127]
val iB = bytes[i + 2].toInt() and 0xff
val iG = bytes[i + 1].toInt() and 0xff
val iR = bytes[i].toInt() and 0xff
// components as floats in sRGB
val srgbFloats = connector.transform(iR / MAX_8_BITS_FLOAT, iG / MAX_8_BITS_FLOAT, iB / MAX_8_BITS_FLOAT)
val srgbR = (srgbFloats[0] * 255.0f + 0.5f).toInt()
val srgbG = (srgbFloats[1] * 255.0f + 0.5f).toInt()
val srgbB = (srgbFloats[2] * 255.0f + 0.5f).toInt()
// keep alpha as it is, in `bytes[i + 3]`
bytes[i + 2] = srgbB.toByte()
bytes[i + 1] = srgbG.toByte()
bytes[i] = srgbR.toByte()
}
}
// convert bytes, without reallocation: // convert bytes, without reallocation:
// - from config RGBA_1010102 to ARGB_8888, // - from config RGBA_1010102 to ARGB_8888,
// - from original color space to sRGB. // - from original color space to sRGB.
@RequiresApi(Build.VERSION_CODES.O) @RequiresApi(Build.VERSION_CODES.O)
private fun rgba1010102toArgb8888(bytes: ByteArray, connector: ColorSpace.Connector) { private fun rgba1010102toArgb8888(bytes: ByteArray, connector: ColorSpace.Connector, start: Int = 0, end: Int = bytes.size) {
val max10Bits = 0x3ff.toFloat() val alphaFactor = 255.0f / MAX_2_BITS_FLOAT
val dstAlpha = 0xff.toByte()
val byteCount = bytes.size for (i in start..<end step BPP_RGBA_1010102) {
for (i in 0..<byteCount step RGBA_1010102_BYTE_SIZE) { // unpacking from RGBA_1010102
// stored as [3,2,1,0] -> [AABBBBBB BBBBGGGG GGGGGGRR RRRRRRRR]
val i3 = bytes[i + 3].toInt() val i3 = bytes[i + 3].toInt()
val i2 = bytes[i + 2].toInt() val i2 = bytes[i + 2].toInt()
val i1 = bytes[i + 1].toInt() val i1 = bytes[i + 1].toInt()
val i0 = bytes[i].toInt() val i0 = bytes[i].toInt()
// unpacking from RGBA_1010102 val iA = ((i3 and 0xc0) shr 6)
// stored as [3,2,1,0] -> [AABBBBBB BBBBGGGG GGGGGGRR RRRRRRRR]
// val iA = ((i3 and 0xc0) shr 6)
val iB = ((i3 and 0x3f) shl 4) or ((i2 and 0xf0) shr 4) val iB = ((i3 and 0x3f) shl 4) or ((i2 and 0xf0) shr 4)
val iG = ((i2 and 0x0f) shl 6) or ((i1 and 0xfc) shr 2) val iG = ((i2 and 0x0f) shl 6) or ((i1 and 0xfc) shr 2)
val iR = ((i1 and 0x03) shl 8) or ((i0 and 0xff) shr 0) val iR = ((i1 and 0x03) shl 8) or ((i0 and 0xff) shr 0)
// components as floats in sRGB // components as floats in sRGB
val srgbFloats = connector.transform(iR / max10Bits, iG / max10Bits, iB / max10Bits) val srgbFloats = connector.transform(iR / MAX_10_BITS_FLOAT, iG / MAX_10_BITS_FLOAT, iB / MAX_10_BITS_FLOAT)
val srgbR = (srgbFloats[0] * 255.0f + 0.5f).toInt() val srgbR = (srgbFloats[0] * 255.0f + 0.5f).toInt()
val srgbG = (srgbFloats[1] * 255.0f + 0.5f).toInt() val srgbG = (srgbFloats[1] * 255.0f + 0.5f).toInt()
val srgbB = (srgbFloats[2] * 255.0f + 0.5f).toInt() val srgbB = (srgbFloats[2] * 255.0f + 0.5f).toInt()
val alpha = (iA * alphaFactor + 0.5f).toInt()
// packing to ARGB_8888 // packing to ARGB_8888
// stored as [3,2,1,0] -> [AAAAAAAA BBBBBBBB GGGGGGGG RRRRRRRR] // stored as [3,2,1,0] -> [AAAAAAAA BBBBBBBB GGGGGGGG RRRRRRRR]
bytes[i + 3] = dstAlpha bytes[i + 3] = alpha.toByte()
bytes[i + 2] = srgbB.toByte() bytes[i + 2] = srgbB.toByte()
bytes[i + 1] = srgbG.toByte() bytes[i + 1] = srgbG.toByte()
bytes[i] = srgbR.toByte() bytes[i] = srgbR.toByte()

15
lib/image_providers/region_provider.dart
View file

@ -48,8 +48,21 @@ class RegionProvider extends ImageProvider<RegionProviderKey> {
if (bytes.isEmpty) { if (bytes.isEmpty) {
throw StateError('$uri ($mimeType) region loading failed'); throw StateError('$uri ($mimeType) region loading failed');
} }
final trailerOffset = bytes.length - 4 * 2;
final trailer = ByteData.sublistView(bytes, trailerOffset);
final bitmapWidth = trailer.getUint32(0);
final bitmapHeight = trailer.getUint32(4);
final buffer = await ui.ImmutableBuffer.fromUint8List(bytes); final buffer = await ui.ImmutableBuffer.fromUint8List(bytes);
return await decode(buffer); final descriptor = ui.ImageDescriptor.raw(
buffer,
width: bitmapWidth,
height: bitmapHeight,
pixelFormat: ui.PixelFormat.rgba8888,
);
return descriptor.instantiateCodec();
} catch (error) { } catch (error) {
// loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF) // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF)
debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error'); debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error');

3
lib/utils/math_utils.dart
View file

@ -4,7 +4,8 @@ import 'dart:ui';
int highestPowerOf2(num x) => x < 1 ? 0 : pow(2, (log(x) / ln2).floor()).toInt(); int highestPowerOf2(num x) => x < 1 ? 0 : pow(2, (log(x) / ln2).floor()).toInt();
num smallestPowerOf2(num x, {bool allowNegativePower = false}) { num smallestPowerOf2(num x, {bool allowNegativePower = false}) {
return x < 1 && !allowNegativePower ? 1 : pow(2, (log(x) / ln2).ceil()); if ((x < 1 && !allowNegativePower) || x <= 0) return 1;
return pow(2, (log(x) / ln2).ceil());
} }
double roundToPrecision(final double value, {required final int decimals}) => (value * pow(10, decimals)).round() / pow(10, decimals); double roundToPrecision(final double value, {required final int decimals}) => (value * pow(10, decimals)).round() / pow(10, decimals);

1
test/utils/math_utils_test.dart
View file

@ -22,6 +22,7 @@ void main() {
expect(smallestPowerOf2(1.5), 2); expect(smallestPowerOf2(1.5), 2);
expect(smallestPowerOf2(0.5, allowNegativePower: true), 0.5); expect(smallestPowerOf2(0.5, allowNegativePower: true), 0.5);
expect(smallestPowerOf2(0.1, allowNegativePower: true), 0.125); expect(smallestPowerOf2(0.1, allowNegativePower: true), 0.125);
expect(smallestPowerOf2(0, allowNegativePower: true), 1);
}); });
test('rounding to a given precision after the decimal', () { test('rounding to a given precision after the decimal', () {