From f850178afde30e55d82b7e9a0f90f9b7b30533e9 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Sun, 2 Mar 2025 13:06:58 +0100 Subject: [PATCH] region decoding: use raw image descriptor in Flutter on decoded bytes from Android --- .../aves/channel/calls/AppAdapterHandler.kt | 4 +- .../aves/channel/calls/EmbeddedDataHandler.kt | 4 +- .../channel/calls/MediaFetchBytesHandler.kt | 7 +- .../channel/calls/fetchers/RegionFetcher.kt | 48 ++++--- .../calls/fetchers/SvgRegionFetcher.kt | 26 ++-- .../calls/fetchers/ThumbnailFetcher.kt | 6 +- .../calls/fetchers/TiffRegionFetcher.kt | 9 +- .../channel/streams/ImageByteStreamHandler.kt | 8 +- .../aves/decoder/VideoThumbnailGlideModule.kt | 25 ++-- .../thibault/aves/utils/BitmapUtils.kt | 119 ++++++++++++++++-- lib/image_providers/region_provider.dart | 15 ++- lib/utils/math_utils.dart | 3 +- test/utils/math_utils_test.dart | 1 + 13 files changed, 205 insertions(+), 70 deletions(-) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt index 577711d68..e9f400ebe 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt @@ -38,7 +38,7 @@ import deckers.thibault.aves.channel.calls.Coresult.Companion.safe import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend import deckers.thibault.aves.model.FieldMap import deckers.thibault.aves.utils.BitmapUtils -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.anyCauseIs import deckers.thibault.aves.utils.getApplicationInfoCompat @@ -175,7 +175,7 @@ class AppAdapterHandler(private val context: Context) : MethodCallHandler { try { val bitmap = withContext(Dispatchers.IO) { target.get() } - data = bitmap?.getBytes(canHaveAlpha = true, recycle = false) + data = bitmap?.getEncodedBytes(canHaveAlpha = true, recycle = false) } catch (e: Exception) { Log.w(LOG_TAG, "failed to decode app icon for packageName=$packageName", e) } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt index 8897e3ab4..f6275dac1 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt @@ -23,7 +23,7 @@ import deckers.thibault.aves.model.FieldMap import deckers.thibault.aves.model.provider.ImageProvider import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider import deckers.thibault.aves.utils.BitmapUtils -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes import deckers.thibault.aves.utils.FileUtils.transferFrom import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.MimeTypes @@ -75,7 +75,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler { val orientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) exif.thumbnailBitmap?.let { bitmap -> TransformationUtils.rotateImageExif(BitmapUtils.getBitmapPool(context), bitmap, orientation)?.let { - it.getBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) } + it.getEncodedBytes(canHaveAlpha = false, recycle = false)?.let { bytes -> thumbnails.add(bytes) } } } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt index 73f2ab19b..bd63c0f33 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt @@ -3,6 +3,7 @@ package deckers.thibault.aves.channel.calls import android.content.Context import android.graphics.Rect import androidx.core.net.toUri +import deckers.thibault.aves.channel.calls.Coresult.Companion.safe import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher @@ -29,7 +30,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { when (call.method) { "getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) } - "getRegion" -> ioScope.launch { safeSuspend(call, result, ::getRegion) } + "getRegion" -> ioScope.launch { safe(call, result, ::getRegion) } else -> result.notImplemented() } } @@ -68,7 +69,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler { ).fetch() } - private suspend fun getRegion(call: MethodCall, result: MethodChannel.Result) { + private fun getRegion(call: MethodCall, result: MethodChannel.Result) { val uri = call.argument("uri")?.toUri() val mimeType = call.argument("mimeType") val pageId = call.argument("pageId") @@ -97,6 +98,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler { imageHeight = imageHeight, result = result, ) + MimeTypes.TIFF -> TiffRegionFetcher(context).fetch( uri = uri, page = pageId ?: 0, @@ -104,6 +106,7 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler { regionRect = regionRect, result = result, ) + else -> regionFetcher.fetch( uri = uri, mimeType = mimeType, diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt index 0b1728a8f..7358d11d2 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt @@ -4,16 +4,17 @@ import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.BitmapRegionDecoder +import android.graphics.ColorSpace import android.graphics.Rect import android.net.Uri +import android.os.Build import android.util.Log import com.bumptech.glide.Glide import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.decoder.MultiPageImage import deckers.thibault.aves.utils.BitmapRegionDecoderCompat import deckers.thibault.aves.utils.BitmapUtils -import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.MathUtils import deckers.thibault.aves.utils.MemoryUtils @@ -33,7 +34,10 @@ class RegionFetcher internal constructor( private val exportUris = HashMap, Uri>() - suspend fun fetch( + // return decoded bytes in ARGB_8888, with trailer bytes: + // - width (int32) + // - height (int32) + fun fetch( uri: Uri, mimeType: String, pageId: Int?, @@ -99,26 +103,37 @@ class RegionFetcher internal constructor( } } - // use `Long` as rect size could be unexpectedly large and go beyond `Int` max - val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * effectiveRect.width() * effectiveRect.height() / effectiveSampleSize + val options = BitmapFactory.Options().apply { + inSampleSize = effectiveSampleSize + // Specifying preferred config and color space avoids the need for conversion afterwards, + // but may prevent decoding (e.g. from RGBA_1010102 to ARGB_8888 on some devices). + inPreferredConfig = PREFERRED_CONFIG + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB) + } + } + + val pixelCount = effectiveRect.width() * effectiveRect.height() / effectiveSampleSize + val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), options.inPreferredConfig) if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { // decoding a region that large would yield an OOM when creating the bitmap result.error("fetch-large-region", "Region too large for uri=$uri regionRect=$regionRect", null) return } - val options = BitmapFactory.Options().apply { - inSampleSize = effectiveSampleSize - } - val bitmap = decoder.decodeRegion(effectiveRect, options) - if (bitmap != null) { - val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) - val recycle = false - var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle) - if (bytes != null && bytes.isEmpty()) { - bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle) + var bitmap = decoder.decodeRegion(effectiveRect, options) + if (bitmap == null) { + // retry without specifying config or color space, + // falling back to custom byte conversion afterwards + options.inPreferredConfig = null + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && options.inPreferredColorSpace != null) { + options.inPreferredColorSpace = null } - bitmap.recycle() + bitmap = decoder.decodeRegion(effectiveRect, options) + } + + val bytes = bitmap?.getDecodedBytes(recycle = true) + if (bytes != null) { result.success(bytes) } else { result.error("fetch-null", "failed to decode region for uri=$uri regionRect=$regionRect", null) @@ -173,5 +188,6 @@ class RegionFetcher internal constructor( companion object { private val LOG_TAG = LogUtils.createTag() + private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888 } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt index d4410030c..9708b1b79 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/SvgRegionFetcher.kt @@ -13,8 +13,8 @@ import com.caverock.androidsvg.SVG import com.caverock.androidsvg.SVGParseException import deckers.thibault.aves.metadata.SVGParserBufferedInputStream import deckers.thibault.aves.metadata.SvgHelper.normalizeSize -import deckers.thibault.aves.utils.BitmapUtils.ARGB_8888_BYTE_SIZE -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils +import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.StorageUtils import io.flutter.plugin.common.MethodChannel @@ -25,7 +25,7 @@ class SvgRegionFetcher internal constructor( ) { private var lastSvgRef: LastSvgRef? = null - suspend fun fetch( + fun fetch( uri: Uri, sizeBytes: Long?, scale: Int, @@ -92,25 +92,25 @@ class SvgRegionFetcher internal constructor( val targetBitmapWidth = regionRect.width() val targetBitmapHeight = regionRect.height() + val canvasWidth = targetBitmapWidth + bleedX * 2 + val canvasHeight = targetBitmapHeight + bleedY * 2 - // use `Long` as rect size could be unexpectedly large and go beyond `Int` max - val targetBitmapSizeBytes: Long = ARGB_8888_BYTE_SIZE.toLong() * targetBitmapWidth * targetBitmapHeight + val config = PREFERRED_CONFIG + val pixelCount = canvasWidth * canvasHeight + val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), config) if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { // decoding a region that large would yield an OOM when creating the bitmap result.error("fetch-read-large-region", "SVG region too large for uri=$uri regionRect=$regionRect", null) return } - var bitmap = createBitmap( - targetBitmapWidth + bleedX * 2, - targetBitmapHeight + bleedY * 2, - Bitmap.Config.ARGB_8888 - ) + var bitmap = createBitmap(canvasWidth, canvasHeight, config) val canvas = Canvas(bitmap) svg.renderToCanvas(canvas, renderOptions) bitmap = Bitmap.createBitmap(bitmap, bleedX, bleedY, targetBitmapWidth, targetBitmapHeight) - result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true)) + val bytes = bitmap.getDecodedBytes(recycle = true) + result.success(bytes) } catch (e: Exception) { result.error("fetch-read-exception", "failed to initialize region decoder for uri=$uri regionRect=$regionRect", e.message) } @@ -120,4 +120,8 @@ class SvgRegionFetcher internal constructor( val uri: Uri, val svg: SVG, ) + + companion object { + private val PREFERRED_CONFIG = Bitmap.Config.ARGB_8888 + } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt index a72a427af..61d4208d5 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt @@ -17,7 +17,7 @@ import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.decoder.MultiPageImage import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.MimeTypes.SVG import deckers.thibault.aves.utils.MimeTypes.isVideo @@ -81,9 +81,9 @@ class ThumbnailFetcher internal constructor( if (bitmap != null) { val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) val recycle = false - var bytes = bitmap.getBytes(canHaveAlpha, quality, recycle) + var bytes = bitmap.getEncodedBytes(canHaveAlpha, quality, recycle) if (bytes != null && bytes.isEmpty()) { - bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, quality, recycle) + bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, quality, recycle) } result.success(bytes) } else { diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt index 3f6f8805f..953f81551 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt @@ -3,7 +3,7 @@ package deckers.thibault.aves.channel.calls.fetchers import android.content.Context import android.graphics.Rect import android.net.Uri -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes import io.flutter.plugin.common.MethodChannel import org.beyka.tiffbitmapfactory.DecodeArea import org.beyka.tiffbitmapfactory.TiffBitmapFactory @@ -11,7 +11,7 @@ import org.beyka.tiffbitmapfactory.TiffBitmapFactory class TiffRegionFetcher internal constructor( private val context: Context, ) { - suspend fun fetch( + fun fetch( uri: Uri, page: Int, sampleSize: Int, @@ -32,8 +32,9 @@ class TiffRegionFetcher internal constructor( inDecodeArea = DecodeArea(regionRect.left, regionRect.top, regionRect.width(), regionRect.height()) } val bitmap = TiffBitmapFactory.decodeFileDescriptor(fd, options) - if (bitmap != null) { - result.success(bitmap.getBytes(canHaveAlpha = true, recycle = true)) + val bytes = bitmap?.getDecodedBytes(recycle = true) + if (bytes != null) { + result.success(bytes) } else { result.error("getRegion-tiff-null", "failed to decode region for uri=$uri page=$page regionRect=$regionRect", null) } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt index ccb9b0626..00fc8de94 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt @@ -10,7 +10,7 @@ import com.bumptech.glide.Glide import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes import deckers.thibault.aves.utils.LogUtils import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MimeTypes @@ -140,9 +140,9 @@ class ImageByteStreamHandler(private val context: Context, private val arguments if (bitmap != null) { val recycle = false val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) - var bytes = bitmap.getBytes(canHaveAlpha, recycle = recycle) + var bytes = bitmap.getEncodedBytes(canHaveAlpha, recycle = recycle) if (bytes != null && bytes.isEmpty()) { - bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getBytes(canHaveAlpha, recycle = recycle) + bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, recycle = recycle) } if (MemoryUtils.canAllocate(sizeBytes)) { success(bytes) @@ -168,7 +168,7 @@ class ImageByteStreamHandler(private val context: Context, private val arguments try { val bitmap = withContext(Dispatchers.IO) { target.get() } if (bitmap != null) { - val bytes = bitmap.getBytes(canHaveAlpha = false, recycle = false) + val bytes = bitmap.getEncodedBytes(canHaveAlpha = false, recycle = false) if (MemoryUtils.canAllocate(sizeBytes)) { success(bytes) } else { diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt index 94a435c8f..d75d21d7b 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt @@ -20,7 +20,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory import com.bumptech.glide.module.LibraryGlideModule import com.bumptech.glide.signature.ObjectKey import deckers.thibault.aves.utils.BitmapUtils -import deckers.thibault.aves.utils.BitmapUtils.getBytes +import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever import kotlinx.coroutines.CoroutineScope @@ -112,7 +112,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt // the returned frame is already rotated according to the video metadata val frame = if (dstWidth > 0 && dstHeight > 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { - val targetBitmapSizeBytes: Long = FORMAT_BYTE_SIZE.toLong() * dstWidth * dstHeight + val pixelCount = dstWidth * dstHeight + val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig()) if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the scaled frame at $dstWidth x $dstHeight") } @@ -122,7 +123,8 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt retriever.getScaledFrameAtTime(timeMicros, option, dstWidth, dstHeight) } } else { - val targetBitmapSizeBytes: Long = (FORMAT_BYTE_SIZE.toLong() * videoWidth * videoHeight).toLong() + val pixelCount = videoWidth * videoHeight + val targetBitmapSizeBytes = BitmapUtils.getExpectedImageSize(pixelCount.toLong(), getPreferredConfig()) if (!MemoryUtils.canAllocate(targetBitmapSizeBytes)) { throw Exception("not enough memory to allocate $targetBitmapSizeBytes bytes for the full frame at $videoWidth x $videoHeight") } @@ -132,7 +134,7 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt retriever.getFrameAtTime(timeMicros, option) } } - bytes = frame?.getBytes(canHaveAlpha = false, recycle = false) + bytes = frame?.getEncodedBytes(canHaveAlpha = false, recycle = false) } if (bytes != null) { @@ -151,8 +153,14 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt } @RequiresApi(Build.VERSION_CODES.P) - private fun getBitmapParams() = MediaMetadataRetriever.BitmapParams().apply { - preferredConfig = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + private fun getBitmapParams(): MediaMetadataRetriever.BitmapParams { + val params = MediaMetadataRetriever.BitmapParams() + params.preferredConfig = this.getPreferredConfig() + return params + } + + private fun getPreferredConfig(): Bitmap.Config { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { // improved precision with the same memory cost as `ARGB_8888` (4 bytes per pixel) // for wide-gamut and HDR content which does not require alpha blending Bitmap.Config.RGBA_1010102 @@ -170,9 +178,4 @@ internal class VideoThumbnailFetcher(private val model: VideoThumbnail, val widt override fun getDataClass(): Class = InputStream::class.java override fun getDataSource(): DataSource = DataSource.LOCAL - - companion object { - // same for either `ARGB_8888` or `RGBA_1010102` - private const val FORMAT_BYTE_SIZE = BitmapUtils.ARGB_8888_BYTE_SIZE - } } \ No newline at end of file diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt b/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt index 1a3acfce9..e3ff40fe9 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt @@ -25,10 +25,81 @@ object BitmapUtils { private val freeBaos = ArrayList() private val mutex = Mutex() - const val ARGB_8888_BYTE_SIZE = 4 - private const val RGBA_1010102_BYTE_SIZE = 4 + private const val INT_BYTE_SIZE = 4 + private const val MAX_2_BITS_FLOAT = 0x3.toFloat() + private const val MAX_8_BITS_FLOAT = 0xff.toFloat() + private const val MAX_10_BITS_FLOAT = 0x3ff.toFloat() - suspend fun Bitmap.getBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean): ByteArray? { + // bytes per pixel with different bitmap config + private const val BPP_ALPHA_8 = 1 + private const val BPP_RGB_565 = 2 + private const val BPP_ARGB_8888 = 4 + private const val BPP_RGBA_1010102 = 4 + private const val BPP_RGBA_F16 = 8 + + private fun getBytePerPixel(config: Bitmap.Config?): Int { + return when (config) { + Bitmap.Config.ALPHA_8 -> BPP_ALPHA_8 + Bitmap.Config.RGB_565 -> BPP_RGB_565 + Bitmap.Config.ARGB_8888 -> BPP_ARGB_8888 + else -> { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && config == Bitmap.Config.RGBA_F16) { + BPP_RGBA_F16 + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && config == Bitmap.Config.RGBA_1010102) { + BPP_RGBA_1010102 + } else { + // default + BPP_ARGB_8888 + } + } + } + } + + fun getExpectedImageSize(pixelCount: Long, config: Bitmap.Config?): Long { + return pixelCount * getBytePerPixel(config) + } + + fun Bitmap.getDecodedBytes(recycle: Boolean): ByteArray? { + if (!MemoryUtils.canAllocate(byteCount)) { + throw Exception("bitmap buffer is $byteCount bytes, which cannot be allocated to a new byte array") + } + + try { + val bytes = ByteBuffer.allocate(byteCount + INT_BYTE_SIZE * 2).apply { + copyPixelsToBuffer(this) + // append bitmap size for use by the caller + putInt(width) + putInt(height) + + rewind() + }.array() + + // convert pixel format and color space, if necessary + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + colorSpace?.let { srcColorSpace -> + val dstColorSpace = ColorSpace.get(ColorSpace.Named.SRGB) + val connector = ColorSpace.connect(srcColorSpace, dstColorSpace) + if (config == Bitmap.Config.ARGB_8888) { + if (srcColorSpace != dstColorSpace) { + argb8888toArgb8888(bytes, connector, end = byteCount) + } + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && config == Bitmap.Config.RGBA_1010102) { + rgba1010102toArgb8888(bytes, connector, end = byteCount) + } + } + } + + // should not be called before accessing color space or other properties + if (recycle) this.recycle() + + return bytes + } catch (e: Exception) { + Log.e(LOG_TAG, "failed to get bytes from bitmap", e) + } + return null + } + + suspend fun Bitmap.getEncodedBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean): ByteArray? { val stream: ByteArrayOutputStream mutex.withLock { // this method is called a lot, so we try and reuse output streams @@ -101,37 +172,59 @@ object BitmapUtils { return null } + @RequiresApi(Build.VERSION_CODES.O) + private fun argb8888toArgb8888(bytes: ByteArray, connector: ColorSpace.Connector, start: Int = 0, end: Int = bytes.size) { + // unpacking from ARGB_8888 and packing to ARGB_8888 + // stored as [3,2,1,0] -> [AAAAAAAA BBBBBBBB GGGGGGGG RRRRRRRR] + for (i in start.. [AABBBBBB BBBBGGGG GGGGGGRR RRRRRRRR] val i3 = bytes[i + 3].toInt() val i2 = bytes[i + 2].toInt() val i1 = bytes[i + 1].toInt() val i0 = bytes[i].toInt() - // unpacking from RGBA_1010102 - // stored as [3,2,1,0] -> [AABBBBBB BBBBGGGG GGGGGGRR RRRRRRRR] -// val iA = ((i3 and 0xc0) shr 6) + val iA = ((i3 and 0xc0) shr 6) val iB = ((i3 and 0x3f) shl 4) or ((i2 and 0xf0) shr 4) val iG = ((i2 and 0x0f) shl 6) or ((i1 and 0xfc) shr 2) val iR = ((i1 and 0x03) shl 8) or ((i0 and 0xff) shr 0) // components as floats in sRGB - val srgbFloats = connector.transform(iR / max10Bits, iG / max10Bits, iB / max10Bits) + val srgbFloats = connector.transform(iR / MAX_10_BITS_FLOAT, iG / MAX_10_BITS_FLOAT, iB / MAX_10_BITS_FLOAT) val srgbR = (srgbFloats[0] * 255.0f + 0.5f).toInt() val srgbG = (srgbFloats[1] * 255.0f + 0.5f).toInt() val srgbB = (srgbFloats[2] * 255.0f + 0.5f).toInt() + val alpha = (iA * alphaFactor + 0.5f).toInt() // packing to ARGB_8888 // stored as [3,2,1,0] -> [AAAAAAAA BBBBBBBB GGGGGGGG RRRRRRRR] - bytes[i + 3] = dstAlpha + bytes[i + 3] = alpha.toByte() bytes[i + 2] = srgbB.toByte() bytes[i + 1] = srgbG.toByte() bytes[i] = srgbR.toByte() diff --git a/lib/image_providers/region_provider.dart b/lib/image_providers/region_provider.dart index dc85ccaa1..89be31014 100644 --- a/lib/image_providers/region_provider.dart +++ b/lib/image_providers/region_provider.dart @@ -48,8 +48,21 @@ class RegionProvider extends ImageProvider { if (bytes.isEmpty) { throw StateError('$uri ($mimeType) region loading failed'); } + + final trailerOffset = bytes.length - 4 * 2; + final trailer = ByteData.sublistView(bytes, trailerOffset); + final bitmapWidth = trailer.getUint32(0); + final bitmapHeight = trailer.getUint32(4); + final buffer = await ui.ImmutableBuffer.fromUint8List(bytes); - return await decode(buffer); + final descriptor = ui.ImageDescriptor.raw( + buffer, + width: bitmapWidth, + height: bitmapHeight, + pixelFormat: ui.PixelFormat.rgba8888, + ); + + return descriptor.instantiateCodec(); } catch (error) { // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF) debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error'); diff --git a/lib/utils/math_utils.dart b/lib/utils/math_utils.dart index 68a3f1ec9..5a5750628 100644 --- a/lib/utils/math_utils.dart +++ b/lib/utils/math_utils.dart @@ -4,7 +4,8 @@ import 'dart:ui'; int highestPowerOf2(num x) => x < 1 ? 0 : pow(2, (log(x) / ln2).floor()).toInt(); num smallestPowerOf2(num x, {bool allowNegativePower = false}) { - return x < 1 && !allowNegativePower ? 1 : pow(2, (log(x) / ln2).ceil()); + if ((x < 1 && !allowNegativePower) || x <= 0) return 1; + return pow(2, (log(x) / ln2).ceil()); } double roundToPrecision(final double value, {required final int decimals}) => (value * pow(10, decimals)).round() / pow(10, decimals); diff --git a/test/utils/math_utils_test.dart b/test/utils/math_utils_test.dart index 7d5989d42..d69871138 100644 --- a/test/utils/math_utils_test.dart +++ b/test/utils/math_utils_test.dart @@ -22,6 +22,7 @@ void main() { expect(smallestPowerOf2(1.5), 2); expect(smallestPowerOf2(0.5, allowNegativePower: true), 0.5); expect(smallestPowerOf2(0.1, allowNegativePower: true), 0.125); + expect(smallestPowerOf2(0, allowNegativePower: true), 1); }); test('rounding to a given precision after the decimal', () {