From d4791df33335f150e9ce57f1c02181578d269c23 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Sun, 2 Mar 2025 18:05:44 +0100 Subject: [PATCH] thumbnail decoding: use raw image descriptor in Flutter on decoded bytes from Android --- .../channel/calls/MediaFetchBytesHandler.kt | 5 +-- .../calls/fetchers/ThumbnailFetcher.kt | 14 ++----- lib/image_providers/region_provider.dart | 23 +++-------- lib/image_providers/thumbnail_provider.dart | 11 +++-- lib/services/app_service.dart | 9 +++- lib/services/media/media_fetch_service.dart | 41 +++++++++++++++---- 6 files changed, 57 insertions(+), 46 deletions(-) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt index bd63c0f33..9286a80d0 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt @@ -4,7 +4,6 @@ import android.content.Context import android.graphics.Rect import androidx.core.net.toUri import deckers.thibault.aves.channel.calls.Coresult.Companion.safe -import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher import deckers.thibault.aves.channel.calls.fetchers.ThumbnailFetcher @@ -29,13 +28,13 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { when (call.method) { - "getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) } + "getThumbnail" -> ioScope.launch { safe(call, result, ::getThumbnail) } "getRegion" -> ioScope.launch { safe(call, result, ::getRegion) } else -> result.notImplemented() } } - private suspend fun getThumbnail(call: MethodCall, result: MethodChannel.Result) { + private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) { val uri = call.argument(EntryFields.URI) val mimeType = call.argument(EntryFields.MIME_TYPE) val dateModifiedMillis = call.argument(EntryFields.DATE_MODIFIED_MILLIS)?.toLong() diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt index 61d4208d5..e8739ba65 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt @@ -15,9 +15,8 @@ import com.bumptech.glide.request.RequestOptions import com.bumptech.glide.signature.ObjectKey import deckers.thibault.aves.decoder.AvesAppGlideModule import deckers.thibault.aves.decoder.MultiPageImage -import deckers.thibault.aves.utils.BitmapUtils import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation -import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes +import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.MimeTypes.SVG import deckers.thibault.aves.utils.MimeTypes.isVideo @@ -49,7 +48,7 @@ class ThumbnailFetcher internal constructor( private val multiPageFetch = pageId != null && MultiPageImage.isSupported(mimeType) private val customFetch = svgFetch || tiffFetch || multiPageFetch - suspend fun fetch() { + fun fetch() { var bitmap: Bitmap? = null var exception: Exception? = null @@ -78,13 +77,8 @@ class ThumbnailFetcher internal constructor( } } - if (bitmap != null) { - val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType) - val recycle = false - var bytes = bitmap.getEncodedBytes(canHaveAlpha, quality, recycle) - if (bytes != null && bytes.isEmpty()) { - bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, quality, recycle) - } + val bytes = bitmap?.getDecodedBytes(recycle = false) + if (bytes != null) { result.success(bytes) } else { var errorDetails: String? = exception?.message diff --git a/lib/image_providers/region_provider.dart b/lib/image_providers/region_provider.dart index 89be31014..f52f8593b 100644 --- a/lib/image_providers/region_provider.dart +++ b/lib/image_providers/region_provider.dart @@ -3,6 +3,7 @@ import 'dart:math'; import 'dart:ui' as ui; import 'package:aves/services/common/services.dart'; +import 'package:aves_report/aves_report.dart'; import 'package:equatable/equatable.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; @@ -33,7 +34,7 @@ class RegionProvider extends ImageProvider { final mimeType = key.mimeType; final pageId = key.pageId; try { - final bytes = await mediaFetchService.getRegion( + final descriptor = await mediaFetchService.getRegion( uri, mimeType, key.rotationDegrees, @@ -45,28 +46,14 @@ class RegionProvider extends ImageProvider { sizeBytes: key.sizeBytes, taskKey: key, ); - if (bytes.isEmpty) { - throw StateError('$uri ($mimeType) region loading failed'); + if (descriptor == null) { + throw UnreportedStateError('$uri ($mimeType) region loading failed'); } - - final trailerOffset = bytes.length - 4 * 2; - final trailer = ByteData.sublistView(bytes, trailerOffset); - final bitmapWidth = trailer.getUint32(0); - final bitmapHeight = trailer.getUint32(4); - - final buffer = await ui.ImmutableBuffer.fromUint8List(bytes); - final descriptor = ui.ImageDescriptor.raw( - buffer, - width: bitmapWidth, - height: bitmapHeight, - pixelFormat: ui.PixelFormat.rgba8888, - ); - return descriptor.instantiateCodec(); } catch (error) { // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF) debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error'); - throw StateError('$mimeType region decoding failed (page $pageId)'); + throw UnreportedStateError('$mimeType region decoding failed (page $pageId)'); } } diff --git a/lib/image_providers/thumbnail_provider.dart b/lib/image_providers/thumbnail_provider.dart index ee36a0a28..cee22ab83 100644 --- a/lib/image_providers/thumbnail_provider.dart +++ b/lib/image_providers/thumbnail_provider.dart @@ -36,7 +36,7 @@ class ThumbnailProvider extends ImageProvider { final mimeType = key.mimeType; final pageId = key.pageId; try { - final bytes = await mediaFetchService.getThumbnail( + final descriptor = await mediaFetchService.getThumbnail( uri: uri, mimeType: mimeType, pageId: pageId, @@ -46,15 +46,14 @@ class ThumbnailProvider extends ImageProvider { extent: key.extent, taskKey: key, ); - if (bytes.isEmpty) { - throw UnreportedStateError('$uri ($mimeType) loading failed'); + if (descriptor == null) { + throw UnreportedStateError('$uri ($mimeType) thumbnail loading failed'); } - final buffer = await ui.ImmutableBuffer.fromUint8List(bytes); - return await decode(buffer); + return descriptor.instantiateCodec(); } catch (error) { // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF) debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error'); - throw UnreportedStateError('$mimeType decoding failed (page $pageId)'); + throw UnreportedStateError('$mimeType thumbnail decoding failed (page $pageId)'); } } diff --git a/lib/services/app_service.dart b/lib/services/app_service.dart index 78ffde0d0..dd0c87180 100644 --- a/lib/services/app_service.dart +++ b/lib/services/app_service.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:ui'; import 'package:aves/geo/uri.dart'; import 'package:aves/model/app_inventory.dart'; @@ -219,7 +220,7 @@ class PlatformAppService implements AppService { Uint8List? iconBytes; if (coverEntry != null) { final size = coverEntry.isVideo ? 0.0 : 256.0; - iconBytes = await mediaFetchService.getThumbnail( + final iconDescriptor = await mediaFetchService.getThumbnail( uri: coverEntry.uri, mimeType: coverEntry.mimeType, pageId: coverEntry.pageId, @@ -228,6 +229,12 @@ class PlatformAppService implements AppService { dateModifiedMillis: coverEntry.dateModifiedMillis, extent: size, ); + if (iconDescriptor != null) { + final codec = await iconDescriptor.instantiateCodec(); + final frameInfo = await codec.getNextFrame(); + final byteData = await frameInfo.image.toByteData(format: ImageByteFormat.png); + iconBytes = byteData?.buffer.asUint8List(); + } } try { await _platform.invokeMethod('pinShortcut', { diff --git a/lib/services/media/media_fetch_service.dart b/lib/services/media/media_fetch_service.dart index f1bb87eda..8e6052c0e 100644 --- a/lib/services/media/media_fetch_service.dart +++ b/lib/services/media/media_fetch_service.dart @@ -11,6 +11,7 @@ import 'package:aves/services/media/byte_receiving_codec.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:streams_channel/streams_channel.dart'; +import 'dart:ui' as ui; abstract class MediaFetchService { Future getEntry(String uri, String? mimeType, {bool allowUnsized = false}); @@ -33,7 +34,7 @@ abstract class MediaFetchService { }); // `rect`: region to decode, with coordinates in reference to `imageSize` - Future getRegion( + Future getRegion( String uri, String mimeType, int rotationDegrees, @@ -47,7 +48,7 @@ abstract class MediaFetchService { int? priority, }); - Future getThumbnail({ + Future getThumbnail({ required String uri, required String mimeType, required int rotationDegrees, @@ -162,7 +163,7 @@ class PlatformMediaFetchService implements MediaFetchService { } @override - Future getRegion( + Future getRegion( String uri, String mimeType, int rotationDegrees, @@ -191,13 +192,16 @@ class PlatformMediaFetchService implements MediaFetchService { 'imageWidth': imageSize.width.toInt(), 'imageHeight': imageSize.height.toInt(), }); - if (result != null) return result as Uint8List; + if (result != null) { + final bytes = result as Uint8List; + return _bytesToCodec(bytes); + } } on PlatformException catch (e, stack) { if (_isUnknownVisual(mimeType)) { await reportService.recordError(e, stack); } } - return Uint8List(0); + return null; }, priority: priority ?? ServiceCallPriority.getRegion, key: taskKey, @@ -205,7 +209,7 @@ class PlatformMediaFetchService implements MediaFetchService { } @override - Future getThumbnail({ + Future getThumbnail({ required String uri, required String mimeType, required int rotationDegrees, @@ -231,13 +235,16 @@ class PlatformMediaFetchService implements MediaFetchService { 'defaultSizeDip': _thumbnailDefaultSize, 'quality': 100, }); - if (result != null) return result as Uint8List; + if (result != null) { + final bytes = result as Uint8List; + return _bytesToCodec(bytes); + } } on PlatformException catch (e, stack) { if (_isUnknownVisual(mimeType)) { await reportService.recordError(e, stack); } } - return Uint8List(0); + return null; }, priority: priority ?? (extent == 0 ? ServiceCallPriority.getFastThumbnail : ServiceCallPriority.getSizedThumbnail), key: taskKey, @@ -264,6 +271,24 @@ class PlatformMediaFetchService implements MediaFetchService { // convenience methods + Future _bytesToCodec(Uint8List bytes) async { + const trailerLength = 4 * 2; + if (bytes.length < trailerLength) return null; + + final trailerOffset = bytes.length - trailerLength; + final trailer = ByteData.sublistView(bytes, trailerOffset); + final bitmapWidth = trailer.getUint32(0); + final bitmapHeight = trailer.getUint32(4); + + final buffer = await ui.ImmutableBuffer.fromUint8List(bytes); + return ui.ImageDescriptor.raw( + buffer, + width: bitmapWidth, + height: bitmapHeight, + pixelFormat: ui.PixelFormat.rgba8888, + ); + } + bool _isUnknownVisual(String mimeType) => !_knownMediaTypes.contains(mimeType) && MimeTypes.isVisual(mimeType); static const Set _knownOpaqueImages = {