thumbnail decoding: use raw image descriptor in Flutter on decoded bytes from Android

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaFetchBytesHandler.kt

@@ -4,7 +4,6 @@ import android.content.Context
 import android.graphics.Rect
 import androidx.core.net.toUri
 import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
-import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
 import deckers.thibault.aves.channel.calls.fetchers.RegionFetcher
 import deckers.thibault.aves.channel.calls.fetchers.SvgRegionFetcher
 import deckers.thibault.aves.channel.calls.fetchers.ThumbnailFetcher
@@ -29,13 +28,13 @@ class MediaFetchBytesHandler(private val context: Context) : MethodCallHandler {
 
     override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
         when (call.method) {
-            "getThumbnail" -> ioScope.launch { safeSuspend(call, result, ::getThumbnail) }
+            "getThumbnail" -> ioScope.launch { safe(call, result, ::getThumbnail) }
             "getRegion" -> ioScope.launch { safe(call, result, ::getRegion) }
             else -> result.notImplemented()
         }
     }
 
-    private suspend fun getThumbnail(call: MethodCall, result: MethodChannel.Result) {
+    private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) {
         val uri = call.argument<String>(EntryFields.URI)
         val mimeType = call.argument<String>(EntryFields.MIME_TYPE)
         val dateModifiedMillis = call.argument<Number>(EntryFields.DATE_MODIFIED_MILLIS)?.toLong()

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt

@@ -15,9 +15,8 @@ import com.bumptech.glide.request.RequestOptions
 import com.bumptech.glide.signature.ObjectKey
 import deckers.thibault.aves.decoder.AvesAppGlideModule
 import deckers.thibault.aves.decoder.MultiPageImage
-import deckers.thibault.aves.utils.BitmapUtils
 import deckers.thibault.aves.utils.BitmapUtils.applyExifOrientation
-import deckers.thibault.aves.utils.BitmapUtils.getEncodedBytes
+import deckers.thibault.aves.utils.BitmapUtils.getDecodedBytes
 import deckers.thibault.aves.utils.MimeTypes
 import deckers.thibault.aves.utils.MimeTypes.SVG
 import deckers.thibault.aves.utils.MimeTypes.isVideo
@@ -49,7 +48,7 @@ class ThumbnailFetcher internal constructor(
     private val multiPageFetch = pageId != null && MultiPageImage.isSupported(mimeType)
     private val customFetch = svgFetch || tiffFetch || multiPageFetch
 
-    suspend fun fetch() {
+    fun fetch() {
         var bitmap: Bitmap? = null
         var exception: Exception? = null
 
@@ -78,13 +77,8 @@ class ThumbnailFetcher internal constructor(
             }
         }
 
-        if (bitmap != null) {
+        val bytes = bitmap?.getDecodedBytes(recycle = false)
-            val canHaveAlpha = MimeTypes.canHaveAlpha(mimeType)
+        if (bytes != null) {
-            val recycle = false
-            var bytes = bitmap.getEncodedBytes(canHaveAlpha, quality, recycle)
-            if (bytes != null && bytes.isEmpty()) {
-                bytes = BitmapUtils.tryPixelFormatConversion(bitmap)?.getEncodedBytes(canHaveAlpha, quality, recycle)
-            }
             result.success(bytes)
         } else {
             var errorDetails: String? = exception?.message

lib/image_providers/region_provider.dart

@@ -3,6 +3,7 @@ import 'dart:math';
 import 'dart:ui' as ui;
 
 import 'package:aves/services/common/services.dart';
+import 'package:aves_report/aves_report.dart';
 import 'package:equatable/equatable.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/widgets.dart';
@@ -33,7 +34,7 @@ class RegionProvider extends ImageProvider<RegionProviderKey> {
     final mimeType = key.mimeType;
     final pageId = key.pageId;
     try {
-      final bytes = await mediaFetchService.getRegion(
+      final descriptor = await mediaFetchService.getRegion(
         uri,
         mimeType,
         key.rotationDegrees,
@@ -45,28 +46,14 @@ class RegionProvider extends ImageProvider<RegionProviderKey> {
         sizeBytes: key.sizeBytes,
         taskKey: key,
       );
-      if (bytes.isEmpty) {
+      if (descriptor == null) {
-        throw StateError('$uri ($mimeType) region loading failed');
+        throw UnreportedStateError('$uri ($mimeType) region loading failed');
       }
-
-      final trailerOffset = bytes.length - 4 * 2;
-      final trailer = ByteData.sublistView(bytes, trailerOffset);
-      final bitmapWidth = trailer.getUint32(0);
-      final bitmapHeight = trailer.getUint32(4);
-
-      final buffer = await ui.ImmutableBuffer.fromUint8List(bytes);
-      final descriptor = ui.ImageDescriptor.raw(
-        buffer,
-        width: bitmapWidth,
-        height: bitmapHeight,
-        pixelFormat: ui.PixelFormat.rgba8888,
-      );
-
       return descriptor.instantiateCodec();
     } catch (error) {
       // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF)
       debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error');
-      throw StateError('$mimeType region decoding failed (page $pageId)');
+      throw UnreportedStateError('$mimeType region decoding failed (page $pageId)');
     }
   }
 

lib/image_providers/thumbnail_provider.dart

@@ -36,7 +36,7 @@ class ThumbnailProvider extends ImageProvider<ThumbnailProviderKey> {
     final mimeType = key.mimeType;
     final pageId = key.pageId;
     try {
-      final bytes = await mediaFetchService.getThumbnail(
+      final descriptor = await mediaFetchService.getThumbnail(
         uri: uri,
         mimeType: mimeType,
         pageId: pageId,
@@ -46,15 +46,14 @@ class ThumbnailProvider extends ImageProvider<ThumbnailProviderKey> {
         extent: key.extent,
         taskKey: key,
       );
-      if (bytes.isEmpty) {
+      if (descriptor == null) {
-        throw UnreportedStateError('$uri ($mimeType) loading failed');
+        throw UnreportedStateError('$uri ($mimeType) thumbnail loading failed');
       }
-      final buffer = await ui.ImmutableBuffer.fromUint8List(bytes);
+      return descriptor.instantiateCodec();
-      return await decode(buffer);
     } catch (error) {
       // loading may fail if the provided MIME type is incorrect (e.g. the Media Store may report a JPEG as a TIFF)
       debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error');
-      throw UnreportedStateError('$mimeType decoding failed (page $pageId)');
+      throw UnreportedStateError('$mimeType thumbnail decoding failed (page $pageId)');
     }
   }
 

lib/services/app_service.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:ui';
 
 import 'package:aves/geo/uri.dart';
 import 'package:aves/model/app_inventory.dart';
@@ -219,7 +220,7 @@ class PlatformAppService implements AppService {
     Uint8List? iconBytes;
     if (coverEntry != null) {
       final size = coverEntry.isVideo ? 0.0 : 256.0;
-      iconBytes = await mediaFetchService.getThumbnail(
+      final iconDescriptor = await mediaFetchService.getThumbnail(
         uri: coverEntry.uri,
         mimeType: coverEntry.mimeType,
         pageId: coverEntry.pageId,
@@ -228,6 +229,12 @@ class PlatformAppService implements AppService {
         dateModifiedMillis: coverEntry.dateModifiedMillis,
         extent: size,
       );
+      if (iconDescriptor != null) {
+        final codec = await iconDescriptor.instantiateCodec();
+        final frameInfo = await codec.getNextFrame();
+        final byteData = await frameInfo.image.toByteData(format: ImageByteFormat.png);
+        iconBytes = byteData?.buffer.asUint8List();
+      }
     }
     try {
       await _platform.invokeMethod('pinShortcut', <String, dynamic>{

lib/services/media/media_fetch_service.dart

@@ -11,6 +11,7 @@ import 'package:aves/services/media/byte_receiving_codec.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:streams_channel/streams_channel.dart';
+import 'dart:ui' as ui;
 
 abstract class MediaFetchService {
   Future<AvesEntry?> getEntry(String uri, String? mimeType, {bool allowUnsized = false});
@@ -33,7 +34,7 @@ abstract class MediaFetchService {
   });
 
   // `rect`: region to decode, with coordinates in reference to `imageSize`
-  Future<Uint8List> getRegion(
+  Future<ui.ImageDescriptor?> getRegion(
     String uri,
     String mimeType,
     int rotationDegrees,
@@ -47,7 +48,7 @@ abstract class MediaFetchService {
     int? priority,
   });
 
-  Future<Uint8List> getThumbnail({
+  Future<ui.ImageDescriptor?> getThumbnail({
     required String uri,
     required String mimeType,
     required int rotationDegrees,
@@ -162,7 +163,7 @@ class PlatformMediaFetchService implements MediaFetchService {
   }
 
   @override
-  Future<Uint8List> getRegion(
+  Future<ui.ImageDescriptor?> getRegion(
     String uri,
     String mimeType,
     int rotationDegrees,
@@ -191,13 +192,16 @@ class PlatformMediaFetchService implements MediaFetchService {
             'imageWidth': imageSize.width.toInt(),
             'imageHeight': imageSize.height.toInt(),
           });
-          if (result != null) return result as Uint8List;
+          if (result != null) {
+            final bytes = result as Uint8List;
+            return _bytesToCodec(bytes);
+          }
         } on PlatformException catch (e, stack) {
           if (_isUnknownVisual(mimeType)) {
             await reportService.recordError(e, stack);
           }
         }
-        return Uint8List(0);
+        return null;
       },
       priority: priority ?? ServiceCallPriority.getRegion,
       key: taskKey,
@@ -205,7 +209,7 @@ class PlatformMediaFetchService implements MediaFetchService {
   }
 
   @override
-  Future<Uint8List> getThumbnail({
+  Future<ui.ImageDescriptor?> getThumbnail({
     required String uri,
     required String mimeType,
     required int rotationDegrees,
@@ -231,13 +235,16 @@ class PlatformMediaFetchService implements MediaFetchService {
             'defaultSizeDip': _thumbnailDefaultSize,
             'quality': 100,
           });
-          if (result != null) return result as Uint8List;
+          if (result != null) {
+            final bytes = result as Uint8List;
+            return _bytesToCodec(bytes);
+          }
         } on PlatformException catch (e, stack) {
           if (_isUnknownVisual(mimeType)) {
             await reportService.recordError(e, stack);
           }
         }
-        return Uint8List(0);
+        return null;
       },
       priority: priority ?? (extent == 0 ? ServiceCallPriority.getFastThumbnail : ServiceCallPriority.getSizedThumbnail),
       key: taskKey,
@@ -264,6 +271,24 @@ class PlatformMediaFetchService implements MediaFetchService {
 
   // convenience methods
 
+  Future<ui.ImageDescriptor?> _bytesToCodec(Uint8List bytes) async {
+    const trailerLength = 4 * 2;
+    if (bytes.length < trailerLength) return null;
+
+    final trailerOffset = bytes.length - trailerLength;
+    final trailer = ByteData.sublistView(bytes, trailerOffset);
+    final bitmapWidth = trailer.getUint32(0);
+    final bitmapHeight = trailer.getUint32(4);
+
+    final buffer = await ui.ImmutableBuffer.fromUint8List(bytes);
+    return ui.ImageDescriptor.raw(
+      buffer,
+      width: bitmapWidth,
+      height: bitmapHeight,
+      pixelFormat: ui.PixelFormat.rgba8888,
+    );
+  }
+
   bool _isUnknownVisual(String mimeType) => !_knownMediaTypes.contains(mimeType) && MimeTypes.isVisual(mimeType);
 
   static const Set<String> _knownOpaqueImages = {