Fabio/aves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

info: access to motion photo video, improved video metadata stream handling

Browse source
This commit is contained in:
Thibault Deckers 2021-04-23 17:09:22 +09:00
parent 7a66df5e37
commit 95b34b753b
20 changed files with 189 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
android/app/libs/fijkplayer-full-release.aar
View file

Binary file not shown.

45
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataHandler.kt
View file

@ -76,6 +76,7 @@ import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import org.beyka.tiffbitmapfactory.TiffBitmapFactory
import java.io.File
import java.io.InputStream
import java.text.ParseException
import java.util.*
import kotlin.math.roundToLong
@ -90,6 +91,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
"getPanoramaInfo" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getPanoramaInfo) }
"getContentResolverProp" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getContentResolverProp) }
"getExifThumbnails" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getExifThumbnails) }
"extractMotionPhotoVideo" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::extractMotionPhotoVideo) }
"extractVideoEmbeddedPicture" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::extractVideoEmbeddedPicture) }
"extractXmpDataProp" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::extractXmpDataProp) }
else -> result.notImplemented()
@ -775,6 +777,41 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
result.success(thumbnails)
}
private fun extractMotionPhotoVideo(call: MethodCall, result: MethodChannel.Result) {
val mimeType = call.argument<String>("mimeType")
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
val sizeBytes = call.argument<Number>("sizeBytes")?.toLong()
if (mimeType == null || uri == null || sizeBytes == null) {
result.error("extractMotionPhotoVideo-args", "failed because of missing arguments", null)
return
}
try {
Metadata.openSafeInputStream(context, uri, mimeType, sizeBytes)?.use { input ->
val metadata = ImageMetadataReader.readMetadata(input)
for (dir in metadata.getDirectoriesOfType(XmpDirectory::class.java)) {
val xmpMeta = dir.xmpMeta
// offset from end
var offsetFromEnd: Int? = null
xmpMeta.getSafeInt(XMP.GCAMERA_SCHEMA_NS, XMP.GCAMERA_VIDEO_OFFSET_PROP_NAME) { offsetFromEnd = it }
if (offsetFromEnd != null) {
StorageUtils.openInputStream(context, uri)?.let { original ->
original.skip(sizeBytes - offsetFromEnd!!)
copyEmbeddedBytes(result, MimeTypes.MP4, original)
}
return
}
}
}
} catch (e: Exception) {
Log.w(LOG_TAG, "failed to extract video from motion photo", e)
} catch (e: NoClassDefFoundError) {
Log.w(LOG_TAG, "failed to extract video from motion photo", e)
}
result.error("extractMotionPhotoVideo-empty", "failed to extract video from motion photo at uri=$uri", null)
}
private fun extractVideoEmbeddedPicture(call: MethodCall, result: MethodChannel.Result) {
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
if (uri == null) {
@ -794,7 +831,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
}
}
embedMimeType?.let { mime ->
copyEmbeddedBytes(bytes, mime, result)
copyEmbeddedBytes(result, mime, bytes.inputStream())
return
}
}
@ -843,7 +880,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
}
}
copyEmbeddedBytes(embedBytes, embedMimeType, result)
copyEmbeddedBytes(result, embedMimeType, embedBytes.inputStream())
return
} catch (e: XMPException) {
result.error("extractXmpDataProp-xmp", "failed to read XMP directory for uri=$uri prop=$dataPropPath", e.message)
@ -859,11 +896,11 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
result.error("extractXmpDataProp-empty", "failed to extract file from XMP uri=$uri prop=$dataPropPath", null)
}
private fun copyEmbeddedBytes(embedBytes: ByteArray, embedMimeType: String, result: MethodChannel.Result) {
private fun copyEmbeddedBytes(result: MethodChannel.Result, embedMimeType: String, embedByteStream: InputStream) {
val embedFile = File.createTempFile("aves", null, context.cacheDir).apply {
deleteOnExit()
outputStream().use { outputStream ->
embedBytes.inputStream().use { inputStream ->
embedByteStream.use { inputStream ->
inputStream.copyTo(outputStream)
}
}

6
android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt
View file

@ -42,6 +42,12 @@ object XMP {
fun isDataPath(path: String) = knownDataPaths.contains(path)
// motion photo
const val GCAMERA_SCHEMA_NS = "http://ns.google.com/photos/1.0/camera/"
const val GCAMERA_VIDEO_OFFSET_PROP_NAME = "GCamera:MicroVideoOffset"
// panorama
// cf https://developers.google.com/streetview/spherical-metadata

1
android/app/src/main/kotlin/deckers/thibault/aves/utils/MimeTypes.kt
View file

@ -39,6 +39,7 @@ object MimeTypes {
private const val MP2T = "video/mp2t"
private const val MP2TS = "video/mp2ts"
const val MP4 = "video/mp4"
private const val WEBM = "video/webm"
fun isImage(mimeType: String?) = mimeType != null && mimeType.startsWith(IMAGE)

3
lib/model/video/metadata.dart
View file

@ -115,7 +115,9 @@ class VideoMetadataFormatter {
save('Channel Layout', _formatChannelLayout(value));
break;
case Keys.codecName:
if (value != 'none') {
save('Format', _formatCodecName(value));
}
break;
case Keys.codecPixelFormat:
if (streamType == StreamTypes.video) {
@ -296,6 +298,7 @@ class VideoMetadataFormatter {
}
class StreamTypes {
static const attachment = 'attachment';
static const audio = 'audio';
static const metadata = 'metadata';
static const subtitle = 'subtitle';

1
lib/ref/xmp.dart
View file

@ -16,6 +16,7 @@ class XMP {
'GettyImagesGIFT': 'Getty Images',
'GIMP': 'GIMP',
'GCamera': 'Google Camera',
'GCreations': 'Google Creations',
'GFocus': 'Google Focus',
'GPano': 'Google Panorama',
'illustrator': 'Illustrator',

17
lib/services/metadata_service.dart
View file

@ -24,6 +24,8 @@ abstract class MetadataService {
Future<List<Uint8List>> getExifThumbnails(AvesEntry entry);
Future<Map> extractMotionPhotoVideo(AvesEntry entry);
Future<Map> extractVideoEmbeddedPicture(String uri);
Future<Map> extractXmpDataProp(AvesEntry entry, String propPath, String propMimeType);
@ -167,6 +169,21 @@ class PlatformMetadataService implements MetadataService {
return [];
}
@override
Future<Map> extractMotionPhotoVideo(AvesEntry entry) async {
try {
final result = await platform.invokeMethod('extractMotionPhotoVideo', <String, dynamic>{
'mimeType': entry.mimeType,
'uri': entry.uri,
'sizeBytes': entry.sizeBytes,
});
return result;
} on PlatformException catch (e) {
debugPrint('extractMotionPhotoVideo failed with code=${e.code}, exception=${e.message}, details=${e.details}');
}
return null;
}
@override
Future<Map> extractVideoEmbeddedPicture(String uri) async {
try {

3
lib/widgets/viewer/info/metadata/metadata_dir_tile.dart
View file

@ -121,6 +121,9 @@ class MetadataDirTile extends StatelessWidget with FeedbackMixin {
Future<void> _openEmbeddedData(BuildContext context, OpenEmbeddedDataNotification notification) async {
Map fields;
switch (notification.source) {
case EmbeddedDataSource.motionPhotoVideo:
fields = await metadataService.extractMotionPhotoVideo(entry);
break;
case EmbeddedDataSource.videoCover:
fields = await metadataService.extractVideoEmbeddedPicture(entry.uri);
break;

24
lib/widgets/viewer/info/metadata/metadata_section.dart
View file

@ -193,6 +193,8 @@ class _MetadataSectionSliverState extends State<MetadataSectionSliver> with Auto
String getTypeText(Map stream) {
final type = stream[Keys.streamType] ?? StreamTypes.unknown;
switch (type) {
case StreamTypes.attachment:
return 'Attachment';
case StreamTypes.audio:
return 'Audio';
case StreamTypes.metadata:
@ -209,8 +211,8 @@ class _MetadataSectionSliverState extends State<MetadataSectionSliver> with Auto
}
final allStreams = (mediaInfo[Keys.streams] as List).cast<Map>();
final unknownStreams = allStreams.where((stream) => stream[Keys.streamType] == StreamTypes.unknown).toList();
final knownStreams = allStreams.whereNot(unknownStreams.contains);
final attachmentStreams = allStreams.where((stream) => stream[Keys.streamType] == StreamTypes.attachment).toList();
final knownStreams = allStreams.whereNot(attachmentStreams.contains);
// display known streams as separate directories (e.g. video, audio, subs)
if (knownStreams.isNotEmpty) {
@ -228,18 +230,18 @@ class _MetadataSectionSliverState extends State<MetadataSectionSliver> with Auto
}
}
// display unknown streams as attachments (e.g. fonts)
if (unknownStreams.isNotEmpty) {
final unknownCodecCount = <String, List<String>>{};
for (final stream in unknownStreams) {
// group attachments by format (e.g. TTF fonts)
if (attachmentStreams.isNotEmpty) {
final formatCount = <String, List<String>>{};
for (final stream in attachmentStreams) {
final codec = (stream[Keys.codecName] as String ?? 'unknown').toUpperCase();
if (!unknownCodecCount.containsKey(codec)) {
unknownCodecCount[codec] = [];
if (!formatCount.containsKey(codec)) {
formatCount[codec] = [];
}
unknownCodecCount[codec].add(stream[Keys.filename]);
formatCount[codec].add(stream[Keys.filename]);
}
if (unknownCodecCount.isNotEmpty) {
final rawTags = unknownCodecCount.map((key, value) {
if (formatCount.isNotEmpty) {
final rawTags = formatCount.map((key, value) {
final count = value.length;
// remove duplicate names, so number of displayed names may not match displayed count
final names = value.where((v) => v != null).toSet().toList()..sort(compareAsciiUpperCase);

50
lib/widgets/viewer/info/metadata/xmp_namespaces.dart
View file

@ -4,21 +4,61 @@ import 'package:aves/utils/constants.dart';
import 'package:aves/utils/string_utils.dart';
import 'package:aves/widgets/common/identity/highlight_title.dart';
import 'package:aves/widgets/viewer/info/common.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/exif.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/google.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/iptc.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/mwg.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/photoshop.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/tiff.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/xmp.dart';
import 'package:collection/collection.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
class XmpNamespace {
final String namespace;
final Map<String, String> rawProps;
const XmpNamespace(this.namespace);
const XmpNamespace(this.namespace, this.rawProps);
factory XmpNamespace.create(String namespace, Map<String, String> rawProps) {
switch (namespace) {
case XmpBasicNamespace.ns:
return XmpBasicNamespace(rawProps);
case XmpExifNamespace.ns:
return XmpExifNamespace(rawProps);
case XmpGAudioNamespace.ns:
return XmpGAudioNamespace(rawProps);
case XmpGCameraNamespace.ns:
return XmpGCameraNamespace(rawProps);
case XmpGDepthNamespace.ns:
return XmpGDepthNamespace(rawProps);
case XmpGImageNamespace.ns:
return XmpGImageNamespace(rawProps);
case XmpIptcCoreNamespace.ns:
return XmpIptcCoreNamespace(rawProps);
case XmpMgwRegionsNamespace.ns:
return XmpMgwRegionsNamespace(rawProps);
case XmpMMNamespace.ns:
return XmpMMNamespace(rawProps);
case XmpNoteNamespace.ns:
return XmpNoteNamespace(rawProps);
case XmpPhotoshopNamespace.ns:
return XmpPhotoshopNamespace(rawProps);
case XmpTiffNamespace.ns:
return XmpTiffNamespace(rawProps);
default:
return XmpNamespace(namespace, rawProps);
}
}
String get displayTitle => XMP.namespaces[namespace] ?? namespace;
List<Widget> buildNamespaceSection({
@required List<MapEntry<String, String>> rawProps,
}) {
final props = rawProps
Map<String, String> get buildProps => rawProps;
List<Widget> buildNamespaceSection() {
final props = buildProps
.entries
.map((kv) {
final prop = XmpProp(kv.key, kv.value);
return extractData(prop) ? null : prop;

2
lib/widgets/viewer/info/metadata/xmp_ns/exif.dart
View file

@ -5,7 +5,7 @@ import 'package:aves/widgets/viewer/info/metadata/xmp_namespaces.dart';
class XmpExifNamespace extends XmpNamespace {
static const ns = 'exif';
XmpExifNamespace() : super(ns);
XmpExifNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'Exif';

40
lib/widgets/viewer/info/metadata/xmp_ns/google.dart
View file

@ -5,7 +5,7 @@ import 'package:aves/widgets/viewer/info/notifications.dart';
import 'package:tuple/tuple.dart';
abstract class XmpGoogleNamespace extends XmpNamespace {
XmpGoogleNamespace(String ns) : super(ns);
XmpGoogleNamespace(String ns, Map<String, String> rawProps) : super(ns, rawProps);
List<Tuple2<String, String>> get dataProps;
@ -34,7 +34,7 @@ abstract class XmpGoogleNamespace extends XmpNamespace {
class XmpGAudioNamespace extends XmpGoogleNamespace {
static const ns = 'GAudio';
XmpGAudioNamespace() : super(ns);
XmpGAudioNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
List<Tuple2<String, String>> get dataProps => [Tuple2('$ns:Data', '$ns:Mime')];
@ -46,7 +46,7 @@ class XmpGAudioNamespace extends XmpGoogleNamespace {
class XmpGDepthNamespace extends XmpGoogleNamespace {
static const ns = 'GDepth';
XmpGDepthNamespace() : super(ns);
XmpGDepthNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
List<Tuple2<String, String>> get dataProps => [
@ -61,7 +61,7 @@ class XmpGDepthNamespace extends XmpGoogleNamespace {
class XmpGImageNamespace extends XmpGoogleNamespace {
static const ns = 'GImage';
XmpGImageNamespace() : super(ns);
XmpGImageNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
List<Tuple2<String, String>> get dataProps => [Tuple2('$ns:Data', '$ns:Mime')];
@ -69,3 +69,35 @@ class XmpGImageNamespace extends XmpGoogleNamespace {
@override
String get displayTitle => 'Google Image';
}
class XmpGCameraNamespace extends XmpNamespace {
static const ns = 'GCamera';
static const videoOffsetKey = 'GCamera:MicroVideoOffset';
static const videoDataKey = 'Data';
bool _isMotionPhoto;
XmpGCameraNamespace(Map<String, String> rawProps) : super(ns, rawProps) {
_isMotionPhoto = rawProps.keys.any((key) => key == videoOffsetKey);
}
@override
Map<String, String> get buildProps {
return _isMotionPhoto
? Map.fromEntries({
MapEntry(videoDataKey, '[skipped]'),
...rawProps.entries,
})
: rawProps;
}
@override
Map<String, InfoLinkHandler> linkifyValues(List<XmpProp> props) {
return {
videoDataKey: InfoLinkHandler(
linkText: (context) => context.l10n.viewerInfoOpenLinkText,
onTap: (context) => OpenEmbeddedDataNotification.motionPhotoVideo().dispatch(context),
),
};
}
}

2
lib/widgets/viewer/info/metadata/xmp_ns/iptc.dart
View file

@ -9,7 +9,7 @@ class XmpIptcCoreNamespace extends XmpNamespace {
final creatorContactInfo = <String, String>{};
XmpIptcCoreNamespace() : super(ns);
XmpIptcCoreNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'IPTC Core';

2
lib/widgets/viewer/info/metadata/xmp_ns/mwg.dart
View file

@ -12,7 +12,7 @@ class XmpMgwRegionsNamespace extends XmpNamespace {
final dimensions = <String, String>{};
final regionList = <int, Map<String, String>>{};
XmpMgwRegionsNamespace() : super(ns);
XmpMgwRegionsNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'Regions';

2
lib/widgets/viewer/info/metadata/xmp_ns/photoshop.dart
View file

@ -5,7 +5,7 @@ import 'package:aves/widgets/viewer/info/metadata/xmp_namespaces.dart';
class XmpPhotoshopNamespace extends XmpNamespace {
static const ns = 'photoshop';
XmpPhotoshopNamespace() : super(ns);
XmpPhotoshopNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'Photoshop';

2
lib/widgets/viewer/info/metadata/xmp_ns/tiff.dart
View file

@ -8,7 +8,7 @@ class XmpTiffNamespace extends XmpNamespace {
@override
String get displayTitle => 'TIFF';
XmpTiffNamespace() : super(ns);
XmpTiffNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String formatValue(XmpProp prop) {

6
lib/widgets/viewer/info/metadata/xmp_ns/xmp.dart
View file

@ -14,7 +14,7 @@ class XmpBasicNamespace extends XmpNamespace {
final thumbnails = <int, Map<String, String>>{};
XmpBasicNamespace() : super(ns);
XmpBasicNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'Basic';
@ -61,7 +61,7 @@ class XmpMMNamespace extends XmpNamespace {
final ingredients = <int, Map<String, String>>{};
final pantry = <int, Map<String, String>>{};
XmpMMNamespace() : super(ns);
XmpMMNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
String get displayTitle => 'Media Management';
@ -114,7 +114,7 @@ class XmpNoteNamespace extends XmpNamespace {
// `xmpNote:HasExtendedXMP` is structural and should not be displayed to users
static const hasExtendedXmp = '$ns:HasExtendedXMP';
XmpNoteNamespace() : super(ns);
XmpNoteNamespace(Map<String, String> rawProps) : super(ns, rawProps);
@override
bool extractData(XmpProp prop) {

48
lib/widgets/viewer/info/metadata/xmp_tile.dart
View file

@ -4,13 +4,6 @@ import 'package:aves/model/entry.dart';
import 'package:aves/ref/xmp.dart';
import 'package:aves/widgets/common/identity/aves_expansion_tile.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_namespaces.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/exif.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/google.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/iptc.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/mwg.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/photoshop.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/tiff.dart';
import 'package:aves/widgets/viewer/info/metadata/xmp_ns/xmp.dart';
import 'package:collection/collection.dart';
import 'package:flutter/material.dart';
@ -36,40 +29,13 @@ class _XmpDirTileState extends State<XmpDirTile> {
@override
Widget build(BuildContext context) {
final sections = SplayTreeMap<XmpNamespace, List<MapEntry<String, String>>>.of(
groupBy(widget.tags.entries, (kv) {
final sections = groupBy(widget.tags.entries, (kv) {
final fullKey = kv.key;
final i = fullKey.indexOf(XMP.propNamespaceSeparator);
final namespace = i == -1 ? '' : fullKey.substring(0, i);
switch (namespace) {
case XmpBasicNamespace.ns:
return XmpBasicNamespace();
case XmpExifNamespace.ns:
return XmpExifNamespace();
case XmpGAudioNamespace.ns:
return XmpGAudioNamespace();
case XmpGDepthNamespace.ns:
return XmpGDepthNamespace();
case XmpGImageNamespace.ns:
return XmpGImageNamespace();
case XmpIptcCoreNamespace.ns:
return XmpIptcCoreNamespace();
case XmpMgwRegionsNamespace.ns:
return XmpMgwRegionsNamespace();
case XmpMMNamespace.ns:
return XmpMMNamespace();
case XmpNoteNamespace.ns:
return XmpNoteNamespace();
case XmpPhotoshopNamespace.ns:
return XmpPhotoshopNamespace();
case XmpTiffNamespace.ns:
return XmpTiffNamespace();
default:
return XmpNamespace(namespace);
}
}),
(a, b) => compareAsciiUpperCase(a.displayTitle, b.displayTitle),
);
return namespace;
}).entries.map((kv) => XmpNamespace.create(kv.key, Map.fromEntries(kv.value))).toList()
..sort((a, b) => compareAsciiUpperCase(a.displayTitle, b.displayTitle));
return AvesExpansionTile(
title: 'XMP',
expandedNotifier: widget.expandedNotifier,
@ -79,11 +45,7 @@ class _XmpDirTileState extends State<XmpDirTile> {
padding: EdgeInsets.only(left: 8, right: 8, bottom: 8),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: sections.entries
.expand((kv) => kv.key.buildNamespaceSection(
rawProps: kv.value,
))
.toList(),
children: sections.expand((section) => section.buildNamespaceSection()).toList(),
),
),
],

6
lib/widgets/viewer/info/notifications.dart
View file

@ -28,7 +28,7 @@ class OpenTempEntryNotification extends Notification {
String toString() => '$runtimeType#${shortHash(this)}{entry=$entry}';
}
enum EmbeddedDataSource { videoCover, xmp }
enum EmbeddedDataSource { motionPhotoVideo, videoCover, xmp }
class OpenEmbeddedDataNotification extends Notification {
final EmbeddedDataSource source;
@ -41,6 +41,10 @@ class OpenEmbeddedDataNotification extends Notification {
this.mimeType,
});
factory OpenEmbeddedDataNotification.motionPhotoVideo() => OpenEmbeddedDataNotification._private(
source: EmbeddedDataSource.motionPhotoVideo,
);
factory OpenEmbeddedDataNotification.videoCover() => OpenEmbeddedDataNotification._private(
source: EmbeddedDataSource.videoCover,
);

2
pubspec.lock
View file

@ -197,7 +197,7 @@ packages:
description:
path: "."
ref: aves
resolved-ref: "0f25874db46d1af6fcfbeb8722915cbc211a10fb"
resolved-ref: "0100934c469f35f575f3f84e17116f13c326f393"
url: "git://github.com/deckerst/fijkplayer.git"
source: git
version: "0.8.7"