From 83530649454ed412c8f3e59a9f381c9bc14c108c Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Sun, 9 Feb 2025 19:47:35 +0100 Subject: [PATCH] identify video location from Apple QuickTime metadata, and 3GPP `loci` atom --- CHANGELOG.md | 1 + .../channel/calls/MetadataFetchHandler.kt | 40 +++++++++++++++++-- .../thibault/aves/metadata/Mp4ParserHelper.kt | 37 +++++++++++------ 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0fb08d7e..abd7cddb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - support for Samsung HEIC motion photos embedding video in sefd box +- Cataloguing: identify video location from Apple QuickTime metadata, and 3GPP `loci` atom ### Fixed diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt index d39452bc5..edccde577 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt @@ -24,6 +24,7 @@ import com.drew.metadata.exif.GpsDirectory import com.drew.metadata.file.FileTypeDirectory import com.drew.metadata.gif.GifAnimationDirectory import com.drew.metadata.iptc.IptcDirectory +import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory import com.drew.metadata.mp4.media.Mp4UuidBoxDirectory import com.drew.metadata.png.PngDirectory import com.drew.metadata.webp.WebpDirectory @@ -102,6 +103,8 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.launch import org.json.JSONObject +import org.mp4parser.boxes.threegpp.ts26244.LocationInformationBox +import org.mp4parser.tools.Path import java.nio.charset.StandardCharsets import java.text.DecimalFormat import java.text.ParseException @@ -450,9 +453,8 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { if (isVideo(mimeType)) { // `metadata-extractor` do not extract custom tags in user data box - val userDataDir = Mp4ParserHelper.getUserData(context, mimeType, uri) - if (userDataDir.isNotEmpty()) { - metadataMap[Metadata.DIR_MP4_USER_DATA] = userDataDir + Mp4ParserHelper.getUserDataBox(context, mimeType, uri)?.let { box -> + metadataMap[Metadata.DIR_MP4_USER_DATA] = Mp4ParserHelper.extractBoxFields(box) } // this is used as fallback when the video metadata cannot be found on the Dart side @@ -544,8 +546,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { val metadataMap = HashMap() getCatalogMetadataByMetadataExtractor(mimeType, uri, path, sizeBytes, metadataMap) + if (isVideo(mimeType) || isHeic(mimeType)) { getMultimediaCatalogMetadataByMediaMetadataRetriever(mimeType, uri, metadataMap) + + // fallback to MP4 `loci` box for location + if (!metadataMap.contains(KEY_LATITUDE) || !metadataMap.contains(KEY_LONGITUDE)) { + Mp4ParserHelper.getUserDataBox(context, mimeType, uri)?.let { userDataBox -> + Path.getPath(userDataBox, LocationInformationBox.TYPE)?.let { locationBox -> + if (!locationBox.isParsed) { + locationBox.parseDetails() + } + metadataMap[KEY_LATITUDE] = locationBox.latitude + metadataMap[KEY_LONGITUDE] = locationBox.longitude + } + } + } } if (isHeic(mimeType)) { @@ -710,6 +726,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } + if (!metadataMap.containsKey(KEY_LATITUDE) || !metadataMap.containsKey(KEY_LONGITUDE)) { + for (dir in metadata.getDirectoriesOfType(QuickTimeMetadataDirectory::class.java)) { + dir.getSafeString(QuickTimeMetadataDirectory.TAG_LOCATION_ISO6709) { locationString -> + val matcher = Metadata.VIDEO_LOCATION_PATTERN.matcher(locationString) + if (matcher.find() && matcher.groupCount() >= 2) { + val latitude = matcher.group(1)?.toDoubleOrNull() + val longitude = matcher.group(2)?.toDoubleOrNull() + if (latitude != null && longitude != null) { + metadataMap[KEY_LATITUDE] = latitude + metadataMap[KEY_LONGITUDE] = longitude + } + } + } + } + } + when (mimeType) { MimeTypes.PNG -> { // date fallback to PNG time chunk @@ -854,7 +886,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { retriever.getSafeDateMillis(MediaMetadataRetriever.METADATA_KEY_DATE) { metadataMap[KEY_DATE_MILLIS] = it } } - if (!metadataMap.containsKey(KEY_LATITUDE)) { + if (!metadataMap.containsKey(KEY_LATITUDE) || !metadataMap.containsKey(KEY_LONGITUDE)) { val locationString = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION) if (locationString != null) { val matcher = Metadata.VIDEO_LOCATION_PATTERN.matcher(locationString) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt index 8ffc7886d..25bac11ec 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt @@ -32,6 +32,7 @@ import org.mp4parser.boxes.iso14496.part12.SegmentIndexBox import org.mp4parser.boxes.iso14496.part12.TrackHeaderBox import org.mp4parser.boxes.iso14496.part12.UserDataBox import org.mp4parser.boxes.threegpp.ts26244.AuthorBox +import org.mp4parser.boxes.threegpp.ts26244.LocationInformationBox import org.mp4parser.support.AbstractBox import org.mp4parser.support.Matrix import org.mp4parser.tools.Path @@ -143,6 +144,7 @@ object Mp4ParserHelper { return false } + // returns the offset and data of the Samsung maker notes box fun getSamsungSefd(context: Context, uri: Uri): Pair? { try { // we can skip uninteresting boxes with a seekable data source @@ -315,13 +317,13 @@ object Mp4ParserHelper { } } - fun getUserData( + fun getUserDataBox( context: Context, mimeType: String, uri: Uri, - ): MutableMap { - val fields = HashMap() - if (mimeType != MimeTypes.MP4) return fields + ): UserDataBox? { + if (mimeType != MimeTypes.MP4) return null + try { // we can skip uninteresting boxes with a seekable data source val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri") @@ -330,10 +332,7 @@ object Mp4ParserHelper { stream.channel.use { channel -> // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device` IsoFile(channel, metadataBoxParser()).use { isoFile -> - val userDataBox = Path.getPath(isoFile.movieBox, UserDataBox.TYPE) - if (userDataBox != null) { - fields.putAll(extractBoxFields(userDataBox)) - } + return Path.getPath(isoFile.movieBox, UserDataBox.TYPE) } } } @@ -343,10 +342,10 @@ object Mp4ParserHelper { } catch (e: Exception) { Log.w(LOG_TAG, "failed to get User Data box by MP4 parser for mimeType=$mimeType uri=$uri", e) } - return fields + return null } - private fun extractBoxFields(container: Container): HashMap { + fun extractBoxFields(container: Container): HashMap { val fields = HashMap() for (box in container.boxes) { if (box is AbstractBox && !box.isParsed) { @@ -360,9 +359,20 @@ object Mp4ParserHelper { is AppleGPSCoordinatesBox -> fields[key] = box.value is AppleItemListBox -> fields.putAll(extractBoxFields(box)) is AppleVariableSignedIntegerBox -> fields[key] = box.value.toString() - is Utf8AppleDataBox -> fields[key] = box.value - is HandlerBox -> {} + is LocationInformationBox -> { + hashMapOf( + "Language" to box.language, + "Name" to box.name, + "Role" to box.role.toString(), + "Longitude" to box.longitude.toString(), + "Latitude" to box.latitude.toString(), + "Altitude" to box.altitude.toString(), + "Astronomical Body" to box.astronomicalBody, + "Additional Notes" to box.additionalNotes, + ).forEach { (k, v) -> fields["$key/$k"] = v } + } + is MetaBox -> { val handlerBox = Path.getPath(box, HandlerBox.TYPE).apply { parseDetails() } when (val handlerType = handlerBox?.handlerType ?: MetaBox.TYPE) { @@ -387,6 +397,8 @@ object Mp4ParserHelper { } } + is Utf8AppleDataBox -> fields[key] = box.value + else -> fields[key] = box.toString() } } @@ -399,6 +411,7 @@ object Mp4ParserHelper { "catg" -> "Category" "covr" -> "Cover Art" "keyw" -> "Keyword" + "loci" -> "Location" "mcvr" -> "Preview Image" "pcst" -> "Podcast" "SDLN" -> "Play Mode"