identify video location from Apple QuickTime metadata, and 3GPP loci atom

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - support for Samsung HEIC motion photos embedding video in sefd box
+- Cataloguing: identify video location from Apple QuickTime metadata, and 3GPP `loci` atom
 
 ### Fixed
 

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt

@@ -24,6 +24,7 @@ import com.drew.metadata.exif.GpsDirectory
 import com.drew.metadata.file.FileTypeDirectory
 import com.drew.metadata.gif.GifAnimationDirectory
 import com.drew.metadata.iptc.IptcDirectory
+import com.drew.metadata.mov.metadata.QuickTimeMetadataDirectory
 import com.drew.metadata.mp4.media.Mp4UuidBoxDirectory
 import com.drew.metadata.png.PngDirectory
 import com.drew.metadata.webp.WebpDirectory
@@ -102,6 +103,8 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.launch
 import org.json.JSONObject
+import org.mp4parser.boxes.threegpp.ts26244.LocationInformationBox
+import org.mp4parser.tools.Path
 import java.nio.charset.StandardCharsets
 import java.text.DecimalFormat
 import java.text.ParseException
@@ -450,9 +453,8 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
 
         if (isVideo(mimeType)) {
             // `metadata-extractor` do not extract custom tags in user data box
-            val userDataDir = Mp4ParserHelper.getUserData(context, mimeType, uri)
-            if (userDataDir.isNotEmpty()) {
-                metadataMap[Metadata.DIR_MP4_USER_DATA] = userDataDir
+            Mp4ParserHelper.getUserDataBox(context, mimeType, uri)?.let { box ->
+                metadataMap[Metadata.DIR_MP4_USER_DATA] = Mp4ParserHelper.extractBoxFields(box)
             }
 
             // this is used as fallback when the video metadata cannot be found on the Dart side
@@ -544,8 +546,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
 
         val metadataMap = HashMap<String, Any>()
         getCatalogMetadataByMetadataExtractor(mimeType, uri, path, sizeBytes, metadataMap)
+
         if (isVideo(mimeType) || isHeic(mimeType)) {
             getMultimediaCatalogMetadataByMediaMetadataRetriever(mimeType, uri, metadataMap)
+
+            // fallback to MP4 `loci` box for location
+            if (!metadataMap.contains(KEY_LATITUDE) || !metadataMap.contains(KEY_LONGITUDE)) {
+                Mp4ParserHelper.getUserDataBox(context, mimeType, uri)?.let { userDataBox ->
+                    Path.getPath<LocationInformationBox>(userDataBox, LocationInformationBox.TYPE)?.let { locationBox ->
+                        if (!locationBox.isParsed) {
+                            locationBox.parseDetails()
+                        }
+                        metadataMap[KEY_LATITUDE] = locationBox.latitude
+                        metadataMap[KEY_LONGITUDE] = locationBox.longitude
+                    }
+                }
+            }
         }
 
         if (isHeic(mimeType)) {
@@ -710,6 +726,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                         }
                     }
 
+                    if (!metadataMap.containsKey(KEY_LATITUDE) || !metadataMap.containsKey(KEY_LONGITUDE)) {
+                        for (dir in metadata.getDirectoriesOfType(QuickTimeMetadataDirectory::class.java)) {
+                            dir.getSafeString(QuickTimeMetadataDirectory.TAG_LOCATION_ISO6709) { locationString ->
+                                val matcher = Metadata.VIDEO_LOCATION_PATTERN.matcher(locationString)
+                                if (matcher.find() && matcher.groupCount() >= 2) {
+                                    val latitude = matcher.group(1)?.toDoubleOrNull()
+                                    val longitude = matcher.group(2)?.toDoubleOrNull()
+                                    if (latitude != null && longitude != null) {
+                                        metadataMap[KEY_LATITUDE] = latitude
+                                        metadataMap[KEY_LONGITUDE] = longitude
+                                    }
+                                }
+                            }
+                        }
+                    }
+
                     when (mimeType) {
                         MimeTypes.PNG -> {
                             // date fallback to PNG time chunk
@@ -854,7 +886,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                 retriever.getSafeDateMillis(MediaMetadataRetriever.METADATA_KEY_DATE) { metadataMap[KEY_DATE_MILLIS] = it }
             }
 
-            if (!metadataMap.containsKey(KEY_LATITUDE)) {
+            if (!metadataMap.containsKey(KEY_LATITUDE) || !metadataMap.containsKey(KEY_LONGITUDE)) {
                 val locationString = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION)
                 if (locationString != null) {
                     val matcher = Metadata.VIDEO_LOCATION_PATTERN.matcher(locationString)

android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt

@@ -32,6 +32,7 @@ import org.mp4parser.boxes.iso14496.part12.SegmentIndexBox
 import org.mp4parser.boxes.iso14496.part12.TrackHeaderBox
 import org.mp4parser.boxes.iso14496.part12.UserDataBox
 import org.mp4parser.boxes.threegpp.ts26244.AuthorBox
+import org.mp4parser.boxes.threegpp.ts26244.LocationInformationBox
 import org.mp4parser.support.AbstractBox
 import org.mp4parser.support.Matrix
 import org.mp4parser.tools.Path
@@ -143,6 +144,7 @@ object Mp4ParserHelper {
         return false
     }
 
+    // returns the offset and data of the Samsung maker notes box
     fun getSamsungSefd(context: Context, uri: Uri): Pair<Long, ByteArray>? {
         try {
             // we can skip uninteresting boxes with a seekable data source
@@ -315,13 +317,13 @@ object Mp4ParserHelper {
         }
     }
 
-    fun getUserData(
+    fun getUserDataBox(
         context: Context,
         mimeType: String,
         uri: Uri,
-    ): MutableMap<String, String> {
-        val fields = HashMap<String, String>()
-        if (mimeType != MimeTypes.MP4) return fields
+    ): UserDataBox? {
+        if (mimeType != MimeTypes.MP4) return null
+
         try {
             // we can skip uninteresting boxes with a seekable data source
             val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
@@ -330,10 +332,7 @@ object Mp4ParserHelper {
                     stream.channel.use { channel ->
                         // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
                         IsoFile(channel, metadataBoxParser()).use { isoFile ->
-                            val userDataBox = Path.getPath<UserDataBox>(isoFile.movieBox, UserDataBox.TYPE)
-                            if (userDataBox != null) {
-                                fields.putAll(extractBoxFields(userDataBox))
-                            }
+                            return Path.getPath(isoFile.movieBox, UserDataBox.TYPE)
                         }
                     }
                 }
@@ -343,10 +342,10 @@ object Mp4ParserHelper {
         } catch (e: Exception) {
             Log.w(LOG_TAG, "failed to get User Data box by MP4 parser for mimeType=$mimeType uri=$uri", e)
         }
-        return fields
+        return null
     }
 
-    private fun extractBoxFields(container: Container): HashMap<String, String> {
+    fun extractBoxFields(container: Container): HashMap<String, String> {
         val fields = HashMap<String, String>()
         for (box in container.boxes) {
             if (box is AbstractBox && !box.isParsed) {
@@ -360,9 +359,20 @@ object Mp4ParserHelper {
                 is AppleGPSCoordinatesBox -> fields[key] = box.value
                 is AppleItemListBox -> fields.putAll(extractBoxFields(box))
                 is AppleVariableSignedIntegerBox -> fields[key] = box.value.toString()
-                is Utf8AppleDataBox -> fields[key] = box.value
-
                 is HandlerBox -> {}
+                is LocationInformationBox -> {
+                    hashMapOf<String, String>(
+                        "Language" to box.language,
+                        "Name" to box.name,
+                        "Role" to box.role.toString(),
+                        "Longitude" to box.longitude.toString(),
+                        "Latitude" to box.latitude.toString(),
+                        "Altitude" to box.altitude.toString(),
+                        "Astronomical Body" to box.astronomicalBody,
+                        "Additional Notes" to box.additionalNotes,
+                    ).forEach { (k, v) -> fields["$key/$k"] = v }
+                }
+
                 is MetaBox -> {
                     val handlerBox = Path.getPath<HandlerBox>(box, HandlerBox.TYPE).apply { parseDetails() }
                     when (val handlerType = handlerBox?.handlerType ?: MetaBox.TYPE) {
@@ -387,6 +397,8 @@ object Mp4ParserHelper {
                     }
                 }
 
+                is Utf8AppleDataBox -> fields[key] = box.value
+
                 else -> fields[key] = box.toString()
             }
         }
@@ -399,6 +411,7 @@ object Mp4ParserHelper {
         "catg" -> "Category"
         "covr" -> "Cover Art"
         "keyw" -> "Keyword"
+        "loci" -> "Location"
         "mcvr" -> "Preview Image"
         "pcst" -> "Podcast"
         "SDLN" -> "Play Mode"