#1423 support for Samsung HEIC motion photos embedding video in sefd box

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## <a id="unreleased"></a>[Unreleased]
 
+### Added
+
+- support for Samsung HEIC motion photos embedding video in sefd box
+
 ## <a id="v1.12.3"></a>[v1.12.3] - 2025-02-06
 
 ### Added

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt

@@ -316,7 +316,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler {
         }
 
         val sb = StringBuilder()
-        if (mimeType == MimeTypes.MP4) {
+        if (mimeType == MimeTypes.MP4 || MimeTypes.isHeic(mimeType)) {
             try {
                 // we can skip uninteresting boxes with a seekable data source
                 val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt

@@ -186,7 +186,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
             return
         }
 
-        MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
+        MultiPage.getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
             val imageSizeBytes = sizeBytes - videoSizeBytes
             StorageUtils.openInputStream(context, uri)?.let { input ->
                 copyEmbeddedBytes(result, mimeType, displayName, input, imageSizeBytes)
@@ -207,11 +207,10 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
             return
         }
 
-        MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
+        MultiPage.getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) ->
-            val videoStartOffset = sizeBytes - videoSizeBytes
             StorageUtils.openInputStream(context, uri)?.let { input ->
-                input.skip(videoStartOffset)
+                input.skip(videoOffset)
-                copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSizeBytes)
+                copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSize)
             }
             return
         }

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt

@@ -6,6 +6,7 @@ import android.media.MediaMetadataRetriever
 import android.net.Uri
 import android.os.Build
 import android.util.Log
+import androidx.core.net.toUri
 import com.adobe.internal.xmp.XMPException
 import com.adobe.internal.xmp.XMPMeta
 import com.adobe.internal.xmp.XMPMetaFactory
@@ -13,6 +14,7 @@ import com.adobe.internal.xmp.options.SerializeOptions
 import com.adobe.internal.xmp.properties.XMPPropertyInfo
 import com.drew.lang.KeyValuePair
 import com.drew.lang.Rational
+import com.drew.lang.SequentialByteArrayReader
 import com.drew.metadata.Tag
 import com.drew.metadata.avi.AviDirectory
 import com.drew.metadata.exif.ExifDirectoryBase
@@ -107,7 +109,6 @@ import java.util.Locale
 import kotlin.math.roundToInt
 import kotlin.math.roundToLong
 import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface
-import androidx.core.net.toUri
 
 class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
     private val ioScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
@@ -470,6 +471,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
             // Android's `MediaExtractor` and `MediaPlayer` cannot be used for details
             // about embedded images as they do not list them as separate tracks
             // and only identify at most one
+        } else if (isHeic(mimeType)) {
+            Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) ->
+                val dir = hashMapOf(
+                    "Size" to bytes.size.toString(),
+                )
+                val reader = SequentialByteArrayReader(bytes).apply {
+                    isMotorolaByteOrder = false
+                }
+                val start = reader.uInt16
+                val tag = reader.uInt16
+                if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
+                    val nameSize = reader.uInt32
+                    dir["Embedded Video Type"] = reader.getString(nameSize.toInt())
+                }
+                metadataMap[Mp4ParserHelper.SAMSUNG_MAKERNOTE_BOX_TYPE] = dir
+            }
         }
 
         if (metadataMap.isNotEmpty()) {
@@ -531,6 +548,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
             getMultimediaCatalogMetadataByMediaMetadataRetriever(mimeType, uri, metadataMap)
         }
 
+        if (isHeic(mimeType)) {
+            val flags = (metadataMap[KEY_FLAGS] ?: 0) as Int
+            if ((flags and MASK_IS_MOTION_PHOTO == 0) && MultiPage.isHeicSefdMotionPhoto(context, uri)) {
+                metadataMap[KEY_FLAGS] = flags or MASK_IS_MULTIPAGE or MASK_IS_MOTION_PHOTO
+            }
+        }
+
         // report success even when empty
         result.success(metadataMap)
     }

android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt

@@ -45,6 +45,16 @@ object Mp4ParserHelper {
     // arbitrary size to detect boxes that may yield an OOM
     private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
 
+    const val SAMSUNG_MAKERNOTE_BOX_TYPE = "sefd"
+    const val SEFD_EMBEDDED_VIDEO_TAG = 0x0a30
+    const val SEFD_MOTION_PHOTO_NAME = "MotionPhoto_Data"
+
+    private val largerTypeWhitelist = listOf(
+        // HEIC motion photo may contain Samsung maker notes in `sefd` box,
+        // including a video larger than the danger threshold
+        SAMSUNG_MAKERNOTE_BOX_TYPE,
+    )
+
     fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List<Pair<Long, ByteArray>> {
         // we can skip uninteresting boxes with a seekable data source
         val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
@@ -133,6 +143,34 @@ object Mp4ParserHelper {
         return false
     }
 
+    fun getSamsungSefd(context: Context, uri: Uri): Pair<Long, ByteArray>? {
+        try {
+            // we can skip uninteresting boxes with a seekable data source
+            val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
+            pfd.use {
+                FileInputStream(it.fileDescriptor).use { stream ->
+                    stream.channel.use { channel ->
+                        IsoFile(channel, metadataBoxParser()).use { isoFile ->
+                            var offset = 0L
+                            for (box in isoFile.boxes) {
+                                if (box is UnknownBox && box.type == SAMSUNG_MAKERNOTE_BOX_TYPE) {
+                                    if (!box.isParsed) {
+                                        box.parseDetails()
+                                    }
+                                    return Pair(offset + 8, box.data.toByteArray()) // skip 8 bytes for box header
+                                }
+                                offset += box.size
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (e: Exception) {
+            Log.w(LOG_TAG, "failed to read sefd box", e)
+        }
+        return null
+    }
+
     // extensions
 
     fun IsoFile.updateLocation(locationIso6709: String?) {
@@ -272,7 +310,7 @@ object Mp4ParserHelper {
         )
         setBoxSkipper { type, size ->
             if (skippedTypes.contains(type)) return@setBoxSkipper true
-            if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
+            if (size > BOX_SIZE_DANGER_THRESHOLD && !largerTypeWhitelist.contains(type)) throw Exception("box (type=$type size=$size) is too large")
             false
         }
     }

android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt

@@ -11,6 +11,7 @@ import android.os.ParcelFileDescriptor
 import android.util.Log
 import com.adobe.internal.xmp.XMPMeta
 import com.drew.imaging.jpeg.JpegSegmentType
+import com.drew.lang.SequentialByteArrayReader
 import com.drew.metadata.exif.ExifDirectoryBase
 import com.drew.metadata.exif.ExifIFD0Directory
 import com.drew.metadata.xmp.XmpDirectory
@@ -37,6 +38,8 @@ import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface
 object MultiPage {
     private val LOG_TAG = LogUtils.createTag<MultiPage>()
 
+    // TODO TLAD more generic support, (e.g. 0x00000014 + `ftyp` + `qt  `)
+    // atom length (variable, e.g. `0x00000018`) + atom type (`ftyp`) + type (variable, e.g. `mp42`, `qt`)
     private val heicMotionPhotoVideoStartIndicator = byteArrayOf(0x00, 0x00, 0x00, 0x18) + "ftypmp42".toByteArray()
 
     // page info
@@ -84,6 +87,22 @@ object MultiPage {
         return tracks
     }
 
+    fun isHeicSefdMotionPhoto(context: Context, uri: Uri): Boolean {
+        Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) ->
+            val reader = SequentialByteArrayReader(bytes).apply {
+                isMotorolaByteOrder = false
+            }
+            val start = reader.uInt16
+            val tag = reader.uInt16
+            if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
+                val nameSize = reader.uInt32
+                val name = reader.getString(nameSize.toInt())
+                return name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME
+            }
+        }
+        return false
+    }
+
     private fun getJpegMpfPrimaryRotation(context: Context, uri: Uri, sizeBytes: Long): Int {
         val mimeType = MimeTypes.JPEG
         var rotationDegrees = 0
@@ -245,8 +264,7 @@ object MultiPage {
 
     fun getMotionPhotoPages(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): ArrayList<FieldMap> {
         val pages = ArrayList<FieldMap>()
-        getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
+        getMotionPhotoVideoInfo(context, uri, mimeType, sizeBytes)?.let { videoInfo ->
-            getTrailerVideoInfo(context, uri, fileSizeBytes = sizeBytes, videoSizeBytes = videoSizeBytes)?.let { videoInfo ->
             // set the original image as the first and default track
             var pageIndex = 0
             pages.add(
@@ -274,11 +292,10 @@ object MultiPage {
                 }
             }
         }
-        }
         return pages
     }
 
-    fun getMotionPhotoVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? {
+    fun getTrailerVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? {
         if (MimeTypes.isHeic(mimeType)) {
             // XMP in HEIC motion photos (as taken with a Samsung Camera v12.0.01.50) indicates an `Item:Length` of 68 bytes for the video.
             // This item does not contain the video itself, but only some kind of metadata (no doc, no spec),
@@ -325,15 +342,25 @@ object MultiPage {
         return offsetFromEnd
     }
 
-    fun getTrailerVideoInfo(context: Context, uri: Uri, fileSizeBytes: Long, videoSizeBytes: Long): MediaFormat? {
+    private fun getMotionPhotoVideoInfo(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): MediaFormat? {
+        getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) ->
+            return getEmbedVideoInfo(context, uri, videoOffset, videoSize)
+        }
+        return null
+    }
+
+    fun getTrailerVideoInfo(context: Context, uri: Uri, fileSize: Long, videoSize: Long): MediaFormat? {
+        return getEmbedVideoInfo(context, uri, videoOffset = fileSize - videoSize, videoSize = videoSize)
+    }
+
+    private fun getEmbedVideoInfo(context: Context, uri: Uri, videoOffset: Long, videoSize: Long): MediaFormat? {
         var format: MediaFormat? = null
         val extractor = MediaExtractor()
         var pfd: ParcelFileDescriptor? = null
         try {
-            val videoStartOffset = fileSizeBytes - videoSizeBytes
             pfd = context.contentResolver.openFileDescriptor(uri, "r")
             pfd?.fileDescriptor?.let { fd ->
-                extractor.setDataSource(fd, videoStartOffset, videoSizeBytes)
+                extractor.setDataSource(fd, videoOffset, videoSize)
                 if (extractor.trackCount > 0) {
                     // only consider the first track to represent the appended video
                     val trackIndex = 0
@@ -353,6 +380,36 @@ object MultiPage {
         return format
     }
 
+    fun getMotionPhotoVideoSizing(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Pair<Long, Long>? {
+        // default to trailer videos
+        getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSize ->
+            val videoOffset = sizeBytes - videoSize
+            return Pair(videoOffset, videoSize)
+        }
+
+        if (MimeTypes.isHeic(mimeType)) {
+            // fallback to video within Samsung SEFD box
+            Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (sefdOffset, bytes) ->
+                val reader = SequentialByteArrayReader(bytes).apply {
+                    isMotorolaByteOrder = false
+                }
+                val start = reader.uInt16
+                val tag = reader.uInt16
+                if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
+                    val nameSize = reader.uInt32
+                    val name = reader.getString(nameSize.toInt())
+                    if (name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME) {
+                        val videoOffset = sefdOffset + reader.position
+                        val videoSize = reader.available().toLong()
+                        return Pair(videoOffset, videoSize)
+                    }
+                }
+            }
+        }
+
+        return null
+    }
+
     fun getTiffPages(context: Context, uri: Uri): ArrayList<FieldMap> {
         fun toMap(pageIndex: Int, options: TiffBitmapFactory.Options): FieldMap {
             return hashMapOf(

android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt

@@ -648,13 +648,13 @@ abstract class ImageProvider {
         val originalFileSize = File(path).length()
         var trailerVideoBytes: ByteArray? = null
         val editableFile = StorageUtils.createTempFile(context).apply {
-            val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
+            val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
-            val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
+            val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
             try {
-                if (videoSize != null && isTrailerVideoValid) {
+                if (trailerVideoSize != null && isTrailerVideoValid) {
                     // handle motion photo and embedded video separately
-                    val imageSize = (originalFileSize - videoSize).toInt()
+                    val imageSize = (originalFileSize - trailerVideoSize).toInt()
-                    val videoByteSize = videoSize.toInt()
+                    val videoByteSize = trailerVideoSize.toInt()
                     trailerVideoBytes = ByteArray(videoByteSize)
 
                     StorageUtils.openInputStream(context, uri)?.let { input ->
@@ -733,13 +733,13 @@ abstract class ImageProvider {
         val originalFileSize = File(path).length()
         var trailerVideoBytes: ByteArray? = null
         val editableFile = StorageUtils.createTempFile(context).apply {
-            val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
+            val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
-            val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
+            val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
             try {
-                if (videoSize != null && isTrailerVideoValid) {
+                if (trailerVideoSize != null && isTrailerVideoValid) {
                     // handle motion photo and embedded video separately
-                    val imageSize = (originalFileSize - videoSize).toInt()
+                    val imageSize = (originalFileSize - trailerVideoSize).toInt()
-                    val videoByteSize = videoSize.toInt()
+                    val videoByteSize = trailerVideoSize.toInt()
                     trailerVideoBytes = ByteArray(videoByteSize)
 
                     StorageUtils.openInputStream(context, uri)?.let { input ->
@@ -899,7 +899,7 @@ abstract class ImageProvider {
         }
 
         val originalFileSize = File(path).length()
-        val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff }
+        val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff }
         val editableFile = StorageUtils.createTempFile(context).apply {
             try {
                 editXmpWithPixy(
@@ -921,7 +921,7 @@ abstract class ImageProvider {
             // copy the edited temporary file back to the original
             editableFile.transferTo(outputStream(context, mimeType, uri, path))
 
-            if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) {
+            if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) {
                 return false
             }
             editableFile.delete()
@@ -1262,15 +1262,15 @@ abstract class ImageProvider {
         callback: ImageOpCallback,
     ) {
         val originalFileSize = File(path).length()
-        val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)
+        val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)
-        if (videoSize == null) {
+        if (trailerVideoSize == null) {
             callback.onFailure(Exception("failed to get trailer video size"))
             return
         }
 
-        val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSizeBytes = originalFileSize, videoSizeBytes = videoSize) != null
+        val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSize = originalFileSize, videoSize = trailerVideoSize) != null
         if (!isTrailerVideoValid) {
-            callback.onFailure(Exception("failed to open trailer video with size=$videoSize"))
+            callback.onFailure(Exception("failed to open trailer video with size=$trailerVideoSize"))
             return
         }
 
@@ -1278,7 +1278,7 @@ abstract class ImageProvider {
             try {
                 val inputStream = StorageUtils.openInputStream(context, uri)
                 // partial copy
-                transferFrom(inputStream, originalFileSize - videoSize)
+                transferFrom(inputStream, originalFileSize - trailerVideoSize)
             } catch (e: Exception) {
                 Log.d(LOG_TAG, "failed to remove trailer video", e)
                 callback.onFailure(e)
@@ -1313,8 +1313,8 @@ abstract class ImageProvider {
         }
 
         val originalFileSize = File(path).length()
-        val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)
+        val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)
-        val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
+        val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
         val editableFile = StorageUtils.createTempFile(context).apply {
             try {
                 outputStream().use { output ->
@@ -1334,7 +1334,7 @@ abstract class ImageProvider {
             // copy the edited temporary file back to the original
             editableFile.transferTo(outputStream(context, mimeType, uri, path))
 
-            if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) {
+            if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) {
                 return
             }
             editableFile.delete()