diff --git a/CHANGELOG.md b/CHANGELOG.md index fcdb05caf..7fbf820f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- support for Samsung HEIC motion photos embedding video in sefd box + ## [v1.12.3] - 2025-02-06 ### Added diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt index 9282ba2be..38135b72f 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt @@ -316,7 +316,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler { } val sb = StringBuilder() - if (mimeType == MimeTypes.MP4) { + if (mimeType == MimeTypes.MP4 || MimeTypes.isHeic(mimeType)) { try { // we can skip uninteresting boxes with a seekable data source val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri") diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt index 46238f70e..8897e3ab4 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt @@ -186,7 +186,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler { return } - MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes -> + MultiPage.getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes -> val imageSizeBytes = sizeBytes - videoSizeBytes StorageUtils.openInputStream(context, uri)?.let { input -> copyEmbeddedBytes(result, mimeType, displayName, input, imageSizeBytes) @@ -207,11 +207,10 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler { return } - MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes -> - val videoStartOffset = sizeBytes - videoSizeBytes + MultiPage.getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) -> StorageUtils.openInputStream(context, uri)?.let { input -> - input.skip(videoStartOffset) - copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSizeBytes) + input.skip(videoOffset) + copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSize) } return } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt index 27aed4bab..d39452bc5 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt @@ -6,6 +6,7 @@ import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Build import android.util.Log +import androidx.core.net.toUri import com.adobe.internal.xmp.XMPException import com.adobe.internal.xmp.XMPMeta import com.adobe.internal.xmp.XMPMetaFactory @@ -13,6 +14,7 @@ import com.adobe.internal.xmp.options.SerializeOptions import com.adobe.internal.xmp.properties.XMPPropertyInfo import com.drew.lang.KeyValuePair import com.drew.lang.Rational +import com.drew.lang.SequentialByteArrayReader import com.drew.metadata.Tag import com.drew.metadata.avi.AviDirectory import com.drew.metadata.exif.ExifDirectoryBase @@ -107,7 +109,6 @@ import java.util.Locale import kotlin.math.roundToInt import kotlin.math.roundToLong import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface -import androidx.core.net.toUri class MetadataFetchHandler(private val context: Context) : MethodCallHandler { private val ioScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) @@ -470,6 +471,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { // Android's `MediaExtractor` and `MediaPlayer` cannot be used for details // about embedded images as they do not list them as separate tracks // and only identify at most one + } else if (isHeic(mimeType)) { + Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) -> + val dir = hashMapOf( + "Size" to bytes.size.toString(), + ) + val reader = SequentialByteArrayReader(bytes).apply { + isMotorolaByteOrder = false + } + val start = reader.uInt16 + val tag = reader.uInt16 + if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) { + val nameSize = reader.uInt32 + dir["Embedded Video Type"] = reader.getString(nameSize.toInt()) + } + metadataMap[Mp4ParserHelper.SAMSUNG_MAKERNOTE_BOX_TYPE] = dir + } } if (metadataMap.isNotEmpty()) { @@ -531,6 +548,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { getMultimediaCatalogMetadataByMediaMetadataRetriever(mimeType, uri, metadataMap) } + if (isHeic(mimeType)) { + val flags = (metadataMap[KEY_FLAGS] ?: 0) as Int + if ((flags and MASK_IS_MOTION_PHOTO == 0) && MultiPage.isHeicSefdMotionPhoto(context, uri)) { + metadataMap[KEY_FLAGS] = flags or MASK_IS_MULTIPAGE or MASK_IS_MOTION_PHOTO + } + } + // report success even when empty result.success(metadataMap) } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt index 2f9801b70..8ffc7886d 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt @@ -45,6 +45,16 @@ object Mp4ParserHelper { // arbitrary size to detect boxes that may yield an OOM private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB + const val SAMSUNG_MAKERNOTE_BOX_TYPE = "sefd" + const val SEFD_EMBEDDED_VIDEO_TAG = 0x0a30 + const val SEFD_MOTION_PHOTO_NAME = "MotionPhoto_Data" + + private val largerTypeWhitelist = listOf( + // HEIC motion photo may contain Samsung maker notes in `sefd` box, + // including a video larger than the danger threshold + SAMSUNG_MAKERNOTE_BOX_TYPE, + ) + fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List> { // we can skip uninteresting boxes with a seekable data source val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri") @@ -133,6 +143,34 @@ object Mp4ParserHelper { return false } + fun getSamsungSefd(context: Context, uri: Uri): Pair? { + try { + // we can skip uninteresting boxes with a seekable data source + val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri") + pfd.use { + FileInputStream(it.fileDescriptor).use { stream -> + stream.channel.use { channel -> + IsoFile(channel, metadataBoxParser()).use { isoFile -> + var offset = 0L + for (box in isoFile.boxes) { + if (box is UnknownBox && box.type == SAMSUNG_MAKERNOTE_BOX_TYPE) { + if (!box.isParsed) { + box.parseDetails() + } + return Pair(offset + 8, box.data.toByteArray()) // skip 8 bytes for box header + } + offset += box.size + } + } + } + } + } + } catch (e: Exception) { + Log.w(LOG_TAG, "failed to read sefd box", e) + } + return null + } + // extensions fun IsoFile.updateLocation(locationIso6709: String?) { @@ -272,7 +310,7 @@ object Mp4ParserHelper { ) setBoxSkipper { type, size -> if (skippedTypes.contains(type)) return@setBoxSkipper true - if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large") + if (size > BOX_SIZE_DANGER_THRESHOLD && !largerTypeWhitelist.contains(type)) throw Exception("box (type=$type size=$size) is too large") false } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt index a0d98a966..adf3e5643 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt @@ -11,6 +11,7 @@ import android.os.ParcelFileDescriptor import android.util.Log import com.adobe.internal.xmp.XMPMeta import com.drew.imaging.jpeg.JpegSegmentType +import com.drew.lang.SequentialByteArrayReader import com.drew.metadata.exif.ExifDirectoryBase import com.drew.metadata.exif.ExifIFD0Directory import com.drew.metadata.xmp.XmpDirectory @@ -37,6 +38,8 @@ import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface object MultiPage { private val LOG_TAG = LogUtils.createTag() + // TODO TLAD more generic support, (e.g. 0x00000014 + `ftyp` + `qt `) + // atom length (variable, e.g. `0x00000018`) + atom type (`ftyp`) + type (variable, e.g. `mp42`, `qt`) private val heicMotionPhotoVideoStartIndicator = byteArrayOf(0x00, 0x00, 0x00, 0x18) + "ftypmp42".toByteArray() // page info @@ -84,6 +87,22 @@ object MultiPage { return tracks } + fun isHeicSefdMotionPhoto(context: Context, uri: Uri): Boolean { + Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) -> + val reader = SequentialByteArrayReader(bytes).apply { + isMotorolaByteOrder = false + } + val start = reader.uInt16 + val tag = reader.uInt16 + if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) { + val nameSize = reader.uInt32 + val name = reader.getString(nameSize.toInt()) + return name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME + } + } + return false + } + private fun getJpegMpfPrimaryRotation(context: Context, uri: Uri, sizeBytes: Long): Int { val mimeType = MimeTypes.JPEG var rotationDegrees = 0 @@ -245,40 +264,38 @@ object MultiPage { fun getMotionPhotoPages(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): ArrayList { val pages = ArrayList() - getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes -> - getTrailerVideoInfo(context, uri, fileSizeBytes = sizeBytes, videoSizeBytes = videoSizeBytes)?.let { videoInfo -> - // set the original image as the first and default track - var pageIndex = 0 - pages.add( - hashMapOf( - KEY_PAGE to pageIndex++, - KEY_MIME_TYPE to mimeType, - KEY_IS_DEFAULT to true, - ) + getMotionPhotoVideoInfo(context, uri, mimeType, sizeBytes)?.let { videoInfo -> + // set the original image as the first and default track + var pageIndex = 0 + pages.add( + hashMapOf( + KEY_PAGE to pageIndex++, + KEY_MIME_TYPE to mimeType, + KEY_IS_DEFAULT to true, ) - // add video tracks from the appended video - videoInfo.getString(MediaFormat.KEY_MIME)?.let { mime -> - if (MimeTypes.isVideo(mime)) { - val page: FieldMap = hashMapOf( - KEY_PAGE to pageIndex++, - KEY_MIME_TYPE to MimeTypes.MP4, - KEY_IS_DEFAULT to false, - ) - videoInfo.getSafeInt(MediaFormat.KEY_WIDTH) { page[KEY_WIDTH] = it } - videoInfo.getSafeInt(MediaFormat.KEY_HEIGHT) { page[KEY_HEIGHT] = it } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - videoInfo.getSafeInt(MediaFormat.KEY_ROTATION) { page[KEY_ROTATION_DEGREES] = it } - } - videoInfo.getSafeLong(MediaFormat.KEY_DURATION) { page[KEY_DURATION] = it / 1000 } - pages.add(page) + ) + // add video tracks from the appended video + videoInfo.getString(MediaFormat.KEY_MIME)?.let { mime -> + if (MimeTypes.isVideo(mime)) { + val page: FieldMap = hashMapOf( + KEY_PAGE to pageIndex++, + KEY_MIME_TYPE to MimeTypes.MP4, + KEY_IS_DEFAULT to false, + ) + videoInfo.getSafeInt(MediaFormat.KEY_WIDTH) { page[KEY_WIDTH] = it } + videoInfo.getSafeInt(MediaFormat.KEY_HEIGHT) { page[KEY_HEIGHT] = it } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + videoInfo.getSafeInt(MediaFormat.KEY_ROTATION) { page[KEY_ROTATION_DEGREES] = it } } + videoInfo.getSafeLong(MediaFormat.KEY_DURATION) { page[KEY_DURATION] = it / 1000 } + pages.add(page) } } } return pages } - fun getMotionPhotoVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? { + fun getTrailerVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? { if (MimeTypes.isHeic(mimeType)) { // XMP in HEIC motion photos (as taken with a Samsung Camera v12.0.01.50) indicates an `Item:Length` of 68 bytes for the video. // This item does not contain the video itself, but only some kind of metadata (no doc, no spec), @@ -325,15 +342,25 @@ object MultiPage { return offsetFromEnd } - fun getTrailerVideoInfo(context: Context, uri: Uri, fileSizeBytes: Long, videoSizeBytes: Long): MediaFormat? { + private fun getMotionPhotoVideoInfo(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): MediaFormat? { + getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) -> + return getEmbedVideoInfo(context, uri, videoOffset, videoSize) + } + return null + } + + fun getTrailerVideoInfo(context: Context, uri: Uri, fileSize: Long, videoSize: Long): MediaFormat? { + return getEmbedVideoInfo(context, uri, videoOffset = fileSize - videoSize, videoSize = videoSize) + } + + private fun getEmbedVideoInfo(context: Context, uri: Uri, videoOffset: Long, videoSize: Long): MediaFormat? { var format: MediaFormat? = null val extractor = MediaExtractor() var pfd: ParcelFileDescriptor? = null try { - val videoStartOffset = fileSizeBytes - videoSizeBytes pfd = context.contentResolver.openFileDescriptor(uri, "r") pfd?.fileDescriptor?.let { fd -> - extractor.setDataSource(fd, videoStartOffset, videoSizeBytes) + extractor.setDataSource(fd, videoOffset, videoSize) if (extractor.trackCount > 0) { // only consider the first track to represent the appended video val trackIndex = 0 @@ -353,6 +380,36 @@ object MultiPage { return format } + fun getMotionPhotoVideoSizing(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Pair? { + // default to trailer videos + getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSize -> + val videoOffset = sizeBytes - videoSize + return Pair(videoOffset, videoSize) + } + + if (MimeTypes.isHeic(mimeType)) { + // fallback to video within Samsung SEFD box + Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (sefdOffset, bytes) -> + val reader = SequentialByteArrayReader(bytes).apply { + isMotorolaByteOrder = false + } + val start = reader.uInt16 + val tag = reader.uInt16 + if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) { + val nameSize = reader.uInt32 + val name = reader.getString(nameSize.toInt()) + if (name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME) { + val videoOffset = sefdOffset + reader.position + val videoSize = reader.available().toLong() + return Pair(videoOffset, videoSize) + } + } + } + } + + return null + } + fun getTiffPages(context: Context, uri: Uri): ArrayList { fun toMap(pageIndex: Int, options: TiffBitmapFactory.Options): FieldMap { return hashMapOf( diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt b/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt index fc46edd70..90b2b3359 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt @@ -648,13 +648,13 @@ abstract class ImageProvider { val originalFileSize = File(path).length() var trailerVideoBytes: ByteArray? = null val editableFile = StorageUtils.createTempFile(context).apply { - val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff } - val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null + val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff } + val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null try { - if (videoSize != null && isTrailerVideoValid) { + if (trailerVideoSize != null && isTrailerVideoValid) { // handle motion photo and embedded video separately - val imageSize = (originalFileSize - videoSize).toInt() - val videoByteSize = videoSize.toInt() + val imageSize = (originalFileSize - trailerVideoSize).toInt() + val videoByteSize = trailerVideoSize.toInt() trailerVideoBytes = ByteArray(videoByteSize) StorageUtils.openInputStream(context, uri)?.let { input -> @@ -733,13 +733,13 @@ abstract class ImageProvider { val originalFileSize = File(path).length() var trailerVideoBytes: ByteArray? = null val editableFile = StorageUtils.createTempFile(context).apply { - val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff } - val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null + val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff } + val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null try { - if (videoSize != null && isTrailerVideoValid) { + if (trailerVideoSize != null && isTrailerVideoValid) { // handle motion photo and embedded video separately - val imageSize = (originalFileSize - videoSize).toInt() - val videoByteSize = videoSize.toInt() + val imageSize = (originalFileSize - trailerVideoSize).toInt() + val videoByteSize = trailerVideoSize.toInt() trailerVideoBytes = ByteArray(videoByteSize) StorageUtils.openInputStream(context, uri)?.let { input -> @@ -899,7 +899,7 @@ abstract class ImageProvider { } val originalFileSize = File(path).length() - val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff } + val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff } val editableFile = StorageUtils.createTempFile(context).apply { try { editXmpWithPixy( @@ -921,7 +921,7 @@ abstract class ImageProvider { // copy the edited temporary file back to the original editableFile.transferTo(outputStream(context, mimeType, uri, path)) - if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) { + if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) { return false } editableFile.delete() @@ -1262,15 +1262,15 @@ abstract class ImageProvider { callback: ImageOpCallback, ) { val originalFileSize = File(path).length() - val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize) - if (videoSize == null) { + val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize) + if (trailerVideoSize == null) { callback.onFailure(Exception("failed to get trailer video size")) return } - val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSizeBytes = originalFileSize, videoSizeBytes = videoSize) != null + val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSize = originalFileSize, videoSize = trailerVideoSize) != null if (!isTrailerVideoValid) { - callback.onFailure(Exception("failed to open trailer video with size=$videoSize")) + callback.onFailure(Exception("failed to open trailer video with size=$trailerVideoSize")) return } @@ -1278,7 +1278,7 @@ abstract class ImageProvider { try { val inputStream = StorageUtils.openInputStream(context, uri) // partial copy - transferFrom(inputStream, originalFileSize - videoSize) + transferFrom(inputStream, originalFileSize - trailerVideoSize) } catch (e: Exception) { Log.d(LOG_TAG, "failed to remove trailer video", e) callback.onFailure(e) @@ -1313,8 +1313,8 @@ abstract class ImageProvider { } val originalFileSize = File(path).length() - val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize) - val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null + val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize) + val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null val editableFile = StorageUtils.createTempFile(context).apply { try { outputStream().use { output -> @@ -1334,7 +1334,7 @@ abstract class ImageProvider { // copy the edited temporary file back to the original editableFile.transferTo(outputStream(context, mimeType, uri, path)) - if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) { + if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) { return } editableFile.delete()