Google Motion Photos: Android 11+ format and grid hover playback

app/Actions/Photo/Pipes/Standalone/PlaceGoogleMotionVideo.php

@@ -10,6 +10,7 @@
 
 use App\DTO\PhotoCreate\StandaloneDTO;
 use App\Image\Files\FlysystemFile;
+use App\Image\Handlers\GoogleMotionPictureHandler;
 
 class PlaceGoogleMotionVideo extends AbstractStandalonePipe
 {
@@ -19,11 +20,7 @@ protected function execute(StandaloneDTO $state, \Closure $next): StandaloneDTO
 		// we must move the preliminary extracted video file next to the
 		// final target file
 		if ($state->tmp_video_file !== null) {
-			$video_target_path =
-				pathinfo($state->target_file->getRelativePath(), PATHINFO_DIRNAME) .
-				'/' .
-				pathinfo($state->target_file->getRelativePath(), PATHINFO_FILENAME) .
-				$state->tmp_video_file->getExtension();
+			$video_target_path = GoogleMotionPictureHandler::videoPathNextTo($state->target_file->getRelativePath());
 			$video_target_file = new FlysystemFile($state->target_file->getDisk(), $video_target_path);
 			$video_target_file->write($state->tmp_video_file->read());
 			$state->photo->live_photo_short_path = $video_target_file->getRelativePath();

app/Console/Commands/ImageProcessing/ExtractMotionPhotos.php

@@ -0,0 +1,223 @@
+<?php
+
+/**
+ * SPDX-License-Identifier: MIT
+ * Copyright (c) 2017-2018 Tobias Reich
+ * Copyright (c) 2018-2026 LycheeOrg.
+ */
+
+namespace App\Console\Commands\ImageProcessing;
+
+use App\Constants\PhotoAlbum as PA;
+use App\Enum\SizeVariantType;
+use App\Events\PhotoSaved;
+use App\Exceptions\MediaFileOperationException;
+use App\Image\Files\FlysystemFile;
+use App\Image\Files\NativeLocalFile;
+use App\Image\Files\TemporaryLocalFile;
+use App\Image\Handlers\GoogleMotionPictureHandler;
+use App\Models\Album;
+use App\Models\Photo;
+use App\Repositories\ConfigManager;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use PHPExif\MotionPhoto\MotionPhotoLocator;
+use Safe\Exceptions\InfoException;
+use function Safe\set_time_limit;
+
+/**
+ * Extracts the video of Google Motion Photos that were imported as plain
+ * stills, before Feature 100 recognised the Motion Photo 1.0 format.
+ *
+ * The original size variant keeps the whole camera file, video included, so
+ * the video is cut from it and stored next to it, as the upload pipeline
+ * does (FR-100-04).
+ */
+class ExtractMotionPhotos extends Command
+{
+	private const MIME_TYPE_JPEG = 'image/jpeg';
+
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'lychee:extract_motion_photos' .
+		' {limit=100 : number of photos to check}' .
+		' {tm=600 : timeout time requirement}' .
+		' {--after= : continue after this photo id (printed by the previous run)}' .
+		' {--album= : only check the photos of this album}' .
+		' {--dry-run : list the motion photos without extracting their video}' .
+		' {--reencode : extract again the video of motion photos that already have one, transcoding HEVC clips to H.264}';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Extract the video of Google Motion Photos imported as plain photos';
+
+	/**
+	 * Execute the console command.
+	 */
+	public function handle(ConfigManager $config_manager): int
+	{
+		$limit = (int) $this->argument('limit');
+		$timeout = (int) $this->argument('tm');
+		$after = $this->option('after');
+		$album_id = $this->option('album');
+		$dry_run = $this->option('dry-run') === true;
+		$reencode = $this->option('reencode') === true;
+
+		try {
+			set_time_limit($timeout);
+		} catch (InfoException) {
+			// Silently do nothing, if `set_time_limit` is denied.
+		}
+
+		if (!$dry_run && !$config_manager->hasFFmpeg()) {
+			$this->error('FFmpeg is required to extract motion photo videos (or use --dry-run).');
+
+			return 1;
+		}
+		if (is_string($album_id) && Album::query()->find($album_id) === null) {
+			$this->error('Album ' . $album_id . ' not found.');
+
+			return 1;
+		}
+
+		$photos = Photo::query()
+			->with(['size_variants' => fn ($r) => $r->where('type', '=', SizeVariantType::ORIGINAL)])
+			->where('type', '=', self::MIME_TYPE_JPEG)
+			// Re-encoding only concerns Google motion photos: an Apple pair has a content id.
+			->when(
+				$reencode,
+				fn ($q) => $q->whereNotNull('live_photo_short_path')->whereNull('live_photo_content_id'),
+				fn ($q) => $q->whereNull('live_photo_short_path'),
+			)
+			->when(is_string($album_id), fn ($q) => $q->whereIn('id', DB::table(PA::PHOTO_ALBUM)->select(PA::PHOTO_ID)->where(PA::ALBUM_ID, '=', $album_id)))
+			// Cursor, not offset: extracted photos leave this set, the others stay in it.
+			->when(is_string($after) && $after !== '', fn ($q) => $q->where('id', '>', $after))
+			->orderBy('id')
+			->limit($limit)
+			->get();
+
+		if ($photos->isEmpty()) {
+			$this->line('No photos require motion photo extraction.');
+
+			return 0;
+		}
+
+		$found = [];
+		$failed = 0;
+		foreach ($photos as $photo) {
+			$is_motion_photo = $this->process($photo, $dry_run);
+			$failed += $is_motion_photo === null ? 1 : 0;
+			if ($is_motion_photo === true) {
+				$found[] = $photo->id;
+				$this->line('Motion photo: ' . $photo->id . ' (' . $photo->title . ')');
+			}
+		}
+
+		PhotoSaved::dispatchIf(!$dry_run && $found !== [], $found);
+
+		$outcome = match (true) {
+			$dry_run => 'found (dry run)',
+			$reencode => 're-extracted',
+			default => 'extracted',
+		};
+		$this->line(sprintf('Checked %d photos: %d motion photos %s, %d failed.', $photos->count(), count($found), $outcome, $failed));
+		if ($photos->count() === $limit) {
+			$this->line('More photos may remain: continue with --after=' . $photos->last()->id);
+			$this->retryHint($failed, $after);
+		}
+
+		return $failed === 0 ? 0 : 1;
+	}
+
+	/**
+	 * @return bool|null true for a motion photo (its video extracted unless `$dry_run`), false for a still, null if the photo could not be processed
+	 */
+	private function process(Photo $photo, bool $dry_run): ?bool
+	{
+		$local_file = null;
+		try {
+			$original = $photo->size_variants->getOriginal()->getFile();
+			if (!$original->exists()) {
+				throw new MediaFileOperationException('original file is missing');
+			}
+
+			$local_file = $this->toLocalFile($original, $photo);
+			$video_length = MotionPhotoLocator::locate($local_file->getRealPath());
+			if ($video_length > 0 && !$dry_run) {
+				$this->extract($photo, $original, $local_file, $video_length);
+			}
+
+			return $video_length > 0;
+		} catch (\Throwable $e) {
+			Log::warning(__METHOD__ . ':' . __LINE__ . ' Motion photo extraction failed for photo ' . $photo->id . ': ' . $e->getMessage());
+			$this->warn('Failed: ' . $photo->id . ' (' . $photo->title . '): ' . $e->getMessage());
+
+			return null;
+		} finally {
+			if ($local_file instanceof TemporaryLocalFile) {
+				$local_file->delete();
+			}
+		}
+	}
+
+	/**
+	 * Continuing with the printed cursor would skip the photos that failed in
+	 * this batch: tell the operator to run the batch again first.
+	 */
+	private function retryHint(int $failed, mixed $after): void
+	{
+		if ($failed === 0) {
+			return;
+		}
+
+		$same_batch = is_string($after) && $after !== '' ? 'with --after=' . $after : 'without --after';
+		$this->line(sprintf('%d photos failed in this batch and would be skipped by that cursor: run it again %s to retry them first.', $failed, $same_batch));
+	}
+
+	/**
+	 * Returns the original itself when it is hosted locally, a downloaded
+	 * temporary copy otherwise (as {@see \App\Image\SizeVariantDefaultFactory}).
+	 */
+	private function toLocalFile(FlysystemFile $original, Photo $photo): NativeLocalFile
+	{
+		if ($original->isLocalFile()) {
+			return $original->toLocalFile();
+		}
+
+		// @codeCoverageIgnoreStart
+		$local_file = new TemporaryLocalFile($original->getOriginalExtension(), $photo->title);
+		$local_file->write($original->read(), false, $photo->type);
+
+		return $local_file;
+		// @codeCoverageIgnoreEnd
+	}
+
+	private function extract(Photo $photo, FlysystemFile $original, NativeLocalFile $local_file, int $video_length): void
+	{
+		$video_file = new TemporaryLocalFile(GoogleMotionPictureHandler::FINAL_VIDEO_FILE_EXTENSION, $local_file->getBasename());
+		try {
+			$handler = new GoogleMotionPictureHandler();
+			$handler->load($local_file, $video_length);
+			$handler->saveVideoStream($video_file);
+
+			// A re-encoded clip replaces the stored one; a new clip goes next to the original.
+			$target_path = $photo->live_photo_short_path ?? GoogleMotionPictureHandler::videoPathNextTo($original->getRelativePath());
+			// Staged and moved over the target, so a failed write never truncates a clip in use.
+			$staged_file = new FlysystemFile($original->getDisk(), $target_path . '.part');
+			$staged_file->write($video_file->read());
+			$staged_file->move($target_path);
+			$photo->live_photo_short_path = $target_path;
+			$photo->save();
+		} finally {
+			$video_file->close();
+			$video_file->delete();
+		}
+	}
+}

app/Image/Handlers/GoogleMotionPictureHandler.php

@@ -15,6 +15,7 @@
 use App\Exceptions\MediaFileOperationException;
 use App\Image\Files\NativeLocalFile;
 use App\Image\Files\TemporaryLocalFile;
+use App\ModelFunctions\H264MOVFormat;
 use App\ModelFunctions\MOVFormat;
 use App\Repositories\ConfigManager;
 use FFMpeg\Exception\ExecutableNotFoundException;
@@ -42,10 +43,22 @@ class GoogleMotionPictureHandler extends VideoHandler
 {
 	public const PRELIMINARY_VIDEO_FILE_EXTENSION = '.mp4';
 	public const FINAL_VIDEO_FILE_EXTENSION = '.mov';
+	private const CODEC_H264 = 'h264';
 
 	/** @var TemporaryLocalFile a working copy of the video stream */
 	protected TemporaryLocalFile $working_copy;
 
+	/**
+	 * Path at which the video of a motion photo is stored: next to the
+	 * original, under the same file name, with the final video extension.
+	 *
+	 * @param string $original_path path of the original size variant, relative to its disk
+	 */
+	public static function videoPathNextTo(string $original_path): string
+	{
+		return pathinfo($original_path, PATHINFO_DIRNAME) . '/' . pathinfo($original_path, PATHINFO_FILENAME) . self::FINAL_VIDEO_FILE_EXTENSION;
+	}
+
 	/**
 	 * @throws MediaFileOperationException
 	 */
@@ -106,16 +119,20 @@ public function load(NativeLocalFile $file, int $video_length = 0): void
 	/**
 	 * Save the video stream to the provided file.
 	 *
+	 * An H.264 stream is copied as is; any other codec (HEVC on current
+	 * Pixel phones) is transcoded to H.264 so that every browser can play it
+	 * (Feature 100, Q-100-09).
+	 *
 	 * @param NativeLocalFile $file the file to write into
 	 *
 	 * @throws MediaFileOperationException
 	 */
 	public function saveVideoStream(NativeLocalFile $file): void
 	{
 		try {
-			$format = new MOVFormat();
+			$format = $this->isH264() ? new MOVFormat() : new H264MOVFormat();
 			// Add additional parameter to extract the first video stream
-			$format->setAdditionalParameters(['-map', '0:0']);
+			$format->setAdditionalParameters([...($format->getAdditionalParameters() ?? []), '-map', '0:0']);
 			$this->video->save($format, $file->getRealPath());
 			// @codeCoverageIgnoreStart
 		} catch (RuntimeException $e) {
@@ -125,4 +142,12 @@ public function saveVideoStream(NativeLocalFile $file): void
 		}
 		// @codeCoverageIgnoreEnd
 	}
+
+	/**
+	 * Whether the first video stream is already H.264.
+	 */
+	private function isH264(): bool
+	{
+		return $this->video->getStreams()->videos()->first()?->get('codec_name') === self::CODEC_H264;
+	}
 }

app/ModelFunctions/H264MOVFormat.php

@@ -0,0 +1,64 @@
+<?php
+
+/**
+ * SPDX-License-Identifier: MIT
+ * Copyright (c) 2017-2018 Tobias Reich
+ * Copyright (c) 2018-2026 LycheeOrg.
+ */
+
+namespace App\ModelFunctions;
+
+use App\Exceptions\Internal\LycheeAssertionError;
+use FFMpeg\Exception\InvalidArgumentException;
+
+/**
+ * Format class for FFMpeg to transcode a video stream to H.264 in a
+ * Quicktime (MOV) container.
+ *
+ * Current Pixel phones record the clip of a Google Motion Photo in HEVC,
+ * which several browsers cannot decode (Feature 100, Q-100-09). H.264 plays
+ * everywhere. Quality is set by a constant rate factor rather than a
+ * bitrate, so a single pass is enough; FFmpeg's default auto-rotation applies
+ * the clip's rotation to the frames.
+ */
+class H264MOVFormat extends MOVFormat
+{
+	public const FFMPEG_VIDEO_CODEC_ID = 'libx264';
+
+	/**
+	 * No bitrate: quality comes from `-crf` (see {@see self::ENCODER_PARAMETERS}).
+	 */
+	protected $kiloBitrate = 0;
+
+	/** @var string[] */
+	private const ENCODER_PARAMETERS = [
+		'-crf', '23',
+		'-preset', 'veryfast',
+		'-pix_fmt', 'yuv420p',
+		'-movflags', '+faststart',
+	];
+
+	public function __construct()
+	{
+		parent::__construct();
+		try {
+			$this
+				->setVideoCodec(self::FFMPEG_VIDEO_CODEC_ID)
+				->setAdditionalParameters(self::ENCODER_PARAMETERS);
+			// @codeCoverageIgnoreStart
+		} catch (InvalidArgumentException $e) {
+			throw LycheeAssertionError::createFromUnexpectedException($e);
+		}
+		// @codeCoverageIgnoreEnd
+	}
+
+	/**
+	 * The parent's `copy` stays available: its constructor sets it first.
+	 *
+	 * @return string[]
+	 */
+	public function getAvailableVideoCodecs(): array
+	{
+		return [...parent::getAvailableVideoCodecs(), self::FFMPEG_VIDEO_CODEC_ID];
+	}
+}

composer.json

@@ -69,7 +69,7 @@
         "league/omnipay": "^3",
         "lychee-org/lycheeverify": "^2.0.7",
         "lychee-org/nestedset": "^12.0.0",
-        "lychee-org/php-exif": "^1.4.0",
+        "lychee-org/php-exif": "^1.5.0",
         "maennchen/zipstream-php": "^2.1 || ^3.1",
         "mavinoo/laravel-batch": "^2.4",
         "moneyphp/money": "^4.7",