Skip to content

Commit 12bb737

Browse files
aminnausinaminnausin
committed
feat: scan external subtitles
1 parent fa7c572 commit 12bb737

5 files changed

Lines changed: 261 additions & 48 deletions

File tree

‎app/Enums/SubtitleSource.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?php
2+
3+
namespace App\Enums;
4+
5+
enum SubtitleSource: string {
6+
case EMBEDDED = 'embedded';
7+
case EXTERNAL = 'external';
8+
9+
public function makeKey(string|int $value): string {
10+
return "{$this->value}:{$value}";
11+
}
12+
}

‎app/Jobs/VerifyFiles.php‎

Lines changed: 105 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ public function handle(TaskService $taskService, SubtitleScanner $subtitleScanne
7979
private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitleScanner) {
8080
$metadataTransactions = [];
8181
$subtitleTransactions = [];
82+
$scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer
8283

8384
$error = false;
8485
$index = 0;
@@ -89,43 +90,46 @@ private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitle
8990

9091
try {
9192
foreach ($this->videos as $video) {
92-
$stored = []; // Metadata from db
93-
$changes = []; // Changes -> stored + changes . length has to be the same for every video so must generate defaults
93+
$baseName = basename($video->path);
94+
$fileName = pathinfo($video->path, PATHINFO_FILENAME);
95+
$compositeId = $video->folder->path . '/' . $baseName;
9496

95-
$compositeId = $video->folder->path . '/' . basename($video->path);
96-
$filePath = str_replace('\\', '/', Storage::disk('public')->path('')) . 'media/' . $video->folder->path . '/' . basename($video->path);
97+
// absolute file path in storage
98+
$filePath = $this->getAbsoluteMediaPath($video);
99+
$folderPath = dirname($filePath);
97100

98101
/**
99102
* @disregard P1013 Undefined method but it actually exists
100103
*/
101-
if (! Storage::disk('public')->fileExists('media/' . $video->folder->path . '/' . basename($video->path))) {
102-
throw new \Exception('Video "media/' . $video->folder->path . '/' . basename($video->path) . '" no longer exists. Index your videos before running this task again.');
104+
if (! Storage::disk('public')->fileExists("media/{$video->folder->path}/{$baseName}")) {
105+
throw new \Exception("File media/{$video->folder->path}/{$baseName} no longer exists. Index your videos before running this task again.");
103106
}
104107

105-
// For use with private storage -> $filePath = str_replace('\\', '/', Storage::path('app/private/')) . 'media/' . $video->folder->path . "/" . basename($video->path);
106-
$this->fileMetaData = is_null($video->uuid) ? $this->getFileMetadata($filePath) : []; // Empty unless uuid is missing or duration is missing
107-
$uuid = $video->uuid ?? ''; // video has ? file has
108+
// enforce loading file metadata if uuid is missing
109+
$this->fileMetaData = is_null($video->uuid) ? $this->getFileMetadata($filePath, "uuid missing") : [];
110+
$uuid = $video->uuid ?? '';
108111

109-
// if the video in db or file does not have a valid uuid, it will add it in both the db and on the file.
110-
if (! Uuid::isValid($uuid ?? '')) {
111-
if (! isset($this->fileMetaData['tags']['uid']) && ! isset($this->fileMetaData['tags']['uuid'])) {
112-
$uuid = Str::uuid()->toString();
113-
$this->embedChain[] = new EmbedUidInMetadata($filePath, $uuid, $this->taskId, $video->id);
114-
} else {
115-
$uuid = $this->fileMetaData['tags']['uuid'];
116-
dump("Found UUID {$uuid}");
117-
$video->update(['uuid' => $uuid]); // If embedding, video is updated in the embed job
118-
}
112+
// handle missing or invalid Uuid
113+
if (! Uuid::isValid($uuid)) {
114+
$uuid = $this->resolveMediaUuid($video, $filePath);
119115
}
120116

121-
$metadata = Metadata::where('uuid', $uuid)->orWhere('composite_id', $compositeId)->first();
117+
$metadata = Metadata::firstOrCreate(
118+
['uuid' => $uuid],
119+
['video_id' => $video->id, 'composite_id' => $compositeId]
120+
);
122121

123-
if (! $metadata) {
124-
$metadata = Metadata::create(['uuid' => $uuid, 'composite_id' => $compositeId, 'video_id' => $video->id]);
125-
}
122+
// $metadata = Metadata::where('uuid', $uuid)->orWhere('composite_id', $compositeId)->first();
126123

127-
$stored = $metadata->toArray();
128-
$fileUpdated = ! is_null($metadata->date_scanned) && filemtime($filePath) > strtotime($metadata->date_scanned);
124+
// if (! $metadata) {
125+
// $metadata = Metadata::create(['uuid' => $uuid, 'composite_id' => $compositeId, 'video_id' => $video->id]);
126+
// }
127+
128+
$stored = $metadata->toArray(); // Metadata from db
129+
$changes = []; // Changes -> stored + changes . length has to be the same for every video so must generate defaults
130+
131+
$lastScannedAt = is_null($metadata->date_scanned) ? 0 : strtotime($metadata->date_scanned);
132+
$fileUpdated = $metadata->date_scanned && filemtime($filePath) > $lastScannedAt;
129133

130134
if (is_null($metadata->uuid) || $fileUpdated) {
131135
$changes['uuid'] = $uuid;
@@ -177,28 +181,69 @@ private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitle
177181
preg_match('!\d+!', $episodeRaw[0] ?? '', $episode);
178182

179183
if (is_null($metadata->duration) || $fileUpdated) {
180-
$this->confirmMetadata($filePath);
184+
$this->confirmMetadata($filePath, "Duration is missing or file was updated");
181185
$duration = $this->fileMetaData['format']['duration'] ?? $this->fileMetaData['streams'][0]['duration'] ?? $metadata->duration;
182186
$changes['duration'] = is_numeric($duration) ? floor($duration) : null;
183187
}
184188

185-
if (is_null($metadata->subtitles_scanned_at) || $fileUpdated) {
186-
$this->confirmMetadata($filePath);
187-
$subtitleStreams = $subtitleScanner->extractSubtitleStreams($this->fileMetaData);
189+
// Embedded Subtitles
190+
191+
$subtitleScanNeeded = is_null($metadata->subtitles_scanned_at) || $fileUpdated;
192+
193+
if ($subtitleScanNeeded) {
194+
$this->confirmMetadata($filePath, "Subtitle scann date is missing or file was updated");
195+
$embeddedSubtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $this->fileMetaData);
196+
197+
foreach ($embeddedSubtitleTransactions as $tx) {
198+
$subtitleTransactions[] = $tx;
199+
}
200+
}
201+
202+
// #region External Subtitles
203+
204+
// check if directory subtitle scan is needed
205+
206+
if (! isset($scannedDirectories[$folderPath])) {
207+
$scannedDirectories[$folderPath] = [
208+
'last_modified' => filemtime($folderPath),
209+
'external_subtitles' => null,
210+
];
211+
}
212+
213+
$dirUpdated = $scannedDirectories[$folderPath]['last_modified'] > $lastScannedAt;
214+
215+
// if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
216+
217+
// Scan each directory once per batch
218+
if (($subtitleScanNeeded || $dirUpdated) && is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
219+
$scannedDirectories[$folderPath]['external_subtitles'] = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);
220+
}
221+
222+
// if subtitles in this directory have been scanned, check for matches
223+
if (! is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
224+
$relevantSubtitles = array_filter(
225+
$scannedDirectories[$folderPath]['external_subtitles'],
226+
fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
227+
);
228+
229+
$externalSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(
230+
$uuid,
231+
$relevantSubtitles
232+
);
188233

189-
if (count($subtitleStreams) > 0) {
190-
$fileSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(
191-
$uuid,
192-
$subtitleStreams
193-
);
194-
$subtitleTransactions = array_merge($subtitleTransactions, $fileSubtitleTransactions);
234+
foreach ($externalSubtitleTransactions as $tx) {
235+
$subtitleTransactions[] = $tx;
195236
}
237+
}
196238

239+
if ($subtitleScanNeeded || $dirUpdated) {
197240
$changes['subtitles_scanned_at'] = now();
198241
}
199242

243+
// #endregion
244+
200245
if (! $is_audio && (is_null($metadata->resolution_height) || is_null($metadata->codec) || $fileUpdated)) {
201-
$this->confirmMetadata($filePath);
246+
$this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {$fileUpdated}");
202247
foreach ($this->fileMetaData['streams'] as $stream) {
203248
if (! isset($stream['codec_type']) || $stream['codec_type'] !== 'video' || ! isset($stream['width'])) {
204249
continue;
@@ -349,7 +394,7 @@ private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitle
349394
$summary = 'Updated ' . count($metadataTransactions) . ' videos from id ' . ($metadataTransactions[0]['video_id']) . ' to ' . ($metadataTransactions[count($metadataTransactions) - 1]['video_id']);
350395

351396
if (count($subtitleTransactions) > 0) {
352-
Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'track_id'], ['language', 'codec']);
397+
Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'codec', 'is_default', 'is_forced', 'external_path']);
353398
$summary .= ' and found ' . count($subtitleTransactions) . ' subtitle track(s)';
354399
}
355400

@@ -360,13 +405,13 @@ private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitle
360405
}
361406
}
362407

363-
public static function getFileMetadata($filePath) {
408+
public static function getFileMetadata($filePath, $reason = "und") {
364409
try {
365410
// ? FFMPEG module with 6 test folders takes 35+ seconds but running the commands through shell takes 18 seconds
366411

367412
$ext = pathinfo($filePath, PATHINFO_EXTENSION);
368413
if (config('app.env') === 'local') {
369-
dump('PULLING METADATA ' . $filePath);
414+
dump('PULLING METADATA ' . $filePath . ' reason: ' . $reason);
370415
}
371416
$command = [
372417
'ffprobe',
@@ -423,14 +468,14 @@ public static function getFileMetadata($filePath) {
423468
}
424469

425470
// Call get metadata when needed
426-
private function confirmMetadata($filePath) {
471+
private function confirmMetadata(string $filePath, string $reason = 'und') {
427472
if (is_null($this->fileMetaData) || count($this->fileMetaData) == 0) {
428-
$this->fileMetaData = $this->getFileMetadata($filePath);
473+
$this->fileMetaData = $this->getFileMetadata($filePath, $reason);
429474
}
430475
}
431476

432477
private function getAudioDescription($filePath) {
433-
$this->confirmMetadata($filePath);
478+
$this->confirmMetadata($filePath, "Get audio description");
434479

435480
$tags = $this->fileMetaData['tags'] ?? [];
436481
$streams = $this->fileMetaData['streams'] ?? [];
@@ -560,4 +605,22 @@ protected function extractMimeType($filePath) {
560605

561606
return $mimeType;
562607
}
608+
609+
protected function getAbsoluteMediaPath($media) {
610+
return str_replace('\\', '/', Storage::disk('public')->path('')) . "media/{$media->folder->path}/" . basename($media->path);
611+
}
612+
613+
protected function resolveMediaUuid($media, $filePath) {
614+
// if the media in db or file does not have a valid uuid, it will add it in both the db and on the file.
615+
if (! isset($this->fileMetaData['tags']['uuid'])) {
616+
$uuid = Str::uuid()->toString();
617+
$this->embedChain[] = new EmbedUidInMetadata($filePath, $uuid, $this->taskId, $media->id);
618+
} else {
619+
$uuid = $this->fileMetaData['tags']['uuid'];
620+
dump("Found UUID on file {$uuid}");
621+
$media->update(['uuid' => $uuid]); // If embedding, media file is updated in the embed job
622+
}
623+
624+
return $uuid;
625+
}
563626
}

‎app/Services/Subtitles/SubtitleScanner.php‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,24 @@
22

33
namespace App\Services\Subtitles;
44

5+
use App\Enums\SubtitleSource;
56
use Illuminate\Support\Facades\Log;
7+
use Illuminate\Support\Facades\Storage;
68

79
// Per Metadata Row
810
class SubtitleScanner {
9-
public function extractSubtitleStreams(array $fileMetadata): array {
11+
12+
public function scanEmbeddedSubtitles(string $uuid, array $fileMetaData): array {
13+
$subtitleStreams = $this->filterSubtitleStreams($fileMetaData);
14+
return count($subtitleStreams) > 0
15+
? $this->buildSubtitleTransactions($uuid, $subtitleStreams)
16+
: [];
17+
}
18+
19+
public function filterSubtitleStreams(array $fileMetaData): array {
1020
return array_filter(
11-
$fileMetadata['streams'] ?? [],
12-
fn ($stream) => ($stream['codec_type'] ?? null) === 'subtitle'
21+
$fileMetaData['streams'] ?? [],
22+
fn($stream) => ($stream['codec_type'] ?? null) === 'subtitle'
1323
);
1424
}
1525

@@ -25,16 +35,91 @@ public function buildSubtitleTransactions(string $uuid, array $streams): array {
2535
continue;
2636
}
2737

28-
$transactions[] = [
38+
$transaction = [
2939
'metadata_uuid' => $uuid,
30-
'track_id' => $stream['index'],
40+
'track_id' => $stream['index'], // is 0 if external
3141
'language' => $stream['tags']['language'] ?? 'und',
3242
'codec' => $stream['codec_name'],
3343
'is_default' => ($stream['disposition']['default'] ?? 0) === 1,
3444
'is_forced' => ($stream['disposition']['forced'] ?? 0) === 1,
45+
'external_path' => null,
46+
'source_key' => SubtitleSource::EMBEDDED->makeKey($stream['index'])
3547
];
48+
49+
if (isset($stream['external_path'])) {
50+
$transaction['external_path'] = $stream['external_path'];
51+
$transaction['source_key'] = SubtitleSource::EXTERNAL->makeKey($stream['external_path']);
52+
}
53+
54+
$transactions[] = $transaction;
3655
}
3756

3857
return $transactions;
3958
}
59+
60+
/**
61+
* @param string $directoryPath the relative directory of the given metadata row /library/folder
62+
* @return array a list of arrays that resemble ffprobe output for each matching subtitle file
63+
*/
64+
public function findExternalSubtitlesInDirectory(string $directoryPath): array {
65+
$extensions = ['srt', 'vtt', 'ass', 'ssa', 'sub'];
66+
$externalStreams = [];
67+
68+
if (config('app.env') === 'local') dump("SCANNING FOR SUBTITLES IN $directoryPath");
69+
70+
foreach ($extensions as $ext) {
71+
// match pattern basename.*.ext (ex/ movie.*.srt)
72+
$pattern = "{$directoryPath}/*.{$ext}";
73+
$files = glob($pattern);
74+
75+
foreach ($files as $file) {
76+
$filename = basename($file);
77+
$parts = explode('.', $filename);
78+
79+
// Parse: movie.en.forced.srt
80+
// parts: [movie, en, forced, srt]
81+
$media_filename = array_shift($parts); // remove related media file name
82+
$extension = array_pop($parts); // remove extension
83+
84+
if (count($parts) < 1) {
85+
continue;
86+
}
87+
88+
$language = $parts[0];
89+
$flags = array_slice($parts, 1);
90+
91+
$isForced = in_array('forced', $flags);
92+
$isDefault = in_array('default', $flags);
93+
94+
// Build ffprobe output
95+
$externalStreams[] = [
96+
'media_filename' => $media_filename,
97+
'index' => 0,
98+
'codec_type' => 'subtitle',
99+
'codec_name' => $this->getCodecFromExtension($extension),
100+
'tags' => [
101+
'language' => $language,
102+
],
103+
'disposition' => [
104+
'forced' => $isForced ? 1 : 0,
105+
'default' => $isDefault ? 1 : 0,
106+
],
107+
'external_path' => str_replace(str_replace('\\', '/', Storage::disk('public')->path('')), '', $file), // store relative location
108+
];
109+
}
110+
}
111+
112+
return $externalStreams;
113+
}
114+
115+
private function getCodecFromExtension(string $ext): string {
116+
return match ($ext) {
117+
'vtt' => 'webvtt',
118+
'srt' => 'subrip',
119+
'ass' => 'ass',
120+
'ssa' => 'ssa',
121+
'sub' => 'dvd_subtitle',
122+
default => 'unknown',
123+
};
124+
}
40125
}

0 commit comments

Comments
 (0)