Merge pull request #2979 from evoxf1/main

JonnyBurger · web-flow · commit d0e57e3f28ff · 2023-10-07T15:06:30.000+02:00
diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx
@@ -133,6 +133,7 @@ export type TRenderAsset = {
 	mediaFrame: number;
 	playbackRate: number;
 	allowAmplificationDuringRender: boolean;
+	toneFrequency: number | null;
 };
 
 export const compositionsRef = React.createRef<{
diff --git a/packages/core/src/audio/AudioForRendering.tsx b/packages/core/src/audio/AudioForRendering.tsx
@@ -51,6 +51,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
 		playbackRate,
 		allowAmplificationDuringRender,
 		onDuration,
+		toneFrequency,
 		_remotionInternalNeedsDurationCalculation,
 		acceptableTimeShiftInSeconds,
 		...nativeProps
@@ -97,6 +98,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
 			mediaFrame: frame,
 			playbackRate: props.playbackRate ?? 1,
 			allowAmplificationDuringRender: allowAmplificationDuringRender ?? false,
+			toneFrequency: toneFrequency ?? null,
 		});
 		return () => unregisterRenderAsset(id);
 	}, [
@@ -112,6 +114,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
 		playbackRate,
 		props.playbackRate,
 		allowAmplificationDuringRender,
+		toneFrequency,
 	]);
 
 	const {src} = props;
diff --git a/packages/core/src/audio/props.ts b/packages/core/src/audio/props.ts
@@ -18,4 +18,5 @@ export type RemotionAudioProps = Omit<
 	acceptableTimeShiftInSeconds?: number;
 	allowAmplificationDuringRender?: boolean;
 	_remotionInternalNeedsDurationCalculation?: boolean;
+	toneFrequency?: number;
 };
diff --git a/packages/core/src/video/OffthreadVideoForRendering.tsx b/packages/core/src/video/OffthreadVideoForRendering.tsx
@@ -26,6 +26,7 @@ export const OffthreadVideoForRendering: React.FC<OffthreadVideoProps> = ({
 	muted,
 	allowAmplificationDuringRender,
 	transparent = false,
+	toneFrequency,
 	...props
 }) => {
 	const absoluteFrame = useTimelinePosition();
@@ -95,6 +96,7 @@ export const OffthreadVideoForRendering: React.FC<OffthreadVideoProps> = ({
 			mediaFrame: frame,
 			playbackRate: playbackRate ?? 1,
 			allowAmplificationDuringRender: allowAmplificationDuringRender ?? false,
+			toneFrequency: toneFrequency ?? null,
 		});
 
 		return () => unregisterRenderAsset(id);
@@ -109,6 +111,7 @@ export const OffthreadVideoForRendering: React.FC<OffthreadVideoProps> = ({
 		absoluteFrame,
 		playbackRate,
 		allowAmplificationDuringRender,
+		toneFrequency,
 	]);
 
 	const currentTime = useMemo(() => {
diff --git a/packages/core/src/video/VideoForDevelopment.tsx b/packages/core/src/video/VideoForDevelopment.tsx
@@ -49,6 +49,7 @@ const VideoForDevelopmentRefForwardingFunction: React.ForwardRefRenderFunction<
 		// @ts-expect-error
 		acceptableTimeShift,
 		acceptableTimeShiftInSeconds,
+		toneFrequency,
 		...nativeProps
 	} = props;
 	if (typeof acceptableTimeShift !== 'undefined') {
diff --git a/packages/core/src/video/VideoForRendering.tsx b/packages/core/src/video/VideoForRendering.tsx
@@ -41,6 +41,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
 		allowAmplificationDuringRender,
 		playbackRate,
 		onDuration,
+		toneFrequency,
 		...props
 	},
 	ref,
@@ -110,6 +111,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
 			mediaFrame: frame,
 			playbackRate: playbackRate ?? 1,
 			allowAmplificationDuringRender: allowAmplificationDuringRender ?? false,
+			toneFrequency: toneFrequency ?? null,
 		});
 
 		return () => unregisterRenderAsset(id);
@@ -124,6 +126,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
 		absoluteFrame,
 		playbackRate,
 		allowAmplificationDuringRender,
+		toneFrequency,
 	]);
 
 	useImperativeHandle(
diff --git a/packages/core/src/video/props.ts b/packages/core/src/video/props.ts
@@ -17,6 +17,7 @@ export type RemotionVideoProps = Omit<
 	playbackRate?: number;
 	acceptableTimeShiftInSeconds?: number;
 	allowAmplificationDuringRender?: boolean;
+	toneFrequency?: number;
 };
 
 type DeprecatedOffthreadVideoProps = {
@@ -37,6 +38,7 @@ export type OffthreadVideoProps = {
 	onError?: React.ReactEventHandler<HTMLVideoElement | HTMLImageElement>;
 	acceptableTimeShiftInSeconds?: number;
 	allowAmplificationDuringRender?: boolean;
+	toneFrequency?: number;
 	transparent?: boolean;
 } & RemotionMainVideoProps &
 	DeprecatedOffthreadVideoProps;
diff --git a/packages/docs/docs/audio.md b/packages/docs/docs/audio.md
@@ -125,13 +125,19 @@ export const MyVideo = () => {
 };
 ```
 
+### `toneFrequency`<AvailableFrom v="4.0.47"/>
+
+Allows you to adjust the pitch of the audio - will only be applied during rendering. It accepts a number between `0.01` and `2`, where `1` represents the original pitch. Values less than `1` will decrease the pitch, while values greater than `1` will increase it.
+For example, a `toneFrequency` of 0.5 would lower the pitch by half, and a `toneFrequency` of `1.5` would increase the pitch by 50%.
+`toneFrequency` should be a positive number between `0.01` (representing the lowest assignable pitch) and `2` (representing the highest assignable pitch). If `toneFrequency` is not specified or null, it defaults to `1`, maintaining the original pitch.
+
 ## `acceptableTimeShiftInSeconds`<AvailableFrom v="3.2.42"/>
 
 In the [Remotion Studio](/docs/terminology#remotion-studio) or in the [Remotion Player](/docs/player), Remotion will seek the audio if it gets too much out of sync with Remotion's internal time - be it due to the audio loading or the page being too slow to keep up in real-time. By default, a seek is triggered if `0.45` seconds of time shift is encountered. Using this prop, you can customize the threshold.
 
 ## `allowAmplificationDuringRender`<AvailableFrom v="3.3.17"/>
 
-Make values for [`volume`](#volume) greater than `1` result in amplification during renders.  
+Make values for [`volume`](#volume) greater than `1` result in amplification during renders.
 In the Remotion Studio, the volume will be limited to `1`, since the browser cannot amplify audio.
 
 ## See also
diff --git a/packages/docs/docs/offthreadvideo.md b/packages/docs/docs/offthreadvideo.md
@@ -64,6 +64,12 @@ With `png`, transparent videos (VP8, VP9, ProRes) can be displayed, however it i
 Make values for [`volume`](/docs/video#volume) greater than `1` result in amplification during renders.  
 During Preview, the volume will be limited to `1`, since the browser cannot amplify audio.
 
+### `toneFrequency`<AvailableFrom v="4.0.47"/>
+
+Allows you to adjust the pitch of the audio - will only be applied during rendering. It accepts a number between `0.01` and `2`, where `1` represents the original pitch. Values less than `1` will decrease the pitch, while values greater than `1` will increase it.
+For example, a `toneFrequency` of 0.5 would lower the pitch by half, and a `toneFrequency` of `1.5` would increase the pitch by 50%.
+`toneFrequency` should be a positive number between `0.01` (representing the lowest assignable pitch) and `2` (representing the highest assignable pitch). If `toneFrequency` is not specified or null, it defaults to `1`, maintaining the original pitch.
+
 ### `onError`
 
 Handle an error playing the video. From v3.3.89, if you pass an `onError` callback, then no exception will be thrown. Previously, the error could not be caught.
diff --git a/packages/docs/docs/video.md b/packages/docs/docs/video.md
@@ -187,6 +187,12 @@ In the [Studio](/docs/terminology#remotion-studio) or in the [Remotion Player](/
 Make values for [`volume`](#volume) greater than `1` result in amplification during renders.  
 During Preview, the volume will be limited to `1`, since the browser cannot amplify audio.
 
+### `toneFrequency`<AvailableFrom v="4.0.47"/>
+
+Allows you to adjust the pitch of the audio - will only be applied during rendering. It accepts a number between `0.01` and `2`, where `1` represents the original pitch. Values less than `1` will decrease the pitch, while values greater than `1` will increase it.
+For example, a `toneFrequency` of 0.5 would lower the pitch by half, and a `toneFrequency` of `1.5` would increase the pitch by 50%.
+`toneFrequency` should be a positive number between `0.01` (representing the lowest assignable pitch) and `2` (representing the highest assignable pitch). If `toneFrequency` is not specified or null, it defaults to `1`, maintaining the original pitch.
+
 ### `onError`
 
 Handle an error playing the video. From v3.3.89, if you pass an `onError` callback, then no exception will be thrown. Previously, the error could not be caught.
diff --git a/packages/renderer/src/assets/calculate-asset-positions.ts b/packages/renderer/src/assets/calculate-asset-positions.ts
@@ -49,6 +49,7 @@ export const calculateAssetPositions = (frames: TRenderAsset[][]): Assets => {
 					volume: [],
 					playbackRate: asset.playbackRate,
 					allowAmplificationDuringRender: asset.allowAmplificationDuringRender,
+					toneFrequency: asset.toneFrequency,
 				});
 			}
 
diff --git a/packages/renderer/src/assets/types.ts b/packages/renderer/src/assets/types.ts
@@ -14,6 +14,7 @@ export type UnsafeAsset = Omit<
 	id: string;
 	playbackRate: number;
 	allowAmplificationDuringRender: boolean;
+	toneFrequency: number | null;
 };
 
 // Volume can either be static, for all frames the same,
diff --git a/packages/renderer/src/calculate-ffmpeg-filters.ts b/packages/renderer/src/calculate-ffmpeg-filters.ts
@@ -34,5 +34,6 @@ export const calculateFfmpegFilter = ({
 		durationInFrames,
 		assetDuration,
 		allowAmplificationDuringRender: asset.allowAmplificationDuringRender,
+		toneFrequency: asset.toneFrequency,
 	});
 };
diff --git a/packages/renderer/src/stringify-ffmpeg-filter.ts b/packages/renderer/src/stringify-ffmpeg-filter.ts
@@ -24,6 +24,7 @@ export const stringifyFfmpegFilter = ({
 	durationInFrames,
 	assetDuration,
 	allowAmplificationDuringRender,
+	toneFrequency,
 }: {
 	trimLeft: number;
 	trimRight: number;
@@ -35,13 +36,20 @@ export const stringifyFfmpegFilter = ({
 	playbackRate: number;
 	assetDuration: number | null;
 	allowAmplificationDuringRender: boolean;
+	toneFrequency: number | null;
 }): FilterWithoutPaddingApplied | null => {
 	const startInVideoSeconds = startInVideo / fps;
 
 	if (assetDuration && trimLeft >= assetDuration) {
 		return null;
 	}
 
+	if (toneFrequency !== null && (toneFrequency <= 0 || toneFrequency > 2)) {
+		throw new Error(
+			'toneFrequency must be a positive number between 0.01 and 2',
+		);
+	}
+
 	const volumeFilter = ffmpegVolumeExpression({
 		volume,
 		fps,
@@ -76,6 +84,9 @@ export const stringifyFfmpegFilter = ({
 				volumeFilter.value === '1'
 					? null
 					: `volume=${volumeFilter.value}:eval=${volumeFilter.eval}`,
+				toneFrequency && toneFrequency !== 1
+					? `asetrate=${DEFAULT_SAMPLE_RATE}*${toneFrequency},aresample=${DEFAULT_SAMPLE_RATE},atempo=1/${toneFrequency}`
+					: null,
 				// For n channels, we delay n + 1 channels.
 				// This is because `ffprobe` for some audio files reports the wrong amount
 				// of channels.
diff --git a/packages/renderer/src/test/asset-calculation.test.tsx b/packages/renderer/src/test/asset-calculation.test.tsx
@@ -40,6 +40,7 @@ test('Should be able to collect assets', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
 
@@ -60,6 +61,7 @@ test('Should get multiple assets', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 	expect(withoutId(assetPositions[1])).toEqual({
 		type: 'audio',
@@ -70,6 +72,7 @@ test('Should get multiple assets', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
 
@@ -94,6 +97,7 @@ test('Should handle jumps inbetween', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 	expect(withoutId(assetPositions[1])).toEqual({
 		type: 'video',
@@ -104,6 +108,7 @@ test('Should handle jumps inbetween', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
 
@@ -125,6 +130,7 @@ test('Should support sequencing', async () => {
 		volume: 1,
 		playbackRate: 1,
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
 
@@ -156,6 +162,7 @@ test('Should calculate volumes correctly', async () => {
 			)
 			.filter((f) => f > 0),
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
 
@@ -197,5 +204,6 @@ test('Should calculate startFrom correctly', async () => {
 			)
 			.filter((i) => i > 0),
 		allowAmplificationDuringRender: false,
+		toneFrequency: null,
 	});
 });
diff --git a/packages/renderer/src/test/asset-compression.test.ts b/packages/renderer/src/test/asset-compression.test.ts
@@ -15,6 +15,7 @@ test('Should compress and uncompress assets', () => {
 				type: 'video' as const,
 				volume: 1,
 				allowAmplificationDuringRender: false,
+				toneFrequency: null,
 			},
 		],
 		[
@@ -27,6 +28,7 @@ test('Should compress and uncompress assets', () => {
 				type: 'video' as const,
 				volume: 1,
 				allowAmplificationDuringRender: false,
+				toneFrequency: null,
 			},
 		],
 	].flat(1);
@@ -50,6 +52,7 @@ test('Should compress and uncompress assets', () => {
 			type: 'video',
 			volume: 1,
 			allowAmplificationDuringRender: false,
+			toneFrequency: null,
 		},
 	]);
 });
diff --git a/packages/renderer/src/test/dont-skip-assets.test.ts b/packages/renderer/src/test/dont-skip-assets.test.ts
@@ -21,6 +21,7 @@ test('Dont skip assets', () => {
 			volume: 1,
 			playbackRate: 1,
 			allowAmplificationDuringRender: false,
+			toneFrequency: null,
 		},
 		{
 			src: 'http://localhost:3000/e15ac5e3d531199ebb1828ca6a99100d.webm',
@@ -32,6 +33,7 @@ test('Dont skip assets', () => {
 			volume: 1,
 			playbackRate: 1,
 			allowAmplificationDuringRender: false,
+			toneFrequency: null,
 		},
 	]);
 });
@@ -51,6 +53,7 @@ const mock: TRenderAsset[][] = new Array(2934)
 						playbackRate: 1,
 						mediaFrame: 0,
 						allowAmplificationDuringRender: false,
+						toneFrequency: null,
 				  }
 				: null,
 			{
@@ -62,6 +65,7 @@ const mock: TRenderAsset[][] = new Array(2934)
 				playbackRate: 1,
 				mediaFrame: k,
 				allowAmplificationDuringRender: false,
+				toneFrequency: null,
 			},
 		].filter(truthy);
 	});
diff --git a/packages/renderer/src/test/ffmpeg-filters.test.ts b/packages/renderer/src/test/ffmpeg-filters.test.ts
@@ -15,6 +15,7 @@ const asset: MediaAsset = {
 	id: '1',
 	playbackRate: 1,
 	allowAmplificationDuringRender: false,
+	toneFrequency: null,
 };
 
 test('Should create a basic filter correctly', () => {
@@ -153,6 +154,7 @@ test('Should calculate pad correctly with a lot of playbackRate', () => {
 				startInVideo: 0,
 				playbackRate: 16,
 				allowAmplificationDuringRender: false,
+				toneFrequency: null,
 			},
 			durationInFrames: 2000,
 			channels: 1,
