diff --git a/packages/models/src/runners/runner-job-private-payload.model.ts b/packages/models/src/runners/runner-job-private-payload.model.ts index c1205984eef..3c9f08de6d1 100644 --- a/packages/models/src/runners/runner-job-private-payload.model.ts +++ b/packages/models/src/runners/runner-job-private-payload.model.ts @@ -15,11 +15,13 @@ export type RunnerJobPrivatePayload = export interface RunnerJobVODWebVideoTranscodingPrivatePayload { videoUUID: string isNewVideo: boolean + deleteInputFileId: number | null } export interface RunnerJobVODAudioMergeTranscodingPrivatePayload { videoUUID: string isNewVideo: boolean + deleteInputFileId: number | null } export interface RunnerJobVODHLSTranscodingPrivatePayload { diff --git a/packages/tests/src/peertube-runner/vod-transcoding.ts b/packages/tests/src/peertube-runner/vod-transcoding.ts index a00b1857abc..f3987faf16e 100644 --- a/packages/tests/src/peertube-runner/vod-transcoding.ts +++ b/packages/tests/src/peertube-runner/vod-transcoding.ts @@ -232,6 +232,45 @@ describe('Test VOD transcoding in peertube-runner program', function () { resolutions: [ 720, 480, 360, 240, 144, 0 ] }) }) + + it('Should not generate an upper resolution than original file', async function () { + this.timeout(120_000) + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + hls: { enabled: true }, + webVideos: { enabled: true }, + resolutions: { + '0p': false, + '144p': false, + '240p': true, + '360p': false, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false + } + } + }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.webm' }) + await waitJobs(servers, { runnerJobs: true }) + + const video = await servers[0].videos.get({ id: uuid }) + const hlsFiles = video.streamingPlaylists[0].files + + expect(video.files).to.have.lengthOf(2) + expect(hlsFiles).to.have.lengthOf(2) + + // eslint-disable-next-line @typescript-eslint/require-array-sort-compare + const resolutions = getAllFiles(video).map(f => f.resolution.id).sort() + expect(resolutions).to.deep.equal([ 240, 240, 480, 480 ]) + }) } before(async function () { diff --git a/packages/tests/src/shared/videos.ts b/packages/tests/src/shared/videos.ts index c8de95bcf1c..62a72b43645 100644 --- a/packages/tests/src/shared/videos.ts +++ b/packages/tests/src/shared/videos.ts @@ -36,6 +36,8 @@ async function completeWebVideoFilesCheck (options: { const transcodingEnabled = serverConfig.transcoding.web_videos.enabled + expect(files).to.have.lengthOf(files.length) + for (const attributeFile of files) { const file = video.files.find(f => f.resolution.id === attributeFile.resolution) expect(file, `resolution ${attributeFile.resolution} does not exist`).not.to.be.undefined diff --git a/server/core/lib/runners/job-handlers/shared/vod-helpers.ts b/server/core/lib/runners/job-handlers/shared/vod-helpers.ts index f7c1b73c90a..d0eb6264f94 100644 --- a/server/core/lib/runners/job-handlers/shared/vod-helpers.ts +++ b/server/core/lib/runners/job-handlers/shared/vod-helpers.ts @@ -8,6 +8,7 @@ import { VideoModel } from '@server/models/video/video.js' import { MVideoFullLight } from '@server/types/models/index.js' import { MRunnerJob } from '@server/types/models/runners/index.js' import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@peertube/peertube-models' +import { lTags } from '@server/lib/object-storage/shared/logger.js' export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { video: MVideoFullLight @@ -28,6 +29,23 @@ export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { videoOutputPath: newVideoFilePath }) + if (privatePayload.deleteInputFileId) { + const inputFile = video.VideoFiles.find(f => f.id === privatePayload.deleteInputFileId) + + if (inputFile) { + await video.removeWebVideoFile(inputFile) + await inputFile.destroy() + + video.VideoFiles = video.VideoFiles.filter(f => f.id !== inputFile.id) + } else { + logger.error( + 'Cannot delete input file %d of video %s: does not exist anymore', + privatePayload.deleteInputFileId, video.uuid, + { ...lTags(video.uuid), privatePayload } + ) + } + } + await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) } diff --git a/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts index e3339f569fa..bcae1b27a94 100644 --- a/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ b/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts @@ -21,6 +21,7 @@ type CreateOptions = { resolution: number fps: number priority: number + deleteInputFileId: number | null dependsOnRunnerJob?: MRunnerJob } @@ -43,7 +44,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo } const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), + ...pick(options, [ 'isNewVideo', 'deleteInputFileId' ]), videoUUID: video.uuid } @@ -81,12 +82,6 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo video.duration = await getVideoStreamDuration(videoFilePath) await video.save() - // We can remove the old audio file - const oldAudioFile = video.VideoFiles[0] - await video.removeWebVideoFile(oldAudioFile) - await oldAudioFile.destroy() - video.VideoFiles = [] - await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) logger.info( diff --git a/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts index a137a5d4803..a23adbc3f98 100644 --- a/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ b/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts @@ -20,6 +20,7 @@ type CreateOptions = { resolution: number fps: number priority: number + deleteInputFileId: number | null dependsOnRunnerJob?: MRunnerJob } @@ -41,7 +42,7 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH } const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), + ...pick(options, [ 'isNewVideo', 'deleteInputFileId' ]), videoUUID: video.uuid } diff --git a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts index 68d84a286b0..914a649fdc1 100644 --- a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts @@ -13,7 +13,7 @@ import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@ import { MRunnerJob } from '@server/types/models/runners/index.js' import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg' import { getTranscodingJobPriority } from '../../transcoding-priority.js' -import { computeResolutionsToTranscode } from '../../transcoding-resolutions.js' +import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions.js' import { AbstractJobBuilder } from './abstract-job-builder.js' /** @@ -52,16 +52,23 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value : await getVideoStreamFPS(videoFilePath, probe) - const maxResolution = await isAudioFile(videoFilePath, probe) + const isAudioInput = await isAudioFile(videoFilePath, probe) + const maxResolution = isAudioInput ? DEFAULT_AUDIO_RESOLUTION - : resolution + : buildOriginalFileResolution(resolution) const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + const deleteInputFileId = isAudioInput || maxResolution !== resolution + ? videoFile.id + : null + + const jobPayload = { video, resolution: maxResolution, fps, isNewVideo, priority, deleteInputFileId } + const mainRunnerJob = videoFile.isAudio() - ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) + ? await new VODAudioMergeTranscodingJobHandler().create(jobPayload) + : await new VODWebVideoTranscodingJobHandler().create(jobPayload) if (CONFIG.TRANSCODING.HLS.ENABLED === true) { await new VODHLSTranscodingJobHandler().create({ @@ -110,12 +117,14 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) + const jobPayload = { video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority, deleteInputFileId: null } + // Process the last resolution before the other ones to prevent concurrency issue // Because low resolutions use the biggest one as ffmpeg input const mainJob = transcodingType === 'hls' // eslint-disable-next-line max-len - ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) + ? await new VODHLSTranscodingJobHandler().create({ ...jobPayload, deleteWebVideoFiles: false }) + : await new VODWebVideoTranscodingJobHandler().create(jobPayload) for (const resolution of childrenResolutions) { const dependsOnRunnerJob = mainJob @@ -141,6 +150,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { fps, isNewVideo, dependsOnRunnerJob, + deleteInputFileId: null, priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) continue @@ -180,6 +190,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { fps, isNewVideo, dependsOnRunnerJob: mainRunnerJob, + deleteInputFileId: null, priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) }) }