Refactor uploadx middlewares

packages/tests/src/api/check-params/index.ts

@@ -9,7 +9,7 @@ import './custom-pages.js'
 import './debug.js'
 import './follows.js'
 import './user-export.js'
-import './user-import.js.js'
+import './user-import.js'
 import './jobs.js'
 import './live.js'
 import './logs.js'

server/core/controllers/api/users/user-imports.ts

@@ -3,7 +3,7 @@ import {
   asyncMiddleware,
   authenticate
 } from '../../../middlewares/index.js'
-import { uploadx } from '@server/lib/uploadx.js'
+import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
 import {
   getLatestImportStatusValidator,
   userImportRequestResumableInitValidator,
@@ -19,30 +19,22 @@ import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
 
 const userImportRouter = express.Router()
 
-userImportRouter.post('/:userId/imports/import-resumable',
-  authenticate,
-  asyncMiddleware(userImportRequestResumableInitValidator),
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
-
-userImportRouter.delete('/:userId/imports/import-resumable',
-  authenticate,
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
-
-userImportRouter.put('/:userId/imports/import-resumable',
-  authenticate,
-  uploadx.upload, // uploadx doesn't next() before the file upload completes
-  asyncMiddleware(userImportRequestResumableValidator),
-  asyncMiddleware(addUserImportResumable)
-)
-
 userImportRouter.get('/:userId/imports/latest',
   authenticate,
   asyncMiddleware(getLatestImportStatusValidator),
   asyncMiddleware(getLatestImport)
 )
 
+setupUploadResumableRoutes({
+  routePath: '/:userId/imports/import-resumable',
+  router: userImportRouter,
+
+  uploadInitAfterMiddlewares: [ asyncMiddleware(userImportRequestResumableInitValidator) ],
+
+  uploadedMiddlewares: [ asyncMiddleware(userImportRequestResumableValidator) ],
+  uploadedController: asyncMiddleware(addUserImportResumable)
+})
+
 // ---------------------------------------------------------------------------
 
 export {

server/core/controllers/api/videos/captions.ts

@@ -47,12 +47,12 @@ async function listVideoCaptions (req: express.Request, res: express.Response) {
 }
 
 async function createVideoCaption (req: express.Request, res: express.Response) {
-  const videoCaptionPhysicalFile = req.files['captionfile'][0]
+  const videoCaptionPhysicalFile: Express.Multer.File = req.files['captionfile'][0]
   const video = res.locals.videoAll
 
   const captionLanguage = req.params.captionLanguage
 
-  const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile })
+  const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile.path })
 
   await sequelizeTypescript.transaction(async t => {
     await federateVideoIfNeeded(video, false, t)

server/core/controllers/api/videos/source.ts

@@ -4,7 +4,7 @@ import { sequelizeTypescript } from '@server/initializers/database.js'
 import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
-import { uploadx } from '@server/lib/uploadx.js'
+import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
 import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { buildNewFile } from '@server/lib/video-file.js'
@@ -35,23 +35,14 @@ videoSourceRouter.get('/:id/source',
   getVideoLatestSource
 )
 
-videoSourceRouter.post('/:id/source/replace-resumable',
-  authenticate,
-  asyncMiddleware(replaceVideoSourceResumableInitValidator),
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
+setupUploadResumableRoutes({
+  routePath: '/:id/source/replace-resumable',
+  router: videoSourceRouter,
 
-videoSourceRouter.delete('/:id/source/replace-resumable',
-  authenticate,
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
-
-videoSourceRouter.put('/:id/source/replace-resumable',
-  authenticate,
-  uploadx.upload, // uploadx doesn't next() before the file upload completes
-  asyncMiddleware(replaceVideoSourceResumableValidator),
-  asyncMiddleware(replaceVideoSourceResumable)
-)
+  uploadInitAfterMiddlewares: [ asyncMiddleware(replaceVideoSourceResumableInitValidator) ],
+  uploadedMiddlewares: [ asyncMiddleware(replaceVideoSourceResumableValidator) ],
+  uploadedController: asyncMiddleware(replaceVideoSourceResumable)
+})
 
 // ---------------------------------------------------------------------------
 

server/core/controllers/api/videos/upload.ts

@@ -1,7 +1,7 @@
 import express from 'express'
 import { getResumableUploadPath } from '@server/helpers/upload.js'
 import { Redis } from '@server/lib/redis.js'
-import { uploadx } from '@server/lib/uploadx.js'
+import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
 import { uuidToShort } from '@peertube/peertube-node-utils'
@@ -45,27 +45,25 @@ uploadRouter.post('/upload',
   asyncRetryTransactionMiddleware(addVideoLegacy)
 )
 
-uploadRouter.post('/upload-resumable',
-  openapiOperationDoc({ operationId: 'uploadResumableInit' }),
-  authenticate,
-  reqVideoFileAddResumable,
-  asyncMiddleware(videosAddResumableInitValidator),
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
+setupUploadResumableRoutes({
+  routePath: '/upload-resumable',
+  router: uploadRouter,
 
-uploadRouter.delete('/upload-resumable',
-  authenticate,
-  asyncMiddleware(deleteUploadResumableCache),
-  (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
-)
+  uploadInitBeforeMiddlewares: [
+    openapiOperationDoc({ operationId: 'uploadResumableInit' }),
+    reqVideoFileAddResumable
+  ],
 
-uploadRouter.put('/upload-resumable',
-  openapiOperationDoc({ operationId: 'uploadResumable' }),
-  authenticate,
-  uploadx.upload, // uploadx doesn't next() before the file upload completes
-  asyncMiddleware(videosAddResumableValidator),
-  asyncMiddleware(addVideoResumable)
-)
+  uploadInitAfterMiddlewares: [ asyncMiddleware(videosAddResumableInitValidator) ],
+
+  uploadDeleteMiddlewares: [ asyncMiddleware(deleteUploadResumableCache) ],
+
+  uploadedMiddlewares: [
+    openapiOperationDoc({ operationId: 'uploadResumable' }),
+    asyncMiddleware(videosAddResumableValidator)
+  ],
+  uploadedController: asyncMiddleware(addVideoResumable)
+})
 
 // ---------------------------------------------------------------------------
 
@@ -110,7 +108,7 @@ async function addVideoResumable (req: express.Request, res: express.Response) {
 async function addVideo (options: {
   req: express.Request
   res: express.Response
-  videoPhysicalFile: express.VideoUploadFile
+  videoPhysicalFile: express.VideoLegacyUploadFile
   videoInfo: VideoCreate
   files: express.UploadFiles
 }) {

server/core/lib/moderation.ts

@@ -1,4 +1,4 @@
-import express, { VideoUploadFile } from 'express'
+import express, { VideoLegacyUploadFile } from 'express'
 import { PathLike } from 'fs-extra/esm'
 import { Transaction } from 'sequelize'
 import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger.js'
@@ -38,7 +38,7 @@ export type AcceptResult = {
 // Stub function that can be filtered by plugins
 function isLocalVideoFileAccepted (object: {
   videoBody: VideoCreate
-  videoFile: VideoUploadFile
+  videoFile: VideoLegacyUploadFile
   user: MUserDefault
 }): AcceptResult {
   return { accepted: true }

server/core/lib/uploadx.ts

@@ -1,13 +1,15 @@
-import express from 'express'
+import express, { Request, Response, NextFunction, RequestHandler } from 'express'
 import { buildLogger } from '@server/helpers/logger.js'
 import { getResumableUploadPath } from '@server/helpers/upload.js'
 import { CONFIG } from '@server/initializers/config.js'
-import { LogLevel, Uploadx } from '@uploadx/core'
+import { FileQuery, LogLevel, Uploadx, Metadata as UploadXMetadata } from '@uploadx/core'
 import { extname } from 'path'
+import { authenticate } from '@server/middlewares/auth.js'
+import { resumableInitValidator } from '@server/middlewares/validators/resumable-upload.js'
 
 const logger = buildLogger('uploadx')
 
-const uploadx = new Uploadx({
+export const uploadx = new Uploadx({
   directory: getResumableUploadPath(),
 
   expiration: { maxAge: undefined, rolling: true },
@@ -32,6 +34,60 @@ const uploadx = new Uploadx({
   filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}`
 })
 
-export {
-  uploadx
+export function safeUploadXCleanup (file: FileQuery) {
+  uploadx.storage.delete(file)
+    .catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
+}
+
+export function buildUploadXFile <T extends UploadXMetadata> (reqBody: T) {
+  return {
+    ...reqBody,
+
+    path: getResumableUploadPath(reqBody.name),
+    filename: reqBody.metadata.filename
+  }
+}
+
+export function setupUploadResumableRoutes (options: {
+  router: express.Router
+  routePath: string
+
+  uploadInitBeforeMiddlewares?: RequestHandler[]
+  uploadInitAfterMiddlewares?: RequestHandler[]
+
+  uploadedMiddlewares?: ((req: Request<any>, res: Response, next: NextFunction) => void)[]
+  uploadedController: (req: Request<any>, res: Response, next: NextFunction) => void
+
+  uploadDeleteMiddlewares?: RequestHandler[]
+}) {
+  const {
+    router,
+    routePath,
+    uploadedMiddlewares = [],
+    uploadedController,
+    uploadInitBeforeMiddlewares = [],
+    uploadInitAfterMiddlewares = [],
+    uploadDeleteMiddlewares = []
+  } = options
+
+  router.post(routePath,
+    authenticate,
+    ...uploadInitBeforeMiddlewares,
+    resumableInitValidator,
+    ...uploadInitAfterMiddlewares,
+    (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+  )
+
+  router.delete(routePath,
+    authenticate,
+    ...uploadDeleteMiddlewares,
+    (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
+  )
+
+  router.put(routePath,
+    authenticate,
+    uploadx.upload, // uploadx doesn't next() before the file upload completes
+    ...uploadedMiddlewares,
+    uploadedController
+  )
 }

server/core/middlewares/validators/index.ts

@@ -16,6 +16,7 @@ export * from './oembed.js'
 export * from './pagination.js'
 export * from './plugins.js'
 export * from './redundancy.js'
+export * from './resumable-upload.js'
 export * from './search.js'
 export * from './server.js'
 export * from './sort.js'

server/core/middlewares/validators/resumable-upload.ts

@@ -0,0 +1,36 @@
+import { logger } from '@server/helpers/logger.js'
+import express from 'express'
+import { body, header } from 'express-validator'
+import { areValidationErrors } from './shared/utils.js'
+import { cleanUpReqFiles } from '@server/helpers/express-utils.js'
+
+export const resumableInitValidator = [
+  body('filename')
+    .exists(),
+
+  header('x-upload-content-length')
+    .isNumeric()
+    .exists()
+    .withMessage('Should specify the file length'),
+  header('x-upload-content-type')
+    .isString()
+    .exists()
+    .withMessage('Should specify the file mimetype'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking resumableInitValidator parameters and headers', {
+      parameters: req.body,
+      headers: req.headers
+    })
+
+    if (areValidationErrors(req, res, { omitLog: true })) return cleanUpReqFiles(req)
+
+    res.locals.uploadVideoFileResumableMetadata = {
+      mimetype: req.headers['x-upload-content-type'] as string,
+      size: +req.headers['x-upload-content-length'],
+      originalname: req.body.filename
+    }
+
+    return next()
+  }
+]

server/core/middlewares/validators/users/user-import.ts

@@ -1,9 +1,7 @@
 import express from 'express'
-import { body, header, param } from 'express-validator'
-import { getResumableUploadPath } from '@server/helpers/upload.js'
-import { uploadx } from '@server/lib/uploadx.js'
+import { param } from 'express-validator'
+import { buildUploadXFile, safeUploadXCleanup } from '@server/lib/uploadx.js'
 import { Metadata as UploadXMetadata } from '@uploadx/core'
-import { logger } from '../../../helpers/logger.js'
 import { areValidationErrors, checkUserIdExist } from '../shared/index.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { HttpStatusCode, ServerErrorCode, UserImportState, UserRight } from '@peertube/peertube-models'
@@ -15,9 +13,8 @@ export const userImportRequestResumableValidator = [
     .isInt().not().isEmpty().withMessage('Should have a valid userId'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
-    const body: express.CustomUploadXFile<UploadXMetadata> = req.body
-    const file = { ...body, path: getResumableUploadPath(body.name), filename: body.metadata.filename }
-    const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
+    const file = buildUploadXFile(req.body as express.CustomUploadXFile<UploadXMetadata>)
+    const cleanup = () => safeUploadXCleanup(file)
 
     if (!await checkUserIdRight(req.params.userId, res)) return cleanup()
 
@@ -40,25 +37,8 @@ export const userImportRequestResumableInitValidator = [
   param('userId')
     .isInt().not().isEmpty().withMessage('Should have a valid userId'),
 
-  body('filename')
-    .exists(),
-
-  header('x-upload-content-length')
-    .isNumeric()
-    .exists()
-    .withMessage('Should specify the file length'),
-  header('x-upload-content-type')
-    .isString()
-    .exists()
-    .withMessage('Should specify the file mimetype'),
-
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
-    logger.debug('Checking userImportRequestResumableInitValidator parameters and headers', {
-      parameters: req.body,
-      headers: req.headers
-    })
-
-    if (areValidationErrors(req, res, { omitLog: true })) return
+    if (areValidationErrors(req, res)) return
 
     if (CONFIG.IMPORT.USERS.ENABLED !== true) {
       return res.fail({
@@ -76,8 +56,9 @@ export const userImportRequestResumableInitValidator = [
 
     if (!await checkUserIdRight(req.params.userId, res)) return
 
+    const fileMetadata = res.locals.uploadVideoFileResumableMetadata
     const user = res.locals.user
-    if (await isUserQuotaValid({ userId: user.id, uploadSize: +req.headers['x-upload-content-length'] }) === false) {
+    if (await isUserQuotaValid({ userId: user.id, uploadSize: fileMetadata.size }) === false) {
       return res.fail({
         message: 'User video quota is exceeded with this import',
         status: HttpStatusCode.PAYLOAD_TOO_LARGE_413,

server/core/middlewares/validators/videos/shared/video-validators.ts

@@ -41,7 +41,7 @@ export async function commonVideoFileChecks (options: {
 export async function isVideoFileAccepted (options: {
   req: express.Request
   res: express.Response
-  videoFile: express.VideoUploadFile
+  videoFile: express.VideoLegacyUploadFile
   hook: Extract<ServerFilterHookName, 'filter:api.video.upload.accept.result' | 'filter:api.video.update-file.accept.result'>
 }) {
   const { req, res, videoFile, hook } = options