diff --git a/.env.sample b/.env.sample index 7dc4976..15c83e6 100644 --- a/.env.sample +++ b/.env.sample @@ -3,7 +3,8 @@ STATIC_SERVE_FOLDER="data/static" LOG_FILE_REQUESTS="data/static/static_log.txt" DB_PATH="./data/db/db.db" RECENT_SEARCHES_FOLDER="./data/recent_searches" -TEST_ARRAY='[ - "one", - "two" +VIDEO_FILES_FOLDERS='[ + "/home/user/videos", + "/mnt/hdd/videos" ]' +VIDEO_FILES_EXT='["*.mkv", "*.mp4", "*.avi"]' diff --git a/package-lock.json b/package-lock.json index 1dea468..4e42eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "rxjs": "^7.2.0", "sqlite": "^4.1.2", "sqlite3": "^5.1.2", + "torrent-name-parser": "^0.6.5", "typeorm": "^0.3.11" }, "devDependencies": { @@ -8809,6 +8810,11 @@ "node": ">=0.6" } }, + "node_modules/torrent-name-parser": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/torrent-name-parser/-/torrent-name-parser-0.6.5.tgz", + "integrity": "sha512-ao92tzD6DjYKN0dedEZxfylL4ZWYaYFgPktKxkEmlsLrF/ia41Sg+l3rJ2XLJufQUR3uTbd7bYMqxbh/kxb1pQ==" + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -16341,6 +16347,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, + "torrent-name-parser": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/torrent-name-parser/-/torrent-name-parser-0.6.5.tgz", + "integrity": "sha512-ao92tzD6DjYKN0dedEZxfylL4ZWYaYFgPktKxkEmlsLrF/ia41Sg+l3rJ2XLJufQUR3uTbd7bYMqxbh/kxb1pQ==" + }, "tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/package.json b/package.json index 0d701df..59a8cb5 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "rxjs": "^7.2.0", "sqlite": "^4.1.2", "sqlite3": "^5.1.2", + "torrent-name-parser": "^0.6.5", "typeorm": "^0.3.11" }, "devDependencies": { diff --git a/src/TryOutModule/try.cotroller.ts b/src/TryOutModule/try.cotroller.ts index b665caf..df57ce3 100644 --- a/src/TryOutModule/try.cotroller.ts +++ b/src/TryOutModule/try.cotroller.ts @@ -1,9 +1,11 @@ import { Controller, Get } from '@nestjs/common'; +import { VideoFilesUpdateService } from '../VideoFiles/FileDbUpdater/video-files-update.service'; @Controller('try') export class TryController { + constructor(readonly videoFilesUpdateService: VideoFilesUpdateService) {} @Get('') main() { - return process.env; + return this.videoFilesUpdateService.updateFsVideoFiles(); } } diff --git a/src/TryOutModule/try.module.ts b/src/TryOutModule/try.module.ts index af92de8..2d5fee1 100644 --- a/src/TryOutModule/try.module.ts +++ b/src/TryOutModule/try.module.ts @@ -1,11 +1,12 @@ import { Module } from '@nestjs/common'; import { TryController } from './try.cotroller'; +import { VideoFilesModule } from '../VideoFiles/video-files.module'; /** * Playground module for playing around */ @Module({ - imports: [], + imports: [VideoFilesModule], controllers: [TryController], providers: [], }) diff --git a/src/VideoFiles/Dto/video-file-path.dto.ts b/src/VideoFiles/Dto/video-file-path.dto.ts new file mode 100644 index 0000000..e9bf76a --- /dev/null +++ b/src/VideoFiles/Dto/video-file-path.dto.ts @@ -0,0 +1,6 @@ +export class VideoFilePathDto { + constructor(readonly path: string, readonly relativePath: string) { + this.path = path; + this.relativePath = relativePath; + } +} diff --git a/src/VideoFiles/FileDbUpdater/video-file-title.service.ts b/src/VideoFiles/FileDbUpdater/video-file-title.service.ts new file mode 100644 index 0000000..ceb651c --- /dev/null +++ b/src/VideoFiles/FileDbUpdater/video-file-title.service.ts @@ -0,0 +1,123 @@ +import { VideoFilePathDto } from '../Dto/video-file-path.dto'; +import { Injectable } from '@nestjs/common'; +import * as tnp from 'torrent-name-parser'; + +//todo - refactor this copy-paste, add types, ect. +//todo - make an interface and add as info/data provider (fileEntity data expander interface) +@Injectable() +export class VideoFileTitleService { + constructor() { + tnp.configure({ sample: /sample/i }, { sample: 'boolean' }); + tnp.configure({ year: /([\[\(]?((?:19[0-9][0-9]|20[0-2][0-9]))[\]\)]?)/ }); + tnp.configure({ container: /MKV|AVI|MP4|mkv|avi|mp4/i }); + } + + extractTitleInfo(filePath: VideoFilePathDto) { + const infos = []; + const arrPath = filePath.relativePath.split('/'); + + for (let i = 0; i < arrPath.length; i++) { + const info = tnp(arrPath[i]); + + //season match from title + if (!info.season) { + let rx = /season ([0-9]{1,2})/i; + let match = info.title.match(rx); + if (match) { + info.season = parseInt(match[1]); + info.title = info.title.replace(rx, '').trim(); + } + + rx = /([0-9]{1,2}) sezonas/i; + match = info.title.match(rx); + if (match) { + info.season = parseInt(match[1]); + info.title = info.title.replace(rx, '').trim(); + } + } + + infos.push(info); + } + + const obj = { + info: undefined, + infos: undefined, + }; + + obj.info = this.combineInfos(infos); + //obj.info.size = convertSizeToHR(obj.size); + obj.infos = infos; + + return obj; + } + + private combineInfos(infos) { + const combined = { + title: undefined, + season: undefined, + episode: undefined, + }; + + for (let i = 0; i < infos.length; i++) { + const info = infos[i]; + + combined.title = combined.title ? combined.title : info.title; + + const season = info.season ? info.season : combined.season; + const episode = info.episode ? info.episode : combined.episode; + + if (season !== undefined) { + combined.season = season; + } + + if (episode !== undefined) { + combined.episode = episode; + } + } + + combined.title = this.removePartNumberFromTitle(combined.title); + combined.title = this.checkForAliases(combined.title); + return combined; + } + + private removePartNumberFromTitle(title) { + const arrTitle = title.split(' '); + if (this.isNumeric(arrTitle[arrTitle.length - 1])) { + arrTitle.pop(); + } + + return arrTitle.join(' '); + } + + private isNumeric(str) { + return /^\d+$/.test(str); + } + + private checkForAliases(title) { + //todo - db or external file + const aliases = [ + ['spider man', 'Spider Man'], + ['spiderman', 'Spider Man'], + ['spider-man', 'Spider Man'], + ['madagascar', 'Madagascar'], + ['ice age', 'Ice Age'], + ['hotel transylvania', 'Hotel Transylvania'], + ['despicable me', 'Despicable Me'], + ['the matrix', 'The Matrix'], + ['frozen', 'Frozen'], + ['garfield', 'Garfield'], + ['harry potter', 'Harry Potter'], + ['fifty shades', 'Fifty Shades'], + ['lilo', 'Lilo and Stitch'], + ]; + + for (let i = 0; i < aliases.length; i++) { + const [pattern, alias] = aliases[i]; + if (title.toLowerCase().indexOf(pattern) > -1) { + return alias; + } + } + + return title; + } +} diff --git a/src/VideoFiles/FileDbUpdater/video-files-saving.service.ts b/src/VideoFiles/FileDbUpdater/video-files-saving.service.ts new file mode 100644 index 0000000..3c3317e --- /dev/null +++ b/src/VideoFiles/FileDbUpdater/video-files-saving.service.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { VideoFilePathDto } from '../Dto/video-file-path.dto'; +import { InjectRepository } from '@nestjs/typeorm'; +import { FileEntity } from '../Entity/file.entity'; +import { Repository } from 'typeorm'; +import { Stats } from 'fs'; +import { stat } from 'fs/promises'; +import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum'; +import { VideoFileTitleService } from './video-file-title.service'; +import { TitleEntity } from '../Entity/title.entity'; +import { TitleTypeEnum } from '../Enum/title-type.enum'; + +@Injectable() +export class VideoFilesSavingService { + constructor( + @InjectRepository(FileEntity) + private readonly fileRepository: Repository, + @InjectRepository(TitleEntity) + private readonly titleRepository: Repository, + private readonly titleService: VideoFileTitleService, + ) {} + + async saveFiles(filePaths: VideoFilePathDto[]): Promise { + for (const filePath of filePaths) { + await this.saveFile(filePath); + } + } + + async saveFile(filePath: VideoFilePathDto): Promise { + const stats: Stats | false = await stat(filePath.path).catch(() => false); + + if (stats === false) { + return; + } + + const fileEntity = await this.fileRepository + .findOneByOrFail({ path: filePath.path }) + .catch(() => new FileEntity()); + + //todo - move to info/mapper service? + fileEntity.path = filePath.path; + fileEntity.deleted = false; + fileEntity.fileName = filePath.path.split('/').pop(); + fileEntity.streamProvider = StreamProviderEnum.fs; + fileEntity.size = stats.size; + + const titleInfo = this.titleService.extractTitleInfo(filePath); + + fileEntity.info = JSON.stringify(titleInfo.info); + fileEntity.infos = JSON.stringify(titleInfo.infos); + fileEntity.season = titleInfo.info.season; + fileEntity.title = await this.findOrCreateTitle(titleInfo); + //todo - ffmpeg duration and other metadata + + await this.fileRepository.save(fileEntity); + } + + //todo - move to repository + private async findOrCreateTitle( + titleInfo: Record>, + ): Promise { + let titleEntity = await this.titleRepository.findOne({ + where: { title: titleInfo.info.title }, //todo - what about titles with same name but can be as show or movie + }); + + if (titleEntity === null) { + titleEntity = new TitleEntity(); + titleEntity.title = titleInfo.info.title; + titleEntity.type = titleInfo.info.season + ? TitleTypeEnum.show + : TitleTypeEnum.movie; + await this.titleRepository.save(titleEntity); + titleEntity = await this.titleRepository.findOne({ + where: { title: titleInfo.info.title }, + }); + } + + return titleEntity; + } +} diff --git a/src/VideoFiles/FileDbUpdater/video-files-scanner.service.ts b/src/VideoFiles/FileDbUpdater/video-files-scanner.service.ts new file mode 100644 index 0000000..0ee9fef --- /dev/null +++ b/src/VideoFiles/FileDbUpdater/video-files-scanner.service.ts @@ -0,0 +1,69 @@ +import { Injectable } from '@nestjs/common'; +import { configService } from '../../config/config.service'; +import { VideoFilesConfig } from '../../config/video-files.config'; +import { spawn } from 'child_process'; + +@Injectable() +export class VideoFilesScannerService { + private readonly config: VideoFilesConfig; + + constructor() { + this.config = configService.getVideoFilesConfig(); + } + + public async scanNewFiles(): Promise< + [paths: string[], pathToRelPath: Record] + > { + let filesInFs = []; + let pathToRelPath = {}; //needed for title/season parsing + for (const videoFilesFolder of this.config.getVideoFilesFolders()) { + const [paths, relPaths] = await this.scanFolderForVideoFiles( + videoFilesFolder, + ).catch(() => [[], {}]); + filesInFs = filesInFs.concat(paths); + pathToRelPath = Object.assign(pathToRelPath, relPaths); + } + + return [filesInFs, pathToRelPath]; + } + + private scanFolderForVideoFiles( + path: string, + ): Promise<[paths: string[], pathToRelPath: Record]> { + return new Promise((resolve, reject) => { + if (path.slice(-1) !== '/') { + path += '/'; + } + + const findOptions = [path, '-name'].concat( + this.config.getVideoFilesExt().join(' -o -name ').split(' '), + ); + + const objCommand = spawn('find', findOptions); + + let strList = ''; + + objCommand.stdout.on('data', (data) => { + strList = strList + data; + }); + + objCommand.stderr.on('data', (data) => { + reject(`stderr: ${data}`); + }); + + objCommand.on('close', (code) => { + if (code !== 0) { + reject(`rejected ${code}`); + return; + } + const arrList = strList.split('\n').filter((item) => item); + const objList = {}; + for (const filePath of arrList) { + objList[filePath] = filePath.replace(path, ''); + } + + resolve([arrList, objList]); + }); + }); + } +} diff --git a/src/VideoFiles/FileDbUpdater/video-files-update.repository.ts b/src/VideoFiles/FileDbUpdater/video-files-update.repository.ts new file mode 100644 index 0000000..07f4720 --- /dev/null +++ b/src/VideoFiles/FileDbUpdater/video-files-update.repository.ts @@ -0,0 +1,46 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { FileEntity } from '../Entity/file.entity'; +import { In, Not, Repository, UpdateResult } from 'typeorm'; +import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum'; + +@Injectable() +export class VideoFilesUpdateRepository { + constructor( + @InjectRepository(FileEntity) + private readonly fileRepository: Repository, + ) {} + + markRemovedFilesAsDeleted(filePaths: string[]): Promise { + return this.fileRepository + .createQueryBuilder() + .update('files') + .set({ deleted: true }) + .where({ + path: Not(In(filePaths)), + deleted: false, + streamProvider: StreamProviderEnum.fs, + }) + .execute(); + } + + getMatchingPaths(filePaths: string[]): Promise { + return new Promise((resolve) => { + this.fileRepository + .find({ + select: ['path'], + where: { + path: In(filePaths), + deleted: false, + streamProvider: StreamProviderEnum.fs, + }, + }) + .then((fileEntitiesInDb) => { + resolve(fileEntitiesInDb.map((obj) => obj.path)); + }) + .catch(() => { + resolve([]); + }); + }); + } +} diff --git a/src/VideoFiles/FileDbUpdater/video-files-update.service.ts b/src/VideoFiles/FileDbUpdater/video-files-update.service.ts new file mode 100644 index 0000000..465dc9a --- /dev/null +++ b/src/VideoFiles/FileDbUpdater/video-files-update.service.ts @@ -0,0 +1,49 @@ +import { Injectable } from '@nestjs/common'; +import { VideoFilesScannerService } from './video-files-scanner.service'; +import { VideoFilePathDto } from '../Dto/video-file-path.dto'; +import { VideoFilesUpdateRepository } from './video-files-update.repository'; +import { VideoFilesSavingService } from './video-files-saving.service'; + +@Injectable() +export class VideoFilesUpdateService { + constructor( + private readonly scanner: VideoFilesScannerService, + private readonly repository: VideoFilesUpdateRepository, + private readonly savingService: VideoFilesSavingService, + ) {} + + async updateFsVideoFiles() { + const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles(); + + await this.repository.markRemovedFilesAsDeleted(filesInFs); + + const filesInDb = await this.repository.getMatchingPaths(filesInFs); + + const filesToAddToDb = filesInFs.filter( + (filePath) => !filesInDb.includes(filePath), + ); + + const newFiles = []; + + for (const fileToAddToDb of filesToAddToDb) { + if (this.checkPathToIgnore(fileToAddToDb)) { + continue; + } + newFiles.push( + new VideoFilePathDto( + fileToAddToDb, + filesToRelativePaths[fileToAddToDb], + ), + ); + } + + await this.savingService.saveFiles(newFiles); + + return newFiles; + } + + private checkPathToIgnore(filePath: string): boolean { + //todo - move to separate provider, include in settings + return filePath.indexOf('AOE1') > -1; + } +} diff --git a/src/VideoFiles/video-files.module.ts b/src/VideoFiles/video-files.module.ts index 53401ab..9c91bf7 100644 --- a/src/VideoFiles/video-files.module.ts +++ b/src/VideoFiles/video-files.module.ts @@ -3,11 +3,23 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { TitleEntity } from './Entity/title.entity'; import { FileEntity } from './Entity/file.entity'; import { VideoFilesProvider } from './video-files.provider'; +import { VideoFilesUpdateService } from './FileDbUpdater/video-files-update.service'; +import { VideoFilesScannerService } from './FileDbUpdater/video-files-scanner.service'; +import { VideoFilesUpdateRepository } from './FileDbUpdater/video-files-update.repository'; +import { VideoFilesSavingService } from './FileDbUpdater/video-files-saving.service'; +import { VideoFileTitleService } from './FileDbUpdater/video-file-title.service'; @Module({ imports: [TypeOrmModule.forFeature([TitleEntity, FileEntity])], controllers: [], - providers: [VideoFilesProvider], - exports: [VideoFilesProvider], + providers: [ + VideoFilesProvider, + VideoFilesUpdateService, + VideoFilesScannerService, + VideoFilesUpdateRepository, + VideoFilesSavingService, + VideoFileTitleService, + ], + exports: [VideoFilesProvider, VideoFilesUpdateService], }) export class VideoFilesModule {} diff --git a/src/VideoFiles/video-files.provider.ts b/src/VideoFiles/video-files.provider.ts index 58bbeb5..34dd289 100644 --- a/src/VideoFiles/video-files.provider.ts +++ b/src/VideoFiles/video-files.provider.ts @@ -27,7 +27,7 @@ export class VideoFilesProvider { return this.titleRepository .findOne({ relations: { files: true }, - where: { id: titleId }, + where: { id: titleId, files: { deleted: false } }, }) .catch(() => null); } diff --git a/src/config/config.service.ts b/src/config/config.service.ts index e86843d..a67ae4e 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -2,12 +2,14 @@ import * as env from 'dotenv'; import { TypeOrmModuleOptions } from '@nestjs/typeorm'; import { TypeOrmConfig } from './type-orm.config'; import { PathsConfig } from './paths.config'; +import { VideoFilesConfig } from './video-files.config'; env.config(); export class ConfigService { private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this); private pathsConfig: PathsConfig = new PathsConfig(this); + private videoFilesConfig: VideoFilesConfig = new VideoFilesConfig(this); public getEnv(key: string): any { return process.env[key]; @@ -26,6 +28,10 @@ export class ConfigService { public getPaths(): PathsConfig { return this.pathsConfig; } + + public getVideoFilesConfig(): VideoFilesConfig { + return this.videoFilesConfig; + } } export const configService = new ConfigService(); diff --git a/src/config/paths.config.ts b/src/config/paths.config.ts index 7020ad3..8d25503 100644 --- a/src/config/paths.config.ts +++ b/src/config/paths.config.ts @@ -18,7 +18,7 @@ export class PathsConfig { return this.getPathByEnvKey('RECENT_SEARCHES_FOLDER'); } - private getPathByEnvKey(key: string) { + public getPathByEnvKey(key: string) { const relPath = this.configService.getEnv(key); return this.getPath(relPath); diff --git a/src/config/video-files.config.ts b/src/config/video-files.config.ts new file mode 100644 index 0000000..0e509a6 --- /dev/null +++ b/src/config/video-files.config.ts @@ -0,0 +1,16 @@ +import { ConfigService } from './config.service'; +import { join } from 'path'; + +export class VideoFilesConfig { + constructor(private readonly configService: ConfigService) { + this.configService = configService; + } + + public getVideoFilesFolders(): string[] { + return JSON.parse(this.configService.getEnv('VIDEO_FILES_FOLDERS')); + } + + public getVideoFilesExt(): string[] { + return JSON.parse(this.configService.getEnv('VIDEO_FILES_EXT')); + } +}