start local fs video files scanner service

.env.sample

@@ -3,7 +3,8 @@ STATIC_SERVE_FOLDER="data/static"
 LOG_FILE_REQUESTS="data/static/static_log.txt"
 DB_PATH="./data/db/db.db"
 RECENT_SEARCHES_FOLDER="./data/recent_searches"
-TEST_ARRAY='[
+VIDEO_FILES_FOLDERS='[
-    "one",
+    "/home/user/videos",
-    "two"
+    "/mnt/hdd/videos"
 ]'
+VIDEO_FILES_EXT='["*.mkv", "*.mp4", "*.avi"]'

package-lock.json

@@ -21,6 +21,7 @@
         "rxjs": "^7.2.0",
         "sqlite": "^4.1.2",
         "sqlite3": "^5.1.2",
+        "torrent-name-parser": "^0.6.5",
         "typeorm": "^0.3.11"
       },
       "devDependencies": {
@@ -8809,6 +8810,11 @@
         "node": ">=0.6"
       }
     },
+    "node_modules/torrent-name-parser": {
+      "version": "0.6.5",
+      "resolved": "https://registry.npmjs.org/torrent-name-parser/-/torrent-name-parser-0.6.5.tgz",
+      "integrity": "sha512-ao92tzD6DjYKN0dedEZxfylL4ZWYaYFgPktKxkEmlsLrF/ia41Sg+l3rJ2XLJufQUR3uTbd7bYMqxbh/kxb1pQ=="
+    },
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@@ -16341,6 +16347,11 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
+    "torrent-name-parser": {
+      "version": "0.6.5",
+      "resolved": "https://registry.npmjs.org/torrent-name-parser/-/torrent-name-parser-0.6.5.tgz",
+      "integrity": "sha512-ao92tzD6DjYKN0dedEZxfylL4ZWYaYFgPktKxkEmlsLrF/ia41Sg+l3rJ2XLJufQUR3uTbd7bYMqxbh/kxb1pQ=="
+    },
     "tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",

package.json

@@ -39,6 +39,7 @@
     "rxjs": "^7.2.0",
     "sqlite": "^4.1.2",
     "sqlite3": "^5.1.2",
+    "torrent-name-parser": "^0.6.5",
     "typeorm": "^0.3.11"
   },
   "devDependencies": {

src/TryOutModule/try.cotroller.ts

@@ -1,9 +1,11 @@
 import { Controller, Get } from '@nestjs/common';
+import { VideoFilesUpdateService } from '../VideoFiles/FileDbUpdater/video-files-update.service';
 
 @Controller('try')
 export class TryController {
+  constructor(readonly videoFilesUpdateService: VideoFilesUpdateService) {}
   @Get('')
   main() {
-    return process.env;
+    return this.videoFilesUpdateService.updateFsVideoFiles();
   }
 }

src/TryOutModule/try.module.ts

@@ -1,11 +1,12 @@
 import { Module } from '@nestjs/common';
 import { TryController } from './try.cotroller';
+import { VideoFilesModule } from '../VideoFiles/video-files.module';
 
 /**
  * Playground module for playing around
  */
 @Module({
-  imports: [],
+  imports: [VideoFilesModule],
   controllers: [TryController],
   providers: [],
 })

src/VideoFiles/Dto/video-file-path.dto.ts

@@ -0,0 +1,6 @@
+export class VideoFilePathDto {
+  constructor(readonly path: string, readonly relativePath: string) {
+    this.path = path;
+    this.relativePath = relativePath;
+  }
+}

src/VideoFiles/FileDbUpdater/video-file-title.service.ts

@@ -0,0 +1,123 @@
+import { VideoFilePathDto } from '../Dto/video-file-path.dto';
+import { Injectable } from '@nestjs/common';
+import * as tnp from 'torrent-name-parser';
+
+//todo - refactor this copy-paste, add types, ect.
+//todo - make an interface and add as info/data provider (fileEntity data expander interface)
+@Injectable()
+export class VideoFileTitleService {
+  constructor() {
+    tnp.configure({ sample: /sample/i }, { sample: 'boolean' });
+    tnp.configure({ year: /([\[\(]?((?:19[0-9][0-9]|20[0-2][0-9]))[\]\)]?)/ });
+    tnp.configure({ container: /MKV|AVI|MP4|mkv|avi|mp4/i });
+  }
+
+  extractTitleInfo(filePath: VideoFilePathDto) {
+    const infos = [];
+    const arrPath = filePath.relativePath.split('/');
+
+    for (let i = 0; i < arrPath.length; i++) {
+      const info = tnp(arrPath[i]);
+
+      //season match from title
+      if (!info.season) {
+        let rx = /season ([0-9]{1,2})/i;
+        let match = info.title.match(rx);
+        if (match) {
+          info.season = parseInt(match[1]);
+          info.title = info.title.replace(rx, '').trim();
+        }
+
+        rx = /([0-9]{1,2}) sezonas/i;
+        match = info.title.match(rx);
+        if (match) {
+          info.season = parseInt(match[1]);
+          info.title = info.title.replace(rx, '').trim();
+        }
+      }
+
+      infos.push(info);
+    }
+
+    const obj = {
+      info: undefined,
+      infos: undefined,
+    };
+
+    obj.info = this.combineInfos(infos);
+    //obj.info.size = convertSizeToHR(obj.size);
+    obj.infos = infos;
+
+    return obj;
+  }
+
+  private combineInfos(infos) {
+    const combined = {
+      title: undefined,
+      season: undefined,
+      episode: undefined,
+    };
+
+    for (let i = 0; i < infos.length; i++) {
+      const info = infos[i];
+
+      combined.title = combined.title ? combined.title : info.title;
+
+      const season = info.season ? info.season : combined.season;
+      const episode = info.episode ? info.episode : combined.episode;
+
+      if (season !== undefined) {
+        combined.season = season;
+      }
+
+      if (episode !== undefined) {
+        combined.episode = episode;
+      }
+    }
+
+    combined.title = this.removePartNumberFromTitle(combined.title);
+    combined.title = this.checkForAliases(combined.title);
+    return combined;
+  }
+
+  private removePartNumberFromTitle(title) {
+    const arrTitle = title.split(' ');
+    if (this.isNumeric(arrTitle[arrTitle.length - 1])) {
+      arrTitle.pop();
+    }
+
+    return arrTitle.join(' ');
+  }
+
+  private isNumeric(str) {
+    return /^\d+$/.test(str);
+  }
+
+  private checkForAliases(title) {
+    //todo - db or external file
+    const aliases = [
+      ['spider man', 'Spider Man'],
+      ['spiderman', 'Spider Man'],
+      ['spider-man', 'Spider Man'],
+      ['madagascar', 'Madagascar'],
+      ['ice age', 'Ice Age'],
+      ['hotel transylvania', 'Hotel Transylvania'],
+      ['despicable me', 'Despicable Me'],
+      ['the matrix', 'The Matrix'],
+      ['frozen', 'Frozen'],
+      ['garfield', 'Garfield'],
+      ['harry potter', 'Harry Potter'],
+      ['fifty shades', 'Fifty Shades'],
+      ['lilo', 'Lilo and Stitch'],
+    ];
+
+    for (let i = 0; i < aliases.length; i++) {
+      const [pattern, alias] = aliases[i];
+      if (title.toLowerCase().indexOf(pattern) > -1) {
+        return alias;
+      }
+    }
+
+    return title;
+  }
+}

src/VideoFiles/FileDbUpdater/video-files-saving.service.ts

@@ -0,0 +1,80 @@
+import { Injectable } from '@nestjs/common';
+import { VideoFilePathDto } from '../Dto/video-file-path.dto';
+import { InjectRepository } from '@nestjs/typeorm';
+import { FileEntity } from '../Entity/file.entity';
+import { Repository } from 'typeorm';
+import { Stats } from 'fs';
+import { stat } from 'fs/promises';
+import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
+import { VideoFileTitleService } from './video-file-title.service';
+import { TitleEntity } from '../Entity/title.entity';
+import { TitleTypeEnum } from '../Enum/title-type.enum';
+
+@Injectable()
+export class VideoFilesSavingService {
+  constructor(
+    @InjectRepository(FileEntity)
+    private readonly fileRepository: Repository<FileEntity>,
+    @InjectRepository(TitleEntity)
+    private readonly titleRepository: Repository<TitleEntity>,
+    private readonly titleService: VideoFileTitleService,
+  ) {}
+
+  async saveFiles(filePaths: VideoFilePathDto[]): Promise<void> {
+    for (const filePath of filePaths) {
+      await this.saveFile(filePath);
+    }
+  }
+
+  async saveFile(filePath: VideoFilePathDto): Promise<void> {
+    const stats: Stats | false = await stat(filePath.path).catch(() => false);
+
+    if (stats === false) {
+      return;
+    }
+
+    const fileEntity = await this.fileRepository
+      .findOneByOrFail({ path: filePath.path })
+      .catch(() => new FileEntity());
+
+    //todo - move to info/mapper service?
+    fileEntity.path = filePath.path;
+    fileEntity.deleted = false;
+    fileEntity.fileName = filePath.path.split('/').pop();
+    fileEntity.streamProvider = StreamProviderEnum.fs;
+    fileEntity.size = stats.size;
+
+    const titleInfo = this.titleService.extractTitleInfo(filePath);
+
+    fileEntity.info = JSON.stringify(titleInfo.info);
+    fileEntity.infos = JSON.stringify(titleInfo.infos);
+    fileEntity.season = titleInfo.info.season;
+    fileEntity.title = await this.findOrCreateTitle(titleInfo);
+    //todo - ffmpeg duration and other metadata
+
+    await this.fileRepository.save(fileEntity);
+  }
+
+  //todo - move to repository
+  private async findOrCreateTitle(
+    titleInfo: Record<string, Record<string, string>>,
+  ): Promise<TitleEntity> {
+    let titleEntity = await this.titleRepository.findOne({
+      where: { title: titleInfo.info.title }, //todo - what about titles with same name but can be as show or movie
+    });
+
+    if (titleEntity === null) {
+      titleEntity = new TitleEntity();
+      titleEntity.title = titleInfo.info.title;
+      titleEntity.type = titleInfo.info.season
+        ? TitleTypeEnum.show
+        : TitleTypeEnum.movie;
+      await this.titleRepository.save(titleEntity);
+      titleEntity = await this.titleRepository.findOne({
+        where: { title: titleInfo.info.title },
+      });
+    }
+
+    return titleEntity;
+  }
+}

src/VideoFiles/FileDbUpdater/video-files-scanner.service.ts

@@ -0,0 +1,69 @@
+import { Injectable } from '@nestjs/common';
+import { configService } from '../../config/config.service';
+import { VideoFilesConfig } from '../../config/video-files.config';
+import { spawn } from 'child_process';
+
+@Injectable()
+export class VideoFilesScannerService {
+  private readonly config: VideoFilesConfig;
+
+  constructor() {
+    this.config = configService.getVideoFilesConfig();
+  }
+
+  public async scanNewFiles(): Promise<
+    [paths: string[], pathToRelPath: Record<string, string>]
+  > {
+    let filesInFs = [];
+    let pathToRelPath = {}; //needed for title/season parsing
+    for (const videoFilesFolder of this.config.getVideoFilesFolders()) {
+      const [paths, relPaths] = await this.scanFolderForVideoFiles(
+        videoFilesFolder,
+      ).catch(() => [[], {}]);
+      filesInFs = filesInFs.concat(paths);
+      pathToRelPath = Object.assign(pathToRelPath, relPaths);
+    }
+
+    return [filesInFs, pathToRelPath];
+  }
+
+  private scanFolderForVideoFiles(
+    path: string,
+  ): Promise<[paths: string[], pathToRelPath: Record<string, string>]> {
+    return new Promise((resolve, reject) => {
+      if (path.slice(-1) !== '/') {
+        path += '/';
+      }
+
+      const findOptions = [path, '-name'].concat(
+        this.config.getVideoFilesExt().join(' -o -name ').split(' '),
+      );
+
+      const objCommand = spawn('find', findOptions);
+
+      let strList = '';
+
+      objCommand.stdout.on('data', (data) => {
+        strList = strList + data;
+      });
+
+      objCommand.stderr.on('data', (data) => {
+        reject(`stderr: ${data}`);
+      });
+
+      objCommand.on('close', (code) => {
+        if (code !== 0) {
+          reject(`rejected ${code}`);
+          return;
+        }
+        const arrList = strList.split('\n').filter((item) => item);
+        const objList = {};
+        for (const filePath of arrList) {
+          objList[filePath] = filePath.replace(path, '');
+        }
+
+        resolve([arrList, objList]);
+      });
+    });
+  }
+}

src/VideoFiles/FileDbUpdater/video-files-update.repository.ts

@@ -0,0 +1,46 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { FileEntity } from '../Entity/file.entity';
+import { In, Not, Repository, UpdateResult } from 'typeorm';
+import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
+
+@Injectable()
+export class VideoFilesUpdateRepository {
+  constructor(
+    @InjectRepository(FileEntity)
+    private readonly fileRepository: Repository<FileEntity>,
+  ) {}
+
+  markRemovedFilesAsDeleted(filePaths: string[]): Promise<UpdateResult> {
+    return this.fileRepository
+      .createQueryBuilder()
+      .update('files')
+      .set({ deleted: true })
+      .where({
+        path: Not(In(filePaths)),
+        deleted: false,
+        streamProvider: StreamProviderEnum.fs,
+      })
+      .execute();
+  }
+
+  getMatchingPaths(filePaths: string[]): Promise<string[]> {
+    return new Promise((resolve) => {
+      this.fileRepository
+        .find({
+          select: ['path'],
+          where: {
+            path: In(filePaths),
+            deleted: false,
+            streamProvider: StreamProviderEnum.fs,
+          },
+        })
+        .then((fileEntitiesInDb) => {
+          resolve(fileEntitiesInDb.map((obj) => obj.path));
+        })
+        .catch(() => {
+          resolve([]);
+        });
+    });
+  }
+}

src/VideoFiles/FileDbUpdater/video-files-update.service.ts

@@ -0,0 +1,49 @@
+import { Injectable } from '@nestjs/common';
+import { VideoFilesScannerService } from './video-files-scanner.service';
+import { VideoFilePathDto } from '../Dto/video-file-path.dto';
+import { VideoFilesUpdateRepository } from './video-files-update.repository';
+import { VideoFilesSavingService } from './video-files-saving.service';
+
+@Injectable()
+export class VideoFilesUpdateService {
+  constructor(
+    private readonly scanner: VideoFilesScannerService,
+    private readonly repository: VideoFilesUpdateRepository,
+    private readonly savingService: VideoFilesSavingService,
+  ) {}
+
+  async updateFsVideoFiles() {
+    const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles();
+
+    await this.repository.markRemovedFilesAsDeleted(filesInFs);
+
+    const filesInDb = await this.repository.getMatchingPaths(filesInFs);
+
+    const filesToAddToDb = filesInFs.filter(
+      (filePath) => !filesInDb.includes(filePath),
+    );
+
+    const newFiles = [];
+
+    for (const fileToAddToDb of filesToAddToDb) {
+      if (this.checkPathToIgnore(fileToAddToDb)) {
+        continue;
+      }
+      newFiles.push(
+        new VideoFilePathDto(
+          fileToAddToDb,
+          filesToRelativePaths[fileToAddToDb],
+        ),
+      );
+    }
+
+    await this.savingService.saveFiles(newFiles);
+
+    return newFiles;
+  }
+
+  private checkPathToIgnore(filePath: string): boolean {
+    //todo - move to separate provider, include in settings
+    return filePath.indexOf('AOE1') > -1;
+  }
+}

src/VideoFiles/video-files.module.ts

@@ -3,11 +3,23 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { TitleEntity } from './Entity/title.entity';
 import { FileEntity } from './Entity/file.entity';
 import { VideoFilesProvider } from './video-files.provider';
+import { VideoFilesUpdateService } from './FileDbUpdater/video-files-update.service';
+import { VideoFilesScannerService } from './FileDbUpdater/video-files-scanner.service';
+import { VideoFilesUpdateRepository } from './FileDbUpdater/video-files-update.repository';
+import { VideoFilesSavingService } from './FileDbUpdater/video-files-saving.service';
+import { VideoFileTitleService } from './FileDbUpdater/video-file-title.service';
 
 @Module({
   imports: [TypeOrmModule.forFeature([TitleEntity, FileEntity])],
   controllers: [],
-  providers: [VideoFilesProvider],
+  providers: [
-  exports: [VideoFilesProvider],
+    VideoFilesProvider,
+    VideoFilesUpdateService,
+    VideoFilesScannerService,
+    VideoFilesUpdateRepository,
+    VideoFilesSavingService,
+    VideoFileTitleService,
+  ],
+  exports: [VideoFilesProvider, VideoFilesUpdateService],
 })
 export class VideoFilesModule {}

src/VideoFiles/video-files.provider.ts

@@ -27,7 +27,7 @@ export class VideoFilesProvider {
     return this.titleRepository
       .findOne({
         relations: { files: true },
-        where: { id: titleId },
+        where: { id: titleId, files: { deleted: false } },
       })
       .catch(() => null);
   }

src/config/config.service.ts

@@ -2,12 +2,14 @@ import * as env from 'dotenv';
 import { TypeOrmModuleOptions } from '@nestjs/typeorm';
 import { TypeOrmConfig } from './type-orm.config';
 import { PathsConfig } from './paths.config';
+import { VideoFilesConfig } from './video-files.config';
 
 env.config();
 
 export class ConfigService {
   private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this);
   private pathsConfig: PathsConfig = new PathsConfig(this);
+  private videoFilesConfig: VideoFilesConfig = new VideoFilesConfig(this);
 
   public getEnv(key: string): any {
     return process.env[key];
@@ -26,6 +28,10 @@ export class ConfigService {
   public getPaths(): PathsConfig {
     return this.pathsConfig;
   }
+
+  public getVideoFilesConfig(): VideoFilesConfig {
+    return this.videoFilesConfig;
+  }
 }
 
 export const configService = new ConfigService();

src/config/paths.config.ts

@@ -18,7 +18,7 @@ export class PathsConfig {
     return this.getPathByEnvKey('RECENT_SEARCHES_FOLDER');
   }
 
-  private getPathByEnvKey(key: string) {
+  public getPathByEnvKey(key: string) {
     const relPath = this.configService.getEnv(key);
 
     return this.getPath(relPath);

src/config/video-files.config.ts

@@ -0,0 +1,16 @@
+import { ConfigService } from './config.service';
+import { join } from 'path';
+
+export class VideoFilesConfig {
+  constructor(private readonly configService: ConfigService) {
+    this.configService = configService;
+  }
+
+  public getVideoFilesFolders(): string[] {
+    return JSON.parse(this.configService.getEnv('VIDEO_FILES_FOLDERS'));
+  }
+
+  public getVideoFilesExt(): string[] {
+    return JSON.parse(this.configService.getEnv('VIDEO_FILES_EXT'));
+  }
+}