refactor video files scanner

2026-07-08 20:37:41 +00:00 · 2023-01-06 15:58:49 +02:00
parent 1631a97417
commit 8d354027fc
24 changed files with 486 additions and 222 deletions
@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddRelativePathColumnToFilesTable1672990196772
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `alter table files
+          add relative_path TEXT;`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `alter table files
+          drop column relative_path;`,
+    );
+  }
+}
@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class RenameLinkomanijaColumnInFilesTable1673010469305
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE files RENAME COLUMN lm to linkomanija;`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE files RENAME COLUMN linkomanija to lm;`,
+    );
+  }
+}
@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddWebtorrentColumnsToFilesTable1673010885172
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `alter table files
+          add webtorrent INTEGER;`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `alter table files
+          drop column webtorrent;`,
+    );
+  }
+}
@@ -15,7 +15,9 @@
        "@nestjs/platform-express": "^9.0.0",
        "@nestjs/serve-static": "^3.0.0",
        "@nestjs/typeorm": "^9.0.1",
+        "@types/fluent-ffmpeg": "^2.1.20",
        "dotenv": "^16.0.3",
+        "fluent-ffmpeg": "^2.1.2",
        "reflect-metadata": "^0.1.13",
        "rimraf": "^3.0.2",
        "rxjs": "^7.2.0",
@@ -2098,6 +2100,14 @@
        "@types/range-parser": "*"
      }
    },
+    "node_modules/@types/fluent-ffmpeg": {
+      "version": "2.1.20",
+      "resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz",
+      "integrity": "sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/graceful-fs": {
      "version": "4.1.5",
      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz",
@@ -2156,8 +2166,7 @@
    "node_modules/@types/node": {
      "version": "16.18.4",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.4.tgz",
-      "integrity": "sha512-9qGjJ5GyShZjUfx2ArBIGM+xExdfLvvaCyQR0t6yRXKPcWCVYF/WemtX/uIU3r7FYECXRXkIiw2Vnhn6y8d+pw==",
-      "devOptional": true
+      "integrity": "sha512-9qGjJ5GyShZjUfx2ArBIGM+xExdfLvvaCyQR0t6yRXKPcWCVYF/WemtX/uIU3r7FYECXRXkIiw2Vnhn6y8d+pw=="
    },
    "node_modules/@types/parse-json": {
      "version": "4.0.0",
@@ -2870,6 +2879,11 @@
      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
      "dev": true
    },
+    "node_modules/async": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
+      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
+    },
    "node_modules/asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -4704,6 +4718,29 @@
      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
      "dev": true
    },
+    "node_modules/fluent-ffmpeg": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+      "integrity": "sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==",
+      "dependencies": {
+        "async": ">=0.2.9",
+        "which": "^1.1.1"
+      },
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
+    "node_modules/fluent-ffmpeg/node_modules/which": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "which": "bin/which"
+      }
+    },
    "node_modules/follow-redirects": {
      "version": "1.15.2",
      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
@@ -5475,8 +5512,7 @@
    "node_modules/isexe": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "devOptional": true
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
    },
    "node_modules/istanbul-lib-coverage": {
      "version": "3.2.0",
@@ -11294,6 +11330,14 @@
        "@types/range-parser": "*"
      }
    },
+    "@types/fluent-ffmpeg": {
+      "version": "2.1.20",
+      "resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz",
+      "integrity": "sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/graceful-fs": {
      "version": "4.1.5",
      "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz",
@@ -11352,8 +11396,7 @@
    "@types/node": {
      "version": "16.18.4",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.4.tgz",
-      "integrity": "sha512-9qGjJ5GyShZjUfx2ArBIGM+xExdfLvvaCyQR0t6yRXKPcWCVYF/WemtX/uIU3r7FYECXRXkIiw2Vnhn6y8d+pw==",
-      "devOptional": true
+      "integrity": "sha512-9qGjJ5GyShZjUfx2ArBIGM+xExdfLvvaCyQR0t6yRXKPcWCVYF/WemtX/uIU3r7FYECXRXkIiw2Vnhn6y8d+pw=="
    },
    "@types/parse-json": {
      "version": "4.0.0",
@@ -11904,6 +11947,11 @@
      "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
      "dev": true
    },
+    "async": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
+      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
+    },
    "asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -13294,6 +13342,25 @@
      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
      "dev": true
    },
+    "fluent-ffmpeg": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
+      "integrity": "sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==",
+      "requires": {
+        "async": ">=0.2.9",
+        "which": "^1.1.1"
+      },
+      "dependencies": {
+        "which": {
+          "version": "1.3.1",
+          "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+          "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        }
+      }
+    },
    "follow-redirects": {
      "version": "1.15.2",
      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
@@ -13840,8 +13907,7 @@
    "isexe": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "devOptional": true
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
    },
    "istanbul-lib-coverage": {
      "version": "3.2.0",
@@ -33,7 +33,9 @@
    "@nestjs/platform-express": "^9.0.0",
    "@nestjs/serve-static": "^3.0.0",
    "@nestjs/typeorm": "^9.0.1",
+    "@types/fluent-ffmpeg": "^2.1.20",
    "dotenv": "^16.0.3",
+    "fluent-ffmpeg": "^2.1.2",
    "reflect-metadata": "^0.1.13",
    "rimraf": "^3.0.2",
    "rxjs": "^7.2.0",
@@ -3,7 +3,7 @@ import ApiResponse from '../Dto/api-response.dto';
 import { KodiApiResponseFactory } from '../kodi-api-response.factory';
 import { StreamerFacade } from '../../Streamer/streamer.facade';
 import { Request, Response } from 'express';
-import { VideoFilesProvider } from '../../VideoFiles/video-files.provider';
+import { VideoFilesFacade } from '../../VideoFiles/video-files.facade';
 import { TitleTypeEnum } from '../../VideoFiles/Enum/title-type.enum';
 import { FileEntity } from '../../VideoFiles/Entity/file.entity';
 import { TitleEntity } from '../../VideoFiles/Entity/title.entity';
@@ -13,7 +13,7 @@ export class AllFilesService {
  constructor(
    private readonly koiApiResponseFactory: KodiApiResponseFactory,
    private readonly streamerFacade: StreamerFacade,
-    private readonly videoTitlesProvider: VideoFilesProvider,
+    private readonly videoFilesFacade: VideoFilesFacade,
  ) {}

  getMenu(): ApiResponse {
@@ -33,7 +33,7 @@ export class AllFilesService {
  }

  async getListOfTitles(type: TitleTypeEnum): Promise<ApiResponse> {
-    const entities = await this.videoTitlesProvider.getListOfTitles(type);
+    const entities = await this.videoFilesFacade.getListOfTitles(type);
    const response = this.koiApiResponseFactory.createApiResponse();
    response.setTitle('All ' + type + 's');
    entities.forEach((entity) => {
@@ -52,7 +52,7 @@ export class AllFilesService {

  async loadTitle(titleId: number, seasonId: number): Promise<ApiResponse> {
    const titleEntity = titleId
-      ? await this.videoTitlesProvider.getTitleWithFiles(titleId)
+      ? await this.videoFilesFacade.getTitleWithFiles(titleId)
      : null;

    if (titleEntity === null) {
@@ -136,7 +136,7 @@ export class AllFilesService {
    if (isNaN(id)) {
      throw new NotFoundException(`id #${fileId} not correct`);
    }
-    const fileEntity = await this.videoTitlesProvider.getFile(id);
+    const fileEntity = await this.videoFilesFacade.getFile(id);

    if (fileEntity === null) {
      throw new NotFoundException(`id #${fileId} not found`);
@@ -1,11 +1,13 @@
 import { Controller, Get } from '@nestjs/common';
-import { VideoFilesUpdateService } from '../VideoFiles/FileDbUpdater/video-files-update.service';
+import { VideoFilesFacade } from '../VideoFiles/video-files.facade';

@Controller('try')
 export class TryController {
-  constructor(readonly videoFilesUpdateService: VideoFilesUpdateService) {}
+  constructor(readonly videoFilesFacade: VideoFilesFacade) {}
+
  @Get('')
  main() {
-    return this.videoFilesUpdateService.updateFsVideoFiles();
+    this.videoFilesFacade.updateFsVideoFiles();
+    return { doing: 'something' };
  }
 }
@@ -1,6 +0,0 @@
-export class VideoFilePathDto {
-  constructor(readonly path: string, readonly relativePath: string) {
-    this.path = path;
-    this.relativePath = relativePath;
-  }
-}
@@ -40,4 +40,17 @@ export class FileEntity {

  @Column({ name: 'stream_provider' })
  streamProvider: StreamProviderEnum;
+
+  @Column({ name: 'relative_path' })
+  relativePath: string;
+
+  //todo - consider if these 3 below should be better organized
+  @Column()
+  transmission: boolean;
+
+  @Column()
+  linkomanija: boolean;
+
+  @Column()
+  webtorrent: boolean;
 }
@@ -1,80 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { VideoFilePathDto } from '../Dto/video-file-path.dto';
-import { InjectRepository } from '@nestjs/typeorm';
-import { FileEntity } from '../Entity/file.entity';
-import { Repository } from 'typeorm';
-import { Stats } from 'fs';
-import { stat } from 'fs/promises';
-import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
-import { VideoFileTitleService } from './video-file-title.service';
-import { TitleEntity } from '../Entity/title.entity';
-import { TitleTypeEnum } from '../Enum/title-type.enum';
-
-@Injectable()
-export class VideoFilesSavingService {
-  constructor(
-    @InjectRepository(FileEntity)
-    private readonly fileRepository: Repository<FileEntity>,
-    @InjectRepository(TitleEntity)
-    private readonly titleRepository: Repository<TitleEntity>,
-    private readonly titleService: VideoFileTitleService,
-  ) {}
-
-  async saveFiles(filePaths: VideoFilePathDto[]): Promise<void> {
-    for (const filePath of filePaths) {
-      await this.saveFile(filePath);
-    }
-  }
-
-  async saveFile(filePath: VideoFilePathDto): Promise<void> {
-    const stats: Stats | false = await stat(filePath.path).catch(() => false);
-
-    if (stats === false) {
-      return;
-    }
-
-    const fileEntity = await this.fileRepository
-      .findOneByOrFail({ path: filePath.path })
-      .catch(() => new FileEntity());
-
-    //todo - move to info/mapper service?
-    fileEntity.path = filePath.path;
-    fileEntity.deleted = false;
-    fileEntity.fileName = filePath.path.split('/').pop();
-    fileEntity.streamProvider = StreamProviderEnum.fs;
-    fileEntity.size = stats.size;
-
-    const titleInfo = this.titleService.extractTitleInfo(filePath);
-
-    fileEntity.info = JSON.stringify(titleInfo.info);
-    fileEntity.infos = JSON.stringify(titleInfo.infos);
-    fileEntity.season = titleInfo.info.season;
-    fileEntity.title = await this.findOrCreateTitle(titleInfo);
-    //todo - ffmpeg duration and other metadata
-
-    await this.fileRepository.save(fileEntity);
-  }
-
-  //todo - move to repository
-  private async findOrCreateTitle(
-    titleInfo: Record<string, Record<string, string>>,
-  ): Promise<TitleEntity> {
-    let titleEntity = await this.titleRepository.findOne({
-      where: { title: titleInfo.info.title }, //todo - what about titles with same name but can be as show or movie
-    });
-
-    if (titleEntity === null) {
-      titleEntity = new TitleEntity();
-      titleEntity.title = titleInfo.info.title;
-      titleEntity.type = titleInfo.info.season
-        ? TitleTypeEnum.show
-        : TitleTypeEnum.movie;
-      await this.titleRepository.save(titleEntity);
-      titleEntity = await this.titleRepository.findOne({
-        where: { title: titleInfo.info.title },
-      });
-    }
-
-    return titleEntity;
-  }
-}
@@ -1,46 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
-import { FileEntity } from '../Entity/file.entity';
-import { In, Not, Repository, UpdateResult } from 'typeorm';
-import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
-
-@Injectable()
-export class VideoFilesUpdateRepository {
-  constructor(
-    @InjectRepository(FileEntity)
-    private readonly fileRepository: Repository<FileEntity>,
-  ) {}
-
-  markRemovedFilesAsDeleted(filePaths: string[]): Promise<UpdateResult> {
-    return this.fileRepository
-      .createQueryBuilder()
-      .update('files')
-      .set({ deleted: true })
-      .where({
-        path: Not(In(filePaths)),
-        deleted: false,
-        streamProvider: StreamProviderEnum.fs,
-      })
-      .execute();
-  }
-
-  getMatchingPaths(filePaths: string[]): Promise<string[]> {
-    return new Promise((resolve) => {
-      this.fileRepository
-        .find({
-          select: ['path'],
-          where: {
-            path: In(filePaths),
-            deleted: false,
-            streamProvider: StreamProviderEnum.fs,
-          },
-        })
-        .then((fileEntitiesInDb) => {
-          resolve(fileEntitiesInDb.map((obj) => obj.path));
-        })
-        .catch(() => {
-          resolve([]);
-        });
-    });
-  }
-}
@@ -1,49 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { VideoFilesScannerService } from './video-files-scanner.service';
-import { VideoFilePathDto } from '../Dto/video-file-path.dto';
-import { VideoFilesUpdateRepository } from './video-files-update.repository';
-import { VideoFilesSavingService } from './video-files-saving.service';
-
-@Injectable()
-export class VideoFilesUpdateService {
-  constructor(
-    private readonly scanner: VideoFilesScannerService,
-    private readonly repository: VideoFilesUpdateRepository,
-    private readonly savingService: VideoFilesSavingService,
-  ) {}
-
-  async updateFsVideoFiles() {
-    const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles();
-
-    await this.repository.markRemovedFilesAsDeleted(filesInFs);
-
-    const filesInDb = await this.repository.getMatchingPaths(filesInFs);
-
-    const filesToAddToDb = filesInFs.filter(
-      (filePath) => !filesInDb.includes(filePath),
-    );
-
-    const newFiles = [];
-
-    for (const fileToAddToDb of filesToAddToDb) {
-      if (this.checkPathToIgnore(fileToAddToDb)) {
-        continue;
-      }
-      newFiles.push(
-        new VideoFilePathDto(
-          fileToAddToDb,
-          filesToRelativePaths[fileToAddToDb],
-        ),
-      );
-    }
-
-    await this.savingService.saveFiles(newFiles);
-
-    return newFiles;
-  }
-
-  private checkPathToIgnore(filePath: string): boolean {
-    //todo - move to separate provider, include in settings
-    return filePath.indexOf('AOE1') > -1;
-  }
-}
@@ -0,0 +1,13 @@
+import { FileEntityExpanderInterface } from './file-entity-expander.interface';
+import { FileEntity } from '../../Entity/file.entity';
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class FileEntityDefaultsExpander implements FileEntityExpanderInterface {
+  expand(fileEntity: FileEntity): Promise<FileEntity> {
+    fileEntity.deleted = false;
+    fileEntity.fileName = fileEntity.path.split('/').pop();
+
+    return Promise.resolve(fileEntity);
+  }
+}
@@ -0,0 +1,19 @@
+import { FileEntityExpanderInterface } from './file-entity-expander.interface';
+import { FileEntity } from '../../Entity/file.entity';
+import * as ffmpeg from 'fluent-ffmpeg';
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class FileEntityDurationExpander implements FileEntityExpanderInterface {
+  expand(fileEntity: FileEntity): Promise<FileEntity> {
+    return new Promise((resolve) => {
+      ffmpeg.ffprobe(fileEntity.path, (err, data) => {
+        if (!err) {
+          //no error handling!
+          fileEntity.duration = data.format.duration;
+        }
+        resolve(fileEntity);
+      });
+    });
+  }
+}
@@ -0,0 +1,5 @@
+import { FileEntity } from '../../Entity/file.entity';
+
+export interface FileEntityExpanderInterface {
+  expand(fileEntity: FileEntity): Promise<FileEntity>;
+}
@@ -0,0 +1,18 @@
+import { FileEntityExpanderInterface } from './file-entity-expander.interface';
+import { FileEntity } from '../../Entity/file.entity';
+import { Stats } from 'fs';
+import { stat } from 'fs/promises';
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class FileEntitySizeExpander implements FileEntityExpanderInterface {
+  async expand(fileEntity: FileEntity): Promise<FileEntity> {
+    const stats: Stats | false = await stat(fileEntity.path).catch(() => false);
+
+    if (stats !== false) {
+      fileEntity.size = stats.size;
+    }
+
+    return fileEntity;
+  }
+}
@@ -1,20 +1,38 @@
-import { VideoFilePathDto } from '../Dto/video-file-path.dto';
 import { Injectable } from '@nestjs/common';
 import * as tnp from 'torrent-name-parser';
+import { FileEntityExpanderInterface } from './file-entity-expander.interface';
+import { FileEntity } from 'src/VideoFiles/Entity/file.entity';
+import { VideoFilesUpdateRepository } from '../video-files-update.repository';
+import { TitleTypeEnum } from '../../Enum/title-type.enum';

 //todo - refactor this copy-paste, add types, ect.
-//todo - make an interface and add as info/data provider (fileEntity data expander interface)
@Injectable()
-export class VideoFileTitleService {
-  constructor() {
+export class FileEntityTitleInfoExpander
+  implements FileEntityExpanderInterface
+{
+  constructor(private readonly repository: VideoFilesUpdateRepository) {
    tnp.configure({ sample: /sample/i }, { sample: 'boolean' });
    tnp.configure({ year: /([\[\(]?((?:19[0-9][0-9]|20[0-2][0-9]))[\]\)]?)/ });
    tnp.configure({ container: /MKV|AVI|MP4|mkv|avi|mp4/i });
  }

-  extractTitleInfo(filePath: VideoFilePathDto) {
+  async expand(fileEntity: FileEntity): Promise<FileEntity> {
+    const titleInfo = this.extractTitleInfo(fileEntity.relativePath);
+
+    fileEntity.info = JSON.stringify(titleInfo.info);
+    fileEntity.infos = JSON.stringify(titleInfo.infos);
+    fileEntity.season = titleInfo.info.season;
+    fileEntity.title = await this.repository.findOrCreateTitle(
+      titleInfo.info.title,
+      titleInfo.info.season ? TitleTypeEnum.show : TitleTypeEnum.movie,
+    );
+
+    return fileEntity;
+  }
+
+  extractTitleInfo(relativePath: string) {
    const infos = [];
-    const arrPath = filePath.relativePath.split('/');
+    const arrPath = relativePath.split('/');

    for (let i = 0; i < arrPath.length; i++) {
      const info = tnp(arrPath[i]);
@@ -0,0 +1,28 @@
+import { FileEntityExpanderInterface } from './file-entity-expander.interface';
+import { Inject, Injectable } from '@nestjs/common';
+import { FileEntity } from '../../Entity/file.entity';
+
+export const FileEntityExpanders = 'FileEntityExpanders';
+
+@Injectable()
+export class FileEntityExpander implements FileEntityExpanderInterface {
+  constructor(
+    @Inject(FileEntityExpanders)
+    private expanders: Array<FileEntityExpanderInterface>,
+  ) {}
+
+  async expand(fileEntity: FileEntity): Promise<FileEntity> {
+    //fileEntity (object) passed byRef, expanders can run in parallel, make sure expanders don't rely on each other's result
+    const expanderPromises = [];
+    for (const expander of this.expanders) {
+      const expanderPromise = expander.expand(fileEntity);
+      expanderPromises.push(expanderPromise);
+    }
+
+    return new Promise((resolve) => {
+      Promise.all(expanderPromises).then(() => {
+        resolve(fileEntity);
+      });
+    });
+  }
+}
@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
-import { configService } from '../../config/config.service';
-import { VideoFilesConfig } from '../../config/video-files.config';
+import { configService } from '../../../config/config.service';
+import { VideoFilesConfig } from '../../../config/video-files.config';
 import { spawn } from 'child_process';

@Injectable()
@@ -0,0 +1,48 @@
+import { Module, Provider } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { TitleEntity } from '../Entity/title.entity';
+import { FileEntity } from '../Entity/file.entity';
+import {
+  FileEntityExpander,
+  FileEntityExpanders,
+} from './Expander/file-entity.expander';
+import { FileEntityTitleInfoExpander } from './Expander/file-entity-title-info.expander';
+import { VideoFilesUpdateService } from './video-files-update.service';
+import { VideoFilesScannerService } from './Scanner/video-files-scanner.service';
+import { VideoFilesUpdateRepository } from './video-files-update.repository';
+import { FileEntitySizeExpander } from './Expander/file-entity-size.expander';
+import { FileEntityDefaultsExpander } from './Expander/file-entity-defaults.expander';
+import { FileEntityDurationExpander } from './Expander/file-entity-duration.expander';
+
+const fileEntityExpanders: Provider<any>[] = [
+  FileEntityExpander,
+  FileEntityDefaultsExpander,
+  FileEntitySizeExpander,
+  FileEntityTitleInfoExpander,
+  FileEntityDurationExpander,
+  {
+    provide: FileEntityExpanders,
+    useFactory: (...expanders) => expanders,
+    inject: [
+      //expanders run in parallel, make sure they don't rely on each other's result
+      //todo - linkomanija, webtorrent, transmission expanders
+      FileEntityDefaultsExpander,
+      FileEntitySizeExpander,
+      FileEntityTitleInfoExpander,
+      FileEntityDurationExpander,
+    ],
+  },
+];
+
+@Module({
+  imports: [TypeOrmModule.forFeature([TitleEntity, FileEntity])],
+  controllers: [],
+  providers: [
+    VideoFilesUpdateService,
+    VideoFilesScannerService,
+    VideoFilesUpdateRepository,
+    ...fileEntityExpanders,
+  ],
+  exports: [VideoFilesUpdateService],
+})
+export class VideoFilesUpdateModule {}
@@ -0,0 +1,100 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { FileEntity } from '../Entity/file.entity';
+import { In, Not, Repository, UpdateResult } from 'typeorm';
+import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
+import { TitleEntity } from '../Entity/title.entity';
+import { TitleTypeEnum } from '../Enum/title-type.enum';
+
+@Injectable()
+export class VideoFilesUpdateRepository {
+  constructor(
+    @InjectRepository(FileEntity)
+    private readonly fileRepository: Repository<FileEntity>,
+    @InjectRepository(TitleEntity)
+    private readonly titleRepository: Repository<TitleEntity>,
+  ) {}
+
+  markRemovedFilesAsDeleted(filePaths: string[]): Promise<UpdateResult> {
+    return this.fileRepository
+      .createQueryBuilder()
+      .update('files')
+      .set({ deleted: true })
+      .where({
+        path: Not(In(filePaths)),
+        deleted: false,
+        streamProvider: StreamProviderEnum.fs,
+      })
+      .execute();
+  }
+
+  getMatchingPaths(filePaths: string[]): Promise<string[]> {
+    return new Promise((resolve) => {
+      this.fileRepository
+        .find({
+          select: ['path'],
+          where: {
+            path: In(filePaths),
+            deleted: false,
+            streamProvider: StreamProviderEnum.fs,
+          },
+        })
+        .then((fileEntitiesInDb) => {
+          resolve(fileEntitiesInDb.map((obj) => obj.path));
+        })
+        .catch(() => {
+          resolve([]);
+        });
+    });
+  }
+
+  public async findOrCreateTitle(
+    title: string,
+    type: TitleTypeEnum,
+    saveIfCreated = true,
+  ): Promise<TitleEntity> {
+    let titleEntity = await this.titleRepository.findOne({
+      where: { title: title, type: type },
+    });
+
+    if (titleEntity === null) {
+      titleEntity = new TitleEntity();
+      titleEntity.title = title;
+      titleEntity.type = type;
+      if (saveIfCreated) {
+        await this.titleRepository.save(titleEntity);
+        titleEntity = await this.titleRepository.findOne({
+          where: { title: title, type: type },
+        });
+      }
+    }
+
+    return titleEntity;
+  }
+
+  public async findOrCreateFile(
+    path: string,
+    saveIfCreated = true,
+  ): Promise<FileEntity> {
+    let fileEntity = await this.fileRepository.findOne({
+      where: { path: path },
+    });
+
+    if (fileEntity === null) {
+      fileEntity = new FileEntity();
+      fileEntity.path = path;
+      if (saveIfCreated) {
+        await this.fileRepository.save(fileEntity);
+        fileEntity = await this.fileRepository.findOne({
+          where: { path: path },
+        });
+      }
+    }
+
+    return fileEntity;
+  }
+
+  public saveFiles(fileEntities: FileEntity[]) {
+    return this.fileRepository.save(fileEntities);
+  }
+}
@@ -0,0 +1,59 @@
+import { Injectable } from '@nestjs/common';
+import { VideoFilesScannerService } from './Scanner/video-files-scanner.service';
+import { VideoFilesUpdateRepository } from './video-files-update.repository';
+import { FileEntityExpander } from './Expander/file-entity.expander';
+import { FileEntity } from '../Entity/file.entity';
+import { StreamProviderEnum } from '../../Streamer/ReadStreamProvider/stream-provider.enum';
+
+@Injectable()
+export class VideoFilesUpdateService {
+  constructor(
+    private readonly scanner: VideoFilesScannerService,
+    private readonly repository: VideoFilesUpdateRepository,
+    private readonly expander: FileEntityExpander,
+  ) {}
+
+  async updateFsVideoFiles(): Promise<void> {
+    const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles();
+
+    await this.repository.markRemovedFilesAsDeleted(filesInFs);
+
+    const filesInDb = await this.repository.getMatchingPaths(filesInFs);
+    const filesToAddToDb = filesInFs.filter(
+      (filePath) => !filesInDb.includes(filePath),
+    );
+
+    const entitiesToSave = [];
+
+    for (const fileToAddToDb of filesToAddToDb) {
+      if (this.checkPathToIgnore(fileToAddToDb)) {
+        continue;
+      }
+
+      const fileEntity = await this.buildFileEntity(
+        fileToAddToDb,
+        filesToRelativePaths[fileToAddToDb],
+      );
+
+      entitiesToSave.push(fileEntity);
+    }
+
+    await this.repository.saveFiles(entitiesToSave);
+  }
+
+  private checkPathToIgnore(filePath: string): boolean {
+    //todo - move to separate provider, include in settings
+    return filePath.indexOf('AOE1') > -1;
+  }
+
+  private async buildFileEntity(
+    path: string,
+    relativePath: string,
+  ): Promise<FileEntity> {
+    const fileEntity = await this.repository.findOrCreateFile(path, false);
+    fileEntity.relativePath = relativePath;
+    fileEntity.streamProvider = StreamProviderEnum.fs;
+
+    return await this.expander.expand(fileEntity);
+  }
+}
@@ -4,14 +4,17 @@ import { Repository } from 'typeorm';
 import { TitleEntity } from './Entity/title.entity';
 import { TitleTypeEnum } from './Enum/title-type.enum';
 import { FileEntity } from './Entity/file.entity';
+import { VideoFilesUpdateService } from './Update/video-files-update.service';

+//todo - refactor provider as facade
@Injectable()
-export class VideoFilesProvider {
+export class VideoFilesFacade {
  constructor(
    @InjectRepository(TitleEntity)
    private readonly titleRepository: Repository<TitleEntity>,
    @InjectRepository(FileEntity)
    private readonly fileRepository: Repository<FileEntity>,
+    private readonly videoFilesUpdateService: VideoFilesUpdateService,
  ) {}

  async getListOfTitles(type: TitleTypeEnum): Promise<TitleEntity[]> {
@@ -37,4 +40,8 @@ export class VideoFilesProvider {
      .findOne({ where: { id: fileId } })
      .catch(() => null);
  }
+
+  updateFsVideoFiles(): void {
+    this.videoFilesUpdateService.updateFsVideoFiles().then();
+  }
 }
@@ -2,24 +2,16 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { TitleEntity } from './Entity/title.entity';
 import { FileEntity } from './Entity/file.entity';
-import { VideoFilesProvider } from './video-files.provider';
-import { VideoFilesUpdateService } from './FileDbUpdater/video-files-update.service';
-import { VideoFilesScannerService } from './FileDbUpdater/video-files-scanner.service';
-import { VideoFilesUpdateRepository } from './FileDbUpdater/video-files-update.repository';
-import { VideoFilesSavingService } from './FileDbUpdater/video-files-saving.service';
-import { VideoFileTitleService } from './FileDbUpdater/video-file-title.service';
+import { VideoFilesFacade } from './video-files.facade';
+import { VideoFilesUpdateModule } from './Update/video-files-update.module';

@Module({
-  imports: [TypeOrmModule.forFeature([TitleEntity, FileEntity])],
-  controllers: [],
-  providers: [
-    VideoFilesProvider,
-    VideoFilesUpdateService,
-    VideoFilesScannerService,
-    VideoFilesUpdateRepository,
-    VideoFilesSavingService,
-    VideoFileTitleService,
+  imports: [
+    TypeOrmModule.forFeature([TitleEntity, FileEntity]),
+    VideoFilesUpdateModule,
  ],
-  exports: [VideoFilesProvider, VideoFilesUpdateService],
+  controllers: [],
+  providers: [VideoFilesFacade],
+  exports: [VideoFilesFacade],
 })
 export class VideoFilesModule {}