move aliases to file storage

README.md

@@ -16,9 +16,9 @@ $ cp .env.example .env
 $ docker-compose up -d
 ```
 visit http://localhost:3000/if (or whatever port you have provided in .env file)
-kodi plugin should be able to connect to this server with the following settings:
+[KODI plugin](https://github.com/sakaljurgis/plugin.video.sklk) should be able to connect to this server with the following settings:
 - api url: http://localhost:3000/api
-- video url: http://localhost:3000 (video url is old relic from previous versions, it is probably not used anymore)
+- video url: http://localhost:3000
 
 ## Migrations (run inside docker container)
 

nest-cli.json

@@ -1,5 +1,14 @@
 {
   "$schema": "https://json.schemastore.org/nest-cli",
   "collection": "@nestjs/schematics",
-  "sourceRoot": "src"
+  "sourceRoot": "src",
+  "compilerOptions": {
+    "assets": [
+      {
+        "include": "KodiApiInterface/View/",
+        "outDir": "dist/src",
+        "watchAssets": true
+      }
+    ]
+  }
 }

package.json

@@ -7,7 +7,7 @@
   "license": "UNLICENSED",
   "scripts": {
     "prebuild": "rimraf dist",
-    "build": "nest build && npm run copy-files",
+    "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "bash ./start.sh",
     "start:dev": "nest start --watch",
@@ -23,8 +23,7 @@
     "typeorm:run-migrations": "npm run typeorm migration:run -- -d ./src/config/type-orm-migrations.config.ts",
     "typeorm:generate-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:generate ./migrations/$npm_config_name",
     "typeorm:create-migration": "npm run typeorm -- migration:create ./migrations/$npm_config_name",
-    "typeorm:revert-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:revert",
-    "copy-files": "copyfiles -u 1 src/**/*.html dist/src/"
+    "typeorm:revert-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:revert"
   },
   "dependencies": {
     "@nestjs/axios": "^1.0.0",
@@ -59,7 +58,6 @@
     "@types/webtorrent": "^0.109.3",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
-    "copyfiles": "^2.4.1",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",

src/DataStorage/data-storage-service.interface.ts

@@ -0,0 +1,4 @@
+export interface DataStorageServiceInterface {
+  get<T>(key: string, fallback: T): Promise<T>;
+  set(key: string, data: unknown): Promise<void>;
+}

src/DataStorage/data-storage.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { JsonFileDataStorageService } from './json-file-data-storage.service';
+
+@Module({
+  imports: [],
+  controllers: [],
+  providers: [JsonFileDataStorageService],
+  exports: [JsonFileDataStorageService],
+})
+export class DataStorageModule {}

src/DataStorage/json-file-data-storage.service.ts

@@ -0,0 +1,56 @@
+import { Injectable } from '@nestjs/common';
+import { join } from 'path';
+import { readFile, writeFile } from 'fs/promises';
+import { configService, ConfigService } from '../config/config.service';
+import { fileExistsSync } from 'tsconfig-paths/lib/filesystem';
+import { DataStorageServiceInterface } from './data-storage-service.interface';
+
+/**
+ * This service is used to store data in json files.
+ * It is cached in memory to save on readFile calls.
+ * Use set(cache = false) to skip caching for large files.
+ */
+@Injectable()
+export class JsonFileDataStorageService implements DataStorageServiceInterface {
+  private readonly data: Record<string, unknown> = {};
+  private readonly config: ConfigService;
+  constructor() {
+    this.config = configService;
+  }
+
+  async get<T>(key: string, fallback: T): Promise<T> {
+    if (this.data[key]) {
+      return this.data[key] as T;
+    }
+
+    const filePath = this.getFilePath(key);
+    if (fileExistsSync(filePath)) {
+      try {
+        const rawData = await readFile(filePath);
+        const data = JSON.parse(rawData.toString());
+        this.data[key] = data;
+
+        return data as T;
+      } catch (e) {
+        console.error(e);
+
+        return fallback;
+      }
+    }
+
+    return fallback;
+  }
+
+  async set(key: string, data: unknown, cache = true): Promise<void> {
+    if (cache) {
+      this.data[key] = data;
+    }
+
+    const filePath = this.getFilePath(key);
+    await writeFile(filePath, JSON.stringify(data, null, 2));
+  }
+
+  private getFilePath(key: string): string {
+    return join(this.config.getPaths().getDbFolderPath(), key + '.json');
+  }
+}

src/KodiApiInterface/kodi-api-interface.controller.ts

@@ -8,6 +8,7 @@ export class KodiApiInterfaceController {
   @Get()
   @Header('content-type', 'text/html')
   async main(@Res() res: Response) {
+    //todo - make the views directory global so no need to edit nest-cli.json every time
     const readable = createReadStream(join(__dirname, './View/if.html'));
     readable.pipe(res);
   }

src/VideoFiles/Update/Expander/file-entity-title-info.expander.ts

@@ -4,13 +4,17 @@ import { FileEntityExpanderInterface } from './file-entity-expander.interface';
 import { FileEntity } from 'src/Shared/Entity/file.entity';
 import { VideoFilesUpdateRepository } from '../video-files-update.repository';
 import { TitleTypeEnum } from '../../../Shared/Enum/title-type.enum';
+import { JsonFileDataStorageService } from '../../../DataStorage/json-file-data-storage.service';
 
 //todo - refactor this copy-paste, add types, ect.
 @Injectable()
 export class FileEntityTitleInfoExpander
   implements FileEntityExpanderInterface
 {
-  constructor(private readonly repository: VideoFilesUpdateRepository) {
+  constructor(
+    private readonly repository: VideoFilesUpdateRepository,
+    private readonly jsonFileDataStorageService: JsonFileDataStorageService,
+  ) {
     tnp.configure({ sample: /sample/i }, { sample: 'boolean' });
     tnp.configure({ year: /([\[\(]?((?:19[0-9][0-9]|20[0-2][0-9]))[\]\)]?)/ });
     tnp.configure({ container: /MKV|AVI|MP4|mkv|avi|mp4/i });
@@ -23,7 +27,7 @@ export class FileEntityTitleInfoExpander
     fileEntity.infos = JSON.stringify(titleInfo.infos);
     fileEntity.season = titleInfo.info.season;
     fileEntity.title = await this.repository.findOrCreateTitle(
-      titleInfo.info.title,
+      await this.checkForAliases(titleInfo.info.title),
       titleInfo.info.season ? TitleTypeEnum.show : TitleTypeEnum.movie,
     );
 
@@ -94,7 +98,6 @@ export class FileEntityTitleInfoExpander
     }
 
     combined.title = this.removePartNumberFromTitle(combined.title);
-    combined.title = this.checkForAliases(combined.title);
     return combined;
   }
 
@@ -111,23 +114,17 @@ export class FileEntityTitleInfoExpander
     return /^\d+$/.test(str);
   }
 
-  private checkForAliases(title) {
-    //todo - db or external file
-    const aliases = [
-      ['spider man', 'Spider Man'],
-      ['spiderman', 'Spider Man'],
-      ['spider-man', 'Spider Man'],
-      ['madagascar', 'Madagascar'],
-      ['ice age', 'Ice Age'],
-      ['hotel transylvania', 'Hotel Transylvania'],
-      ['despicable me', 'Despicable Me'],
-      ['the matrix', 'The Matrix'],
-      ['frozen', 'Frozen'],
-      ['garfield', 'Garfield'],
-      ['harry potter', 'Harry Potter'],
-      ['fifty shades', 'Fifty Shades'],
-      ['lilo', 'Lilo and Stitch'],
-    ];
+  /**
+   * Check if title matches any of the aliases and return the alias if it does
+   * @param title
+   * @private
+   */
+  private async checkForAliases(title) {
+    type Alias = [string, string];
+    const aliases: Alias[] = await this.jsonFileDataStorageService.get<Alias[]>(
+      'aliases',
+      [],
+    );
 
     for (let i = 0; i < aliases.length; i++) {
       const [pattern, alias] = aliases[i];

src/VideoFiles/Update/Scanner/video-files-scanner.service.ts

@@ -11,7 +11,7 @@ export class VideoFilesScannerService {
     this.config = configService.getVideoFilesConfig();
   }
 
-  public async scanNewFiles(): Promise<
+  public async scanFsForVideoFiles(): Promise<
     [paths: string[], pathToRelPath: Record<string, string>]
   > {
     let filesInFs = [];

src/VideoFiles/Update/video-files-update.module.ts

@@ -15,6 +15,7 @@ import { FileEntityDefaultsExpander } from './Expander/file-entity-defaults.expa
 import { FileEntityDurationExpander } from './Expander/file-entity-duration.expander';
 import { FileEntityTransmissionExpander } from './Expander/file-entity-transmission.expander';
 import { TorrentSeedClientModule } from '../../Torrent/SeedClient/torrent-seed-client.module';
+import { DataStorageModule } from '../../DataStorage/data-storage.module';
 
 const fileEntityExpanders: Provider<any>[] = [
   FileEntityExpander,
@@ -41,6 +42,7 @@ const fileEntityExpanders: Provider<any>[] = [
   imports: [
     TypeOrmModule.forFeature([TitleEntity, FileEntity]),
     TorrentSeedClientModule,
+    DataStorageModule,
   ],
   controllers: [],
   providers: [

src/VideoFiles/Update/video-files-update.service.ts

@@ -18,17 +18,27 @@ export class VideoFilesUpdateService {
     private readonly expander: FileEntityExpander,
   ) {}
 
-  async updateFsVideoFiles(): Promise<void> {
-    const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles();
+  /**
+   * Scan for new files and remove deleted.
+   * Refresh is used to force a refresh of all files
+   *   i.e. existing files db will be parsed on new rules (e.g. new aliases)
+   * @param refresh
+   */
+  async updateFsVideoFiles(refresh = false): Promise<void> {
+    const [filesInFs, filesToRelativePaths] =
+      await this.scanner.scanFsForVideoFiles();
 
     await this.repository.markRemovedFilesAsDeleted(filesInFs);
 
-    const filesInDb = await this.repository.getMatchingPaths(filesInFs);
-    const filesToAddToDb = filesInFs.filter(
-      (filePath) =>
-        !filesInDb.includes(filePath) && !this.isPathToIgnore(filePath),
-    );
+    let filesToAddToDb: string[] = filesInFs;
 
+    if (!refresh) {
+      const filesInDb = await this.repository.getMatchingPaths(filesInFs);
+      filesToAddToDb = filesInFs.filter(
+        (filePath) =>
+          !filesInDb.includes(filePath) && !this.isPathToIgnore(filePath),
+      );
+    }
     const entitiesToSave = [];
 
     for (const fileToAddToDb of filesToAddToDb) {
@@ -38,6 +48,7 @@ export class VideoFilesUpdateService {
       );
 
       //check if download is complete
+      //todo - figure out if this makes sense
       if (
         fileEntity.streamProvider === StreamProviderEnum.wt &&
         fileEntity.torrent &&

src/config/config.service.ts

@@ -8,8 +8,8 @@ import { RecentSearchesConfig } from './recent-searches.config';
 env.config();
 
 export class ConfigService {
-  private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this);
   private pathsConfig: PathsConfig = new PathsConfig(this);
+  private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this);
   private videoFilesConfig: VideoFilesConfig = new VideoFilesConfig(this);
   private recentSearches: RecentSearchesConfig = new RecentSearchesConfig(this);
 

src/config/paths.config.ts

@@ -23,6 +23,10 @@ export class PathsConfig {
     return this.getPath(relPath);
   }
 
+  public getDbFolderPath(): string {
+    return '/srv/data/db';
+  }
+
   private getPath(relPath: string) {
     if (relPath[0] === '/') {
       //this is an absolute path

src/config/type-orm.config.ts

@@ -9,7 +9,10 @@ import { join } from 'path';
 export class TypeOrmConfig {
   constructor(configService: ConfigService) {
     this.type = 'sqlite';
-    this.database = join('/srv/data/db', configService.getEnv('DB_FILENAME'));
+    this.database = join(
+      configService.getPaths().getDbFolderPath(),
+      configService.getEnv('DB_FILENAME'),
+    );
     this.entities = [LrtCategory, TitleEntity, FileEntity, TorrentEntity];
   }
   type: 'sqlite';

start.sh

@@ -10,5 +10,8 @@ if [ "$NODE_ENV" == "production" ]; then
     npm run start:prod
 else
   echo "Starting app in development mode"
+  echo "Running prebuild"
+  npm run prebuild
+  echo "Starting app"
   npm run start:dev
 fi