From e707fa838b647b5a4bb7b1419ba3eee72562643f Mon Sep 17 00:00:00 2001 From: Jurgis Sakalauskas Date: Mon, 30 Oct 2023 11:30:16 +0200 Subject: [PATCH] move aliases to file storage --- README.md | 4 +- nest-cli.json | 11 +++- package.json | 6 +- .../data-storage-service.interface.ts | 4 ++ src/DataStorage/data-storage.module.ts | 10 ++++ .../json-file-data-storage.service.ts | 56 +++++++++++++++++++ .../kodi-api-interface.controller.ts | 1 + .../file-entity-title-info.expander.ts | 37 ++++++------ .../Scanner/video-files-scanner.service.ts | 2 +- .../Update/video-files-update.module.ts | 2 + .../Update/video-files-update.service.ts | 25 ++++++--- src/config/config.service.ts | 2 +- src/config/paths.config.ts | 4 ++ src/config/type-orm.config.ts | 5 +- start.sh | 3 + 15 files changed, 135 insertions(+), 37 deletions(-) create mode 100644 src/DataStorage/data-storage-service.interface.ts create mode 100644 src/DataStorage/data-storage.module.ts create mode 100644 src/DataStorage/json-file-data-storage.service.ts diff --git a/README.md b/README.md index a9b78ab..bbf1b55 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ $ cp .env.example .env $ docker-compose up -d ``` visit http://localhost:3000/if (or whatever port you have provided in .env file) -kodi plugin should be able to connect to this server with the following settings: +[KODI plugin](https://github.com/sakaljurgis/plugin.video.sklk) should be able to connect to this server with the following settings: - api url: http://localhost:3000/api -- video url: http://localhost:3000 (video url is old relic from previous versions, it is probably not used anymore) +- video url: http://localhost:3000 ## Migrations (run inside docker container) diff --git a/nest-cli.json b/nest-cli.json index 2566481..f53cb4d 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -1,5 +1,14 @@ { "$schema": "https://json.schemastore.org/nest-cli", "collection": "@nestjs/schematics", - "sourceRoot": "src" + "sourceRoot": "src", + "compilerOptions": { + "assets": [ + { + "include": "KodiApiInterface/View/", + "outDir": "dist/src", + "watchAssets": true + } + ] + } } diff --git a/package.json b/package.json index 14f11a3..d00f3fb 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "license": "UNLICENSED", "scripts": { "prebuild": "rimraf dist", - "build": "nest build && npm run copy-files", + "build": "nest build", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "start": "bash ./start.sh", "start:dev": "nest start --watch", @@ -23,8 +23,7 @@ "typeorm:run-migrations": "npm run typeorm migration:run -- -d ./src/config/type-orm-migrations.config.ts", "typeorm:generate-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:generate ./migrations/$npm_config_name", "typeorm:create-migration": "npm run typeorm -- migration:create ./migrations/$npm_config_name", - "typeorm:revert-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:revert", - "copy-files": "copyfiles -u 1 src/**/*.html dist/src/" + "typeorm:revert-migration": "npm run typeorm -- -d ./src/config/type-orm-migrations.config.ts migration:revert" }, "dependencies": { "@nestjs/axios": "^1.0.0", @@ -59,7 +58,6 @@ "@types/webtorrent": "^0.109.3", "@typescript-eslint/eslint-plugin": "^5.0.0", "@typescript-eslint/parser": "^5.0.0", - "copyfiles": "^2.4.1", "eslint": "^8.0.1", "eslint-config-prettier": "^8.3.0", "eslint-plugin-prettier": "^4.0.0", diff --git a/src/DataStorage/data-storage-service.interface.ts b/src/DataStorage/data-storage-service.interface.ts new file mode 100644 index 0000000..a737d70 --- /dev/null +++ b/src/DataStorage/data-storage-service.interface.ts @@ -0,0 +1,4 @@ +export interface DataStorageServiceInterface { + get(key: string, fallback: T): Promise; + set(key: string, data: unknown): Promise; +} diff --git a/src/DataStorage/data-storage.module.ts b/src/DataStorage/data-storage.module.ts new file mode 100644 index 0000000..0799dd2 --- /dev/null +++ b/src/DataStorage/data-storage.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { JsonFileDataStorageService } from './json-file-data-storage.service'; + +@Module({ + imports: [], + controllers: [], + providers: [JsonFileDataStorageService], + exports: [JsonFileDataStorageService], +}) +export class DataStorageModule {} diff --git a/src/DataStorage/json-file-data-storage.service.ts b/src/DataStorage/json-file-data-storage.service.ts new file mode 100644 index 0000000..7032ed8 --- /dev/null +++ b/src/DataStorage/json-file-data-storage.service.ts @@ -0,0 +1,56 @@ +import { Injectable } from '@nestjs/common'; +import { join } from 'path'; +import { readFile, writeFile } from 'fs/promises'; +import { configService, ConfigService } from '../config/config.service'; +import { fileExistsSync } from 'tsconfig-paths/lib/filesystem'; +import { DataStorageServiceInterface } from './data-storage-service.interface'; + +/** + * This service is used to store data in json files. + * It is cached in memory to save on readFile calls. + * Use set(cache = false) to skip caching for large files. + */ +@Injectable() +export class JsonFileDataStorageService implements DataStorageServiceInterface { + private readonly data: Record = {}; + private readonly config: ConfigService; + constructor() { + this.config = configService; + } + + async get(key: string, fallback: T): Promise { + if (this.data[key]) { + return this.data[key] as T; + } + + const filePath = this.getFilePath(key); + if (fileExistsSync(filePath)) { + try { + const rawData = await readFile(filePath); + const data = JSON.parse(rawData.toString()); + this.data[key] = data; + + return data as T; + } catch (e) { + console.error(e); + + return fallback; + } + } + + return fallback; + } + + async set(key: string, data: unknown, cache = true): Promise { + if (cache) { + this.data[key] = data; + } + + const filePath = this.getFilePath(key); + await writeFile(filePath, JSON.stringify(data, null, 2)); + } + + private getFilePath(key: string): string { + return join(this.config.getPaths().getDbFolderPath(), key + '.json'); + } +} diff --git a/src/KodiApiInterface/kodi-api-interface.controller.ts b/src/KodiApiInterface/kodi-api-interface.controller.ts index 25271d1..4fb1f66 100644 --- a/src/KodiApiInterface/kodi-api-interface.controller.ts +++ b/src/KodiApiInterface/kodi-api-interface.controller.ts @@ -8,6 +8,7 @@ export class KodiApiInterfaceController { @Get() @Header('content-type', 'text/html') async main(@Res() res: Response) { + //todo - make the views directory global so no need to edit nest-cli.json every time const readable = createReadStream(join(__dirname, './View/if.html')); readable.pipe(res); } diff --git a/src/VideoFiles/Update/Expander/file-entity-title-info.expander.ts b/src/VideoFiles/Update/Expander/file-entity-title-info.expander.ts index bcced57..570d014 100644 --- a/src/VideoFiles/Update/Expander/file-entity-title-info.expander.ts +++ b/src/VideoFiles/Update/Expander/file-entity-title-info.expander.ts @@ -4,13 +4,17 @@ import { FileEntityExpanderInterface } from './file-entity-expander.interface'; import { FileEntity } from 'src/Shared/Entity/file.entity'; import { VideoFilesUpdateRepository } from '../video-files-update.repository'; import { TitleTypeEnum } from '../../../Shared/Enum/title-type.enum'; +import { JsonFileDataStorageService } from '../../../DataStorage/json-file-data-storage.service'; //todo - refactor this copy-paste, add types, ect. @Injectable() export class FileEntityTitleInfoExpander implements FileEntityExpanderInterface { - constructor(private readonly repository: VideoFilesUpdateRepository) { + constructor( + private readonly repository: VideoFilesUpdateRepository, + private readonly jsonFileDataStorageService: JsonFileDataStorageService, + ) { tnp.configure({ sample: /sample/i }, { sample: 'boolean' }); tnp.configure({ year: /([\[\(]?((?:19[0-9][0-9]|20[0-2][0-9]))[\]\)]?)/ }); tnp.configure({ container: /MKV|AVI|MP4|mkv|avi|mp4/i }); @@ -23,7 +27,7 @@ export class FileEntityTitleInfoExpander fileEntity.infos = JSON.stringify(titleInfo.infos); fileEntity.season = titleInfo.info.season; fileEntity.title = await this.repository.findOrCreateTitle( - titleInfo.info.title, + await this.checkForAliases(titleInfo.info.title), titleInfo.info.season ? TitleTypeEnum.show : TitleTypeEnum.movie, ); @@ -94,7 +98,6 @@ export class FileEntityTitleInfoExpander } combined.title = this.removePartNumberFromTitle(combined.title); - combined.title = this.checkForAliases(combined.title); return combined; } @@ -111,23 +114,17 @@ export class FileEntityTitleInfoExpander return /^\d+$/.test(str); } - private checkForAliases(title) { - //todo - db or external file - const aliases = [ - ['spider man', 'Spider Man'], - ['spiderman', 'Spider Man'], - ['spider-man', 'Spider Man'], - ['madagascar', 'Madagascar'], - ['ice age', 'Ice Age'], - ['hotel transylvania', 'Hotel Transylvania'], - ['despicable me', 'Despicable Me'], - ['the matrix', 'The Matrix'], - ['frozen', 'Frozen'], - ['garfield', 'Garfield'], - ['harry potter', 'Harry Potter'], - ['fifty shades', 'Fifty Shades'], - ['lilo', 'Lilo and Stitch'], - ]; + /** + * Check if title matches any of the aliases and return the alias if it does + * @param title + * @private + */ + private async checkForAliases(title) { + type Alias = [string, string]; + const aliases: Alias[] = await this.jsonFileDataStorageService.get( + 'aliases', + [], + ); for (let i = 0; i < aliases.length; i++) { const [pattern, alias] = aliases[i]; diff --git a/src/VideoFiles/Update/Scanner/video-files-scanner.service.ts b/src/VideoFiles/Update/Scanner/video-files-scanner.service.ts index 69bec4d..e7584bb 100644 --- a/src/VideoFiles/Update/Scanner/video-files-scanner.service.ts +++ b/src/VideoFiles/Update/Scanner/video-files-scanner.service.ts @@ -11,7 +11,7 @@ export class VideoFilesScannerService { this.config = configService.getVideoFilesConfig(); } - public async scanNewFiles(): Promise< + public async scanFsForVideoFiles(): Promise< [paths: string[], pathToRelPath: Record] > { let filesInFs = []; diff --git a/src/VideoFiles/Update/video-files-update.module.ts b/src/VideoFiles/Update/video-files-update.module.ts index 71fe1d7..260cf47 100644 --- a/src/VideoFiles/Update/video-files-update.module.ts +++ b/src/VideoFiles/Update/video-files-update.module.ts @@ -15,6 +15,7 @@ import { FileEntityDefaultsExpander } from './Expander/file-entity-defaults.expa import { FileEntityDurationExpander } from './Expander/file-entity-duration.expander'; import { FileEntityTransmissionExpander } from './Expander/file-entity-transmission.expander'; import { TorrentSeedClientModule } from '../../Torrent/SeedClient/torrent-seed-client.module'; +import { DataStorageModule } from '../../DataStorage/data-storage.module'; const fileEntityExpanders: Provider[] = [ FileEntityExpander, @@ -41,6 +42,7 @@ const fileEntityExpanders: Provider[] = [ imports: [ TypeOrmModule.forFeature([TitleEntity, FileEntity]), TorrentSeedClientModule, + DataStorageModule, ], controllers: [], providers: [ diff --git a/src/VideoFiles/Update/video-files-update.service.ts b/src/VideoFiles/Update/video-files-update.service.ts index b3261c5..7a23f08 100644 --- a/src/VideoFiles/Update/video-files-update.service.ts +++ b/src/VideoFiles/Update/video-files-update.service.ts @@ -18,17 +18,27 @@ export class VideoFilesUpdateService { private readonly expander: FileEntityExpander, ) {} - async updateFsVideoFiles(): Promise { - const [filesInFs, filesToRelativePaths] = await this.scanner.scanNewFiles(); + /** + * Scan for new files and remove deleted. + * Refresh is used to force a refresh of all files + * i.e. existing files db will be parsed on new rules (e.g. new aliases) + * @param refresh + */ + async updateFsVideoFiles(refresh = false): Promise { + const [filesInFs, filesToRelativePaths] = + await this.scanner.scanFsForVideoFiles(); await this.repository.markRemovedFilesAsDeleted(filesInFs); - const filesInDb = await this.repository.getMatchingPaths(filesInFs); - const filesToAddToDb = filesInFs.filter( - (filePath) => - !filesInDb.includes(filePath) && !this.isPathToIgnore(filePath), - ); + let filesToAddToDb: string[] = filesInFs; + if (!refresh) { + const filesInDb = await this.repository.getMatchingPaths(filesInFs); + filesToAddToDb = filesInFs.filter( + (filePath) => + !filesInDb.includes(filePath) && !this.isPathToIgnore(filePath), + ); + } const entitiesToSave = []; for (const fileToAddToDb of filesToAddToDb) { @@ -38,6 +48,7 @@ export class VideoFilesUpdateService { ); //check if download is complete + //todo - figure out if this makes sense if ( fileEntity.streamProvider === StreamProviderEnum.wt && fileEntity.torrent && diff --git a/src/config/config.service.ts b/src/config/config.service.ts index 4444bbe..1b06ce6 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -8,8 +8,8 @@ import { RecentSearchesConfig } from './recent-searches.config'; env.config(); export class ConfigService { - private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this); private pathsConfig: PathsConfig = new PathsConfig(this); + private typeOrmConfig: TypeOrmConfig = new TypeOrmConfig(this); private videoFilesConfig: VideoFilesConfig = new VideoFilesConfig(this); private recentSearches: RecentSearchesConfig = new RecentSearchesConfig(this); diff --git a/src/config/paths.config.ts b/src/config/paths.config.ts index 9b40d3c..52ff921 100644 --- a/src/config/paths.config.ts +++ b/src/config/paths.config.ts @@ -23,6 +23,10 @@ export class PathsConfig { return this.getPath(relPath); } + public getDbFolderPath(): string { + return '/srv/data/db'; + } + private getPath(relPath: string) { if (relPath[0] === '/') { //this is an absolute path diff --git a/src/config/type-orm.config.ts b/src/config/type-orm.config.ts index d0ef9b3..0eded03 100644 --- a/src/config/type-orm.config.ts +++ b/src/config/type-orm.config.ts @@ -9,7 +9,10 @@ import { join } from 'path'; export class TypeOrmConfig { constructor(configService: ConfigService) { this.type = 'sqlite'; - this.database = join('/srv/data/db', configService.getEnv('DB_FILENAME')); + this.database = join( + configService.getPaths().getDbFolderPath(), + configService.getEnv('DB_FILENAME'), + ); this.entities = [LrtCategory, TitleEntity, FileEntity, TorrentEntity]; } type: 'sqlite'; diff --git a/start.sh b/start.sh index add5cb1..5c6d093 100644 --- a/start.sh +++ b/start.sh @@ -10,5 +10,8 @@ if [ "$NODE_ENV" == "production" ]; then npm run start:prod else echo "Starting app in development mode" + echo "Running prebuild" + npm run prebuild + echo "Starting app" npm run start:dev fi