krakalas/kobo
1
0
Fork 0
mirror of https://github.com/sakaljurgis/kobo.git synced 2026-07-08 22:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

add delay

Browse Source
This commit is contained in:
Jurgis Sakalauskas
2023-12-30 19:28:33 +02:00
parent 4327050e0b
commit e52c378245
2 changed files with 77 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/elasticsearch/index.ts
+2 -2
View File
@@ -4,9 +4,9 @@ interface DataType {
path: string;
title: string;
creator: string;
fileId: string;
fileId: number;
[key: string]: string;
[key: string]: string | number;
}
export class ElasticSearch {
src/routes/scan.ts
+65 -34
View File
@@ -8,6 +8,7 @@ import { exec } from "child_process";
import * as epubMetadata from "../libs/epub-metadata";
import { elasticSearch as es } from "../libs/elasticsearch";
import { booksRepository } from "../repository/books-repository";
import { setTimeout as sleep } from 'node:timers/promises'
export const scanRouter = express.Router();
@@ -36,15 +37,18 @@ scanRouter.get('/', async (req, res) => {
});
const results = (await Promise.all(findPromises)).flat();
console.log("found", results.length, "files, running one by one");
//get metadata
const metadatas = await Promise.all(
results.map(result => epubMetadata(result))
);
await es.ping();
await es.createIndex();
// map es to documents
// @ts-ignore
const metaWithPaths = metadatas.map((metadata, i) => {
res.render('empty', {
body: `scan complete, indexing ${results.length} files`,
});
for (const path of results) {
try {
const metadata = await epubMetadata(path);
const title = metadata.title?.text ?? metadata.title;
let creator = metadata.creator?.text ?? metadata.creator;
let err = (metadata.title && metadata.creator) ? undefined : metadata;
@@ -52,38 +56,65 @@ scanRouter.get('/', async (req, res) => {
creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
}
err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
const id =booksRepository.getId(results[i]);
return {
index: i,
const id = booksRepository.getId(path);
console.log("adding", id, path);
if (!err) {
const record = {
path,
fileId: id,
path: results[i],
title,
creator,
err,
}
});
await es.ping();
await es.createIndex();
for (const metaWithPath of metaWithPaths) {
if (metaWithPath.err) {
console.log("ignoring", metaWithPath.index)
continue;
await es.addDocument(record);
await sleep(25);
}
await es.addDocument({
path: metaWithPath.path,
title: metaWithPath.title,
creator: metaWithPath.creator,
fileId: metaWithPath.fileId,
});
}
//return metaWithPaths;
//res.end(JSON.stringify(metaWithPaths));
//res.end(`scan complete, ${metaWithPaths.length} total books`);
res.render('empty', {
body: `scan complete, ${metaWithPaths.length} total books`,
});
} catch (e) {
console.log("error getting metadata", path);
}
}
// //get metadata
// const metadatas = await Promise.all(
// results.map(result => epubMetadata(result))
// );
//
// // map es to documents
// // @ts-ignore
// const metaWithPaths = metadatas.map((metadata, i) => {
// const title = metadata.title?.text ?? metadata.title;
// let creator = metadata.creator?.text ?? metadata.creator;
// let err = (metadata.title && metadata.creator) ? undefined : metadata;
// if (Array.isArray(creator)) {
// creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
// }
// err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
// const id =booksRepository.getId(results[i]);
//
// return {
// index: i,
// fileId: id,
// path: results[i],
// title,
// creator,
// err,
// }
// });
//
// await es.ping();
// await es.createIndex();
//
// for (const metaWithPath of metaWithPaths) {
// if (metaWithPath.err) {
// console.log("ignoring", metaWithPath.index)
// continue;
// }
// await es.addDocument({
// path: metaWithPath.path,
// title: metaWithPath.title,
// creator: metaWithPath.creator,
// fileId: metaWithPath.fileId,
// });
// }
});