add delay

src/libs/elasticsearch/index.ts

@@ -4,9 +4,9 @@ interface DataType {
   path: string;
   title: string;
   creator: string;
-  fileId: string;
+  fileId: number;
 
-  [key: string]: string;
+  [key: string]: string | number;
 }
 
 export class ElasticSearch {

src/routes/scan.ts

@@ -8,6 +8,7 @@ import { exec } from "child_process";
 import * as epubMetadata from "../libs/epub-metadata";
 import { elasticSearch as es } from "../libs/elasticsearch";
 import { booksRepository } from "../repository/books-repository";
+import { setTimeout as sleep } from 'node:timers/promises'
 
 export const scanRouter = express.Router();
 
@@ -36,54 +37,84 @@ scanRouter.get('/', async (req, res) => {
   });
 
   const results = (await Promise.all(findPromises)).flat();
-
+  console.log("found", results.length, "files, running one by one");
-  //get metadata
-  const metadatas = await Promise.all(
-    results.map(result => epubMetadata(result))
-  );
-
-  // map es to documents
-  // @ts-ignore
-  const metaWithPaths = metadatas.map((metadata, i) => {
-    const title = metadata.title?.text ?? metadata.title;
-    let creator = metadata.creator?.text ?? metadata.creator;
-    let err = (metadata.title && metadata.creator) ? undefined : metadata;
-    if (Array.isArray(creator)) {
-      creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
-    }
-    err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
-    const id =booksRepository.getId(results[i]);
-
-    return {
-      index: i,
-      fileId: id,
-      path: results[i],
-      title,
-      creator,
-      err,
-    }
-  });
 
   await es.ping();
   await es.createIndex();
 
-  for (const metaWithPath of metaWithPaths) {
-    if (metaWithPath.err) {
-      console.log("ignoring", metaWithPath.index)
-      continue;
-    }
-    await es.addDocument({
-      path: metaWithPath.path,
-      title: metaWithPath.title,
-      creator: metaWithPath.creator,
-      fileId: metaWithPath.fileId,
-    });
-  }
-  //return metaWithPaths;
-  //res.end(JSON.stringify(metaWithPaths));
-  //res.end(`scan complete, ${metaWithPaths.length} total books`);
-
   res.render('empty', {
-    body: `scan complete, ${metaWithPaths.length} total books`,
+    body: `scan complete, indexing ${results.length} files`,
   });
+
+  for (const path of results) {
+    try {
+      const metadata = await epubMetadata(path);
+      const title = metadata.title?.text ?? metadata.title;
+      let creator = metadata.creator?.text ?? metadata.creator;
+      let err = (metadata.title && metadata.creator) ? undefined : metadata;
+      if (Array.isArray(creator)) {
+        creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
+      }
+      err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
+      const id = booksRepository.getId(path);
+      console.log("adding", id, path);
+      if (!err) {
+        const record = {
+          path,
+          fileId: id,
+          title,
+          creator,
+        }
+
+        await es.addDocument(record);
+        await sleep(25);
+      }
+
+    } catch (e) {
+      console.log("error getting metadata", path);
+    }
+  }
+
+  // //get metadata
+  // const metadatas = await Promise.all(
+  //   results.map(result => epubMetadata(result))
+  // );
+  //
+  // // map es to documents
+  // // @ts-ignore
+  // const metaWithPaths = metadatas.map((metadata, i) => {
+  //   const title = metadata.title?.text ?? metadata.title;
+  //   let creator = metadata.creator?.text ?? metadata.creator;
+  //   let err = (metadata.title && metadata.creator) ? undefined : metadata;
+  //   if (Array.isArray(creator)) {
+  //     creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
+  //   }
+  //   err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
+  //   const id =booksRepository.getId(results[i]);
+  //
+  //   return {
+  //     index: i,
+  //     fileId: id,
+  //     path: results[i],
+  //     title,
+  //     creator,
+  //     err,
+  //   }
+  // });
+  //
+  // await es.ping();
+  // await es.createIndex();
+  //
+  // for (const metaWithPath of metaWithPaths) {
+  //   if (metaWithPath.err) {
+  //     console.log("ignoring", metaWithPath.index)
+  //     continue;
+  //   }
+  //   await es.addDocument({
+  //     path: metaWithPath.path,
+  //     title: metaWithPath.title,
+  //     creator: metaWithPath.creator,
+  //     fileId: metaWithPath.fileId,
+  //   });
+  // }
 });