diff --git a/.env.sample b/.env.sample index 766252b..b3a197a 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,6 @@ PORT=3000 NODE_ENV=development +BOOKS_FOLDER_1=/home/user/books +BOOKS_FOLDER_2='/home/user/Calibre Library' +ES_HOST=search +ES_PORT=9200 diff --git a/docker-compose.yaml b/docker-compose.yaml index c3e81e5..cb11046 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,7 +6,24 @@ services: volumes: - .:/srv/app/ - ./data:/data + - $BOOKS_FOLDER_1:$BOOKS_FOLDER_1 + - $BOOKS_FOLDER_2:$BOOKS_FOLDER_2 env_file: - .env ports: - $PORT:$PORT + + search: + image: elasticsearch:8.11.3 + environment: + - "discovery.type=single-node" + - "xpack.security.enabled=false" +# - "ES_JAVA_OPTS=-Xms256m -Xmx256m" + deploy: + resources: + limits: +# cpus: '0.001' + memory: 512M + ports: + - 9200:9200 +# - 9300:9300 diff --git a/package-lock.json b/package-lock.json index ea3cdcf..49f0ade 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,11 +5,15 @@ "packages": { "": { "dependencies": { + "@elastic/elasticsearch": "^8.11.0", "@extractus/article-extractor": "^7.3.1", "cheerio": "^1.0.0-rc.12", "express": "^4.18.2", "express-handlebars": "^7.1.2", - "ts-node": "^10.9.1" + "fs-promise": "^0.3.1", + "jszip": "^2.5.0", + "ts-node": "^10.9.1", + "xml-mapping": "^1.7.0" }, "devDependencies": { "@types/express": "^4.17.17", @@ -218,6 +222,49 @@ "node": ">=12" } }, + "node_modules/@elastic/elasticsearch": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.11.0.tgz", + "integrity": "sha512-1UEQFdGLuKdROLJnMTjegasRM3X9INm/PVADoIVgdTfuv6DeJ17UMuNwYSkCrLrC0trLjjGV4YganpbJJX/VLg==", + "dependencies": { + "@elastic/transport": "^8.4.0", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@elastic/elasticsearch/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "node_modules/@elastic/transport": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.0.tgz", + "integrity": "sha512-Yb3fDa7yGD0ca3uMbL64M3vM1cE5h5uHmBcTjkdB4VpCasRNKSd09iDpwqX8zX1tbBtxcaKYLceKthWvPeIxTw==", + "dependencies": { + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^5.22.1" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@elastic/transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/@elastic/transport/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -289,6 +336,14 @@ "node": ">= 15" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "engines": { + "node": ">=14" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", @@ -929,6 +984,11 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/any-promise": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-0.1.0.tgz", + "integrity": "sha512-lqzY9o+BbeGHRCOyxQkt/Tgvz0IZhTmQiA+LxQW8wSNpcTbj8K+0cZiSEvbpNZZP9/11Gy7dnLO3GNWUXO4d1g==" + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -1385,7 +1445,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -2426,6 +2485,15 @@ "node": ">= 0.6" } }, + "node_modules/fs-promise": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-0.3.1.tgz", + "integrity": "sha512-JjkAd4+JaA8VTL1vmX54f7xz6AgBZ9VA6mXlIvN8eJMJGZMVyJ6fdRyjwCP0pIuEkWkM0XcbVz/4F6sFu7OdQg==", + "deprecated": "Use mz or fs-extra^3.0 with Promise Support", + "dependencies": { + "any-promise": "~0.1.0" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2670,6 +2738,14 @@ "node": ">=10" } }, + "node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "engines": { + "node": ">=14" + } + }, "node_modules/html-escaper": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", @@ -3060,6 +3136,14 @@ "node": ">=6" } }, + "node_modules/jszip": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-2.7.0.tgz", + "integrity": "sha512-JIsRKRVC3gTRo2vM4Wy9WBC3TRcfnIZU8k65Phi3izkvPH975FowRYtKGT6PxevA0XnJ/yO8b0QwV0ydVyQwfw==", + "dependencies": { + "pako": "~1.0.2" + } + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -3336,8 +3420,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mute-stream": { "version": "0.0.8", @@ -3661,6 +3744,11 @@ "node": ">=6" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -4294,6 +4382,16 @@ "postcss": "^8.3.11" } }, + "node_modules/sax": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-0.4.3.tgz", + "integrity": "sha512-WvnHpLKMuEsJFV3LXuvxKY4sLdKev2tIeUxn+ljlQAhpx4ZEmJOW+nEa0uERX7XvfxoimvXvEqeo94p2jXoL6g==" + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -4829,6 +4927,17 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/undici": { + "version": "5.28.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", + "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -5028,6 +5137,26 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/xml-mapping": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/xml-mapping/-/xml-mapping-1.7.2.tgz", + "integrity": "sha512-r6HWaifjTUZPGUj/MI006DKpbItA3EsbhXQw30KX8QkyU/+CfNKxEMuHjqxdapzdl5zbYQPebUUZPhZyvDA1+w==", + "dependencies": { + "sax": "=0.4.3", + "xml-writer": ">=1.7.0" + }, + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/xml-writer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/xml-writer/-/xml-writer-1.7.0.tgz", + "integrity": "sha512-elFVMRiV5jb59fbc87zzVa0C01QLBEWP909mRuWqFqrYC5wNTH5QW4AaKMNv7d6zAsuOulkD7wnztZNLQW0Nfg==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -5222,6 +5351,47 @@ "@jridgewell/trace-mapping": "0.3.9" } }, + "@elastic/elasticsearch": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.11.0.tgz", + "integrity": "sha512-1UEQFdGLuKdROLJnMTjegasRM3X9INm/PVADoIVgdTfuv6DeJ17UMuNwYSkCrLrC0trLjjGV4YganpbJJX/VLg==", + "requires": { + "@elastic/transport": "^8.4.0", + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + } + } + }, + "@elastic/transport": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.0.tgz", + "integrity": "sha512-Yb3fDa7yGD0ca3uMbL64M3vM1cE5h5uHmBcTjkdB4VpCasRNKSd09iDpwqX8zX1tbBtxcaKYLceKthWvPeIxTw==", + "requires": { + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^5.22.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + } + } + }, "@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -5272,6 +5442,11 @@ "sanitize-html": "2.11.0" } }, + "@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==" + }, "@humanwhocodes/config-array": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", @@ -5735,6 +5910,11 @@ "color-convert": "^2.0.1" } }, + "any-promise": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-0.1.0.tgz", + "integrity": "sha512-lqzY9o+BbeGHRCOyxQkt/Tgvz0IZhTmQiA+LxQW8wSNpcTbj8K+0cZiSEvbpNZZP9/11Gy7dnLO3GNWUXO4d1g==" + }, "anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -6084,7 +6264,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "requires": { "ms": "2.1.2" } @@ -6830,6 +7009,14 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, + "fs-promise": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-0.3.1.tgz", + "integrity": "sha512-JjkAd4+JaA8VTL1vmX54f7xz6AgBZ9VA6mXlIvN8eJMJGZMVyJ6fdRyjwCP0pIuEkWkM0XcbVz/4F6sFu7OdQg==", + "requires": { + "any-promise": "~0.1.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -6996,6 +7183,11 @@ "lru-cache": "^6.0.0" } }, + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + }, "html-escaper": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", @@ -7271,6 +7463,14 @@ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, + "jszip": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-2.7.0.tgz", + "integrity": "sha512-JIsRKRVC3gTRo2vM4Wy9WBC3TRcfnIZU8k65Phi3izkvPH975FowRYtKGT6PxevA0XnJ/yO8b0QwV0ydVyQwfw==", + "requires": { + "pako": "~1.0.2" + } + }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -7474,8 +7674,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mute-stream": { "version": "0.0.8", @@ -7699,6 +7898,11 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -8119,6 +8323,16 @@ "postcss": "^8.3.11" } }, + "sax": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-0.4.3.tgz", + "integrity": "sha512-WvnHpLKMuEsJFV3LXuvxKY4sLdKev2tIeUxn+ljlQAhpx4ZEmJOW+nEa0uERX7XvfxoimvXvEqeo94p2jXoL6g==" + }, + "secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -8509,6 +8723,14 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "undici": { + "version": "5.28.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", + "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", + "requires": { + "@fastify/busboy": "^2.0.0" + } + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -8652,6 +8874,20 @@ "signal-exit": "^3.0.7" } }, + "xml-mapping": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/xml-mapping/-/xml-mapping-1.7.2.tgz", + "integrity": "sha512-r6HWaifjTUZPGUj/MI006DKpbItA3EsbhXQw30KX8QkyU/+CfNKxEMuHjqxdapzdl5zbYQPebUUZPhZyvDA1+w==", + "requires": { + "sax": "=0.4.3", + "xml-writer": ">=1.7.0" + } + }, + "xml-writer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/xml-writer/-/xml-writer-1.7.0.tgz", + "integrity": "sha512-elFVMRiV5jb59fbc87zzVa0C01QLBEWP909mRuWqFqrYC5wNTH5QW4AaKMNv7d6zAsuOulkD7wnztZNLQW0Nfg==" + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/package.json b/package.json index 8625352..71557f2 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,14 @@ "start": "bash ./start.sh" }, "dependencies": { + "@elastic/elasticsearch": "^8.11.0", "@extractus/article-extractor": "^7.3.1", "cheerio": "^1.0.0-rc.12", "express": "^4.18.2", "express-handlebars": "^7.1.2", - "ts-node": "^10.9.1" + "fs-promise": "^0.3.1", + "jszip": "^2.5.0", + "ts-node": "^10.9.1", + "xml-mapping": "^1.7.0" } } diff --git a/src/index.ts b/src/index.ts index 839894a..0a2911e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,8 @@ import {searchRouter} from "./routes/search"; import {join} from "path"; import {addArticleRouter} from "./routes/add-article"; import {articlesRouter} from "./routes/articles"; +import { booksRouter } from "./routes/books"; +import { scanRouter } from "./routes/scan"; const PORT = process.env.PORT || 3000; const app = express(); @@ -31,6 +33,8 @@ app.use("/", indexRouter); app.use("/search", searchRouter); app.use("/add-article", addArticleRouter); app.use("/articles", articlesRouter); +app.use("/books", booksRouter); +app.use("/scan", scanRouter); app.listen(PORT, () => { console.log(`Server is running on port ${PORT}`); diff --git a/src/libs/elasticsearch/index.ts b/src/libs/elasticsearch/index.ts new file mode 100644 index 0000000..5b3f41b --- /dev/null +++ b/src/libs/elasticsearch/index.ts @@ -0,0 +1,90 @@ +import { Client, errors } from '@elastic/elasticsearch'; + +interface DataType { + path: string; + title: string; + creator: string; + fileId: string; + + [key: string]: string; +} + +export class ElasticSearch { + private readonly client: Client; + + constructor(private readonly indexName: string = 'books') { + this.client = new Client({ + node: `http://${process.env.ES_HOST}:${process.env.ES_PORT || 9200}`, + }); + } + + async ping() { + return this.client.ping(); + } + + async createIndex() { + const exists = await this.client.indices.exists({ + index: this.indexName, + }); + + if (exists) { + return; + } + + await this.client.indices.create({ + index: this.indexName, + mappings: { + properties: { + path: { type: 'text' }, + title: { type: 'text', analyzer: 'autocomplete', search_analyzer: 'standard' }, + creator: { type: 'text', analyzer: 'autocomplete', search_analyzer: 'standard' }, + fileId: { type: 'integer' }, + }, + }, + settings: { + max_ngram_diff: 19, + analysis: { + filter: { + autocomplete_filter: { + type: 'ngram', + min_gram: 1, + max_gram: 20, + }, + }, + analyzer: { + autocomplete: { + filter: ['lowercase', 'autocomplete_filter', "asciifolding"], + type: 'custom', + tokenizer: 'standard', + }, + }, + }, + number_of_replicas: 1, + }, + }); + }; + + async addDocument(payload: DataType) { + return this.client.index({ + index: this.indexName, + id: payload.path, + document: payload + }); + }; + + async search(query: string, limit: number, offset: number) { + return this.client.search({ + index: this.indexName, + query: { + multi_match: { + query, + fields: ['title', 'creator'], + }, + }, + size: limit, + from: offset, + }); + }; +} + +export const elasticSearch = new ElasticSearch(); diff --git a/src/libs/epub-metadata/index.js b/src/libs/epub-metadata/index.js new file mode 100755 index 0000000..8ce0f4d --- /dev/null +++ b/src/libs/epub-metadata/index.js @@ -0,0 +1,158 @@ +/** + * this is a fork of https://www.npmjs.com/package/epub-metadata + * just copy-pasted the code from the original source and modified it to work with my epub files + */ + +var path = require('path'), + + fsp = require('fs-promise'), + JsZip = require('jszip'), + xmlMapping = require('xml-mapping') +const {isObject, isArray} = require("util"); + + +function cleanUpMetadata (key, value) { + if (value === undefined) { + value = null; + } + // Remove namespace part from keys + if (typeof key === 'string' && /\$/.test(key)) { + var newKey = key.split('$')[1] + this[newKey] = value + delete this[key] + key = newKey + } + + // Make text property the actual value + if (isObject(value) && value.hasOwnProperty('$text') && + Object.keys(value).length === 1) { + this[key] = value.$text + } + + return value +} + +function loadDcMetadata (json, metadata) { + if (!json.package?.metadata) { + return; + } + for (var key in json.package.metadata) { + if (key.search('dc') === 0) { + !function () { + var newKey = key.replace(/^dc\$/, '') + metadata[newKey] = json.package.metadata[key] + }() + } + } +} + +function rewriteMetaElements (json) { + if (!json.package?.metadata?.meta) { + return + } + if (!isArray(json.package.metadata.meta)) { + json.package.metadata.meta = [json.package.metadata.meta]; + } + + json.package.metadata.meta.forEach(function (meta) { + json.package.metadata['dc$' + meta.name] = meta.content + }) + delete json.package.metadata.meta +} + +function getCoverImagePath (contentPath, coverId, json) { + if (!json.package?.manifest?.item) { + return; + } + var href = '' + json.package.manifest.item.some(function (item) { + if (item.id === coverId) { + href = path.join(contentPath, '..', item.href) + return true + } + }) + return href +} + +module.exports = function (epubPath) { + + return fsp + .readFile(epubPath) + .then(function (fileContent) { + + var epub = new JsZip(fileContent), + metadata = {}, + contentPaths = [ + 'content.opf', + 'OEBPS/content.opf' + ], + contentPath, + json, + xml + + contentPaths.some(function (contentFilePath) { + xml = epub.file(contentFilePath) + contentPath = contentFilePath + return Boolean(xml) + }) + + if (!xml) + throw new Error( + 'The software was not able to locate ' + + 'a content.opf file in ' + epubPath + ) + + json = xmlMapping.load(xml.asText(), {longTag: true}) + // return json; + if (!json.package) { + json.package = json.ns0$package; + //ns0$metadata, ns0$manifest, ns0$spine, ns0$guide + json.package.metadata = json.package.ns0$metadata ?? json.package.metadata; + json.package.manifest = json.package.ns0$manifest ?? json.package.manifest; + json.package.spine = json.package.ns0$spine ?? json.package.spine; + json.package.guide = json.package.ns0$guide ?? json.package.guide; + //ns1$title, ns1$rights, ns1$identifier + json.package.metadata.dc$title = json.package.metadata.ns1$title ?? json.package.metadata.dc$title; + json.package.metadata.dc$rights = json.package.metadata.ns1$rights ?? json.package.metadata.dc$rights; + json.package.metadata.dc$identifier = json.package.metadata.ns1$identifier ?? json.package.metadata.dc$identifier; + } + + rewriteMetaElements(json) + + loadDcMetadata(json, metadata) + + // This is a little hack to iterate recursively over an object + JSON.stringify(metadata, cleanUpMetadata) + + // Use keys for identifiers + if (!Array.isArray(metadata.identifier)) + metadata.identifier = [metadata.identifier] + + metadata.identifier.forEach(function (identifier) { + if (identifier) { + if (typeof identifier === 'object') { + identifier.text = identifier.text || 'unk' + identifier.scheme = identifier.scheme || identifier.id ||identifier.text + + metadata[identifier.scheme.toLowerCase()] = identifier.text + } + if (typeof identifier === 'string') { + metadata[identifier] = identifier + } + } + }) + + delete metadata.identifier + + metadata.coverPath = getCoverImagePath( + contentPath, + metadata.cover, + json + ) + + return metadata + }) + .catch(function (error) { + console.error(error.stack) + }) +} diff --git a/src/repository/books-repository.ts b/src/repository/books-repository.ts new file mode 100644 index 0000000..06685c2 --- /dev/null +++ b/src/repository/books-repository.ts @@ -0,0 +1,50 @@ +import {existsSync, readFileSync} from "fs"; +import {writeFile} from "fs/promises"; + +class BooksRepository { + private readonly idByPath: Record = {}; + private counter: number = 1; + private saveDbTimeout: NodeJS.Timeout | null = null; + + constructor() { + if (!existsSync('/data/books.json')) { + return; + } + + const rawData = readFileSync('/data/books.json', 'utf-8'); + const data = JSON.parse(rawData); + this.idByPath = data.idByPath ?? {}; + this.counter = data.counter ?? 1; + } + + public getId(path: string): number { + if (this.idByPath[path]) { + return this.idByPath[path]; + } + + const id = this.counter++; + this.idByPath[path] = id; + // delay save, save only once in 5 seconds + if (!this.saveDbTimeout) { + this.saveDbTimeout = setTimeout(() => { + this.saveDbTimeout = null; + this.saveDb(); + }, 5000); + } + + return id; + } + + public getPath(id: number): string | undefined { + return Object.entries(this.idByPath).find(([path, _id]) => _id == id )?.[0]; + } + + private async saveDb(): Promise { + await writeFile('/data/books.json', JSON.stringify({ + idByPath: this.idByPath, + counter: this.counter, + }, null, 2)); + } +} + +export const booksRepository = new BooksRepository(); diff --git a/src/routes/books.ts b/src/routes/books.ts new file mode 100644 index 0000000..09a3f27 --- /dev/null +++ b/src/routes/books.ts @@ -0,0 +1,19 @@ +import * as express from 'express'; +import { booksRepository } from "../repository/books-repository"; + +export const booksRouter = express.Router(); + +booksRouter.get('/', (req, res) => { + res.render('books-index', { + title: `books`, + }); +}); + +booksRouter.get('/:id', (req, res) => { + const bookPath = booksRepository.getPath(Number(req.params.id)); + if (bookPath) { + return res.download(bookPath); + } + + throw new Error('book not found'); +}); diff --git a/src/routes/scan.ts b/src/routes/scan.ts new file mode 100644 index 0000000..8a639ac --- /dev/null +++ b/src/routes/scan.ts @@ -0,0 +1,89 @@ +/** + * scan for new books, currently rescan and index everything, delete is not done + */ + +import * as express from 'express'; +import { exec } from "child_process"; +// @ts-ignore +import * as epubMetadata from "../libs/epub-metadata"; +import { elasticSearch as es } from "../libs/elasticsearch"; +import { booksRepository } from "../repository/books-repository"; + +export const scanRouter = express.Router(); + +scanRouter.get('/', async (req, res) => { + // find epubs + const findPromises: Promise[] = []; + const paths = Object.keys(process.env) + .filter(key => key.startsWith('BOOKS_FOLDER_')) + .map(key => process.env[key]) + .filter((path): path is string => !!path && path.length > 0); + + if (!paths) { + res.send("oh well.."); + return; + } + paths.forEach(path => { + findPromises.push(new Promise((resolve, reject) => { + exec(`find "${path}" -name "*.epub"`, (err, stdout, stderr) => { + if (err) { + reject(err); + } + const files = stdout.split('\n').filter((file): file is string => !!file && file.length > 0); + resolve(files); + }); + })); + }); + + const results = (await Promise.all(findPromises)).flat(); + + //get metadata + const metadatas = await Promise.all( + results.map(result => epubMetadata(result)) + ); + + // map es to documents + // @ts-ignore + const metaWithPaths = metadatas.map((metadata, i) => { + const title = metadata.title?.text ?? metadata.title; + let creator = metadata.creator?.text ?? metadata.creator; + let err = (metadata.title && metadata.creator) ? undefined : metadata; + if (Array.isArray(creator)) { + creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator; + } + err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined); + const id =booksRepository.getId(results[i]); + + return { + index: i, + fileId: id, + path: results[i], + title, + creator, + err, + } + }); + + await es.ping(); + await es.createIndex(); + + for (const metaWithPath of metaWithPaths) { + if (metaWithPath.err) { + console.log("ignoring", metaWithPath.index) + continue; + } + await es.addDocument({ + path: metaWithPath.path, + title: metaWithPath.title, + creator: metaWithPath.creator, + fileId: metaWithPath.fileId, + }); + } + //return metaWithPaths; + //res.end(JSON.stringify(metaWithPaths)); + //res.end(`scan complete, ${metaWithPaths.length} total books`); + + res.render('empty', { + body: `scan complete, ${metaWithPaths.length} total books`, + }); +}); diff --git a/src/routes/search.ts b/src/routes/search.ts index 2cced2e..bf71345 100644 --- a/src/routes/search.ts +++ b/src/routes/search.ts @@ -1,30 +1,38 @@ import * as express from 'express'; -import {RelativeUrl} from "../helpers/relative-url.helper"; -import {generatePagination} from "../helpers/generate-pagination"; +import { RelativeUrl } from "../helpers/relative-url.helper"; +import { generatePagination } from "../helpers/generate-pagination"; +import { elasticSearch as es } from "../libs/elasticsearch"; export const searchRouter = express.Router(); -const searchQueries = [ - "1. first entry", - "2. second entry", - "3. third entry", - "4. fourth entry", - "5. fifth entry", - "6. sixth entry", - "7. seventh entry", - "8. eighth entry", - "9. ninth entry", - "10. tenth entry", - "11. eleventh entry", -]; - -searchRouter.get('/', (req, res) => { +searchRouter.get('/', async (req, res) => { const searchQuery = req.query.query; + if (!searchQuery) { + res.render('search', { + title: `search`, + searchResults: [], + query: '', + pagesList: [], + }); + return; + } + const page: number = parseInt(req.query.page as string) || 1; const perPage = 6; - const searchResults = searchQueries.slice((page - 1) * perPage, page * perPage); - const pages = Math.ceil(searchQueries.length / perPage); + const offset = (page - 1) * perPage; + let pages = 1; + + const result = await es.search(searchQuery as string, perPage, offset); + const searchResults = result.hits.hits.map((hit: any) => { + return { + title: `${hit._source.creator} - ${hit._source.title}`, + id: hit._source.fileId, + } + }); + if (result.hits.total && typeof result.hits.total === "object" && "value" in result.hits.total) { + pages = Math.ceil(result.hits.total.value / perPage); + } const url = new RelativeUrl(req.originalUrl); diff --git a/src/views/books-index.hbs b/src/views/books-index.hbs new file mode 100644 index 0000000..4e7d983 --- /dev/null +++ b/src/views/books-index.hbs @@ -0,0 +1,16 @@ + +
Back to main
+ + +
+

books

+ +
By Author Last Name
+ + +
By Author First Name
+ + +
By Title
+ +
diff --git a/src/views/empty.hbs b/src/views/empty.hbs new file mode 100644 index 0000000..4291133 --- /dev/null +++ b/src/views/empty.hbs @@ -0,0 +1,8 @@ +

+ {{body}} +

+ + + +
Back to main
+ diff --git a/src/views/index.hbs b/src/views/index.hbs index b24dc5c..741cd2b 100644 --- a/src/views/index.hbs +++ b/src/views/index.hbs @@ -4,9 +4,12 @@ - -
Browse all
- + + + + +
Scan all files
+
diff --git a/src/views/search.hbs b/src/views/search.hbs index 9d61314..86c6629 100644 --- a/src/views/search.hbs +++ b/src/views/search.hbs @@ -1,9 +1,14 @@ - +
+ + +
Back to main
{{#each searchResults}} -
- {{this}} -
+ +
+ {{ this.id }}. {{this.title}} +
+ {{/each}}