krakalas/kobo
1
0
Fork 0
mirror of https://github.com/sakaljurgis/kobo.git synced 2026-07-08 22:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

add search

Browse Source
This commit is contained in:
Jurgis Sakalauskas
2023-12-30 18:31:54 +02:00
parent ca4b4eddc0
commit 4327050e0b
15 changed files with 745 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.sample
+4
View File
@@ -1,2 +1,6 @@
PORT=3000 PORT=3000
NODE_ENV=development NODE_ENV=development
BOOKS_FOLDER_1=/home/user/books
BOOKS_FOLDER_2='/home/user/Calibre Library'
ES_HOST=search
ES_PORT=9200
docker-compose.yaml
+17
View File
@@ -6,7 +6,24 @@ services:
volumes: volumes:
- .:/srv/app/ - .:/srv/app/
- ./data:/data - ./data:/data
- $BOOKS_FOLDER_1:$BOOKS_FOLDER_1
- $BOOKS_FOLDER_2:$BOOKS_FOLDER_2
env_file: env_file:
- .env - .env
ports: ports:
- $PORT:$PORT - $PORT:$PORT
search:
image: elasticsearch:8.11.3
environment:
- "discovery.type=single-node"
- "xpack.security.enabled=false"
# - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
deploy:
resources:
limits:
# cpus: '0.001'
memory: 512M
ports:
- 9200:9200
# - 9300:9300
package-lock.json
Generated
+243 -7
View File
@@ -5,11 +5,15 @@
"packages": { "packages": {
"": { "": {
"dependencies": { "dependencies": {
"@elastic/elasticsearch": "^8.11.0",
"@extractus/article-extractor": "^7.3.1", "@extractus/article-extractor": "^7.3.1",
"cheerio": "^1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"express": "^4.18.2", "express": "^4.18.2",
"express-handlebars": "^7.1.2", "express-handlebars": "^7.1.2",
"ts-node": "^10.9.1" "fs-promise": "^0.3.1",
"jszip": "^2.5.0",
"ts-node": "^10.9.1",
"xml-mapping": "^1.7.0"
}, },
"devDependencies": { "devDependencies": {
"@types/express": "^4.17.17", "@types/express": "^4.17.17",
@@ -218,6 +222,49 @@
"node": ">=12" "node": ">=12"
} }
}, },
"node_modules/@elastic/elasticsearch": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.11.0.tgz",
"integrity": "sha512-1UEQFdGLuKdROLJnMTjegasRM3X9INm/PVADoIVgdTfuv6DeJ17UMuNwYSkCrLrC0trLjjGV4YganpbJJX/VLg==",
"dependencies": {
"@elastic/transport": "^8.4.0",
"tslib": "^2.4.0"
},
"engines": {
"node": ">=18"
}
},
"node_modules/@elastic/elasticsearch/node_modules/tslib": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
},
"node_modules/@elastic/transport": {
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.0.tgz",
"integrity": "sha512-Yb3fDa7yGD0ca3uMbL64M3vM1cE5h5uHmBcTjkdB4VpCasRNKSd09iDpwqX8zX1tbBtxcaKYLceKthWvPeIxTw==",
"dependencies": {
"debug": "^4.3.4",
"hpagent": "^1.0.0",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0",
"tslib": "^2.4.0",
"undici": "^5.22.1"
},
"engines": {
"node": ">=16"
}
},
"node_modules/@elastic/transport/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/@elastic/transport/node_modules/tslib": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
},
"node_modules/@eslint-community/eslint-utils": { "node_modules/@eslint-community/eslint-utils": {
"version": "4.4.0", "version": "4.4.0",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
@@ -289,6 +336,14 @@
"node": ">= 15" "node": ">= 15"
} }
}, },
"node_modules/@fastify/busboy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
"integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
"engines": {
"node": ">=14"
}
},
"node_modules/@humanwhocodes/config-array": { "node_modules/@humanwhocodes/config-array": {
"version": "0.11.10", "version": "0.11.10",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",
@@ -929,6 +984,11 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1" "url": "https://github.com/chalk/ansi-styles?sponsor=1"
} }
}, },
"node_modules/any-promise": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-0.1.0.tgz",
"integrity": "sha512-lqzY9o+BbeGHRCOyxQkt/Tgvz0IZhTmQiA+LxQW8wSNpcTbj8K+0cZiSEvbpNZZP9/11Gy7dnLO3GNWUXO4d1g=="
},
"node_modules/anymatch": { "node_modules/anymatch": {
"version": "3.1.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@@ -1385,7 +1445,6 @@
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dev": true,
"dependencies": { "dependencies": {
"ms": "2.1.2" "ms": "2.1.2"
}, },
@@ -2426,6 +2485,15 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/fs-promise": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-0.3.1.tgz",
"integrity": "sha512-JjkAd4+JaA8VTL1vmX54f7xz6AgBZ9VA6mXlIvN8eJMJGZMVyJ6fdRyjwCP0pIuEkWkM0XcbVz/4F6sFu7OdQg==",
"deprecated": "Use mz or fs-extra^3.0 with Promise Support",
"dependencies": {
"any-promise": "~0.1.0"
}
},
"node_modules/fs.realpath": { "node_modules/fs.realpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -2670,6 +2738,14 @@
"node": ">=10" "node": ">=10"
} }
}, },
"node_modules/hpagent": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz",
"integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==",
"engines": {
"node": ">=14"
}
},
"node_modules/html-escaper": { "node_modules/html-escaper": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
@@ -3060,6 +3136,14 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/jszip": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/jszip/-/jszip-2.7.0.tgz",
"integrity": "sha512-JIsRKRVC3gTRo2vM4Wy9WBC3TRcfnIZU8k65Phi3izkvPH975FowRYtKGT6PxevA0XnJ/yO8b0QwV0ydVyQwfw==",
"dependencies": {
"pako": "~1.0.2"
}
},
"node_modules/kind-of": { "node_modules/kind-of": {
"version": "6.0.3", "version": "6.0.3",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -3336,8 +3420,7 @@
"node_modules/ms": { "node_modules/ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
"dev": true
}, },
"node_modules/mute-stream": { "node_modules/mute-stream": {
"version": "0.0.8", "version": "0.0.8",
@@ -3661,6 +3744,11 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/pako": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
},
"node_modules/parent-module": { "node_modules/parent-module": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -4294,6 +4382,16 @@
"postcss": "^8.3.11" "postcss": "^8.3.11"
} }
}, },
"node_modules/sax": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/sax/-/sax-0.4.3.tgz",
"integrity": "sha512-WvnHpLKMuEsJFV3LXuvxKY4sLdKev2tIeUxn+ljlQAhpx4ZEmJOW+nEa0uERX7XvfxoimvXvEqeo94p2jXoL6g=="
},
"node_modules/secure-json-parse": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
},
"node_modules/semver": { "node_modules/semver": {
"version": "7.5.4", "version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@@ -4829,6 +4927,17 @@
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
"dev": true "dev": true
}, },
"node_modules/undici": {
"version": "5.28.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz",
"integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==",
"dependencies": {
"@fastify/busboy": "^2.0.0"
},
"engines": {
"node": ">=14.0"
}
},
"node_modules/unpipe": { "node_modules/unpipe": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -5028,6 +5137,26 @@
"node": "^12.13.0 || ^14.15.0 || >=16.0.0" "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
} }
}, },
"node_modules/xml-mapping": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/xml-mapping/-/xml-mapping-1.7.2.tgz",
"integrity": "sha512-r6HWaifjTUZPGUj/MI006DKpbItA3EsbhXQw30KX8QkyU/+CfNKxEMuHjqxdapzdl5zbYQPebUUZPhZyvDA1+w==",
"dependencies": {
"sax": "=0.4.3",
"xml-writer": ">=1.7.0"
},
"engines": {
"node": ">=0.2.0"
}
},
"node_modules/xml-writer": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/xml-writer/-/xml-writer-1.7.0.tgz",
"integrity": "sha512-elFVMRiV5jb59fbc87zzVa0C01QLBEWP909mRuWqFqrYC5wNTH5QW4AaKMNv7d6zAsuOulkD7wnztZNLQW0Nfg==",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/yallist": { "node_modules/yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -5222,6 +5351,47 @@
"@jridgewell/trace-mapping": "0.3.9" "@jridgewell/trace-mapping": "0.3.9"
} }
}, },
"@elastic/elasticsearch": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.11.0.tgz",
"integrity": "sha512-1UEQFdGLuKdROLJnMTjegasRM3X9INm/PVADoIVgdTfuv6DeJ17UMuNwYSkCrLrC0trLjjGV4YganpbJJX/VLg==",
"requires": {
"@elastic/transport": "^8.4.0",
"tslib": "^2.4.0"
},
"dependencies": {
"tslib": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
}
}
},
"@elastic/transport": {
"version": "8.4.0",
"resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.0.tgz",
"integrity": "sha512-Yb3fDa7yGD0ca3uMbL64M3vM1cE5h5uHmBcTjkdB4VpCasRNKSd09iDpwqX8zX1tbBtxcaKYLceKthWvPeIxTw==",
"requires": {
"debug": "^4.3.4",
"hpagent": "^1.0.0",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0",
"tslib": "^2.4.0",
"undici": "^5.22.1"
},
"dependencies": {
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"tslib": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
}
}
},
"@eslint-community/eslint-utils": { "@eslint-community/eslint-utils": {
"version": "4.4.0", "version": "4.4.0",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
@@ -5272,6 +5442,11 @@
"sanitize-html": "2.11.0" "sanitize-html": "2.11.0"
} }
}, },
"@fastify/busboy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
"integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA=="
},
"@humanwhocodes/config-array": { "@humanwhocodes/config-array": {
"version": "0.11.10", "version": "0.11.10",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",
@@ -5735,6 +5910,11 @@
"color-convert": "^2.0.1" "color-convert": "^2.0.1"
} }
}, },
"any-promise": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-0.1.0.tgz",
"integrity": "sha512-lqzY9o+BbeGHRCOyxQkt/Tgvz0IZhTmQiA+LxQW8wSNpcTbj8K+0cZiSEvbpNZZP9/11Gy7dnLO3GNWUXO4d1g=="
},
"anymatch": { "anymatch": {
"version": "3.1.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@@ -6084,7 +6264,6 @@
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dev": true,
"requires": { "requires": {
"ms": "2.1.2" "ms": "2.1.2"
} }
@@ -6830,6 +7009,14 @@
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q=="
}, },
"fs-promise": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/fs-promise/-/fs-promise-0.3.1.tgz",
"integrity": "sha512-JjkAd4+JaA8VTL1vmX54f7xz6AgBZ9VA6mXlIvN8eJMJGZMVyJ6fdRyjwCP0pIuEkWkM0XcbVz/4F6sFu7OdQg==",
"requires": {
"any-promise": "~0.1.0"
}
},
"fs.realpath": { "fs.realpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -6996,6 +7183,11 @@
"lru-cache": "^6.0.0" "lru-cache": "^6.0.0"
} }
}, },
"hpagent": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz",
"integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA=="
},
"html-escaper": { "html-escaper": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
@@ -7271,6 +7463,14 @@
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true "dev": true
}, },
"jszip": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/jszip/-/jszip-2.7.0.tgz",
"integrity": "sha512-JIsRKRVC3gTRo2vM4Wy9WBC3TRcfnIZU8k65Phi3izkvPH975FowRYtKGT6PxevA0XnJ/yO8b0QwV0ydVyQwfw==",
"requires": {
"pako": "~1.0.2"
}
},
"kind-of": { "kind-of": {
"version": "6.0.3", "version": "6.0.3",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -7474,8 +7674,7 @@
"ms": { "ms": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
"dev": true
}, },
"mute-stream": { "mute-stream": {
"version": "0.0.8", "version": "0.0.8",
@@ -7699,6 +7898,11 @@
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
"dev": true "dev": true
}, },
"pako": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
},
"parent-module": { "parent-module": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -8119,6 +8323,16 @@
"postcss": "^8.3.11" "postcss": "^8.3.11"
} }
}, },
"sax": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/sax/-/sax-0.4.3.tgz",
"integrity": "sha512-WvnHpLKMuEsJFV3LXuvxKY4sLdKev2tIeUxn+ljlQAhpx4ZEmJOW+nEa0uERX7XvfxoimvXvEqeo94p2jXoL6g=="
},
"secure-json-parse": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
},
"semver": { "semver": {
"version": "7.5.4", "version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
@@ -8509,6 +8723,14 @@
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
"dev": true "dev": true
}, },
"undici": {
"version": "5.28.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz",
"integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==",
"requires": {
"@fastify/busboy": "^2.0.0"
}
},
"unpipe": { "unpipe": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -8652,6 +8874,20 @@
"signal-exit": "^3.0.7" "signal-exit": "^3.0.7"
} }
}, },
"xml-mapping": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/xml-mapping/-/xml-mapping-1.7.2.tgz",
"integrity": "sha512-r6HWaifjTUZPGUj/MI006DKpbItA3EsbhXQw30KX8QkyU/+CfNKxEMuHjqxdapzdl5zbYQPebUUZPhZyvDA1+w==",
"requires": {
"sax": "=0.4.3",
"xml-writer": ">=1.7.0"
}
},
"xml-writer": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/xml-writer/-/xml-writer-1.7.0.tgz",
"integrity": "sha512-elFVMRiV5jb59fbc87zzVa0C01QLBEWP909mRuWqFqrYC5wNTH5QW4AaKMNv7d6zAsuOulkD7wnztZNLQW0Nfg=="
},
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
package.json
+5 -1
View File
@@ -20,10 +20,14 @@
"start": "bash ./start.sh" "start": "bash ./start.sh"
}, },
"dependencies": { "dependencies": {
"@elastic/elasticsearch": "^8.11.0",
"@extractus/article-extractor": "^7.3.1", "@extractus/article-extractor": "^7.3.1",
"cheerio": "^1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"express": "^4.18.2", "express": "^4.18.2",
"express-handlebars": "^7.1.2", "express-handlebars": "^7.1.2",
"ts-node": "^10.9.1" "fs-promise": "^0.3.1",
"jszip": "^2.5.0",
"ts-node": "^10.9.1",
"xml-mapping": "^1.7.0"
} }
} }
src/index.ts
+4
View File
@@ -5,6 +5,8 @@ import {searchRouter} from "./routes/search";
import {join} from "path"; import {join} from "path";
import {addArticleRouter} from "./routes/add-article"; import {addArticleRouter} from "./routes/add-article";
import {articlesRouter} from "./routes/articles"; import {articlesRouter} from "./routes/articles";
import { booksRouter } from "./routes/books";
import { scanRouter } from "./routes/scan";
const PORT = process.env.PORT || 3000; const PORT = process.env.PORT || 3000;
const app = express(); const app = express();
@@ -31,6 +33,8 @@ app.use("/", indexRouter);
app.use("/search", searchRouter); app.use("/search", searchRouter);
app.use("/add-article", addArticleRouter); app.use("/add-article", addArticleRouter);
app.use("/articles", articlesRouter); app.use("/articles", articlesRouter);
app.use("/books", booksRouter);
app.use("/scan", scanRouter);
app.listen(PORT, () => { app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}`); console.log(`Server is running on port ${PORT}`);
src/libs/elasticsearch/index.ts
+90
View File
@@ -0,0 +1,90 @@
import { Client, errors } from '@elastic/elasticsearch';
interface DataType {
path: string;
title: string;
creator: string;
fileId: string;
[key: string]: string;
}
export class ElasticSearch {
private readonly client: Client;
constructor(private readonly indexName: string = 'books') {
this.client = new Client({
node: `http://${process.env.ES_HOST}:${process.env.ES_PORT || 9200}`,
});
}
async ping() {
return this.client.ping();
}
async createIndex() {
const exists = await this.client.indices.exists({
index: this.indexName,
});
if (exists) {
return;
}
await this.client.indices.create({
index: this.indexName,
mappings: {
properties: {
path: { type: 'text' },
title: { type: 'text', analyzer: 'autocomplete', search_analyzer: 'standard' },
creator: { type: 'text', analyzer: 'autocomplete', search_analyzer: 'standard' },
fileId: { type: 'integer' },
},
},
settings: {
max_ngram_diff: 19,
analysis: {
filter: {
autocomplete_filter: {
type: 'ngram',
min_gram: 1,
max_gram: 20,
},
},
analyzer: {
autocomplete: {
filter: ['lowercase', 'autocomplete_filter', "asciifolding"],
type: 'custom',
tokenizer: 'standard',
},
},
},
number_of_replicas: 1,
},
});
};
async addDocument(payload: DataType) {
return this.client.index({
index: this.indexName,
id: payload.path,
document: payload
});
};
async search(query: string, limit: number, offset: number) {
return this.client.search({
index: this.indexName,
query: {
multi_match: {
query,
fields: ['title', 'creator'],
},
},
size: limit,
from: offset,
});
};
}
export const elasticSearch = new ElasticSearch();
src/libs/epub-metadata/index.js
Executable
+158
View File
@@ -0,0 +1,158 @@
/**
* this is a fork of https://www.npmjs.com/package/epub-metadata
* just copy-pasted the code from the original source and modified it to work with my epub files
*/
var path = require('path'),
fsp = require('fs-promise'),
JsZip = require('jszip'),
xmlMapping = require('xml-mapping')
const {isObject, isArray} = require("util");
function cleanUpMetadata (key, value) {
if (value === undefined) {
value = null;
}
// Remove namespace part from keys
if (typeof key === 'string' && /\$/.test(key)) {
var newKey = key.split('$')[1]
this[newKey] = value
delete this[key]
key = newKey
}
// Make text property the actual value
if (isObject(value) && value.hasOwnProperty('$text') &&
Object.keys(value).length === 1) {
this[key] = value.$text
}
return value
}
function loadDcMetadata (json, metadata) {
if (!json.package?.metadata) {
return;
}
for (var key in json.package.metadata) {
if (key.search('dc') === 0) {
!function () {
var newKey = key.replace(/^dc\$/, '')
metadata[newKey] = json.package.metadata[key]
}()
}
}
}
function rewriteMetaElements (json) {
if (!json.package?.metadata?.meta) {
return
}
if (!isArray(json.package.metadata.meta)) {
json.package.metadata.meta = [json.package.metadata.meta];
}
json.package.metadata.meta.forEach(function (meta) {
json.package.metadata['dc$' + meta.name] = meta.content
})
delete json.package.metadata.meta
}
function getCoverImagePath (contentPath, coverId, json) {
if (!json.package?.manifest?.item) {
return;
}
var href = ''
json.package.manifest.item.some(function (item) {
if (item.id === coverId) {
href = path.join(contentPath, '..', item.href)
return true
}
})
return href
}
module.exports = function (epubPath) {
return fsp
.readFile(epubPath)
.then(function (fileContent) {
var epub = new JsZip(fileContent),
metadata = {},
contentPaths = [
'content.opf',
'OEBPS/content.opf'
],
contentPath,
json,
xml
contentPaths.some(function (contentFilePath) {
xml = epub.file(contentFilePath)
contentPath = contentFilePath
return Boolean(xml)
})
if (!xml)
throw new Error(
'The software was not able to locate ' +
'a content.opf file in ' + epubPath
)
json = xmlMapping.load(xml.asText(), {longTag: true})
// return json;
if (!json.package) {
json.package = json.ns0$package;
//ns0$metadata, ns0$manifest, ns0$spine, ns0$guide
json.package.metadata = json.package.ns0$metadata ?? json.package.metadata;
json.package.manifest = json.package.ns0$manifest ?? json.package.manifest;
json.package.spine = json.package.ns0$spine ?? json.package.spine;
json.package.guide = json.package.ns0$guide ?? json.package.guide;
//ns1$title, ns1$rights, ns1$identifier
json.package.metadata.dc$title = json.package.metadata.ns1$title ?? json.package.metadata.dc$title;
json.package.metadata.dc$rights = json.package.metadata.ns1$rights ?? json.package.metadata.dc$rights;
json.package.metadata.dc$identifier = json.package.metadata.ns1$identifier ?? json.package.metadata.dc$identifier;
}
rewriteMetaElements(json)
loadDcMetadata(json, metadata)
// This is a little hack to iterate recursively over an object
JSON.stringify(metadata, cleanUpMetadata)
// Use keys for identifiers
if (!Array.isArray(metadata.identifier))
metadata.identifier = [metadata.identifier]
metadata.identifier.forEach(function (identifier) {
if (identifier) {
if (typeof identifier === 'object') {
identifier.text = identifier.text || 'unk'
identifier.scheme = identifier.scheme || identifier.id ||identifier.text
metadata[identifier.scheme.toLowerCase()] = identifier.text
}
if (typeof identifier === 'string') {
metadata[identifier] = identifier
}
}
})
delete metadata.identifier
metadata.coverPath = getCoverImagePath(
contentPath,
metadata.cover,
json
)
return metadata
})
.catch(function (error) {
console.error(error.stack)
})
}
src/repository/books-repository.ts
+50
View File
@@ -0,0 +1,50 @@
import {existsSync, readFileSync} from "fs";
import {writeFile} from "fs/promises";
class BooksRepository {
private readonly idByPath: Record<string, number> = {};
private counter: number = 1;
private saveDbTimeout: NodeJS.Timeout | null = null;
constructor() {
if (!existsSync('/data/books.json')) {
return;
}
const rawData = readFileSync('/data/books.json', 'utf-8');
const data = JSON.parse(rawData);
this.idByPath = data.idByPath ?? {};
this.counter = data.counter ?? 1;
}
public getId(path: string): number {
if (this.idByPath[path]) {
return this.idByPath[path];
}
const id = this.counter++;
this.idByPath[path] = id;
// delay save, save only once in 5 seconds
if (!this.saveDbTimeout) {
this.saveDbTimeout = setTimeout(() => {
this.saveDbTimeout = null;
this.saveDb();
}, 5000);
}
return id;
}
public getPath(id: number): string | undefined {
return Object.entries(this.idByPath).find(([path, _id]) => _id == id )?.[0];
}
private async saveDb(): Promise<void> {
await writeFile('/data/books.json', JSON.stringify({
idByPath: this.idByPath,
counter: this.counter,
}, null, 2));
}
}
export const booksRepository = new BooksRepository();
src/routes/books.ts
+19
View File
@@ -0,0 +1,19 @@
import * as express from 'express';
import { booksRepository } from "../repository/books-repository";
export const booksRouter = express.Router();
booksRouter.get('/', (req, res) => {
res.render('books-index', {
title: `books`,
});
});
booksRouter.get('/:id', (req, res) => {
const bookPath = booksRepository.getPath(Number(req.params.id));
if (bookPath) {
return res.download(bookPath);
}
throw new Error('book not found');
});
src/routes/scan.ts
+89
View File
@@ -0,0 +1,89 @@
/**
* scan for new books, currently rescan and index everything, delete is not done
*/
import * as express from 'express';
import { exec } from "child_process";
// @ts-ignore
import * as epubMetadata from "../libs/epub-metadata";
import { elasticSearch as es } from "../libs/elasticsearch";
import { booksRepository } from "../repository/books-repository";
export const scanRouter = express.Router();
scanRouter.get('/', async (req, res) => {
// find epubs
const findPromises: Promise<string[]>[] = [];
const paths = Object.keys(process.env)
.filter(key => key.startsWith('BOOKS_FOLDER_'))
.map(key => process.env[key])
.filter((path): path is string => !!path && path.length > 0);
if (!paths) {
res.send("oh well..");
return;
}
paths.forEach(path => {
findPromises.push(new Promise((resolve, reject) => {
exec(`find "${path}" -name "*.epub"`, (err, stdout, stderr) => {
if (err) {
reject(err);
}
const files = stdout.split('\n').filter((file): file is string => !!file && file.length > 0);
resolve(files);
});
}));
});
const results = (await Promise.all(findPromises)).flat();
//get metadata
const metadatas = await Promise.all(
results.map(result => epubMetadata(result))
);
// map es to documents
// @ts-ignore
const metaWithPaths = metadatas.map((metadata, i) => {
const title = metadata.title?.text ?? metadata.title;
let creator = metadata.creator?.text ?? metadata.creator;
let err = (metadata.title && metadata.creator) ? undefined : metadata;
if (Array.isArray(creator)) {
creator = creator.find(c => c.role === 'aut')?.text ?? creator[0]?.text ?? creator;
}
err = err ?? (Array.isArray(title) || Array.isArray(creator) ? metadata : undefined);
const id =booksRepository.getId(results[i]);
return {
index: i,
fileId: id,
path: results[i],
title,
creator,
err,
}
});
await es.ping();
await es.createIndex();
for (const metaWithPath of metaWithPaths) {
if (metaWithPath.err) {
console.log("ignoring", metaWithPath.index)
continue;
}
await es.addDocument({
path: metaWithPath.path,
title: metaWithPath.title,
creator: metaWithPath.creator,
fileId: metaWithPath.fileId,
});
}
//return metaWithPaths;
//res.end(JSON.stringify(metaWithPaths));
//res.end(`scan complete, ${metaWithPaths.length} total books`);
res.render('empty', {
body: `scan complete, ${metaWithPaths.length} total books`,
});
});
src/routes/search.ts
+25 -17
View File
@@ -1,30 +1,38 @@
import * as express from 'express'; import * as express from 'express';
import { RelativeUrl } from "../helpers/relative-url.helper"; import { RelativeUrl } from "../helpers/relative-url.helper";
import { generatePagination } from "../helpers/generate-pagination"; import { generatePagination } from "../helpers/generate-pagination";
import { elasticSearch as es } from "../libs/elasticsearch";
export const searchRouter = express.Router(); export const searchRouter = express.Router();
const searchQueries = [ searchRouter.get('/', async (req, res) => {
"1. first entry",
"2. second entry",
"3. third entry",
"4. fourth entry",
"5. fifth entry",
"6. sixth entry",
"7. seventh entry",
"8. eighth entry",
"9. ninth entry",
"10. tenth entry",
"11. eleventh entry",
];
searchRouter.get('/', (req, res) => {
const searchQuery = req.query.query; const searchQuery = req.query.query;
if (!searchQuery) {
res.render('search', {
title: `search`,
searchResults: [],
query: '',
pagesList: [],
});
return;
}
const page: number = parseInt(req.query.page as string) || 1; const page: number = parseInt(req.query.page as string) || 1;
const perPage = 6; const perPage = 6;
const searchResults = searchQueries.slice((page - 1) * perPage, page * perPage); const offset = (page - 1) * perPage;
const pages = Math.ceil(searchQueries.length / perPage); let pages = 1;
const result = await es.search(searchQuery as string, perPage, offset);
const searchResults = result.hits.hits.map((hit: any) => {
return {
title: `${hit._source.creator} - ${hit._source.title}`,
id: hit._source.fileId,
}
});
if (result.hits.total && typeof result.hits.total === "object" && "value" in result.hits.total) {
pages = Math.ceil(result.hits.total.value / perPage);
}
const url = new RelativeUrl(req.originalUrl); const url = new RelativeUrl(req.originalUrl);
src/views/books-index.hbs
+16
View File
@@ -0,0 +1,16 @@
<a href="/">
<div class="item">Back to main</div>
</a>
<div>
<h2>books</h2>
<a href="/books/by-author-last-name">
<div class="item">By Author Last Name</div>
</a>
<a href="/books/by-author-first-name">
<div class="item">By Author First Name</div>
</a>
<a href="/books/by-title">
<div class="item">By Title</div>
</a>
</div>
src/views/empty.hbs
+8
View File
@@ -0,0 +1,8 @@
<p>
{{body}}
</p>
<a href="/">
<div class="item">Back to main</div>
</a>
src/views/index.hbs
+5 -2
View File
@@ -4,8 +4,11 @@
<input class="kobo-input" type="text" name="query" value="{{query}}" class="col-7" tabindex="-1"> <input class="kobo-input" type="text" name="query" value="{{query}}" class="col-7" tabindex="-1">
<input class="kobo-input" type="submit" value="search"/> <input class="kobo-input" type="submit" value="search"/>
</form> </form>
<a href="books"> <!-- <a href="books">-->
<div class="item">Browse all</div> <!-- <div class="item">Browse all</div>-->
<!-- </a>-->
<a href="scan">
<div class="item">Scan all files</div>
</a> </a>
</div> </div>
src/views/search.hbs
+7 -2
View File
@@ -1,9 +1,14 @@
<form method="get" action="search">
<input class="kobo-input" type="text" name="query" value="{{query}}" class="col-7" tabindex="-1">
<input class="kobo-input" type="submit" value="search"/>
</form>
<a href="/"><div class="item">Back to main</div></a> <a href="/"><div class="item">Back to main</div></a>
<div> <div>
{{#each searchResults}} {{#each searchResults}}
<a href="/books/{{this.id}}">
<div class="item"> <div class="item">
{{this}} {{ this.id }}. {{this.title}}
</div> </div>
</a>
{{/each}} {{/each}}
</div> </div>