mirror of
https://github.com/sakaljurgis/nntp-worker.git
synced 2026-07-08 21:27:41 +00:00
initial
This commit is contained in:
Jurgis Sakalauskas
@@ -0,0 +1,2 @@
|
||||
pb
|
||||
node_modules
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
pb
|
||||
.env
|
||||
Generated
+8
@@ -0,0 +1,8 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
Generated
+35
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="pb_data" uuid="fb5e281b-919a-41d1-99d7-327d76109d5d">
|
||||
<driver-ref>sqlite.xerial</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
|
||||
<jdbc-url>jdbc:sqlite:$PROJECT_DIR$/pb/pb_data/data.db</jdbc-url>
|
||||
<working-dir>$ProjectFileDir$</working-dir>
|
||||
<libraries>
|
||||
<library>
|
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.39.2/sqlite-jdbc-3.39.2.jar</url>
|
||||
</library>
|
||||
</libraries>
|
||||
</data-source>
|
||||
<data-source source="LOCAL" name="pb_data_aux" uuid="549267bf-9f6d-488d-ad42-a8fa02e9c6c7">
|
||||
<driver-ref>sqlite.xerial</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
|
||||
<jdbc-url>jdbc:sqlite:$PROJECT_DIR$/pb/pb_data/auxiliary.db</jdbc-url>
|
||||
<working-dir>$ProjectFileDir$</working-dir>
|
||||
<libraries>
|
||||
<library>
|
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.jar</url>
|
||||
</library>
|
||||
<library>
|
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar</url>
|
||||
</library>
|
||||
<library>
|
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.39.2/sqlite-jdbc-3.39.2.jar</url>
|
||||
</library>
|
||||
</libraries>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
Generated
+8
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/nntp-client.iml" filepath="$PROJECT_DIR$/.idea/nntp-client.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
Generated
+8
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="WEB_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
Generated
+19
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="MessDetectorOptionsConfiguration">
|
||||
<option name="transferred" value="true" />
|
||||
</component>
|
||||
<component name="PHPCSFixerOptionsConfiguration">
|
||||
<option name="transferred" value="true" />
|
||||
</component>
|
||||
<component name="PHPCodeSnifferOptionsConfiguration">
|
||||
<option name="highlightLevel" value="WARNING" />
|
||||
<option name="transferred" value="true" />
|
||||
</component>
|
||||
<component name="PhpStanOptionsConfiguration">
|
||||
<option name="transferred" value="true" />
|
||||
</component>
|
||||
<component name="PsalmOptionsConfiguration">
|
||||
<option name="transferred" value="true" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM node:20
|
||||
RUN apt-get install python3 -y
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
COPY mailparser/package*.json ./mailparser/
|
||||
RUN npm install
|
||||
COPY . .
|
||||
EXPOSE 3000
|
||||
CMD [ "npm", "start"]
|
||||
@@ -0,0 +1,96 @@
|
||||
import { NntpClient, type NntpClientConfig } from './worker/nntp-client';
|
||||
import { Repository } from './worker/repository';
|
||||
import { downloadAndProcessGroup, updateGroupsList } from './worker/groups';
|
||||
import { updateLastReplyDate } from './worker/last-reply-date';
|
||||
import { assignThread } from './worker/assign-thread';
|
||||
import http from 'http';
|
||||
import dotenv from 'dotenv';
|
||||
dotenv.config();
|
||||
|
||||
async function main() {
|
||||
const { PB_URL, PB_USER, PB_PASS, NNTP_SERVER } = process.env;
|
||||
if (!PB_URL || !PB_USER || !PB_PASS || !NNTP_SERVER) {
|
||||
throw new Error('Missing environment variables');
|
||||
}
|
||||
|
||||
const repository = new Repository(PB_URL, PB_USER, PB_PASS);
|
||||
await repository.connect();
|
||||
|
||||
const config: NntpClientConfig = {
|
||||
server: NNTP_SERVER,
|
||||
};
|
||||
const client = new NntpClient(config);
|
||||
await client.connect();
|
||||
|
||||
const test = false;
|
||||
if (test) {
|
||||
await client.selectGroup('omnitel.books');
|
||||
const parsedArticle = await client.getArticle(2)
|
||||
await client.disconnect();
|
||||
console.log(parsedArticle);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const groups = await updateGroupsList(repository, client);
|
||||
const groupNames = groups.map((group) => group.group);
|
||||
|
||||
let numDownloaded = 0;
|
||||
|
||||
for (const groupName of groupNames) {
|
||||
console.log(`Downloading and processing group ${groupName}`);
|
||||
const numDownloadedInGroup = await downloadAndProcessGroup(groups, groupName, repository, client);
|
||||
console.log(`Downloaded and processed ${numDownloadedInGroup} articles in group ${groupName}`);
|
||||
numDownloaded += numDownloadedInGroup;
|
||||
}
|
||||
|
||||
await client.disconnect();
|
||||
|
||||
console.log(`Downloaded and processed ${numDownloaded} articles in total`);
|
||||
|
||||
if (numDownloaded > 0) {
|
||||
await updateLastReplyDate(repository);
|
||||
await assignThread(repository);
|
||||
}
|
||||
|
||||
return numDownloaded;
|
||||
}
|
||||
|
||||
const errors: unknown[] = [];
|
||||
const successes: string[] = [];
|
||||
|
||||
async function runWorker() {
|
||||
try {
|
||||
const numDownloaded = await main();
|
||||
successes.push(`On ${new Date().toISOString()} processed ${numDownloaded}`);
|
||||
if (numDownloaded > 0) {
|
||||
setTimeout(runWorker, 5 * 60 * 1000);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
setTimeout(runWorker, 10 * 60 * 1000);
|
||||
} catch (e: unknown) {
|
||||
console.log(e);
|
||||
errors.push([new Date().toISOString(), e]);
|
||||
}
|
||||
}
|
||||
|
||||
http.createServer((req, res) => {
|
||||
|
||||
if (req.url === '/restart') {
|
||||
runWorker();
|
||||
res.writeHead(302, {'Location': '/'});
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(200, {'Content-Type': 'text/text'});
|
||||
const workerRunning = errors.length === 0;
|
||||
res.write('Worker running: ' + workerRunning + '\n');
|
||||
res.write('Successes:\n' + successes.join('\n'));
|
||||
res.write('\n\nErrors:\n' + errors.join('\n'));
|
||||
res.end();
|
||||
|
||||
}).listen(3000);
|
||||
|
||||
runWorker();
|
||||
@@ -0,0 +1 @@
|
||||
*.js text eol=lf
|
||||
@@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
upgrade: true,
|
||||
reject: [
|
||||
// api changes, check and fix
|
||||
'eslint',
|
||||
'grunt-eslint'
|
||||
]
|
||||
};
|
||||
@@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
printWidth: 160,
|
||||
tabWidth: 4,
|
||||
singleQuote: true,
|
||||
endOfLine: 'lf',
|
||||
trailingComma: 'none',
|
||||
arrowParens: 'avoid'
|
||||
};
|
||||
@@ -0,0 +1,52 @@
|
||||
# Changelog
|
||||
|
||||
## [3.7.2](https://github.com/nodemailer/mailparser/compare/v3.7.1...v3.7.2) (2024-11-29)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **deps:** Bumped deps to fix issue with missing whitespace ([92884d0](https://github.com/nodemailer/mailparser/commit/92884d0619efa77042ecc35fbc887e93a59e5a93))
|
||||
|
||||
## [3.7.1](https://github.com/nodemailer/mailparser/compare/v3.7.0...v3.7.1) (2024-04-25)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **deps:** Replaced 'punycode' with 'punycode.js' module ([4a15157](https://github.com/nodemailer/mailparser/commit/4a15157dc9a815aa0e756d9e6ae0e8631842c447))
|
||||
|
||||
## [3.7.0](https://github.com/nodemailer/mailparser/compare/v3.6.9...v3.7.0) (2024-04-01)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **events:** Emit a new headerLines event to gain access the raw headers ([#364](https://github.com/nodemailer/mailparser/issues/364)) ([d33d7ec](https://github.com/nodemailer/mailparser/commit/d33d7ec4b8e32a4eb7a9a664cec5fdb545c274af))
|
||||
|
||||
## [3.6.9](https://github.com/nodemailer/mailparser/compare/v3.6.8...v3.6.9) (2024-02-29)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **deps:** Bumped deps ([db842ad](https://github.com/nodemailer/mailparser/commit/db842addd36e2fe94d0c4b466da80719a36f47ac))
|
||||
|
||||
## [3.6.8](https://github.com/nodemailer/mailparser/compare/v3.6.7...v3.6.8) (2024-02-29)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **punycode:** Fixes [#355](https://github.com/nodemailer/mailparser/issues/355) Deprecation warning of the punycode module ([#356](https://github.com/nodemailer/mailparser/issues/356)) ([0f35330](https://github.com/nodemailer/mailparser/commit/0f35330c87d715d38e8c853ae6c2f64d098b971d))
|
||||
|
||||
## [3.6.7](https://github.com/nodemailer/mailparser/compare/v3.6.6...v3.6.7) (2024-02-01)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* :arrow_up: update nodemailer dependency to resolve security issue GHSA-9h6g-pr28-7cqp ([#357](https://github.com/nodemailer/mailparser/issues/357)) ([8bc4225](https://github.com/nodemailer/mailparser/commit/8bc42251fca6f538ece599f0a5bebe09b0aeff4f))
|
||||
|
||||
## [3.6.6](https://github.com/nodemailer/mailparser/compare/v3.6.5...v3.6.6) (2024-01-04)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **deploy:** added auto-deployment ([d6eb56f](https://github.com/nodemailer/mailparser/commit/d6eb56fe09fe8b415e5bbf2e53704f6788ca0fee))
|
||||
* Fix produced text address list string according to rfc 2822 ([#340](https://github.com/nodemailer/mailparser/issues/340)) ([6bae600](https://github.com/nodemailer/mailparser/commit/6bae600a3f4a0452ee7ca43634a11939de7bcc6d))
|
||||
* **test:** updated test matrix (18, 20, 21) ([a2ba9c2](https://github.com/nodemailer/mailparser/commit/a2ba9c236dcd7f990c9d53a386ffaa5b564181b3))
|
||||
@@ -0,0 +1,16 @@
|
||||
Copyright (c) 2020 - 2021 Andris Reinman
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,31 @@
|
||||
# mailparser
|
||||
|
||||
|
||||
|
||||
Advanced email parser for Node.js. Everything is handled as a stream which should make it able to parse even very large messages (100MB+) with relatively low overhead.
|
||||
|
||||
## Looking for a front-end solution?
|
||||
|
||||
_mailparser_ is Node.js only library, so you can't use it reliably in the front-end or bundle with WebPack. If you do need a solution to parse emails in the front-end then use [PostalMime](https://www.npmjs.com/package/postal-mime).
|
||||
|
||||
## Installation
|
||||
|
||||
First install the module from npm:
|
||||
|
||||
```
|
||||
$ npm install mailparser
|
||||
```
|
||||
|
||||
next import the `mailparser` object into your script:
|
||||
|
||||
```js
|
||||
const mailparser = require('mailparser');
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
See [mailparser homepage](https://nodemailer.com/extras/mailparser/) for documentation and terms.
|
||||
|
||||
### License
|
||||
|
||||
Licensed under MIT
|
||||
@@ -0,0 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
const MailParser = require('./lib/mail-parser');
|
||||
const simpleParser = require('./lib/simple-parser');
|
||||
|
||||
module.exports = {
|
||||
MailParser,
|
||||
simpleParser
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,137 @@
|
||||
'use strict';
|
||||
|
||||
const MailParser = require('./mail-parser.js');
|
||||
|
||||
module.exports = (input, options, callback) => {
|
||||
if (input === null || input === undefined) {
|
||||
throw new TypeError('Input cannot be null or undefined.');
|
||||
}
|
||||
|
||||
if (!callback && typeof options === 'function') {
|
||||
callback = options;
|
||||
options = false;
|
||||
}
|
||||
|
||||
let promise;
|
||||
if (!callback) {
|
||||
promise = new Promise((resolve, reject) => {
|
||||
callback = callbackPromise(resolve, reject);
|
||||
});
|
||||
}
|
||||
|
||||
options = options || {};
|
||||
let keepCidLinks = !!options.keepCidLinks;
|
||||
|
||||
let mail = {
|
||||
attachments: []
|
||||
};
|
||||
|
||||
let parser = new MailParser(options);
|
||||
|
||||
parser.on('error', err => {
|
||||
callback(err);
|
||||
});
|
||||
|
||||
parser.on('headers', headers => {
|
||||
mail.headers = headers;
|
||||
mail.headerLines = parser.headerLines;
|
||||
});
|
||||
|
||||
let reading = false;
|
||||
let reader = () => {
|
||||
reading = true;
|
||||
|
||||
let data = parser.read();
|
||||
|
||||
if (data === null) {
|
||||
reading = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.type === 'text') {
|
||||
Object.keys(data).forEach(key => {
|
||||
if (['text', 'html', 'textAsHtml'].includes(key)) {
|
||||
mail[key] = data[key];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (data.type === 'attachment') {
|
||||
mail.attachments.push(data);
|
||||
|
||||
let chunks = [];
|
||||
let chunklen = 0;
|
||||
data.content.on('readable', () => {
|
||||
let chunk;
|
||||
while ((chunk = data.content.read()) !== null) {
|
||||
chunks.push(chunk);
|
||||
chunklen += chunk.length;
|
||||
}
|
||||
});
|
||||
|
||||
data.content.on('end', () => {
|
||||
data.content = Buffer.concat(chunks, chunklen);
|
||||
data.release();
|
||||
reader();
|
||||
});
|
||||
} else {
|
||||
reader();
|
||||
}
|
||||
};
|
||||
|
||||
parser.on('readable', () => {
|
||||
if (!reading) {
|
||||
reader();
|
||||
}
|
||||
});
|
||||
|
||||
parser.on('end', () => {
|
||||
['subject', 'references', 'date', 'to', 'from', 'to', 'cc', 'bcc', 'message-id', 'in-reply-to', 'reply-to'].forEach(key => {
|
||||
if (mail.headers && mail.headers.has(key)) {
|
||||
mail[key.replace(/-([a-z])/g, (m, c) => c.toUpperCase())] = mail.headers.get(key);
|
||||
}
|
||||
});
|
||||
|
||||
if (keepCidLinks) {
|
||||
return callback(null, mail);
|
||||
}
|
||||
parser.updateImageLinks(
|
||||
(attachment, done) => done(false, 'data:' + attachment.contentType + ';base64,' + attachment.content.toString('base64')),
|
||||
(err, html) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
mail.html = html;
|
||||
|
||||
callback(null, mail);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
if (typeof input === 'string') {
|
||||
parser.end(Buffer.from(input));
|
||||
} else if (Buffer.isBuffer(input)) {
|
||||
parser.end(input);
|
||||
} else {
|
||||
input
|
||||
.once('error', err => {
|
||||
input.destroy();
|
||||
parser.destroy();
|
||||
callback(err);
|
||||
})
|
||||
.pipe(parser);
|
||||
}
|
||||
|
||||
return promise;
|
||||
};
|
||||
|
||||
function callbackPromise(resolve, reject) {
|
||||
return function (...args) {
|
||||
let err = args.shift();
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(...args);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const Transform = require('stream').Transform;
|
||||
|
||||
class StreamHash extends Transform {
|
||||
constructor(attachment, algo) {
|
||||
super();
|
||||
this.attachment = attachment;
|
||||
this.algo = (algo || 'md5').toLowerCase();
|
||||
this.hash = crypto.createHash(algo);
|
||||
this.byteCount = 0;
|
||||
}
|
||||
|
||||
_transform(chunk, encoding, done) {
|
||||
this.hash.update(chunk);
|
||||
this.byteCount += chunk.length;
|
||||
done(null, chunk);
|
||||
}
|
||||
|
||||
_flush(done) {
|
||||
this.attachment.checksum = this.hash.digest('hex');
|
||||
this.attachment.size = this.byteCount;
|
||||
done();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = StreamHash;
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"name": "mailparser",
|
||||
"version": "3.7.2",
|
||||
"description": "Parse e-mails",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "grunt",
|
||||
"update": "rm -rf node_modules package-lock.json && ncu -u && npm install"
|
||||
},
|
||||
"author": "Andris Reinman",
|
||||
"contributors": [
|
||||
{
|
||||
"name": "Peter Salomonsen",
|
||||
"email": "petersalomonsen@runbox.com",
|
||||
"url": "https://github.com/petersalomonsen"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"encoding-japanese": "2.2.0",
|
||||
"he": "1.2.0",
|
||||
"html-to-text": "9.0.5",
|
||||
"iconv-lite": "0.6.3",
|
||||
"libmime": "5.3.6",
|
||||
"linkify-it": "5.0.0",
|
||||
"mailsplit": "5.4.2",
|
||||
"nodemailer": "6.9.16",
|
||||
"punycode.js": "2.3.1",
|
||||
"tlds": "1.255.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"ajv": "8.17.1",
|
||||
"eslint": "8.57.0",
|
||||
"eslint-config-nodemailer": "1.2.0",
|
||||
"eslint-config-prettier": "9.1.0",
|
||||
"grunt": "1.6.1",
|
||||
"grunt-cli": "1.5.0",
|
||||
"grunt-contrib-nodeunit": "5.0.0",
|
||||
"grunt-eslint": "24.3.0",
|
||||
"iconv": "3.0.1",
|
||||
"random-message": "1.1.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/nodemailer/mailparser.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/nodemailer/mailparser/issues"
|
||||
}
|
||||
}
|
||||
Generated
+6942
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "nntp-client",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"workspaces": [
|
||||
"mailparser"
|
||||
],
|
||||
"scripts": {
|
||||
"start": "npx tsx index.ts"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@types/node": "^22.13.14",
|
||||
"dotenv": "^16.5.0",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"pocketbase": "^0.25.2",
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.8.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/mailparser": "^3.4.5"
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
// "jsx": "preserve", /* Specify what JSX code is generated. */
|
||||
// "libReplacement": true, /* Enable lib replacement. */
|
||||
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
|
||||
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
|
||||
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
|
||||
|
||||
/* Modules */
|
||||
"module": "commonjs", /* Specify what module code is generated. */
|
||||
// "rootDir": "./", /* Specify the root folder within your source files. */
|
||||
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
|
||||
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
|
||||
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
|
||||
// "rewriteRelativeImportExtensions": true, /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
|
||||
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
|
||||
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
|
||||
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
|
||||
// "noUncheckedSideEffectImports": true, /* Check side effect imports. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files. */
|
||||
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
|
||||
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
|
||||
// "outDir": "./", /* Specify an output folder for all emitted files. */
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
|
||||
// "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
|
||||
// "erasableSyntaxOnly": true, /* Do not allow runtime constructs that are not part of ECMAScript. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true, /* Enable all strict type-checking options. */
|
||||
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
|
||||
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
|
||||
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
|
||||
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
|
||||
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
|
||||
// "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
|
||||
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
|
||||
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
|
||||
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
|
||||
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
|
||||
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
|
||||
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true /* Skip type checking all .d.ts files. */
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { Repository } from './repository';
|
||||
import { NntpClient } from './nntp-client';
|
||||
import { checkPartialContentType, mapParsedMailToArticle } from './mapper';
|
||||
import { ArticleRecord, Attachment } from './types';
|
||||
import { parseAttachments } from './attachments';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { checkAndCombinePartial } from './partial';
|
||||
import { truncateText } from './truncate';
|
||||
import { extractAllUUEncodedAttachments } from './uudecode';
|
||||
|
||||
export async function downloadAndProcessArticle(repository: Repository, groupName: string, articleNumber: number, nntpClient: NntpClient) {
|
||||
const groupId = await repository.getGroupId(groupName);
|
||||
if (!groupId) {
|
||||
console.log(`Group ${groupName} not found in local database`);
|
||||
return;
|
||||
}
|
||||
|
||||
const groupArticleNumber = await repository.findGroupArticleNumber(groupName, articleNumber);
|
||||
|
||||
if (groupArticleNumber) {
|
||||
console.log(`Article ${articleNumber} already exists in local database`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (nntpClient.getSelectedGroup() !== groupName) {
|
||||
await nntpClient.selectGroup(groupName);
|
||||
}
|
||||
|
||||
|
||||
const articleMail = await nntpClient.getArticle(articleNumber).catch(e => {
|
||||
console.log(e);
|
||||
return null;
|
||||
});
|
||||
|
||||
if (!articleMail) {
|
||||
console.log('No article found');
|
||||
return;
|
||||
}
|
||||
|
||||
const partial = checkPartialContentType(articleMail.headers);
|
||||
// restart from here after combining partials
|
||||
let attachments: Attachment[] = await parseAttachments(articleMail, partial);
|
||||
let article = mapParsedMailToArticle(articleMail, attachments);
|
||||
|
||||
|
||||
|
||||
if (partial) {
|
||||
const filePath = `./partials/${partial.id}-part_${partial.number}-article.json`
|
||||
writeFileSync(filePath, JSON.stringify(article, null, 2));
|
||||
console.log(`Partial article saved to ${filePath}`);
|
||||
}
|
||||
|
||||
if (partial) {
|
||||
const combined = await checkAndCombinePartial(partial);
|
||||
if (!combined) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO - this is duplicate code from let attachments = saveAttachments
|
||||
attachments = await parseAttachments(combined, null);
|
||||
article = mapParsedMailToArticle(combined, attachments);
|
||||
}
|
||||
|
||||
if (!article.text) {
|
||||
article.text = '';
|
||||
}
|
||||
|
||||
// check and extract uuencoded attachments
|
||||
const { text: newText, attachments: moreAttachments } = await extractAllUUEncodedAttachments(article.text);
|
||||
|
||||
attachments = attachments.concat(moreAttachments);
|
||||
article.text = newText;
|
||||
|
||||
article.text = article.text.trim();
|
||||
|
||||
const groupsRecordsIds = await repository.findGroupsIdsByNames(article.groups);
|
||||
const parentRecordId = article.parent ? await repository.findArticleIdByArticleId(article.parent) : undefined;
|
||||
const referencesIds = await repository.findArticleIdsByArticleIds(article.references);
|
||||
|
||||
const isRoot = article.references.length === 0;
|
||||
const { text, isTruncated, fullTextFile } = truncateText(article.text, isRoot)
|
||||
|
||||
const data: ArticleRecord = {
|
||||
...article,
|
||||
isRoot,
|
||||
groups: groupsRecordsIds,
|
||||
text,
|
||||
isTruncated,
|
||||
fullTextFile,
|
||||
references: referencesIds,
|
||||
parent: parentRecordId ?? null,
|
||||
guessRootId: article.references[0] ?? null,
|
||||
files: attachments.map((a)=>a.file),
|
||||
}
|
||||
|
||||
try {
|
||||
const createRecord = await repository.createArticle(data);
|
||||
|
||||
console.log('Record created:', createRecord.id);
|
||||
|
||||
await repository.assignArticleToGroupsMessages(createRecord.id, article.inGroups);
|
||||
} catch (e) {
|
||||
console.log(data, article);
|
||||
throw e;
|
||||
}
|
||||
|
||||
console.log('Done');
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Repository } from './repository';
|
||||
|
||||
export async function assignThread(repository: Repository) {
|
||||
// root
|
||||
let rootWithoutThread = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&isRoot=true',
|
||||
})
|
||||
console.log(`Updating root thread for ${rootWithoutThread.totalItems} articles. Total pages: ${rootWithoutThread.totalPages}`);
|
||||
|
||||
while (rootWithoutThread.items.length > 0) {
|
||||
for (const item of rootWithoutThread.items) {
|
||||
await repository.updateArticle(item.id, { thread: item.id });
|
||||
}
|
||||
|
||||
rootWithoutThread = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&isRoot=true',
|
||||
})
|
||||
console.log(`Updating root thread for ${rootWithoutThread.totalItems} articles. Total pages: ${rootWithoutThread.totalPages}`);
|
||||
}
|
||||
|
||||
// replies
|
||||
let replyWithoutThreadWithRoot = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&references.isRoot?=true',
|
||||
expand: 'references',
|
||||
})
|
||||
console.log(`Updating reply thread for ${replyWithoutThreadWithRoot.totalItems} articles. Total pages: ${replyWithoutThreadWithRoot.totalPages}`);
|
||||
|
||||
while (replyWithoutThreadWithRoot.items.length > 0) {
|
||||
for (const item of replyWithoutThreadWithRoot.items) {
|
||||
let rootId: string | null = null;
|
||||
// @ts-ignore
|
||||
for (const ref of item.expand.references) {
|
||||
if (ref.isRoot) {
|
||||
rootId = ref.id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (rootId) {
|
||||
await repository.updateArticle(item.id, { thread: rootId });
|
||||
}
|
||||
}
|
||||
|
||||
replyWithoutThreadWithRoot = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&references.isRoot?=true',
|
||||
expand: 'references',
|
||||
})
|
||||
console.log(`Updating reply thread for ${replyWithoutThreadWithRoot.totalItems} articles. Total pages: ${replyWithoutThreadWithRoot.totalPages}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Attachment, PartialMessageMime } from './types';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { type ParsedMail } from 'mailparser';
|
||||
|
||||
export const parseAttachments = async (article: ParsedMail, partial: PartialMessageMime['params'] | null) => {
|
||||
let attachments: Attachment[] = [];
|
||||
if (!article.attachments) {
|
||||
return attachments;
|
||||
}
|
||||
|
||||
article.attachments.map((attachment) => {
|
||||
if (partial) {
|
||||
const fileName = `${partial.id}-part_${partial.number}-attachment`;
|
||||
const filePath = `./partials/${fileName}`
|
||||
writeFileSync(filePath, attachment.content);
|
||||
}
|
||||
|
||||
attachments.push({
|
||||
file: partial ? undefined : new File([attachment.content], attachment.filename ?? crypto.randomUUID(), { type: attachment.contentType }),
|
||||
fileName: attachment.filename ?? '',
|
||||
size: attachment.size,
|
||||
checksum: attachment.checksum,
|
||||
contentType: attachment.contentType,
|
||||
})
|
||||
});
|
||||
|
||||
return attachments;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Repository } from './repository';
|
||||
import { NntpClient } from './nntp-client';
|
||||
import { downloadAndProcessArticle } from './article';
|
||||
|
||||
export const updateGroupsList = async (repository: Repository, nntpClient: NntpClient) => {
|
||||
const groupsById = await repository.getGroupsIds();
|
||||
const groups = await nntpClient.listGroups();
|
||||
const groupsRecords = await repository.getAllGroups();
|
||||
const filteredGroups: typeof groups = [];
|
||||
|
||||
const createGroupPromises = groups.map(async (group) => {
|
||||
if (!group.group.startsWith('omnitel.')) {
|
||||
return;
|
||||
}
|
||||
if (group.group === 'omnitel.binaries') {
|
||||
return;
|
||||
}
|
||||
filteredGroups.push(group);
|
||||
|
||||
if (!groupsById.has(group.group)) {
|
||||
return await repository.createGroup(group.group);
|
||||
}
|
||||
|
||||
return await repository.updateGroupData({
|
||||
name: group.group,
|
||||
last: group.last,
|
||||
first: group.first,
|
||||
})
|
||||
})
|
||||
|
||||
await Promise.all(createGroupPromises);
|
||||
|
||||
return filteredGroups;
|
||||
}
|
||||
|
||||
export const downloadAndProcessGroup = async (
|
||||
groupsList: Awaited<ReturnType<typeof updateGroupsList>>,
|
||||
groupName: string,
|
||||
repository: Repository,
|
||||
nntpClient: NntpClient
|
||||
) => {
|
||||
const group = groupsList.find((g) => g.group === groupName);
|
||||
|
||||
if (!group) {
|
||||
console.log(`Group ${groupName} not found`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
const lastSynced = await repository.getGroupLastSynced(groupName);
|
||||
|
||||
if (lastSynced >= group.last) {
|
||||
console.log(`Group ${groupName} already synced`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const articleNumberStart = lastSynced + 1;
|
||||
const articleNumberEnd = group.last;
|
||||
let numDownloaded = 0;
|
||||
|
||||
for (let articleNumber = articleNumberStart; articleNumber <= articleNumberEnd; articleNumber++) {
|
||||
await downloadAndProcessArticle(repository, groupName, articleNumber, nntpClient);
|
||||
await repository.updateGroupLastSynced(groupName, articleNumber);
|
||||
numDownloaded++;
|
||||
}
|
||||
|
||||
return numDownloaded;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Repository } from './repository';
|
||||
|
||||
export async function updateLastReplyDate(repository: Repository) {
|
||||
let mismatched = await repository.getLastReplyMismatchedArticles(100);
|
||||
console.log(`Updating last reply date for ${mismatched.totalItems} articles. Total pages: ${mismatched.totalPages}`);
|
||||
while (mismatched.items.length > 0) {
|
||||
for (const item of mismatched.items) {
|
||||
await repository.updateArticle(item.id, {
|
||||
lastReplyDate: item.calculatedLastReplyDate,
|
||||
numReplies: item.calculatedNumReplies,
|
||||
});
|
||||
}
|
||||
|
||||
mismatched = await repository.getLastReplyMismatchedArticles(100);
|
||||
console.log(`Updating last reply date for ${mismatched.totalItems} articles. Total pages: ${mismatched.totalPages}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Headers, ParsedMail, StructuredHeader } from 'mailparser'
|
||||
import { Attachment, PartialMessageMime } from './types';
|
||||
|
||||
export const checkPartialContentType = (headers: Headers): PartialMessageMime['params'] | null => {
|
||||
const contentType = headers.get('content-type') as StructuredHeader | undefined;
|
||||
if (!contentType || !contentType.value || !contentType.value.includes('message/partial')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const params = contentType.params;
|
||||
|
||||
if (!params || !params.number || !params.total || !params.id) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
number: parseInt(params.number),
|
||||
total: parseInt(params.total),
|
||||
id: params.id,
|
||||
};
|
||||
}
|
||||
|
||||
export const mapParsedMailToArticle = (article: ParsedMail, attachments: Attachment[]) => {
|
||||
let references: string[] | string = article.headers.get('references') as string[] | string || [];
|
||||
if (typeof references === 'string') {
|
||||
references = [references];
|
||||
}
|
||||
|
||||
const groups = article.headers.get('newsgroups') as string;
|
||||
const xref = article.headers.get('xref') as string;
|
||||
const parent = article.headers.get('in-reply-to') as string ?? references[references.length - 1];
|
||||
|
||||
return {
|
||||
groups: groups.split(','),
|
||||
subject: article.subject,
|
||||
from: article.from?.text,
|
||||
date: article.date,
|
||||
articleId: article.messageId,
|
||||
inGroups: xref.split(' ').map((group: string) => {
|
||||
return {
|
||||
groupName: group.split(':')[0],
|
||||
messageNumber: group.split(':')[1],
|
||||
}
|
||||
}),
|
||||
references,
|
||||
parent,
|
||||
text: article.text,
|
||||
attachments: attachments,
|
||||
headers: Object.fromEntries(article.headers),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
//http://www.tcpipguide.com/free/t_NNTPCommands-2.htm
|
||||
import { Socket } from 'net';
|
||||
import { EventEmitter } from 'events';
|
||||
import { simpleParser, MailParser } from 'mailparser'
|
||||
import { writeFileSync } from 'fs';
|
||||
|
||||
export interface NntpClientConfig {
|
||||
server: string;
|
||||
port?: number;
|
||||
}
|
||||
|
||||
const commandsDef = {
|
||||
connect: {
|
||||
command: '',
|
||||
responseCode: '200',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
listGroups: {
|
||||
command: 'LIST',
|
||||
responseCode: '215',
|
||||
completeIndicator: '.\r\n',
|
||||
},
|
||||
selectGroup: {
|
||||
command: 'GROUP',
|
||||
responseCode: '211',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
disconnect: {
|
||||
command: 'QUIT',
|
||||
responseCode: '205',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
getArticle: {
|
||||
command: 'ARTICLE',
|
||||
responseCode: '220',
|
||||
completeIndicator: '\r\n.\r\n',
|
||||
},
|
||||
}
|
||||
|
||||
type Command = keyof typeof commandsDef;
|
||||
|
||||
interface SendCommandResult {
|
||||
command: Command;
|
||||
status: 'success' | 'error';
|
||||
statusLine: string;
|
||||
responseString: string;
|
||||
responseBuffer: Buffer;
|
||||
}
|
||||
|
||||
export class NntpClient {
|
||||
private config: NntpClientConfig;
|
||||
private client: Socket;
|
||||
private currentCommand: Command | null = null;
|
||||
private currentResponseString = '';
|
||||
private currentResponseBuffer: Buffer;
|
||||
private eventEmitter: EventEmitter;
|
||||
private selectedGroup: string | null = null;
|
||||
|
||||
constructor(config: NntpClientConfig) {
|
||||
this.config = config;
|
||||
this.client = new Socket();
|
||||
this.eventEmitter = new EventEmitter();
|
||||
|
||||
this.currentResponseBuffer = Buffer.alloc(0);
|
||||
this.setupClient();
|
||||
}
|
||||
|
||||
private setupClient() {
|
||||
this.client.on('data', (data)=> {
|
||||
this.currentResponseBuffer = Buffer.concat([this.currentResponseBuffer, data]);
|
||||
this.currentResponseString += data.toString();
|
||||
if (!this.currentCommand) {
|
||||
console.error(`Received data without a command: ${this.currentResponseString}`);
|
||||
throw new Error('No command is currently being executed');
|
||||
}
|
||||
const currentCommandDef = commandsDef[this.currentCommand];
|
||||
|
||||
// todo - timeout
|
||||
if (this.currentResponseString.endsWith(currentCommandDef.completeIndicator) || !this.currentResponseString.startsWith(currentCommandDef.responseCode)) {
|
||||
const statusLine = this.currentResponseString.split('\r\n')[0];
|
||||
console.log(`Command ${this.currentCommand} completed with status:`, statusLine);
|
||||
this.eventEmitter.emit(`${this.currentCommand}`, {
|
||||
command: this.currentCommand,
|
||||
status: this.currentResponseString.startsWith(currentCommandDef.responseCode) ? 'success' : 'error',
|
||||
statusLine,
|
||||
responseString: this.currentResponseString,
|
||||
responseBuffer: Buffer.from(this.currentResponseBuffer),
|
||||
});
|
||||
this.currentResponseBuffer = Buffer.alloc(0);
|
||||
this.currentResponseString = '';
|
||||
this.currentCommand = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async connect() {
|
||||
// todo - handle disconnect (by host)
|
||||
// todo - add connected status
|
||||
this.currentCommand = 'connect';
|
||||
return new Promise((resolve, reject) => {
|
||||
this.client.connect(this.config.port || 119, this.config.server);
|
||||
this.client.on('error', reject);
|
||||
|
||||
this.eventEmitter.once(`${this.currentCommand}`, (data) => {
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
await this.sendCommand('disconnect');
|
||||
}
|
||||
|
||||
private async sendCommand(command: Command, params?: string): Promise<SendCommandResult> {
|
||||
console.log(`Sending command ${command} with params ${params}`);
|
||||
if (this.currentCommand) {
|
||||
// todo - queue
|
||||
return Promise.reject({
|
||||
status: 'error',
|
||||
message: 'Another command is currently being executed',
|
||||
});
|
||||
}
|
||||
this.currentCommand = command;
|
||||
const commandDef = commandsDef[command];
|
||||
this.client.write(`${commandDef.command + (params ? ' ' + params : '')}\r\n`);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.eventEmitter.once(`${this.currentCommand}`, (data) => {
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async listGroups() {
|
||||
const result = await this.sendCommand('listGroups');
|
||||
|
||||
return result.responseString.split('\r\n').slice(1, -2).map((line) => {
|
||||
const [group, last, first, flag] = line.split(' ');
|
||||
return {
|
||||
group,
|
||||
last: parseInt(last),
|
||||
first: parseInt(first),
|
||||
flag,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
getSelectedGroup() {
|
||||
return this.selectedGroup;
|
||||
}
|
||||
|
||||
async selectGroup(group: string) {
|
||||
const result = await this.sendCommand('selectGroup', group);
|
||||
|
||||
if (result.status === 'error') {
|
||||
return {
|
||||
status: result.status,
|
||||
message: result.statusLine,
|
||||
}
|
||||
}
|
||||
|
||||
const [statusCode, totalEstimate, first, last, groupName] = result.statusLine.split(' ');
|
||||
this.selectedGroup = groupName;
|
||||
|
||||
return {
|
||||
statusCode,
|
||||
status: result.status,
|
||||
totalEstimate: parseInt(totalEstimate),
|
||||
first: parseInt(first),
|
||||
last: parseInt(last),
|
||||
groupName,
|
||||
}
|
||||
}
|
||||
|
||||
async getArticle(articleId: number) {
|
||||
if (!this.selectedGroup) {
|
||||
throw new Error('No group selected. Please select a group before fetching articles.');
|
||||
}
|
||||
const result = await this.sendCommand('getArticle', articleId.toString());
|
||||
|
||||
if (result.status === 'error') {
|
||||
throw new Error(`Error fetching article: ${result.statusLine}`);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// remove 1st and last line from buffer, do not convert to string (to retain encoding)
|
||||
const buffer = result.responseBuffer.slice(result.responseBuffer.indexOf('\r\n') + 2, result.responseBuffer.lastIndexOf('\r\n.\r\n') + 2);
|
||||
|
||||
|
||||
const parsed = await simpleParser(buffer, {
|
||||
skipTextToHtml: true,
|
||||
});
|
||||
|
||||
// console.log({
|
||||
// final: parsed.text,
|
||||
// })
|
||||
|
||||
// if (articleId == 395) {
|
||||
// writeFileSync('./err/article.mail', result.responseBuffer);
|
||||
// console.log(parsed.text?.length);
|
||||
// process.exit(1);
|
||||
// }
|
||||
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { PartialMessageMime } from './types';
|
||||
import { stat, readFile, unlink } from 'fs/promises';
|
||||
import { simpleParser } from 'mailparser';
|
||||
|
||||
export const checkAndCombinePartial = async (partial: PartialMessageMime['params']) => {
|
||||
const allStats = await Promise.allSettled(
|
||||
[...
|
||||
Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, i) => stat(`./partials/${partial.id}-part_${i + 1}-attachment`)),
|
||||
...Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, i) => stat(`./partials/${partial.id}-part_${i + 1}-article.json`)),
|
||||
]
|
||||
);
|
||||
const allExists = allStats.every((result) => result.status === 'fulfilled' && result.value.isFile());
|
||||
|
||||
if (!allExists) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//combine all partials
|
||||
const filesReadPromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, index) => {
|
||||
console.log(`Reading file ./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
return readFile(`./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
})
|
||||
|
||||
const files = await Promise.all(filesReadPromises);
|
||||
|
||||
const file = Buffer.concat(files);
|
||||
|
||||
const parsed = await simpleParser(file);
|
||||
const xrefs = new Set<string>();
|
||||
let articleId = '';
|
||||
const articlesReadPromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map(async (_, index) => {
|
||||
console.log(`Reading file ./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
const articleJson = await readFile(`./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
const article = JSON.parse(articleJson.toString());
|
||||
if (index === 0) {
|
||||
articleId = article.articleId;
|
||||
}
|
||||
if (article && article.headers && article.headers.xref) {
|
||||
return article.headers.xref as string;
|
||||
}
|
||||
})
|
||||
|
||||
const articlesXrefs = await Promise.all(articlesReadPromises);
|
||||
|
||||
articlesXrefs.forEach((xref) => {
|
||||
if (xref) {
|
||||
xref.split(' ').forEach(xre => xrefs.add(xre))
|
||||
}
|
||||
})
|
||||
|
||||
parsed.headers.set('xref', Array.from(xrefs).join(' '));
|
||||
|
||||
if (!articleId) {
|
||||
throw new Error(`articleId not found for ${partial.id}`);
|
||||
}
|
||||
|
||||
const deletePromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map(async (_, index) => {
|
||||
console.log(`Deleting file ./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
await unlink(`./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
console.log(`Deleting file ./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
await unlink(`./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
})
|
||||
|
||||
await Promise.all(deletePromises);
|
||||
parsed.headers.set('messageId', articleId);
|
||||
parsed.messageId = articleId;
|
||||
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import Pocketbase, { RecordModel } from 'pocketbase';
|
||||
import { ArticleRecord, Attachment, GroupRecord } from './types';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { writeFileSync } from 'fs';
|
||||
|
||||
export class Repository {
|
||||
private readonly client: Pocketbase;
|
||||
private groupToId: Map<string, string> | null = null;
|
||||
constructor(
|
||||
url: string,
|
||||
private readonly username: string,
|
||||
private readonly password: string
|
||||
) {
|
||||
this.client = new Pocketbase(url);
|
||||
}
|
||||
|
||||
async connect() {
|
||||
return await this.client.collection('_superusers').authWithPassword(
|
||||
this.username,
|
||||
this.password,
|
||||
{ autoRefreshThreshold: 30 * 60 }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* key groupName
|
||||
* value groupId
|
||||
* todo - this can be cached
|
||||
*/
|
||||
async getGroupsIds() {
|
||||
if (!this.groupToId) {
|
||||
const groupToId = new Map<string, string>();
|
||||
(await this.client.collection('groups').getFullList()).forEach((group) => {
|
||||
groupToId.set(group.name, group.id);
|
||||
});
|
||||
|
||||
this.groupToId = groupToId;
|
||||
}
|
||||
|
||||
return this.groupToId;
|
||||
}
|
||||
|
||||
async getAllGroups() {
|
||||
return await this.client.collection<GroupRecord & RecordModel>('groups').getFullList()
|
||||
}
|
||||
|
||||
async updateGroupData(groupData: Partial<GroupRecord>) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
if (!groupData.name) {
|
||||
return;
|
||||
}
|
||||
const groupId = await this.getGroupId(groupData.name);
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return await groupsCollection.update(groupId, groupData);
|
||||
}
|
||||
|
||||
async findGroupArticleNumber(groupName: string, articleNumber: number) {
|
||||
const groupId = await this.getGroupId(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return await this.client
|
||||
.collection('groupArticleNumber')
|
||||
.getFirstListItem(`group="${groupId}" && number="${articleNumber}"`)
|
||||
.catch(() => null);
|
||||
}
|
||||
|
||||
async getGroupId(groupName: string) {
|
||||
const groupsIds = await this.getGroupsIds();
|
||||
|
||||
return groupsIds.get(groupName);
|
||||
}
|
||||
|
||||
async findGroupsIdsByNames(groupNames: string[]) {
|
||||
if (groupNames.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groups = await groupsCollection.getList(1, 100, {
|
||||
filter: groupNames.map((name: string) => `name="${name}"`).join('||'),
|
||||
});
|
||||
|
||||
return groups.items.map((group) => group.id);
|
||||
}
|
||||
|
||||
async findArticleIdByArticleId(articleId: string) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
const articleRecord = await articlesCollection.getFirstListItem(`articleId="${articleId}"`).catch(() => undefined);
|
||||
|
||||
return articleRecord?.id;
|
||||
}
|
||||
|
||||
async findArticleIdsByArticleIds(articleIds: string[]) {
|
||||
if (articleIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
const articles = await articlesCollection.getList(1, 100, {
|
||||
filter: articleIds.map((name: string) => `articleId="${name}"`).join('||'),
|
||||
}).catch(() => ({ items: [] }));
|
||||
|
||||
return articles.items.map((art) => art.id);
|
||||
}
|
||||
|
||||
async assignArticleToGroupsMessages(id: string, groupsMessageNumbers: { groupName: string; messageNumber: string }[]) {
|
||||
const availableGroups = await this.getGroupsIds();
|
||||
|
||||
const promises = groupsMessageNumbers.map(async (inGroup) => {
|
||||
const groupId = availableGroups.get(inGroup.groupName);
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
console.log(`Adding article ${id} to group ${inGroup.groupName} with number ${inGroup.messageNumber}`);
|
||||
return await this.client
|
||||
.collection('groupArticleNumber')
|
||||
.create({
|
||||
group: groupId,
|
||||
article: id,
|
||||
number: inGroup.messageNumber
|
||||
}, {requestKey: `${groupId}-${id}-${inGroup.messageNumber}`});
|
||||
});
|
||||
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
|
||||
async createArticle(articleRecord: ArticleRecord) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
return await articlesCollection.create<ArticleRecord & RecordModel>(articleRecord);
|
||||
}
|
||||
|
||||
async updateArticle(articleId: string, articleRecord: Partial<ArticleRecord>) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
return await articlesCollection.update(articleId, articleRecord);
|
||||
}
|
||||
|
||||
getArticlesCollection() {
|
||||
return this.client.collection('articles');
|
||||
}
|
||||
|
||||
async getLastReplyMismatchedArticles(numItems = 100) {
|
||||
const lastReplyCollection = this.client.collection('lastReply');
|
||||
|
||||
return await lastReplyCollection.getList<{id: string, calculatedLastReplyDate: string, calculatedNumReplies: number}>(1,numItems, {
|
||||
filter: 'lastReplyDate!=calculatedLastReplyDate||numReplies!=calculatedNumReplies',
|
||||
});
|
||||
}
|
||||
|
||||
async createGroup(groupName: string) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
this.groupToId = null;
|
||||
|
||||
return await groupsCollection.create({ name: groupName, numLastSynced: 0 }, { requestKey: groupName });
|
||||
}
|
||||
|
||||
async updateGroupLastSynced(groupName: string, lastUpdatedNumber: number) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groupIds = await this.getGroupsIds();
|
||||
const groupId = groupIds.get(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return await groupsCollection.update(groupId, { numLastSynced: lastUpdatedNumber });
|
||||
}
|
||||
|
||||
async getGroupLastSynced(groupName: string): Promise<number> {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groupIds = await this.getGroupsIds();
|
||||
const groupId = groupIds.get(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const group = await groupsCollection.getOne(groupId)
|
||||
|
||||
return group.numLastSynced;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
export function truncateText(originalText: string | undefined, isRoot: boolean) {
|
||||
|
||||
const allowedLength = isRoot ? 8000 : 4000;
|
||||
const level3Threshold = 1000; // if more than this, truncate L3 (>>>) quotes
|
||||
const level2Threshold = 1000; // if after truncation more than this, truncate L2 (>>) quotes
|
||||
const level1Threshold = 2500; // if after truncation more than this, truncate L1 (>) quotes
|
||||
|
||||
originalText = originalText ?? '';
|
||||
let txt = originalText ?? '';
|
||||
|
||||
let isTruncated = false;
|
||||
|
||||
if (txt.length > level3Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 3);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
if (txt.length > level2Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 2);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
if (txt.length > level1Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 1);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
const truncateMore = txt.length > allowedLength;
|
||||
const fullTextFile = isTruncated || truncateMore ? new File([originalText], 'article.txt', { type: 'text/plain' }) : null;
|
||||
const text = truncateMore ? txt.substring(0, allowedLength) : txt;
|
||||
|
||||
return {
|
||||
text,
|
||||
isTruncated,
|
||||
fullTextFile,
|
||||
}
|
||||
}
|
||||
|
||||
/* todo - check if needs a blank row in between
|
||||
test test test
|
||||
> ok this is one level
|
||||
>> another one
|
||||
>
|
||||
> back to one
|
||||
|
||||
done
|
||||
*/
|
||||
function truncateQuotes(text: string, isTruncated: boolean, level: number) {
|
||||
const lines = text.split('\n');
|
||||
const levelString = '>'.repeat(level);
|
||||
let currentlyInNestedQuote = false;
|
||||
const truncatedText = lines.map((line) => {
|
||||
return line.trim();
|
||||
}).filter((line) => {
|
||||
if (line.startsWith(levelString)) {
|
||||
isTruncated = true;
|
||||
if (currentlyInNestedQuote) {
|
||||
return false;
|
||||
}
|
||||
currentlyInNestedQuote = true;
|
||||
return true;
|
||||
}
|
||||
currentlyInNestedQuote = false;
|
||||
return true;
|
||||
}).map((line) => {
|
||||
if (line.startsWith(levelString)) {
|
||||
return `${line.substring(0,49)} ...[truncated]`;
|
||||
}
|
||||
return line;
|
||||
}).join('\n');
|
||||
|
||||
return {
|
||||
truncatedText,
|
||||
isTruncated,
|
||||
}
|
||||
}
|
||||
|
||||
// function fixLineQuotes(text: string): string {
|
||||
// // Only fix quote markers at the beginning of the string
|
||||
// const match = text.match(/^((?:>\s*)+)(.*)$/);
|
||||
//
|
||||
// if (!match) {
|
||||
// return text;
|
||||
// }
|
||||
//
|
||||
// const [, quotePart, contentPart] = match;
|
||||
//
|
||||
// // Count the number of > characters in the quote part
|
||||
// const quoteCount = (quotePart.match(/>/g) || []).length;
|
||||
//
|
||||
// // Create a normalized quote prefix
|
||||
// const normalizedPrefix = '>'.repeat(quoteCount);
|
||||
//
|
||||
// // Add a space if there's content after the quotes
|
||||
// return contentPart.length > 0 ?
|
||||
// `${normalizedPrefix} ${contentPart}` :
|
||||
// normalizedPrefix;
|
||||
// }
|
||||
|
||||
// function test() {
|
||||
// const tests = [
|
||||
// 'test this out',
|
||||
// '>> > test3 with some addi > tional',
|
||||
// '>> > test3 with some addi >>> tional',
|
||||
// '>> > test3 with some addi > >> tional',
|
||||
// '>> > test3 with some addi > > > tional',
|
||||
// '>> test',
|
||||
// '>>test',
|
||||
// '> test',
|
||||
// '>test',
|
||||
// '>> test',
|
||||
// '>>test',
|
||||
// '>> > test',
|
||||
// '>> >test',
|
||||
// '>> > > test',
|
||||
// '>> > >test',
|
||||
// '>> > > > test',
|
||||
// '>> > > >test',
|
||||
// '>> > > > > test',
|
||||
// '>> > > > >test',
|
||||
// '>> > > > > > test',
|
||||
// '>> > > > > >test',
|
||||
// `> >> what about
|
||||
// > >> multiline`
|
||||
// ]
|
||||
// tests.forEach((test) => {
|
||||
// console.log({
|
||||
// test,
|
||||
// resl: fixLineQuotes(test),
|
||||
// });
|
||||
// })
|
||||
// }
|
||||
// test();
|
||||
@@ -0,0 +1,42 @@
|
||||
export interface Attachment {
|
||||
fileName: string;
|
||||
file?: File;
|
||||
size: number;
|
||||
checksum: string;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
export interface PartialMessageMime {
|
||||
value: string;
|
||||
params: { number: number; total: number; id: string }
|
||||
}
|
||||
|
||||
export interface ArticleRecord {
|
||||
// from nntp
|
||||
articleId?: string;
|
||||
subject?: string;
|
||||
from?: string;
|
||||
text?: string;
|
||||
headers: Record<string, unknown>;
|
||||
date?: Date;
|
||||
isRoot: boolean;
|
||||
isTruncated: boolean;
|
||||
fullTextFile: File | null;
|
||||
fullHtmlFile?: File;
|
||||
// from nntp and remaped from repo
|
||||
groups: string[];
|
||||
files: (File | undefined)[];
|
||||
references: string[];
|
||||
parent: string | null;
|
||||
guessRootId?: string;
|
||||
lastReplyDate?: string;
|
||||
numReplies?: number;
|
||||
thread?: string;
|
||||
}
|
||||
|
||||
export interface GroupRecord {
|
||||
name: string,
|
||||
numLastSynced: number
|
||||
first: number,
|
||||
last: number,
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// import { readFileSync, writeFileSync } from 'fs';
|
||||
import { Attachment } from './types';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
function uudecode(uuencodedText: string) {
|
||||
// Regular expressions to identify UUencoded content
|
||||
const beginRegex = /^begin\s+(\d+)\s+(\S+)\s*$/;
|
||||
const endRegex = /^end\s*$/;
|
||||
const uuLineRegex = /^[M-` ]/;
|
||||
|
||||
// Split input into lines
|
||||
const lines = uuencodedText.split(/\r?\n/);
|
||||
|
||||
let collecting = false;
|
||||
let fileName = null;
|
||||
let fileMode = null;
|
||||
let encodedData = [];
|
||||
|
||||
// Extract the UUencoded portion
|
||||
for (const line of lines) {
|
||||
// Check for beginning of UUencoded data
|
||||
const beginMatch = line.match(beginRegex);
|
||||
if (beginMatch) {
|
||||
collecting = true;
|
||||
fileMode = beginMatch[1];
|
||||
fileName = beginMatch[2];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for end of UUencoded data
|
||||
if (endRegex.test(line)) {
|
||||
collecting = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Collect UUencoded lines
|
||||
if (collecting && uuLineRegex.test(line)) {
|
||||
encodedData.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Actual UUdecoding
|
||||
let decodedData = new Uint8Array(encodedData.reduce((acc, line) => {
|
||||
if (line.length <= 1) return acc; // Skip empty lines
|
||||
|
||||
// First character of each line indicates the length
|
||||
const length = (line.charCodeAt(0) - 32) & 0x3F;
|
||||
if (length === 0) return acc; // Skip zero-length lines
|
||||
|
||||
// Process the line in groups of 4 characters
|
||||
const bytes = [];
|
||||
for (let i = 1; i < line.length; i += 4) {
|
||||
if (i + 3 >= line.length) break;
|
||||
|
||||
// Decode each group of 4 characters into 3 bytes
|
||||
const chars = line.slice(i, i + 4).split('').map(c => (c.charCodeAt(0) - 32) & 0x3F);
|
||||
|
||||
if (bytes.length < length) bytes.push((chars[0] << 2) | (chars[1] >> 4));
|
||||
if (bytes.length < length) bytes.push(((chars[1] & 0x0F) << 4) | (chars[2] >> 2));
|
||||
if (bytes.length < length) bytes.push(((chars[2] & 0x03) << 6) | chars[3]);
|
||||
}
|
||||
|
||||
return acc.concat(bytes);
|
||||
}, [] as number[]));
|
||||
|
||||
return {
|
||||
fileName,
|
||||
fileMode,
|
||||
data: decodedData
|
||||
};
|
||||
}
|
||||
|
||||
export async function extractAllUUEncodedAttachments(messageText: string) {
|
||||
const beginRegex = /^begin\s+(\d+)\s+(\S+)\s*$/mg;
|
||||
const results = [];
|
||||
let match;
|
||||
const positions: [number, number][] = [];
|
||||
|
||||
// Find all "begin" markers
|
||||
while ((match = beginRegex.exec(messageText)) !== null) {
|
||||
|
||||
const startPos = match.index;
|
||||
const endPos = messageText.indexOf("\nend", startPos);
|
||||
|
||||
if (endPos !== -1) {
|
||||
const encodedSection = messageText.substring(startPos, endPos + 4);
|
||||
const decoded = uudecode(encodedSection);
|
||||
if (decoded.data.length > 0) {
|
||||
results.push(decoded);
|
||||
}
|
||||
|
||||
positions.push([startPos, endPos + 4]);
|
||||
}
|
||||
}
|
||||
|
||||
// remove the decoded data from the text message
|
||||
let text = '';
|
||||
if (positions.length === 0) {
|
||||
text = messageText;
|
||||
}
|
||||
|
||||
if (positions.length > 1) {
|
||||
// remove one by one in reverse order
|
||||
text = messageText;
|
||||
for (let i = positions.length - 1; i >= 0; i--) {
|
||||
const [startPos, endPos] = positions[i];
|
||||
text = text.substring(0, startPos) + text.substring(endPos);
|
||||
}
|
||||
}
|
||||
|
||||
// for (let i = 0; i < positions.length; i++) {
|
||||
// const [startPos, endPos] = positions[i];
|
||||
// text += messageText.substring(0, startPos) + messageText.substring(endPos);
|
||||
// }
|
||||
|
||||
const attachmentsPromises: Promise<Attachment>[] = results.map(async (result) => {
|
||||
return {
|
||||
file: new File([result.data], result.fileName ?? ''),
|
||||
fileName: result.fileName ?? '',
|
||||
size: result.data.length,
|
||||
// md5 of result.data
|
||||
checksum: crypto.createHash('md5').update(result.data).digest('hex'),
|
||||
contentType: 'application/octet-stream', // todo - extract from
|
||||
}
|
||||
})
|
||||
|
||||
const attachments = await Promise.all(attachmentsPromises);
|
||||
|
||||
|
||||
return {
|
||||
attachments,
|
||||
positions,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
// async function test() {
|
||||
// const articleString = readFileSync('../err/article_2.json').toString();
|
||||
// const article = JSON.parse(articleString);
|
||||
//
|
||||
// const result = await extractAllUUEncodedAttachments(article.text);
|
||||
// article.text = result.text;
|
||||
// writeFileSync('../err/article_3.json', JSON.stringify(article, null, 2));
|
||||
// for (const attachment of result.attachments) {
|
||||
// if (attachment.file) {
|
||||
// writeFileSync(`../err/${attachment.fileName}`, Buffer.from((await attachment.file.arrayBuffer())));
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// console.log(result);
|
||||
// }
|
||||
//
|
||||
// test();
|
||||
Reference in New Issue
Block a user