mirror of
https://github.com/sakaljurgis/nntp-worker.git
synced 2026-07-08 21:27:41 +00:00
initial
This commit is contained in:
Jurgis Sakalauskas
@@ -0,0 +1,108 @@
|
||||
import { Repository } from './repository';
|
||||
import { NntpClient } from './nntp-client';
|
||||
import { checkPartialContentType, mapParsedMailToArticle } from './mapper';
|
||||
import { ArticleRecord, Attachment } from './types';
|
||||
import { parseAttachments } from './attachments';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { checkAndCombinePartial } from './partial';
|
||||
import { truncateText } from './truncate';
|
||||
import { extractAllUUEncodedAttachments } from './uudecode';
|
||||
|
||||
export async function downloadAndProcessArticle(repository: Repository, groupName: string, articleNumber: number, nntpClient: NntpClient) {
|
||||
const groupId = await repository.getGroupId(groupName);
|
||||
if (!groupId) {
|
||||
console.log(`Group ${groupName} not found in local database`);
|
||||
return;
|
||||
}
|
||||
|
||||
const groupArticleNumber = await repository.findGroupArticleNumber(groupName, articleNumber);
|
||||
|
||||
if (groupArticleNumber) {
|
||||
console.log(`Article ${articleNumber} already exists in local database`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (nntpClient.getSelectedGroup() !== groupName) {
|
||||
await nntpClient.selectGroup(groupName);
|
||||
}
|
||||
|
||||
|
||||
const articleMail = await nntpClient.getArticle(articleNumber).catch(e => {
|
||||
console.log(e);
|
||||
return null;
|
||||
});
|
||||
|
||||
if (!articleMail) {
|
||||
console.log('No article found');
|
||||
return;
|
||||
}
|
||||
|
||||
const partial = checkPartialContentType(articleMail.headers);
|
||||
// restart from here after combining partials
|
||||
let attachments: Attachment[] = await parseAttachments(articleMail, partial);
|
||||
let article = mapParsedMailToArticle(articleMail, attachments);
|
||||
|
||||
|
||||
|
||||
if (partial) {
|
||||
const filePath = `./partials/${partial.id}-part_${partial.number}-article.json`
|
||||
writeFileSync(filePath, JSON.stringify(article, null, 2));
|
||||
console.log(`Partial article saved to ${filePath}`);
|
||||
}
|
||||
|
||||
if (partial) {
|
||||
const combined = await checkAndCombinePartial(partial);
|
||||
if (!combined) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO - this is duplicate code from let attachments = saveAttachments
|
||||
attachments = await parseAttachments(combined, null);
|
||||
article = mapParsedMailToArticle(combined, attachments);
|
||||
}
|
||||
|
||||
if (!article.text) {
|
||||
article.text = '';
|
||||
}
|
||||
|
||||
// check and extract uuencoded attachments
|
||||
const { text: newText, attachments: moreAttachments } = await extractAllUUEncodedAttachments(article.text);
|
||||
|
||||
attachments = attachments.concat(moreAttachments);
|
||||
article.text = newText;
|
||||
|
||||
article.text = article.text.trim();
|
||||
|
||||
const groupsRecordsIds = await repository.findGroupsIdsByNames(article.groups);
|
||||
const parentRecordId = article.parent ? await repository.findArticleIdByArticleId(article.parent) : undefined;
|
||||
const referencesIds = await repository.findArticleIdsByArticleIds(article.references);
|
||||
|
||||
const isRoot = article.references.length === 0;
|
||||
const { text, isTruncated, fullTextFile } = truncateText(article.text, isRoot)
|
||||
|
||||
const data: ArticleRecord = {
|
||||
...article,
|
||||
isRoot,
|
||||
groups: groupsRecordsIds,
|
||||
text,
|
||||
isTruncated,
|
||||
fullTextFile,
|
||||
references: referencesIds,
|
||||
parent: parentRecordId ?? null,
|
||||
guessRootId: article.references[0] ?? null,
|
||||
files: attachments.map((a)=>a.file),
|
||||
}
|
||||
|
||||
try {
|
||||
const createRecord = await repository.createArticle(data);
|
||||
|
||||
console.log('Record created:', createRecord.id);
|
||||
|
||||
await repository.assignArticleToGroupsMessages(createRecord.id, article.inGroups);
|
||||
} catch (e) {
|
||||
console.log(data, article);
|
||||
throw e;
|
||||
}
|
||||
|
||||
console.log('Done');
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Repository } from './repository';
|
||||
|
||||
export async function assignThread(repository: Repository) {
|
||||
// root
|
||||
let rootWithoutThread = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&isRoot=true',
|
||||
})
|
||||
console.log(`Updating root thread for ${rootWithoutThread.totalItems} articles. Total pages: ${rootWithoutThread.totalPages}`);
|
||||
|
||||
while (rootWithoutThread.items.length > 0) {
|
||||
for (const item of rootWithoutThread.items) {
|
||||
await repository.updateArticle(item.id, { thread: item.id });
|
||||
}
|
||||
|
||||
rootWithoutThread = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&isRoot=true',
|
||||
})
|
||||
console.log(`Updating root thread for ${rootWithoutThread.totalItems} articles. Total pages: ${rootWithoutThread.totalPages}`);
|
||||
}
|
||||
|
||||
// replies
|
||||
let replyWithoutThreadWithRoot = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&references.isRoot?=true',
|
||||
expand: 'references',
|
||||
})
|
||||
console.log(`Updating reply thread for ${replyWithoutThreadWithRoot.totalItems} articles. Total pages: ${replyWithoutThreadWithRoot.totalPages}`);
|
||||
|
||||
while (replyWithoutThreadWithRoot.items.length > 0) {
|
||||
for (const item of replyWithoutThreadWithRoot.items) {
|
||||
let rootId: string | null = null;
|
||||
// @ts-ignore
|
||||
for (const ref of item.expand.references) {
|
||||
if (ref.isRoot) {
|
||||
rootId = ref.id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (rootId) {
|
||||
await repository.updateArticle(item.id, { thread: rootId });
|
||||
}
|
||||
}
|
||||
|
||||
replyWithoutThreadWithRoot = await repository.getArticlesCollection().getList(1, 100, {
|
||||
filter: 'thread=null&&references.isRoot?=true',
|
||||
expand: 'references',
|
||||
})
|
||||
console.log(`Updating reply thread for ${replyWithoutThreadWithRoot.totalItems} articles. Total pages: ${replyWithoutThreadWithRoot.totalPages}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Attachment, PartialMessageMime } from './types';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { type ParsedMail } from 'mailparser';
|
||||
|
||||
export const parseAttachments = async (article: ParsedMail, partial: PartialMessageMime['params'] | null) => {
|
||||
let attachments: Attachment[] = [];
|
||||
if (!article.attachments) {
|
||||
return attachments;
|
||||
}
|
||||
|
||||
article.attachments.map((attachment) => {
|
||||
if (partial) {
|
||||
const fileName = `${partial.id}-part_${partial.number}-attachment`;
|
||||
const filePath = `./partials/${fileName}`
|
||||
writeFileSync(filePath, attachment.content);
|
||||
}
|
||||
|
||||
attachments.push({
|
||||
file: partial ? undefined : new File([attachment.content], attachment.filename ?? crypto.randomUUID(), { type: attachment.contentType }),
|
||||
fileName: attachment.filename ?? '',
|
||||
size: attachment.size,
|
||||
checksum: attachment.checksum,
|
||||
contentType: attachment.contentType,
|
||||
})
|
||||
});
|
||||
|
||||
return attachments;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Repository } from './repository';
|
||||
import { NntpClient } from './nntp-client';
|
||||
import { downloadAndProcessArticle } from './article';
|
||||
|
||||
export const updateGroupsList = async (repository: Repository, nntpClient: NntpClient) => {
|
||||
const groupsById = await repository.getGroupsIds();
|
||||
const groups = await nntpClient.listGroups();
|
||||
const groupsRecords = await repository.getAllGroups();
|
||||
const filteredGroups: typeof groups = [];
|
||||
|
||||
const createGroupPromises = groups.map(async (group) => {
|
||||
if (!group.group.startsWith('omnitel.')) {
|
||||
return;
|
||||
}
|
||||
if (group.group === 'omnitel.binaries') {
|
||||
return;
|
||||
}
|
||||
filteredGroups.push(group);
|
||||
|
||||
if (!groupsById.has(group.group)) {
|
||||
return await repository.createGroup(group.group);
|
||||
}
|
||||
|
||||
return await repository.updateGroupData({
|
||||
name: group.group,
|
||||
last: group.last,
|
||||
first: group.first,
|
||||
})
|
||||
})
|
||||
|
||||
await Promise.all(createGroupPromises);
|
||||
|
||||
return filteredGroups;
|
||||
}
|
||||
|
||||
export const downloadAndProcessGroup = async (
|
||||
groupsList: Awaited<ReturnType<typeof updateGroupsList>>,
|
||||
groupName: string,
|
||||
repository: Repository,
|
||||
nntpClient: NntpClient
|
||||
) => {
|
||||
const group = groupsList.find((g) => g.group === groupName);
|
||||
|
||||
if (!group) {
|
||||
console.log(`Group ${groupName} not found`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
const lastSynced = await repository.getGroupLastSynced(groupName);
|
||||
|
||||
if (lastSynced >= group.last) {
|
||||
console.log(`Group ${groupName} already synced`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const articleNumberStart = lastSynced + 1;
|
||||
const articleNumberEnd = group.last;
|
||||
let numDownloaded = 0;
|
||||
|
||||
for (let articleNumber = articleNumberStart; articleNumber <= articleNumberEnd; articleNumber++) {
|
||||
await downloadAndProcessArticle(repository, groupName, articleNumber, nntpClient);
|
||||
await repository.updateGroupLastSynced(groupName, articleNumber);
|
||||
numDownloaded++;
|
||||
}
|
||||
|
||||
return numDownloaded;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Repository } from './repository';
|
||||
|
||||
export async function updateLastReplyDate(repository: Repository) {
|
||||
let mismatched = await repository.getLastReplyMismatchedArticles(100);
|
||||
console.log(`Updating last reply date for ${mismatched.totalItems} articles. Total pages: ${mismatched.totalPages}`);
|
||||
while (mismatched.items.length > 0) {
|
||||
for (const item of mismatched.items) {
|
||||
await repository.updateArticle(item.id, {
|
||||
lastReplyDate: item.calculatedLastReplyDate,
|
||||
numReplies: item.calculatedNumReplies,
|
||||
});
|
||||
}
|
||||
|
||||
mismatched = await repository.getLastReplyMismatchedArticles(100);
|
||||
console.log(`Updating last reply date for ${mismatched.totalItems} articles. Total pages: ${mismatched.totalPages}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Headers, ParsedMail, StructuredHeader } from 'mailparser'
|
||||
import { Attachment, PartialMessageMime } from './types';
|
||||
|
||||
export const checkPartialContentType = (headers: Headers): PartialMessageMime['params'] | null => {
|
||||
const contentType = headers.get('content-type') as StructuredHeader | undefined;
|
||||
if (!contentType || !contentType.value || !contentType.value.includes('message/partial')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const params = contentType.params;
|
||||
|
||||
if (!params || !params.number || !params.total || !params.id) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
number: parseInt(params.number),
|
||||
total: parseInt(params.total),
|
||||
id: params.id,
|
||||
};
|
||||
}
|
||||
|
||||
export const mapParsedMailToArticle = (article: ParsedMail, attachments: Attachment[]) => {
|
||||
let references: string[] | string = article.headers.get('references') as string[] | string || [];
|
||||
if (typeof references === 'string') {
|
||||
references = [references];
|
||||
}
|
||||
|
||||
const groups = article.headers.get('newsgroups') as string;
|
||||
const xref = article.headers.get('xref') as string;
|
||||
const parent = article.headers.get('in-reply-to') as string ?? references[references.length - 1];
|
||||
|
||||
return {
|
||||
groups: groups.split(','),
|
||||
subject: article.subject,
|
||||
from: article.from?.text,
|
||||
date: article.date,
|
||||
articleId: article.messageId,
|
||||
inGroups: xref.split(' ').map((group: string) => {
|
||||
return {
|
||||
groupName: group.split(':')[0],
|
||||
messageNumber: group.split(':')[1],
|
||||
}
|
||||
}),
|
||||
references,
|
||||
parent,
|
||||
text: article.text,
|
||||
attachments: attachments,
|
||||
headers: Object.fromEntries(article.headers),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
//http://www.tcpipguide.com/free/t_NNTPCommands-2.htm
|
||||
import { Socket } from 'net';
|
||||
import { EventEmitter } from 'events';
|
||||
import { simpleParser, MailParser } from 'mailparser'
|
||||
import { writeFileSync } from 'fs';
|
||||
|
||||
export interface NntpClientConfig {
|
||||
server: string;
|
||||
port?: number;
|
||||
}
|
||||
|
||||
const commandsDef = {
|
||||
connect: {
|
||||
command: '',
|
||||
responseCode: '200',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
listGroups: {
|
||||
command: 'LIST',
|
||||
responseCode: '215',
|
||||
completeIndicator: '.\r\n',
|
||||
},
|
||||
selectGroup: {
|
||||
command: 'GROUP',
|
||||
responseCode: '211',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
disconnect: {
|
||||
command: 'QUIT',
|
||||
responseCode: '205',
|
||||
completeIndicator: '\r\n',
|
||||
},
|
||||
getArticle: {
|
||||
command: 'ARTICLE',
|
||||
responseCode: '220',
|
||||
completeIndicator: '\r\n.\r\n',
|
||||
},
|
||||
}
|
||||
|
||||
type Command = keyof typeof commandsDef;
|
||||
|
||||
interface SendCommandResult {
|
||||
command: Command;
|
||||
status: 'success' | 'error';
|
||||
statusLine: string;
|
||||
responseString: string;
|
||||
responseBuffer: Buffer;
|
||||
}
|
||||
|
||||
export class NntpClient {
|
||||
private config: NntpClientConfig;
|
||||
private client: Socket;
|
||||
private currentCommand: Command | null = null;
|
||||
private currentResponseString = '';
|
||||
private currentResponseBuffer: Buffer;
|
||||
private eventEmitter: EventEmitter;
|
||||
private selectedGroup: string | null = null;
|
||||
|
||||
constructor(config: NntpClientConfig) {
|
||||
this.config = config;
|
||||
this.client = new Socket();
|
||||
this.eventEmitter = new EventEmitter();
|
||||
|
||||
this.currentResponseBuffer = Buffer.alloc(0);
|
||||
this.setupClient();
|
||||
}
|
||||
|
||||
private setupClient() {
|
||||
this.client.on('data', (data)=> {
|
||||
this.currentResponseBuffer = Buffer.concat([this.currentResponseBuffer, data]);
|
||||
this.currentResponseString += data.toString();
|
||||
if (!this.currentCommand) {
|
||||
console.error(`Received data without a command: ${this.currentResponseString}`);
|
||||
throw new Error('No command is currently being executed');
|
||||
}
|
||||
const currentCommandDef = commandsDef[this.currentCommand];
|
||||
|
||||
// todo - timeout
|
||||
if (this.currentResponseString.endsWith(currentCommandDef.completeIndicator) || !this.currentResponseString.startsWith(currentCommandDef.responseCode)) {
|
||||
const statusLine = this.currentResponseString.split('\r\n')[0];
|
||||
console.log(`Command ${this.currentCommand} completed with status:`, statusLine);
|
||||
this.eventEmitter.emit(`${this.currentCommand}`, {
|
||||
command: this.currentCommand,
|
||||
status: this.currentResponseString.startsWith(currentCommandDef.responseCode) ? 'success' : 'error',
|
||||
statusLine,
|
||||
responseString: this.currentResponseString,
|
||||
responseBuffer: Buffer.from(this.currentResponseBuffer),
|
||||
});
|
||||
this.currentResponseBuffer = Buffer.alloc(0);
|
||||
this.currentResponseString = '';
|
||||
this.currentCommand = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async connect() {
|
||||
// todo - handle disconnect (by host)
|
||||
// todo - add connected status
|
||||
this.currentCommand = 'connect';
|
||||
return new Promise((resolve, reject) => {
|
||||
this.client.connect(this.config.port || 119, this.config.server);
|
||||
this.client.on('error', reject);
|
||||
|
||||
this.eventEmitter.once(`${this.currentCommand}`, (data) => {
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
await this.sendCommand('disconnect');
|
||||
}
|
||||
|
||||
private async sendCommand(command: Command, params?: string): Promise<SendCommandResult> {
|
||||
console.log(`Sending command ${command} with params ${params}`);
|
||||
if (this.currentCommand) {
|
||||
// todo - queue
|
||||
return Promise.reject({
|
||||
status: 'error',
|
||||
message: 'Another command is currently being executed',
|
||||
});
|
||||
}
|
||||
this.currentCommand = command;
|
||||
const commandDef = commandsDef[command];
|
||||
this.client.write(`${commandDef.command + (params ? ' ' + params : '')}\r\n`);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.eventEmitter.once(`${this.currentCommand}`, (data) => {
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async listGroups() {
|
||||
const result = await this.sendCommand('listGroups');
|
||||
|
||||
return result.responseString.split('\r\n').slice(1, -2).map((line) => {
|
||||
const [group, last, first, flag] = line.split(' ');
|
||||
return {
|
||||
group,
|
||||
last: parseInt(last),
|
||||
first: parseInt(first),
|
||||
flag,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
getSelectedGroup() {
|
||||
return this.selectedGroup;
|
||||
}
|
||||
|
||||
async selectGroup(group: string) {
|
||||
const result = await this.sendCommand('selectGroup', group);
|
||||
|
||||
if (result.status === 'error') {
|
||||
return {
|
||||
status: result.status,
|
||||
message: result.statusLine,
|
||||
}
|
||||
}
|
||||
|
||||
const [statusCode, totalEstimate, first, last, groupName] = result.statusLine.split(' ');
|
||||
this.selectedGroup = groupName;
|
||||
|
||||
return {
|
||||
statusCode,
|
||||
status: result.status,
|
||||
totalEstimate: parseInt(totalEstimate),
|
||||
first: parseInt(first),
|
||||
last: parseInt(last),
|
||||
groupName,
|
||||
}
|
||||
}
|
||||
|
||||
async getArticle(articleId: number) {
|
||||
if (!this.selectedGroup) {
|
||||
throw new Error('No group selected. Please select a group before fetching articles.');
|
||||
}
|
||||
const result = await this.sendCommand('getArticle', articleId.toString());
|
||||
|
||||
if (result.status === 'error') {
|
||||
throw new Error(`Error fetching article: ${result.statusLine}`);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// remove 1st and last line from buffer, do not convert to string (to retain encoding)
|
||||
const buffer = result.responseBuffer.slice(result.responseBuffer.indexOf('\r\n') + 2, result.responseBuffer.lastIndexOf('\r\n.\r\n') + 2);
|
||||
|
||||
|
||||
const parsed = await simpleParser(buffer, {
|
||||
skipTextToHtml: true,
|
||||
});
|
||||
|
||||
// console.log({
|
||||
// final: parsed.text,
|
||||
// })
|
||||
|
||||
// if (articleId == 395) {
|
||||
// writeFileSync('./err/article.mail', result.responseBuffer);
|
||||
// console.log(parsed.text?.length);
|
||||
// process.exit(1);
|
||||
// }
|
||||
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { PartialMessageMime } from './types';
|
||||
import { stat, readFile, unlink } from 'fs/promises';
|
||||
import { simpleParser } from 'mailparser';
|
||||
|
||||
export const checkAndCombinePartial = async (partial: PartialMessageMime['params']) => {
|
||||
const allStats = await Promise.allSettled(
|
||||
[...
|
||||
Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, i) => stat(`./partials/${partial.id}-part_${i + 1}-attachment`)),
|
||||
...Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, i) => stat(`./partials/${partial.id}-part_${i + 1}-article.json`)),
|
||||
]
|
||||
);
|
||||
const allExists = allStats.every((result) => result.status === 'fulfilled' && result.value.isFile());
|
||||
|
||||
if (!allExists) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//combine all partials
|
||||
const filesReadPromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map((_, index) => {
|
||||
console.log(`Reading file ./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
return readFile(`./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
})
|
||||
|
||||
const files = await Promise.all(filesReadPromises);
|
||||
|
||||
const file = Buffer.concat(files);
|
||||
|
||||
const parsed = await simpleParser(file);
|
||||
const xrefs = new Set<string>();
|
||||
let articleId = '';
|
||||
const articlesReadPromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map(async (_, index) => {
|
||||
console.log(`Reading file ./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
const articleJson = await readFile(`./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
const article = JSON.parse(articleJson.toString());
|
||||
if (index === 0) {
|
||||
articleId = article.articleId;
|
||||
}
|
||||
if (article && article.headers && article.headers.xref) {
|
||||
return article.headers.xref as string;
|
||||
}
|
||||
})
|
||||
|
||||
const articlesXrefs = await Promise.all(articlesReadPromises);
|
||||
|
||||
articlesXrefs.forEach((xref) => {
|
||||
if (xref) {
|
||||
xref.split(' ').forEach(xre => xrefs.add(xre))
|
||||
}
|
||||
})
|
||||
|
||||
parsed.headers.set('xref', Array.from(xrefs).join(' '));
|
||||
|
||||
if (!articleId) {
|
||||
throw new Error(`articleId not found for ${partial.id}`);
|
||||
}
|
||||
|
||||
const deletePromises = Array(partial.total)
|
||||
.fill(null)
|
||||
.map(async (_, index) => {
|
||||
console.log(`Deleting file ./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
await unlink(`./partials/${partial.id}-part_${index + 1}-article.json`)
|
||||
console.log(`Deleting file ./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
await unlink(`./partials/${partial.id}-part_${index + 1}-attachment`)
|
||||
})
|
||||
|
||||
await Promise.all(deletePromises);
|
||||
parsed.headers.set('messageId', articleId);
|
||||
parsed.messageId = articleId;
|
||||
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import Pocketbase, { RecordModel } from 'pocketbase';
|
||||
import { ArticleRecord, Attachment, GroupRecord } from './types';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { writeFileSync } from 'fs';
|
||||
|
||||
export class Repository {
|
||||
private readonly client: Pocketbase;
|
||||
private groupToId: Map<string, string> | null = null;
|
||||
constructor(
|
||||
url: string,
|
||||
private readonly username: string,
|
||||
private readonly password: string
|
||||
) {
|
||||
this.client = new Pocketbase(url);
|
||||
}
|
||||
|
||||
async connect() {
|
||||
return await this.client.collection('_superusers').authWithPassword(
|
||||
this.username,
|
||||
this.password,
|
||||
{ autoRefreshThreshold: 30 * 60 }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* key groupName
|
||||
* value groupId
|
||||
* todo - this can be cached
|
||||
*/
|
||||
async getGroupsIds() {
|
||||
if (!this.groupToId) {
|
||||
const groupToId = new Map<string, string>();
|
||||
(await this.client.collection('groups').getFullList()).forEach((group) => {
|
||||
groupToId.set(group.name, group.id);
|
||||
});
|
||||
|
||||
this.groupToId = groupToId;
|
||||
}
|
||||
|
||||
return this.groupToId;
|
||||
}
|
||||
|
||||
async getAllGroups() {
|
||||
return await this.client.collection<GroupRecord & RecordModel>('groups').getFullList()
|
||||
}
|
||||
|
||||
async updateGroupData(groupData: Partial<GroupRecord>) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
if (!groupData.name) {
|
||||
return;
|
||||
}
|
||||
const groupId = await this.getGroupId(groupData.name);
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return await groupsCollection.update(groupId, groupData);
|
||||
}
|
||||
|
||||
async findGroupArticleNumber(groupName: string, articleNumber: number) {
|
||||
const groupId = await this.getGroupId(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return await this.client
|
||||
.collection('groupArticleNumber')
|
||||
.getFirstListItem(`group="${groupId}" && number="${articleNumber}"`)
|
||||
.catch(() => null);
|
||||
}
|
||||
|
||||
async getGroupId(groupName: string) {
|
||||
const groupsIds = await this.getGroupsIds();
|
||||
|
||||
return groupsIds.get(groupName);
|
||||
}
|
||||
|
||||
async findGroupsIdsByNames(groupNames: string[]) {
|
||||
if (groupNames.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groups = await groupsCollection.getList(1, 100, {
|
||||
filter: groupNames.map((name: string) => `name="${name}"`).join('||'),
|
||||
});
|
||||
|
||||
return groups.items.map((group) => group.id);
|
||||
}
|
||||
|
||||
async findArticleIdByArticleId(articleId: string) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
const articleRecord = await articlesCollection.getFirstListItem(`articleId="${articleId}"`).catch(() => undefined);
|
||||
|
||||
return articleRecord?.id;
|
||||
}
|
||||
|
||||
async findArticleIdsByArticleIds(articleIds: string[]) {
|
||||
if (articleIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
const articles = await articlesCollection.getList(1, 100, {
|
||||
filter: articleIds.map((name: string) => `articleId="${name}"`).join('||'),
|
||||
}).catch(() => ({ items: [] }));
|
||||
|
||||
return articles.items.map((art) => art.id);
|
||||
}
|
||||
|
||||
async assignArticleToGroupsMessages(id: string, groupsMessageNumbers: { groupName: string; messageNumber: string }[]) {
|
||||
const availableGroups = await this.getGroupsIds();
|
||||
|
||||
const promises = groupsMessageNumbers.map(async (inGroup) => {
|
||||
const groupId = availableGroups.get(inGroup.groupName);
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
console.log(`Adding article ${id} to group ${inGroup.groupName} with number ${inGroup.messageNumber}`);
|
||||
return await this.client
|
||||
.collection('groupArticleNumber')
|
||||
.create({
|
||||
group: groupId,
|
||||
article: id,
|
||||
number: inGroup.messageNumber
|
||||
}, {requestKey: `${groupId}-${id}-${inGroup.messageNumber}`});
|
||||
});
|
||||
|
||||
return await Promise.all(promises);
|
||||
}
|
||||
|
||||
async createArticle(articleRecord: ArticleRecord) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
return await articlesCollection.create<ArticleRecord & RecordModel>(articleRecord);
|
||||
}
|
||||
|
||||
async updateArticle(articleId: string, articleRecord: Partial<ArticleRecord>) {
|
||||
const articlesCollection = this.client.collection('articles');
|
||||
|
||||
return await articlesCollection.update(articleId, articleRecord);
|
||||
}
|
||||
|
||||
getArticlesCollection() {
|
||||
return this.client.collection('articles');
|
||||
}
|
||||
|
||||
async getLastReplyMismatchedArticles(numItems = 100) {
|
||||
const lastReplyCollection = this.client.collection('lastReply');
|
||||
|
||||
return await lastReplyCollection.getList<{id: string, calculatedLastReplyDate: string, calculatedNumReplies: number}>(1,numItems, {
|
||||
filter: 'lastReplyDate!=calculatedLastReplyDate||numReplies!=calculatedNumReplies',
|
||||
});
|
||||
}
|
||||
|
||||
async createGroup(groupName: string) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
this.groupToId = null;
|
||||
|
||||
return await groupsCollection.create({ name: groupName, numLastSynced: 0 }, { requestKey: groupName });
|
||||
}
|
||||
|
||||
async updateGroupLastSynced(groupName: string, lastUpdatedNumber: number) {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groupIds = await this.getGroupsIds();
|
||||
const groupId = groupIds.get(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return await groupsCollection.update(groupId, { numLastSynced: lastUpdatedNumber });
|
||||
}
|
||||
|
||||
async getGroupLastSynced(groupName: string): Promise<number> {
|
||||
const groupsCollection = this.client.collection('groups');
|
||||
const groupIds = await this.getGroupsIds();
|
||||
const groupId = groupIds.get(groupName);
|
||||
|
||||
if (!groupId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const group = await groupsCollection.getOne(groupId)
|
||||
|
||||
return group.numLastSynced;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
export function truncateText(originalText: string | undefined, isRoot: boolean) {
|
||||
|
||||
const allowedLength = isRoot ? 8000 : 4000;
|
||||
const level3Threshold = 1000; // if more than this, truncate L3 (>>>) quotes
|
||||
const level2Threshold = 1000; // if after truncation more than this, truncate L2 (>>) quotes
|
||||
const level1Threshold = 2500; // if after truncation more than this, truncate L1 (>) quotes
|
||||
|
||||
originalText = originalText ?? '';
|
||||
let txt = originalText ?? '';
|
||||
|
||||
let isTruncated = false;
|
||||
|
||||
if (txt.length > level3Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 3);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
if (txt.length > level2Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 2);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
if (txt.length > level1Threshold && !isRoot) {
|
||||
const truncateResult = truncateQuotes(txt, isTruncated, 1);
|
||||
isTruncated = truncateResult.isTruncated;
|
||||
txt = truncateResult.truncatedText;
|
||||
}
|
||||
|
||||
const truncateMore = txt.length > allowedLength;
|
||||
const fullTextFile = isTruncated || truncateMore ? new File([originalText], 'article.txt', { type: 'text/plain' }) : null;
|
||||
const text = truncateMore ? txt.substring(0, allowedLength) : txt;
|
||||
|
||||
return {
|
||||
text,
|
||||
isTruncated,
|
||||
fullTextFile,
|
||||
}
|
||||
}
|
||||
|
||||
/* todo - check if needs a blank row in between
|
||||
test test test
|
||||
> ok this is one level
|
||||
>> another one
|
||||
>
|
||||
> back to one
|
||||
|
||||
done
|
||||
*/
|
||||
function truncateQuotes(text: string, isTruncated: boolean, level: number) {
|
||||
const lines = text.split('\n');
|
||||
const levelString = '>'.repeat(level);
|
||||
let currentlyInNestedQuote = false;
|
||||
const truncatedText = lines.map((line) => {
|
||||
return line.trim();
|
||||
}).filter((line) => {
|
||||
if (line.startsWith(levelString)) {
|
||||
isTruncated = true;
|
||||
if (currentlyInNestedQuote) {
|
||||
return false;
|
||||
}
|
||||
currentlyInNestedQuote = true;
|
||||
return true;
|
||||
}
|
||||
currentlyInNestedQuote = false;
|
||||
return true;
|
||||
}).map((line) => {
|
||||
if (line.startsWith(levelString)) {
|
||||
return `${line.substring(0,49)} ...[truncated]`;
|
||||
}
|
||||
return line;
|
||||
}).join('\n');
|
||||
|
||||
return {
|
||||
truncatedText,
|
||||
isTruncated,
|
||||
}
|
||||
}
|
||||
|
||||
// function fixLineQuotes(text: string): string {
|
||||
// // Only fix quote markers at the beginning of the string
|
||||
// const match = text.match(/^((?:>\s*)+)(.*)$/);
|
||||
//
|
||||
// if (!match) {
|
||||
// return text;
|
||||
// }
|
||||
//
|
||||
// const [, quotePart, contentPart] = match;
|
||||
//
|
||||
// // Count the number of > characters in the quote part
|
||||
// const quoteCount = (quotePart.match(/>/g) || []).length;
|
||||
//
|
||||
// // Create a normalized quote prefix
|
||||
// const normalizedPrefix = '>'.repeat(quoteCount);
|
||||
//
|
||||
// // Add a space if there's content after the quotes
|
||||
// return contentPart.length > 0 ?
|
||||
// `${normalizedPrefix} ${contentPart}` :
|
||||
// normalizedPrefix;
|
||||
// }
|
||||
|
||||
// function test() {
|
||||
// const tests = [
|
||||
// 'test this out',
|
||||
// '>> > test3 with some addi > tional',
|
||||
// '>> > test3 with some addi >>> tional',
|
||||
// '>> > test3 with some addi > >> tional',
|
||||
// '>> > test3 with some addi > > > tional',
|
||||
// '>> test',
|
||||
// '>>test',
|
||||
// '> test',
|
||||
// '>test',
|
||||
// '>> test',
|
||||
// '>>test',
|
||||
// '>> > test',
|
||||
// '>> >test',
|
||||
// '>> > > test',
|
||||
// '>> > >test',
|
||||
// '>> > > > test',
|
||||
// '>> > > >test',
|
||||
// '>> > > > > test',
|
||||
// '>> > > > >test',
|
||||
// '>> > > > > > test',
|
||||
// '>> > > > > >test',
|
||||
// `> >> what about
|
||||
// > >> multiline`
|
||||
// ]
|
||||
// tests.forEach((test) => {
|
||||
// console.log({
|
||||
// test,
|
||||
// resl: fixLineQuotes(test),
|
||||
// });
|
||||
// })
|
||||
// }
|
||||
// test();
|
||||
@@ -0,0 +1,42 @@
|
||||
export interface Attachment {
|
||||
fileName: string;
|
||||
file?: File;
|
||||
size: number;
|
||||
checksum: string;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
export interface PartialMessageMime {
|
||||
value: string;
|
||||
params: { number: number; total: number; id: string }
|
||||
}
|
||||
|
||||
export interface ArticleRecord {
|
||||
// from nntp
|
||||
articleId?: string;
|
||||
subject?: string;
|
||||
from?: string;
|
||||
text?: string;
|
||||
headers: Record<string, unknown>;
|
||||
date?: Date;
|
||||
isRoot: boolean;
|
||||
isTruncated: boolean;
|
||||
fullTextFile: File | null;
|
||||
fullHtmlFile?: File;
|
||||
// from nntp and remaped from repo
|
||||
groups: string[];
|
||||
files: (File | undefined)[];
|
||||
references: string[];
|
||||
parent: string | null;
|
||||
guessRootId?: string;
|
||||
lastReplyDate?: string;
|
||||
numReplies?: number;
|
||||
thread?: string;
|
||||
}
|
||||
|
||||
export interface GroupRecord {
|
||||
name: string,
|
||||
numLastSynced: number
|
||||
first: number,
|
||||
last: number,
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// import { readFileSync, writeFileSync } from 'fs';
|
||||
import { Attachment } from './types';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
function uudecode(uuencodedText: string) {
|
||||
// Regular expressions to identify UUencoded content
|
||||
const beginRegex = /^begin\s+(\d+)\s+(\S+)\s*$/;
|
||||
const endRegex = /^end\s*$/;
|
||||
const uuLineRegex = /^[M-` ]/;
|
||||
|
||||
// Split input into lines
|
||||
const lines = uuencodedText.split(/\r?\n/);
|
||||
|
||||
let collecting = false;
|
||||
let fileName = null;
|
||||
let fileMode = null;
|
||||
let encodedData = [];
|
||||
|
||||
// Extract the UUencoded portion
|
||||
for (const line of lines) {
|
||||
// Check for beginning of UUencoded data
|
||||
const beginMatch = line.match(beginRegex);
|
||||
if (beginMatch) {
|
||||
collecting = true;
|
||||
fileMode = beginMatch[1];
|
||||
fileName = beginMatch[2];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for end of UUencoded data
|
||||
if (endRegex.test(line)) {
|
||||
collecting = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Collect UUencoded lines
|
||||
if (collecting && uuLineRegex.test(line)) {
|
||||
encodedData.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Actual UUdecoding
|
||||
let decodedData = new Uint8Array(encodedData.reduce((acc, line) => {
|
||||
if (line.length <= 1) return acc; // Skip empty lines
|
||||
|
||||
// First character of each line indicates the length
|
||||
const length = (line.charCodeAt(0) - 32) & 0x3F;
|
||||
if (length === 0) return acc; // Skip zero-length lines
|
||||
|
||||
// Process the line in groups of 4 characters
|
||||
const bytes = [];
|
||||
for (let i = 1; i < line.length; i += 4) {
|
||||
if (i + 3 >= line.length) break;
|
||||
|
||||
// Decode each group of 4 characters into 3 bytes
|
||||
const chars = line.slice(i, i + 4).split('').map(c => (c.charCodeAt(0) - 32) & 0x3F);
|
||||
|
||||
if (bytes.length < length) bytes.push((chars[0] << 2) | (chars[1] >> 4));
|
||||
if (bytes.length < length) bytes.push(((chars[1] & 0x0F) << 4) | (chars[2] >> 2));
|
||||
if (bytes.length < length) bytes.push(((chars[2] & 0x03) << 6) | chars[3]);
|
||||
}
|
||||
|
||||
return acc.concat(bytes);
|
||||
}, [] as number[]));
|
||||
|
||||
return {
|
||||
fileName,
|
||||
fileMode,
|
||||
data: decodedData
|
||||
};
|
||||
}
|
||||
|
||||
export async function extractAllUUEncodedAttachments(messageText: string) {
|
||||
const beginRegex = /^begin\s+(\d+)\s+(\S+)\s*$/mg;
|
||||
const results = [];
|
||||
let match;
|
||||
const positions: [number, number][] = [];
|
||||
|
||||
// Find all "begin" markers
|
||||
while ((match = beginRegex.exec(messageText)) !== null) {
|
||||
|
||||
const startPos = match.index;
|
||||
const endPos = messageText.indexOf("\nend", startPos);
|
||||
|
||||
if (endPos !== -1) {
|
||||
const encodedSection = messageText.substring(startPos, endPos + 4);
|
||||
const decoded = uudecode(encodedSection);
|
||||
if (decoded.data.length > 0) {
|
||||
results.push(decoded);
|
||||
}
|
||||
|
||||
positions.push([startPos, endPos + 4]);
|
||||
}
|
||||
}
|
||||
|
||||
// remove the decoded data from the text message
|
||||
let text = '';
|
||||
if (positions.length === 0) {
|
||||
text = messageText;
|
||||
}
|
||||
|
||||
if (positions.length > 1) {
|
||||
// remove one by one in reverse order
|
||||
text = messageText;
|
||||
for (let i = positions.length - 1; i >= 0; i--) {
|
||||
const [startPos, endPos] = positions[i];
|
||||
text = text.substring(0, startPos) + text.substring(endPos);
|
||||
}
|
||||
}
|
||||
|
||||
// for (let i = 0; i < positions.length; i++) {
|
||||
// const [startPos, endPos] = positions[i];
|
||||
// text += messageText.substring(0, startPos) + messageText.substring(endPos);
|
||||
// }
|
||||
|
||||
const attachmentsPromises: Promise<Attachment>[] = results.map(async (result) => {
|
||||
return {
|
||||
file: new File([result.data], result.fileName ?? ''),
|
||||
fileName: result.fileName ?? '',
|
||||
size: result.data.length,
|
||||
// md5 of result.data
|
||||
checksum: crypto.createHash('md5').update(result.data).digest('hex'),
|
||||
contentType: 'application/octet-stream', // todo - extract from
|
||||
}
|
||||
})
|
||||
|
||||
const attachments = await Promise.all(attachmentsPromises);
|
||||
|
||||
|
||||
return {
|
||||
attachments,
|
||||
positions,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
// async function test() {
|
||||
// const articleString = readFileSync('../err/article_2.json').toString();
|
||||
// const article = JSON.parse(articleString);
|
||||
//
|
||||
// const result = await extractAllUUEncodedAttachments(article.text);
|
||||
// article.text = result.text;
|
||||
// writeFileSync('../err/article_3.json', JSON.stringify(article, null, 2));
|
||||
// for (const attachment of result.attachments) {
|
||||
// if (attachment.file) {
|
||||
// writeFileSync(`../err/${attachment.fileName}`, Buffer.from((await attachment.file.arrayBuffer())));
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// console.log(result);
|
||||
// }
|
||||
//
|
||||
// test();
|
||||
Reference in New Issue
Block a user