nntp-worker/worker/article.ts

import { Repository } from './repository';
import { NntpClient } from './nntp-client';
import { checkPartialContentType, mapParsedMailToArticle } from './mapper';
import { ArticleRecord, Attachment } from './types';
import { parseAttachments } from './attachments';
import { writeFileSync } from 'fs';
import { checkAndCombinePartial } from './partial';
import { truncateText } from './truncate';
import { extractAllUUEncodedAttachments } from './uudecode';

export async function downloadAndProcessArticle(repository: Repository, groupName: string, articleNumber: number, nntpClient: NntpClient) {
  const groupId = await repository.getGroupId(groupName);
  if (!groupId) {
    console.log(`Group ${groupName} not found in local database`);
    return;
  }

  const groupArticleNumber = await repository.findGroupArticleNumber(groupName, articleNumber);

  if (groupArticleNumber) {
    console.log(`Article ${articleNumber} already exists in local database`);
    return;
  }

  if (nntpClient.getSelectedGroup() !== groupName) {
    await nntpClient.selectGroup(groupName);
  }


  const articleMail = await nntpClient.getArticle(articleNumber).catch(e => {
    console.log(e);
    return null;
  });

  if (!articleMail) {
    console.log('No article found');
    return;
  }

  const partial = checkPartialContentType(articleMail.headers);
  // restart from here after combining partials
  let attachments: Attachment[] = await parseAttachments(articleMail, partial);
  let article = mapParsedMailToArticle(articleMail, attachments);



  if (partial) {
    const filePath = `./partials/${partial.id}-part_${partial.number}-article.json`
    writeFileSync(filePath, JSON.stringify(article, null, 2));
    console.log(`Partial article saved to ${filePath}`);
  }

  if (partial) {
    const combined = await checkAndCombinePartial(partial);
    if (!combined) {
      return;
    }

    // TODO - this is duplicate code from let attachments = saveAttachments
    attachments = await parseAttachments(combined, null);
    article = mapParsedMailToArticle(combined, attachments);
  }

  if (!article.text) {
    article.text = '';
  }

  // check and extract uuencoded attachments
  const { text: newText, attachments: moreAttachments } = await extractAllUUEncodedAttachments(article.text);

  attachments = attachments.concat(moreAttachments);
  article.text = newText;

  article.text = article.text.trim();

  const groupsRecordsIds = await repository.findGroupsIdsByNames(article.groups);
  const parentRecordId = article.parent ? await repository.findArticleIdByArticleId(article.parent) : undefined;
  const referencesIds = await repository.findArticleIdsByArticleIds(article.references);

  const isRoot = article.references.length === 0;
  const { text, isTruncated, fullTextFile } = truncateText(article.text, isRoot)

  const data: ArticleRecord = {
    ...article,
    isRoot,
    groups: groupsRecordsIds,
    text,
    isTruncated,
    fullTextFile,
    references: referencesIds,
    parent: parentRecordId ?? null,
    guessRootId: article.references[0] ?? null,
    files: attachments.map((a)=>a.file),
  }

  try {
    const createRecord = await repository.createArticle(data);

    console.log('Record created:', createRecord.id);

    await repository.assignArticleToGroupsMessages(createRecord.id, article.inGroups);
  } catch (e) {
    console.log(data, article);
    throw e;
  }

  console.log('Done');
}