From 0b20b545ff34d71521995a78101b37b4b4be9c24 Mon Sep 17 00:00:00 2001 From: Jurgis Sakalauskas Date: Fri, 17 Oct 2025 17:04:41 +0300 Subject: [PATCH] Cache LLM item import details and normalize attributes --- server/src/controllers/items-controller.ts | 77 ++------ server/src/db/urls-repository.ts | 22 +++ server/src/services/item-import-service.ts | 214 +++++++++++++++++++++ 3 files changed, 256 insertions(+), 57 deletions(-) create mode 100644 server/src/services/item-import-service.ts diff --git a/server/src/controllers/items-controller.ts b/server/src/controllers/items-controller.ts index a114538..ec54b4f 100644 --- a/server/src/controllers/items-controller.ts +++ b/server/src/controllers/items-controller.ts @@ -9,6 +9,7 @@ import { import { createUrl, findUrlByValue, + updateUrlAttributes, updateUrlBodyText } from '../db/urls-repository.js'; import { findImageById } from '../db/images-repository.js'; @@ -23,6 +24,10 @@ import { } from '../validation/items.js'; import { readUrlMarkdown } from '../services/url-reader-service.js'; import { normalizeUrl } from '../utils/url-normalizer.js'; +import { + extractItemImportDetails, + parseCachedItemImportDetails +} from '../services/item-import-service.js'; export const listItems = async (req: Request, res: Response) => { const projectId = parseUuid(req.params.projectId, 'projectId'); @@ -150,16 +155,10 @@ export const deleteItem = async (req: Request, res: Response) => { }; export const importItemFromUrl = async (req: Request, res: Response) => { - await new Promise((resolve) => { - setTimeout(resolve, 1000); - }) const projectId = parseUuid(req.params.projectId, 'projectId'); - // projectId is validated but not otherwise used for the mock response. void projectId; const { url } = parseItemImportPayload(req.body); - let hostname: string | null = null; - let lastPathSegment: string | null = null; let normalizedUrl = url; try { @@ -198,62 +197,26 @@ export const importItemFromUrl = async (req: Request, res: Response) => { } } - try { - const parsed = new URL(normalizedUrl); - hostname = parsed.hostname.replace(/^www\./, ''); - const pathSegments = parsed.pathname.split('/').filter(Boolean); - if (pathSegments.length) { - lastPathSegment = pathSegments[pathSegments.length - 1]; + const cachedDetails = parseCachedItemImportDetails(urlRecord?.attributes); + const details = cachedDetails + ? cachedDetails + : await extractItemImportDetails({ normalizedUrl, markdown }); + + if (!cachedDetails) { + if (urlRecord) { + const updated = await updateUrlAttributes(urlRecord.id, details); + if (updated) { + urlRecord = updated; + } + } else { + urlRecord = await createUrl({ url: normalizedDbUrl, bodyText: markdown, attributes: details }); } - } catch { - // non-standard URLs fall back to generic mock content - } - - const manufacturerBase = hostname ? hostname.split('.')[0] : 'imported'; - const manufacturer = - manufacturerBase.charAt(0).toUpperCase() + manufacturerBase.slice(1); - - const cleanedSegment = - lastPathSegment?.replace(/[-_]/g, ' ').replace(/\s+/g, ' ').trim() ?? 'Item'; - const model = cleanedSegment.length ? cleanedSegment : 'Imported Item'; - - const imageSeedBase = cleanedSegment.length - ? cleanedSegment.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') - : manufacturer.toLowerCase(); - const normalizedSeed = imageSeedBase.length ? imageSeedBase : 'imported-item'; - const imageSeeds = Array.from({ length: 3 }, (_, index) => `${normalizedSeed}-${index + 1}`); - const images = imageSeeds.map((seed) => ({ - url: `https://picsum.photos/seed/${encodeURIComponent(seed)}/800/600` - })); - - const importedAt = new Date().toISOString(); - const attributes: Record = { - originUrl: normalizedUrl, - importedAt, - condition: 'New', - availability: 'In stock', - dpi: 1200, - imageCount: images.length, - urlReader: { - provider: 'jina.ai', - format: 'markdown', - fetchedAt: importedAt, - content: markdown - } - }; - - if (hostname) { - attributes.domain = hostname; } res.json({ data: { - manufacturer, - model, - note: null, - attributes, - sourceUrl: normalizedUrl, - images + ...details, + sourceUrl: normalizedUrl } }); }; diff --git a/server/src/db/urls-repository.ts b/server/src/db/urls-repository.ts index bdc286e..a5c6eaf 100644 --- a/server/src/db/urls-repository.ts +++ b/server/src/db/urls-repository.ts @@ -112,3 +112,25 @@ export const updateUrlBodyText = async ( return mapUrlRow(result.rows[0]); }; + +export const updateUrlAttributes = async ( + id: string, + attributes: unknown +): Promise => { + const result = await query( + ` + UPDATE urls + SET attributes = $2, + updated_at = NOW() + WHERE id = $1 + RETURNING id, url, body_text, attributes, has_price, created_at, updated_at + `, + [id, attributes ?? {}] + ); + + if (result.rows.length === 0) { + return null; + } + + return mapUrlRow(result.rows[0]); +}; diff --git a/server/src/services/item-import-service.ts b/server/src/services/item-import-service.ts new file mode 100644 index 0000000..269a6e3 --- /dev/null +++ b/server/src/services/item-import-service.ts @@ -0,0 +1,214 @@ +import { requestStructuredLlmResponse } from './llm-service.js'; + +interface LlmImportExtractionResult { + manufacturer: string | null; + model: string | null; + note?: string | null; + attributes?: Record | null; + images?: Array<{ url?: string | null; alt?: string | null }> | null; +} + +const toSnakeCaseKey = (value: string): string => { + return value + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .replace(/[\s-]+/g, '_') + .replace(/[^a-zA-Z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, '') + .toLowerCase(); +}; + +const normalizeAttributes = (input: unknown): Record => { + if (!input || typeof input !== 'object' || Array.isArray(input)) { + return {}; + } + + const normalized: Record = {}; + + for (const [rawKey, rawValue] of Object.entries(input)) { + if (typeof rawKey !== 'string') { + continue; + } + + const trimmed = rawKey.trim(); + if (!trimmed) { + continue; + } + + const snakeCase = toSnakeCaseKey(trimmed); + if (!snakeCase) { + continue; + } + + normalized[snakeCase] = rawValue; + } + + return normalized; +}; + +export interface ItemImportDetails { + manufacturer: string | null; + model: string | null; + note: string | null; + attributes: Record; + images: Array<{ url: string }>; +} + +const sanitizeItemImportDetails = (raw: unknown): ItemImportDetails => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + return { + manufacturer: null, + model: null, + note: null, + attributes: {}, + images: [] + }; + } + + const manufacturerValue = (raw as { manufacturer?: unknown }).manufacturer; + const modelValue = (raw as { model?: unknown }).model; + const noteValue = (raw as { note?: unknown }).note; + const attributesValue = (raw as { attributes?: unknown }).attributes; + const imagesValue = (raw as { images?: unknown }).images; + + const manufacturer = + typeof manufacturerValue === 'string' + ? manufacturerValue.trim() || null + : manufacturerValue === null + ? null + : null; + + const model = + typeof modelValue === 'string' ? modelValue.trim() || null : modelValue === null ? null : null; + + const note = + typeof noteValue === 'string' ? noteValue.trim() || null : noteValue === null ? null : null; + + const attributes = normalizeAttributes(attributesValue); + + const images: Array<{ url: string }> = []; + if (Array.isArray(imagesValue)) { + const seen = new Set(); + for (const candidate of imagesValue) { + if (!candidate || typeof candidate !== 'object') { + continue; + } + + const urlValue = (candidate as { url?: unknown }).url; + if (typeof urlValue !== 'string') { + continue; + } + + const trimmed = urlValue.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + + seen.add(trimmed); + images.push({ url: trimmed }); + } + } + + return { + manufacturer, + model, + note, + attributes, + images + }; +}; + +export const parseCachedItemImportDetails = (raw: unknown): ItemImportDetails | null => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + return null; + } + + const details = sanitizeItemImportDetails(raw); + + if ( + details.manufacturer === null && + details.model === null && + Object.keys(details.attributes).length === 0 && + details.images.length === 0 && + details.note === null + ) { + return null; + } + + return details; +}; + +export const extractItemImportDetails = async ({ + normalizedUrl, + markdown +}: { + normalizedUrl: string; + markdown: string; +}): Promise => { + const llmExtraction = await requestStructuredLlmResponse({ + systemPrompt: + 'You extract structured product information from markdown content. ' + + 'Return only facts explicitly supported by the text. Never invent details.', + prompt: [ + 'Extract product details for cataloging.', + `Source URL: ${normalizedUrl}`, + 'Provide the best available manufacturer, model, concise note (if applicable), up to 12 key attributes, and any image URLs present.', + 'Markdown content:', + '```markdown', + markdown, + '```' + ].join('\n'), + responseSchema: { + name: 'product_import', + schema: { + type: 'object', + additionalProperties: false, + properties: { + manufacturer: { + type: ['string', 'null'], + description: + 'Manufacturer or brand responsible for the product. Use null when not stated.' + }, + model: { + type: ['string', 'null'], + description: 'Product model or name. Use null when not stated.' + }, + note: { + type: ['string', 'null'], + description: 'Optional short note or summary gleaned from the content.' + }, + attributes: { + type: 'object', + description: + 'Key-value attributes derived from the content (e.g., price, dimensions, materials).', + additionalProperties: { + anyOf: [ + { type: 'string' }, + { type: 'number' }, + { type: 'integer' }, + { type: 'boolean' }, + { type: 'null' } + ] + } + }, + images: { + type: 'array', + description: 'Direct image URLs found in the content.', + items: { + type: 'object', + additionalProperties: false, + properties: { + url: { type: 'string', format: 'uri' }, + alt: { type: ['string', 'null'] } + }, + required: ['url'] + } + } + }, + required: ['manufacturer', 'model', 'attributes', 'images'] + } + } + }); + + return sanitizeItemImportDetails(llmExtraction); +};