From d5cac734aa68a9c3051e7b34962d894bec796167 Mon Sep 17 00:00:00 2001 From: Jurgis Sakalauskas Date: Fri, 17 Oct 2025 19:20:57 +0300 Subject: [PATCH] Improve import attribute hints --- server/src/controllers/items-controller.ts | 29 +++++- server/src/db/items-repository.ts | 109 +++++++++++++++++++++ server/src/services/item-import-service.ts | 85 +++++++++++++--- 3 files changed, 206 insertions(+), 17 deletions(-) diff --git a/server/src/controllers/items-controller.ts b/server/src/controllers/items-controller.ts index ec54b4f..78e4928 100644 --- a/server/src/controllers/items-controller.ts +++ b/server/src/controllers/items-controller.ts @@ -3,6 +3,7 @@ import { createItem as createItemRepo, deleteItem as deleteItemRepo, getItemById, + getProjectAttributeExamples, listItems as listItemsRepo, updateItem as updateItemRepo } from '../db/items-repository.js'; @@ -26,8 +27,10 @@ import { readUrlMarkdown } from '../services/url-reader-service.js'; import { normalizeUrl } from '../utils/url-normalizer.js'; import { extractItemImportDetails, - parseCachedItemImportDetails + parseCachedItemImportDetails, + ItemImportDetails } from '../services/item-import-service.js'; +import { getProjectById } from '../db/projects-repository.js'; export const listItems = async (req: Request, res: Response) => { const projectId = parseUuid(req.params.projectId, 'projectId'); @@ -156,7 +159,11 @@ export const deleteItem = async (req: Request, res: Response) => { export const importItemFromUrl = async (req: Request, res: Response) => { const projectId = parseUuid(req.params.projectId, 'projectId'); - void projectId; + const project = await getProjectById(projectId); + + if (!project) { + throw new HttpError(404, 'Project not found'); + } const { url } = parseItemImportPayload(req.body); let normalizedUrl = url; @@ -198,9 +205,21 @@ export const importItemFromUrl = async (req: Request, res: Response) => { } const cachedDetails = parseCachedItemImportDetails(urlRecord?.attributes); - const details = cachedDetails - ? cachedDetails - : await extractItemImportDetails({ normalizedUrl, markdown }); + let details: ItemImportDetails; + + if (cachedDetails) { + details = cachedDetails; + } else { + const attributeExamples = await getProjectAttributeExamples(projectId); + details = await extractItemImportDetails({ + normalizedUrl, + markdown, + attributeHints: { + projectAttributeNames: project.attributes, + projectAttributeExamples: attributeExamples + } + }); + } if (!cachedDetails) { if (urlRecord) { diff --git a/server/src/db/items-repository.ts b/server/src/db/items-repository.ts index 097932e..ce5b4cc 100644 --- a/server/src/db/items-repository.ts +++ b/server/src/db/items-repository.ts @@ -353,3 +353,112 @@ export const deleteItem = async (id: string): Promise => { return (result.rowCount ?? 0) > 0; }; + +export interface ProjectAttributeExample { + key: string; + sampleValues: string[]; +} + +export interface ProjectAttributeExampleOptions { + itemLimit?: number; + attributeLimit?: number; + valuesPerAttribute?: number; +} + +export const getProjectAttributeExamples = async ( + projectId: string, + options?: ProjectAttributeExampleOptions +): Promise => { + const itemLimit = Math.max(1, Math.min(options?.itemLimit ?? 50, 200)); + const attributeLimit = Math.max(1, Math.min(options?.attributeLimit ?? 15, 100)); + const valuesPerAttribute = Math.max(1, Math.min(options?.valuesPerAttribute ?? 5, 10)); + + const result = await query<{ attributes: Record }>( + ` + SELECT attributes + FROM items + WHERE project_id = $1 + ORDER BY updated_at DESC + LIMIT $2 + `, + [projectId, itemLimit] + ); + + const attributeOrder: string[] = []; + const valueMap = new Map>(); + + const formatValue = (value: unknown): string | null => { + if (value === null || value === undefined) { + return null; + } + + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + + return trimmed.length > 120 ? `${trimmed.slice(0, 117)}...` : trimmed; + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + + if (value instanceof Date) { + return value.toISOString(); + } + + try { + const json = JSON.stringify(value); + if (!json || json === '[]' || json === '{}') { + return null; + } + + return json.length > 120 ? `${json.slice(0, 117)}...` : json; + } catch { + return null; + } + }; + + for (const row of result.rows) { + const attributes = row.attributes ?? {}; + + for (const [rawKey, rawValue] of Object.entries(attributes)) { + if (typeof rawKey !== 'string') { + continue; + } + + const key = rawKey.trim(); + if (!key) { + continue; + } + + let valueSet = valueMap.get(key); + if (!valueSet) { + if (attributeOrder.length >= attributeLimit) { + continue; + } + valueSet = new Set(); + valueMap.set(key, valueSet); + attributeOrder.push(key); + } + + if (valueSet.size >= valuesPerAttribute) { + continue; + } + + const formattedValue = formatValue(rawValue); + if (!formattedValue || valueSet.has(formattedValue)) { + continue; + } + + valueSet.add(formattedValue); + } + } + + return attributeOrder.map((key) => ({ + key, + sampleValues: Array.from(valueMap.get(key) ?? []) + })); +}; diff --git a/server/src/services/item-import-service.ts b/server/src/services/item-import-service.ts index c861170..d527bd3 100644 --- a/server/src/services/item-import-service.ts +++ b/server/src/services/item-import-service.ts @@ -8,6 +8,16 @@ interface LlmImportExtractionResult { images?: Array<{ url?: string | null; alt?: string | null }> | null; } +interface AttributeHintSample { + key: string; + sampleValues: string[]; +} + +interface ProjectAttributeHints { + projectAttributeNames?: string[]; + projectAttributeExamples?: AttributeHintSample[]; +} + const toSnakeCaseKey = (value: string): string => { return value .replace(/([a-z0-9])([A-Z])/g, '$1_$2') @@ -140,24 +150,75 @@ export const parseCachedItemImportDetails = (raw: unknown): ItemImportDetails | export const extractItemImportDetails = async ({ normalizedUrl, - markdown + markdown, + attributeHints }: { normalizedUrl: string; markdown: string; + attributeHints?: ProjectAttributeHints; }): Promise => { + const preferredAttributeNames = (attributeHints?.projectAttributeNames ?? []) + .map((name) => name.trim()) + .filter((name) => name.length > 0) + .slice(0, 12); + + const attributeExampleLines = (attributeHints?.projectAttributeExamples ?? []) + .map((example) => { + const key = example.key.trim(); + if (!key) { + return null; + } + + const values = example.sampleValues + .map((value) => value.trim()) + .filter((value) => value.length > 0) + .slice(0, 3); + + if (values.length === 0) { + return null; + } + + return `- ${key}: ${values.join(' | ')}`; + }) + .filter((line): line is string => Boolean(line)) + .slice(0, 10); + + const promptSections: string[] = [ + 'Extract product details for cataloging.', + `Source URL: ${normalizedUrl}` + ]; + + if (preferredAttributeNames.length > 0 || attributeExampleLines.length > 0) { + promptSections.push('Project attribute context:'); + + if (preferredAttributeNames.length > 0) { + promptSections.push(`Preferred attribute keys: ${preferredAttributeNames.join(', ')}`); + } + + if (attributeExampleLines.length > 0) { + promptSections.push('Existing item attribute examples:'); + promptSections.push(...attributeExampleLines); + } + + promptSections.push( + 'When the content maps to these attributes, reuse the exact key. Only introduce new keys when necessary.' + ); + } + + promptSections.push( + 'Provide the best available manufacturer, model, concise note (if applicable), key attributes, and any image URLs present.', + 'Markdown content:', + '```markdown', + markdown, + '```' + ); + const llmExtraction = await requestStructuredLlmResponse({ systemPrompt: - 'You extract structured product information from markdown content. ' + + 'You extract structured product information for a catalog from markdown content. ' + + 'Use the provided project attribute context to align attribute names. ' + 'Return only facts explicitly supported by the text. Never invent details.', - prompt: [ - 'Extract product details for cataloging.', - `Source URL: ${normalizedUrl}`, - 'Provide the best available manufacturer, model, concise note (if applicable), key attributes, and any image URLs present.', - 'Markdown content:', - '```markdown', - markdown, - '```' - ].join('\n'), + prompt: promptSections.join('\n'), responseSchema: { name: 'product_import', schema: { @@ -180,7 +241,7 @@ export const extractItemImportDetails = async ({ attributes: { type: 'object', description: - 'Key-value attributes derived from the content (e.g., price, dimensions, materials).', + 'Key-value attributes derived from the content (e.g., price, dimensions, materials). Prefer attribute keys that match the provided project context.', additionalProperties: { anyOf: [ { type: 'string' },