diff --git a/server/src/config/env.ts b/server/src/config/env.ts index 0583fa2..69d8d01 100644 --- a/server/src/config/env.ts +++ b/server/src/config/env.ts @@ -9,10 +9,19 @@ const rootDir = path.resolve(__dirname, '../../..'); config({ path: path.resolve(rootDir, '.env') }); const port = Number(process.env.SERVER_PORT ?? process.env.PORT ?? 3000); +const urlReaderBaseUrl = (process.env.URL_READER_BASE_URL ?? 'https://r.jina.ai').replace( + /\/$/, + '' +); +const urlReaderApiKey = process.env.URL_READER_API_KEY ?? ''; export const env = { port, apiBasePath: process.env.API_BASE_PATH ?? '/api', databaseUrl: process.env.DATABASE_URL ?? '', - appName: process.env.APP_NAME ?? 'BestChoice' + appName: process.env.APP_NAME ?? 'BestChoice', + urlReader: { + baseUrl: urlReaderBaseUrl, + apiKey: urlReaderApiKey + } }; diff --git a/server/src/controllers/items-controller.ts b/server/src/controllers/items-controller.ts index 6abad34..8cce246 100644 --- a/server/src/controllers/items-controller.ts +++ b/server/src/controllers/items-controller.ts @@ -16,6 +16,7 @@ import { parseItemStatusFilter, parseItemUpdatePayload } from '../validation/items.js'; +import { readUrlMarkdown } from '../services/url-reader-service.js'; export const listItems = async (req: Request, res: Response) => { const projectId = parseUuid(req.params.projectId, 'projectId'); @@ -151,19 +152,31 @@ export const importItemFromUrl = async (req: Request, res: Response) => { void projectId; const { url } = parseItemImportPayload(req.body); - const loweredUrl = url.toLowerCase(); + let hostname: string | null = null; + let lastPathSegment: string | null = null; + let normalizedUrl = url; - if (loweredUrl.includes('fail') || loweredUrl.includes('error')) { + try { + normalizedUrl = new URL(url).toString(); + } catch { + // If the URL constructor fails here, readUrlMarkdown will surface the error. + } + + let markdown: string; + try { + markdown = await readUrlMarkdown(normalizedUrl); + } catch (error) { + if (error instanceof HttpError) { + throw error; + } throw new HttpError( 422, 'We could not read that URL. Please enter the item details manually.' ); } - let hostname: string | null = null; - let lastPathSegment: string | null = null; try { - const parsed = new URL(url); + const parsed = new URL(normalizedUrl); hostname = parsed.hostname.replace(/^www\./, ''); const pathSegments = parsed.pathname.split('/').filter(Boolean); if (pathSegments.length) { @@ -190,13 +203,20 @@ export const importItemFromUrl = async (req: Request, res: Response) => { url: `https://picsum.photos/seed/${encodeURIComponent(seed)}/800/600` })); + const importedAt = new Date().toISOString(); const attributes: Record = { - originUrl: url, - importedAt: new Date().toISOString(), + originUrl: normalizedUrl, + importedAt, condition: 'New', availability: 'In stock', dpi: 1200, - imageCount: images.length + imageCount: images.length, + urlReader: { + provider: 'jina.ai', + format: 'markdown', + fetchedAt: importedAt, + content: markdown + } }; if (hostname) { @@ -209,6 +229,7 @@ export const importItemFromUrl = async (req: Request, res: Response) => { model, note: null, attributes, + sourceUrl: normalizedUrl, images } }); diff --git a/server/src/services/url-reader-service.ts b/server/src/services/url-reader-service.ts new file mode 100644 index 0000000..d1b1fbb --- /dev/null +++ b/server/src/services/url-reader-service.ts @@ -0,0 +1,73 @@ +import { env } from '../config/env.js'; +import { HttpError } from '../errors/http-error.js'; + +const ensureHttpUrl = (rawUrl: string): string => { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new HttpError(400, 'Enter a valid absolute URL.'); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new HttpError(400, 'URL reader only supports http and https URLs.'); + } + + return parsed.toString(); +}; + +const buildReaderUrl = (targetUrl: string): string => { + const base = env.urlReader.baseUrl || 'https://r.jina.ai'; + return `${base}/${targetUrl}`; +}; + +export const readUrlMarkdown = async (rawUrl: string): Promise => { + if (typeof fetch !== 'function') { + throw new HttpError(500, 'URL reader requires fetch support but it is unavailable.'); + } + + if (!env.urlReader.apiKey) { + throw new HttpError(500, 'URL reader service is not configured.'); + } + + const normalizedTargetUrl = ensureHttpUrl(rawUrl); + const requestUrl = buildReaderUrl(normalizedTargetUrl); + + let response: Awaited>; + try { + response = await fetch(requestUrl, { + headers: { + Authorization: `Bearer ${env.urlReader.apiKey}` + } + }); + } catch (error) { + throw new HttpError( + 502, + 'We could not reach the URL reader service. Please try again later.' + ); + } + + if (response.status === 404) { + throw new HttpError( + 422, + 'We could not read that URL. Please ensure it is publicly accessible.' + ); + } + + if (!response.ok) { + throw new HttpError( + 502, + 'The URL reader service returned an unexpected response. Please try again later.' + ); + } + + const markdown = await response.text(); + if (!markdown.trim()) { + throw new HttpError( + 422, + 'The URL reader returned empty content for that URL. Please fill in the details manually.' + ); + } + + return markdown; +};