add url reader

server/src/config/env.ts

@@ -9,10 +9,19 @@ const rootDir = path.resolve(__dirname, '../../..');
 config({ path: path.resolve(rootDir, '.env') });
 
 const port = Number(process.env.SERVER_PORT ?? process.env.PORT ?? 3000);
+const urlReaderBaseUrl = (process.env.URL_READER_BASE_URL ?? 'https://r.jina.ai').replace(
+  /\/$/,
+  ''
+);
+const urlReaderApiKey = process.env.URL_READER_API_KEY ?? '';
 
 export const env = {
   port,
   apiBasePath: process.env.API_BASE_PATH ?? '/api',
   databaseUrl: process.env.DATABASE_URL ?? '',
-  appName: process.env.APP_NAME ?? 'BestChoice'
+  appName: process.env.APP_NAME ?? 'BestChoice',
+  urlReader: {
+    baseUrl: urlReaderBaseUrl,
+    apiKey: urlReaderApiKey
+  }
 };

server/src/controllers/items-controller.ts

@@ -16,6 +16,7 @@ import {
   parseItemStatusFilter,
   parseItemUpdatePayload
 } from '../validation/items.js';
+import { readUrlMarkdown } from '../services/url-reader-service.js';
 
 export const listItems = async (req: Request, res: Response) => {
   const projectId = parseUuid(req.params.projectId, 'projectId');
@@ -151,19 +152,31 @@ export const importItemFromUrl = async (req: Request, res: Response) => {
   void projectId;
 
   const { url } = parseItemImportPayload(req.body);
-  const loweredUrl = url.toLowerCase();
+  let hostname: string | null = null;
+  let lastPathSegment: string | null = null;
+  let normalizedUrl = url;
 
-  if (loweredUrl.includes('fail') || loweredUrl.includes('error')) {
+  try {
+    normalizedUrl = new URL(url).toString();
+  } catch {
+    // If the URL constructor fails here, readUrlMarkdown will surface the error.
+  }
+
+  let markdown: string;
+  try {
+    markdown = await readUrlMarkdown(normalizedUrl);
+  } catch (error) {
+    if (error instanceof HttpError) {
+      throw error;
+    }
     throw new HttpError(
       422,
       'We could not read that URL. Please enter the item details manually.'
     );
   }
 
-  let hostname: string | null = null;
-  let lastPathSegment: string | null = null;
   try {
-    const parsed = new URL(url);
+    const parsed = new URL(normalizedUrl);
     hostname = parsed.hostname.replace(/^www\./, '');
     const pathSegments = parsed.pathname.split('/').filter(Boolean);
     if (pathSegments.length) {
@@ -190,13 +203,20 @@ export const importItemFromUrl = async (req: Request, res: Response) => {
     url: `https://picsum.photos/seed/${encodeURIComponent(seed)}/800/600`
   }));
 
+  const importedAt = new Date().toISOString();
   const attributes: Record<string, unknown> = {
-    originUrl: url,
-    importedAt: new Date().toISOString(),
+    originUrl: normalizedUrl,
+    importedAt,
     condition: 'New',
     availability: 'In stock',
     dpi: 1200,
-    imageCount: images.length
+    imageCount: images.length,
+    urlReader: {
+      provider: 'jina.ai',
+      format: 'markdown',
+      fetchedAt: importedAt,
+      content: markdown
+    }
   };
 
   if (hostname) {
@@ -209,6 +229,7 @@ export const importItemFromUrl = async (req: Request, res: Response) => {
       model,
       note: null,
       attributes,
+      sourceUrl: normalizedUrl,
       images
     }
   });

server/src/services/url-reader-service.ts

@@ -0,0 +1,73 @@
+import { env } from '../config/env.js';
+import { HttpError } from '../errors/http-error.js';
+
+const ensureHttpUrl = (rawUrl: string): string => {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    throw new HttpError(400, 'Enter a valid absolute URL.');
+  }
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new HttpError(400, 'URL reader only supports http and https URLs.');
+  }
+
+  return parsed.toString();
+};
+
+const buildReaderUrl = (targetUrl: string): string => {
+  const base = env.urlReader.baseUrl || 'https://r.jina.ai';
+  return `${base}/${targetUrl}`;
+};
+
+export const readUrlMarkdown = async (rawUrl: string): Promise<string> => {
+  if (typeof fetch !== 'function') {
+    throw new HttpError(500, 'URL reader requires fetch support but it is unavailable.');
+  }
+
+  if (!env.urlReader.apiKey) {
+    throw new HttpError(500, 'URL reader service is not configured.');
+  }
+
+  const normalizedTargetUrl = ensureHttpUrl(rawUrl);
+  const requestUrl = buildReaderUrl(normalizedTargetUrl);
+
+  let response: Awaited<ReturnType<typeof fetch>>;
+  try {
+    response = await fetch(requestUrl, {
+      headers: {
+        Authorization: `Bearer ${env.urlReader.apiKey}`
+      }
+    });
+  } catch (error) {
+    throw new HttpError(
+      502,
+      'We could not reach the URL reader service. Please try again later.'
+    );
+  }
+
+  if (response.status === 404) {
+    throw new HttpError(
+      422,
+      'We could not read that URL. Please ensure it is publicly accessible.'
+    );
+  }
+
+  if (!response.ok) {
+    throw new HttpError(
+      502,
+      'The URL reader service returned an unexpected response. Please try again later.'
+    );
+  }
+
+  const markdown = await response.text();
+  if (!markdown.trim()) {
+    throw new HttpError(
+      422,
+      'The URL reader returned empty content for that URL. Please fill in the details manually.'
+    );
+  }
+
+  return markdown;
+};