Merge pull request #1418 from andrew-bierman/fix/items_import

fix import from csv

packages/app/modules/item/components/ImportForm.tsx

@@ -89,6 +89,16 @@ export const ImportForm: FC<ImportFormProps> = ({
     if (newValue) setSelectedType(newValue);
   };
 
+  const stripBOM = (content: string) => {
+    if (content.startsWith('\uFEFF')) {
+      console.log('BOM detected and removed');
+    }
+    const cleanedContent = content
+      .replace(/^\uFEFF/, '')
+      .replace(/^\xEF\xBB\xBF/, '');
+    return cleanedContent;
+  };
+
   const handleItemImport = async () => {
     setIsImporting(true);
     try {
@@ -110,7 +120,7 @@ export const ImportForm: FC<ImportFormProps> = ({
             if (file) {
               const base64Content = file.uri.split(',')[1];
               if (base64Content) {
-                fileContent = atob(base64Content);
+                fileContent = stripBOM(atob(base64Content));
               } else {
                 throw new Error('No file content available');
               }
@@ -125,7 +135,7 @@ export const ImportForm: FC<ImportFormProps> = ({
             (res as DocumentPicker.DocumentPickerSuccessResult).assets?.[0]
               ?.uri || '',
           );
-          fileContent = await response.text();
+          fileContent = stripBOM(await response.text());
         }
 
         if (currentpage === 'items') {

server/src/controllers/item/importItemsGlobal.ts

@@ -1,8 +1,5 @@
 import { type Context } from 'hono';
-import {
-  addItemGlobalService,
-  bulkAddItemsGlobalService,
-} from '../../services/item/item.service';
+import { bulkAddItemsGlobalService } from '../../services/item/item.service';
 import { protectedProcedure } from '../../trpc';
 import * as validator from '@packrat/validations';
 import Papa from 'papaparse';
@@ -11,70 +8,72 @@ import { ItemCategoryEnum } from '../../utils/itemCategory';
 export const importItemsGlobal = async (c: Context) => {
   try {
     const { content, ownerId } = await c.req.json();
+    const validHeaders = [
+      'name',
+      'weight',
+      'weight_unit',
+      'category',
+      'image_urls',
+      'sku',
+      'product_url',
+      'description',
+      'techs',
+      'seller',
+    ] as const;
 
     return new Promise((resolve, reject) => {
-      Papa.parse(content, {
+      Papa.parse<Record<string, unknown>>(content, {
         header: true,
         complete: async function (results) {
-          const expectedHeaders = [
-            'Name',
-            'Weight',
-            'Unit',
-            'Quantity',
-            'Category',
-            'image_urls',
-          ];
           const parsedHeaders = results.meta.fields ?? [];
           try {
-            const allHeadersPresent = expectedHeaders.every((header) =>
-              (parsedHeaders as string[]).includes(header),
+            const presentValidHeaders = parsedHeaders.filter((header) =>
+              validHeaders.includes(header as (typeof validHeaders)[number]),
             );
-            if (!allHeadersPresent) {
+
+            const invalidHeaders = presentValidHeaders.filter(
+              (header) =>
+                !validHeaders.includes(header as (typeof validHeaders)[number]),
+            );
+
+            if (invalidHeaders.length > 0) {
               return reject(
-                new Error('CSV does not contain all the expected Item headers'),
+                new Error(
+                  `Invalid header format for: ${invalidHeaders.join(', ')}`,
+                ),
               );
             }
 
-            for (const [index, item] of results.data.entries()) {
-              const row = item as {
-                Name: string;
-                Weight: string;
-                Unit: string;
-                Category: string;
-                image_urls?: string;
-                sku?: string;
-                product_url?: string;
-                description?: string;
-                seller?: string;
-                techs?: string;
-              };
-              if (
-                index === results.data.length - 1 &&
-                Object.values(row).every((value) => value === '')
-              ) {
-                continue;
-              }
+            const lastRawItem = results.data[results.data.length - 1];
+            if (
+              lastRawItem &&
+              Object.values(lastRawItem).every((value) => value === '')
+            ) {
+              results.data.pop();
+            }
 
-              await addItemGlobalService(
-                {
-                  name: row.Name,
-                  weight: Number(row.Weight),
-                  unit: row.Unit,
-                  type: row.Category as 'Food' | 'Water' | 'Essentials',
-                  ownerId,
-                  image_urls: row.image_urls,
+            const errors: Error[] = [];
+            const createdItems = await bulkAddItemsGlobalService(
+              sanitizeItemsIterator(results.data, ownerId),
+              c.executionCtx,
+              {
+                onItemCreationError: (error) => {
+                  errors.push(error);
                 },
-                c.executionCtx,
-              );
-            }
-            resolve('items');
+              },
+            );
+
+            return resolve({
+              status: 'success',
+              items: createdItems,
+              errorsCount: errors.length,
+              errors,
+            });
           } catch (error) {
+            console.error(error);
             reject(new Error(`Failed to add items: ${error.message}`));
           }
         },
-        error: function (error) {
-          reject(new Error(`Error parsing CSV file: ${error.message}`));
-        },
       });
     })
       .then((result) => c.json({ result }, 200))
@@ -97,6 +96,11 @@ function* sanitizeItemsIterator(
   for (let idx = 0; idx < csvRawItems.length; idx++) {
     const item = csvRawItems[idx];
 
+    // Ignore items with weight -1
+    if (Number(item?.weight) === -1) {
+      continue;
+    }
+
     const productDetailsStr = String(item?.techs ?? '')
       .replace(/'([^']*)'\s*:/g, '"$1":') // Replace single quotes keys with double quotes.
       .replace(/:\s*'([^']*)'/g, ': "$1"') // Replace single quotes values with double quotes.
@@ -106,24 +110,23 @@ function* sanitizeItemsIterator(
         return String.fromCharCode(codePoint);
       });
 
-    console.log(`${idx} / ${csvRawItems.length}`);
     let parsedProductDetails:
       | validator.AddItemGlobalType['productDetails']
       | null = null;
     try {
       parsedProductDetails = JSON.parse(productDetailsStr);
     } catch (e) {
       console.log(
-        `${productDetailsStr}\nFailed to parse product details for item ${item?.Name ?? 'unknown'}: ${e.message}`,
+        `${productDetailsStr}\nFailed to parse product details for item ${item?.name ?? 'unknown'}: ${e.message}`,
       );
       throw e;
     }
 
     const validatedItem: validator.AddItemGlobalType = {
-      name: String(item?.Name ?? ''),
-      weight: Number(item?.Weight ?? 0),
-      unit: String(item?.Unit ?? ''),
-      type: String(item?.Category ?? '') as ItemCategoryEnum,
+      name: String(item?.name ?? ''),
+      weight: Number(item?.weight ?? 0),
+      unit: String(item?.weight_unit ?? ''),
+      type: String(item?.category ?? '') as ItemCategoryEnum,
       ownerId,
       image_urls: item?.image_urls ? String(item.image_urls) : undefined,
       sku: item?.sku ? String(item.sku) : undefined,
@@ -139,36 +142,48 @@ function* sanitizeItemsIterator(
     yield validatedItem;
   }
 }
+
 export function importItemsGlobalRoute() {
-  const expectedHeaders = [
-    'Name',
-    'Weight',
-    'Unit',
-    'Category',
+  const validHeaders = [
+    'name',
+    'weight',
+    'weight_unit',
+    'category',
     'image_urls',
     'sku',
     'product_url',
     'description',
     'techs',
     'seller',
   ] as const;
+
   return protectedProcedure
     .input(validator.importItemsGlobal)
     .mutation(async (opts) => {
       const { content, ownerId } = opts.input;
       return new Promise((resolve, reject) => {
-        Papa.parse<Record<(typeof expectedHeaders)[number], unknown>>(content, {
+        Papa.parse<Record<string, unknown>>(content, {
           header: true,
           complete: async function (results) {
             const parsedHeaders = results.meta.fields ?? [];
             try {
-              const allHeadersPresent = expectedHeaders.every((header) =>
-                (parsedHeaders as string[]).includes(header),
+              // Only validate headers that are present in our validHeaders list
+              const presentValidHeaders = parsedHeaders.filter((header) =>
+                validHeaders.includes(header as (typeof validHeaders)[number]),
               );
-              if (!allHeadersPresent) {
+
+              // Check if any present valid headers are malformed
+              const invalidHeaders = presentValidHeaders.filter(
+                (header) =>
+                  !validHeaders.includes(
+                    header as (typeof validHeaders)[number],
+                  ),
+              );
+
+              if (invalidHeaders.length > 0) {
                 return reject(
                   new Error(
-                    'CSV does not contain all the expected Item headers',
+                    `Invalid header format for: ${invalidHeaders.join(', ')}`,
                   ),
                 );
               }

server/src/services/item/addItemGlobalService.ts

@@ -7,7 +7,10 @@ import {
 } from '../../db/schema';
 import { Item as ItemClass } from '../../drizzle/methods/Item';
 import { ItemCategory } from '../../drizzle/methods/itemcategory';
-import { ItemCategory as categories } from '../../utils/itemCategory';
+import {
+  ItemCategory as categories,
+  getCategoryOrDefault,
+} from '../../utils/itemCategory';
 import { VectorClient } from '../../vector/client';
 import { convertWeight, SMALLEST_WEIGHT_UNIT } from '../../utils/convertWeight';
 import { summarizeItem } from '../../utils/item';
@@ -24,20 +27,14 @@ export const addItemGlobalService = async (
   item: validator.AddItemGlobalType,
   executionCtx?: ExecutionContext,
 ): Promise<ItemWithCategory> => {
-  let category: InsertItemCategory | null;
-  if (!categories.includes(item.type)) {
-    const error = new Error(
-      `[${item.sku}#${item.name}]: Category must be one of: ${categories.join(', ')}`,
-    );
-    throw error;
-  }
+  const categoryType = getCategoryOrDefault(item.type);
 
   const itemClass = new ItemClass();
   const itemCategoryClass = new ItemCategory();
-  category =
-    (await itemCategoryClass.findItemCategory({ name: item.type })) || null;
+  let category =
+    (await itemCategoryClass.findItemCategory({ name: categoryType })) || null;
   if (!category) {
-    category = await itemCategoryClass.create({ name: item.type });
+    category = await itemCategoryClass.create({ name: categoryType });
   }
 
   const newItem = (await itemClass.create({

server/src/utils/itemCategory.ts

@@ -4,3 +4,10 @@ export enum ItemCategoryEnum {
   WATER = 'Water',
   ESSENTIALS = 'Essentials',
 }
+
+export function getCategoryOrDefault(category: string): ItemCategoryEnum {
+  if (ItemCategory.includes(category as any)) {
+    return category as ItemCategoryEnum;
+  }
+  return ItemCategoryEnum.ESSENTIALS;
+}