diff --git a/App.tsx b/App.tsx index 643d8a6..43dcf7a 100644 --- a/App.tsx +++ b/App.tsx @@ -5,7 +5,7 @@ import Dashboard from './components/Dashboard'; import FilterBar from './components/FilterBar'; import AIChat from './components/AIChat'; import CrazeLogo from './components/CrazeLogo'; -import { processCSV, processExcel, filterData, aggregateData, processAdsCSV, processAdsExcel, mergeSalesAndAdsData, processTrafficExcel, processStockExcel, filterAdsData, calculateForecastViewData, processVendorStockExcel, processUKInventoryExcel, calculateVelocityMap, getUniqueValues, processForecastExcel, processBuyBoxExcel } from './services/dataProcessor'; +import { processCSV, processExcel, filterData, aggregateData, processAdsCSV, processAdsExcel, mergeSalesAndAdsData, processTrafficExcel, processStockExcel, filterAdsData, calculateForecastViewData, processVendorStockExcel, processUKInventoryExcel, calculateVelocityMap, getUniqueValues, processForecastExcel, processBuyBoxExcel, processVendorCSV } from './services/dataProcessor'; import { SalesRecord, FilterState, AggregatedData, AdsRecord, TrafficRecord, ForecastRecord, ProductForecastData } from './types'; import { queryGemini } from './services/geminiService'; import { ChartIcon, TableIcon, UploadIcon, DownloadIcon, CloseIcon, TrendingIcon, MegaphoneIcon } from './components/Icons'; @@ -438,19 +438,39 @@ const App: React.FC = () => { } }; - // Handle uploaded Vendor CSV (sends to Supabase via API) + // Handle uploaded Vendor CSV (sends to Supabase via API with batching) const handleVendorUpload = async (file: File) => { setSyncing(true); try { - const text = await file.text(); - const response = await fetch('/api/upload-vendor-data', { - method: 'POST', - headers: { 'Content-Type': 'text/csv' }, - body: text, - }); - const result = await response.json(); - if (!response.ok) throw new Error(result.error); - alert(`Vendor data uploaded: ${result.rowsUpserted} rows processed.`); + console.log("[App] Parsing Vendor CSV on client..."); + const rows = await processVendorCSV(file); + + if (rows.length === 0) { + alert("No valid rows found in CSV."); + return; + } + + console.log(`[App] Uploading ${rows.length} rows in batches...`); + + const BATCH_SIZE = 500; + let totalUpserted = 0; + + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + const batch = rows.slice(i, i + BATCH_SIZE); + const response = await fetch('/api/upload-vendor-data', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(batch), + }); + + const result = await response.json(); + if (!response.ok) throw new Error(result.error || `Batch ${i / BATCH_SIZE + 1} failed`); + + totalUpserted += result.rowsUpserted; + console.log(`[App] Batch ${Math.floor(i / BATCH_SIZE) + 1} complete. Total: ${totalUpserted}`); + } + + alert(`Vendor data uploaded: ${totalUpserted} rows processed successfully.`); setIsDataModalOpen(false); } catch (error: any) { console.error("Failed to upload vendor data", error); diff --git a/api/upload-vendor-data.ts b/api/upload-vendor-data.ts index a1a8112..089e838 100644 --- a/api/upload-vendor-data.ts +++ b/api/upload-vendor-data.ts @@ -1,6 +1,7 @@ import type { VercelRequest, VercelResponse } from '@vercel/node'; import { createClient } from '@supabase/supabase-js'; import Papa from 'papaparse'; +import { VendorDailyRow } from '../types'; const supabase = createClient( process.env.SUPABASE_URL || '', @@ -34,43 +35,51 @@ export default async function handler(req: VercelRequest, res: VercelResponse) { if (req.method !== 'POST') return res.status(405).json({ error: 'Method not allowed' }); try { - const csvText = typeof req.body === 'string' ? req.body : req.body?.toString() || ''; + let rows: VendorDailyRow[] = []; - if (!csvText.trim()) { - return res.status(400).json({ error: 'Empty CSV body' }); + // Check if the body is already parsed JSON (array of rows) + if (Array.isArray(req.body)) { + console.log(`[upload-vendor-data] Received ${req.body.length} rows as JSON`); + rows = req.body; + } else { + const csvText = typeof req.body === 'string' ? req.body : req.body?.toString() || ''; + + if (!csvText.trim()) { + return res.status(400).json({ error: 'Empty CSV body' }); + } + + const parsed = Papa.parse(csvText, { + header: true, + skipEmptyLines: true, + }); + + if (parsed.errors.length > 0) { + console.error('[upload-vendor-data] Parse errors:', parsed.errors.slice(0, 5)); + } + + rows = parsed.data + .filter(row => row['Date'] && row['Market'] && row['ASIN']) + .map(row => ({ + date: row['Date'], + market: row['Market'], + asin: row['ASIN'], + product_title: row['Product Title'] || null, + tags: row['Tags'] || null, + bsr_top_rank: parseIntSafe(row['Top Level Category (Rank)']), + bsr_top_category: row['Top Level Category (Name)'] || null, + bsr_detail_rank: parseIntSafe(row['Detail Level Category (Rank)']), + bsr_detail_category: row['Detail Level Category (Name)'] || null, + avg_rating: parseEUNumber(row['Average Rating']), + num_reviews: parseIntSafe(row['Number of Reviews']), + buybox_owner: row['Buybox Seller Name'] || null, + buybox_price: parseEUNumber(row['Buybox Price']), + amazon_has_buybox: row['Amazon Has Buybox'] === '1', + glance_views: parseIntSafe(row['Glance Views']), + })); } - const parsed = Papa.parse(csvText, { - header: true, - skipEmptyLines: true, - }); - - if (parsed.errors.length > 0) { - console.error('[upload-vendor-data] Parse errors:', parsed.errors.slice(0, 5)); - } - - const rows = parsed.data - .filter(row => row['Date'] && row['Market'] && row['ASIN']) - .map(row => ({ - date: row['Date'], - market: row['Market'], - asin: row['ASIN'], - product_title: row['Product Title'] || null, - tags: row['Tags'] || null, - bsr_top_rank: parseIntSafe(row['Top Level Category (Rank)']), - bsr_top_category: row['Top Level Category (Name)'] || null, - bsr_detail_rank: parseIntSafe(row['Detail Level Category (Rank)']), - bsr_detail_category: row['Detail Level Category (Name)'] || null, - avg_rating: parseEUNumber(row['Average Rating']), - num_reviews: parseIntSafe(row['Number of Reviews']), - buybox_owner: row['Buybox Seller Name'] || null, - buybox_price: parseEUNumber(row['Buybox Price']), - amazon_has_buybox: row['Amazon Has Buybox'] === '1', - glance_views: parseIntSafe(row['Glance Views']), - })); - if (rows.length === 0) { - return res.status(400).json({ error: 'No valid rows found in CSV' }); + return res.status(400).json({ error: 'No valid rows found' }); } // Upsert in batches of 500 @@ -94,7 +103,6 @@ export default async function handler(req: VercelRequest, res: VercelResponse) { res.status(200).json({ success: true, - rowsParsed: parsed.data.length, rowsUpserted: totalUpserted, }); } catch (error: any) { diff --git a/services/dataProcessor.ts b/services/dataProcessor.ts index 1772cf7..98dabfe 100644 --- a/services/dataProcessor.ts +++ b/services/dataProcessor.ts @@ -1,7 +1,57 @@ -import { SalesRecord, AdsRecord, TrafficRecord, CombinedKPIs, FilterState, AggregatedData, LineGrowthMetric, ItemGrowthMetric, SeasonalityPoint, YearlySplitData, PivotRow, YearlyData, TimeSeriesData, ComparisonTimeSeriesPoint, ForecastRecord, MonthlyForecastPoint, ProductForecastData } from '../types'; +import { SalesRecord, AdsRecord, TrafficRecord, CombinedKPIs, FilterState, AggregatedData, LineGrowthMetric, ItemGrowthMetric, SeasonalityPoint, YearlySplitData, PivotRow, YearlyData, TimeSeriesData, ComparisonTimeSeriesPoint, ForecastRecord, MonthlyForecastPoint, ProductForecastData, VendorCSVRow, VendorDailyRow } from '../types'; import * as XLSX from 'xlsx'; import Papa from 'papaparse'; +/** + * Parses Vendor CSV and returns mapped VendorDailyRow items. + */ +export const processVendorCSV = (fileOrContent: File | string): Promise => { + return new Promise((resolve, reject) => { + const config = { + header: true, + skipEmptyLines: true, + complete: (results: any) => { + const rows = results.data + .filter((row: any) => row['Date'] && row['Market'] && row['ASIN']) + .map((row: any) => ({ + date: row['Date'], + market: row['Market'], + asin: row['ASIN'], + product_title: row['Product Title'] || null, + tags: row['Tags'] || null, + bsr_top_rank: parseIntSafe(row['Top Level Category (Rank)']), + bsr_top_category: row['Top Level Category (Name)'] || null, + bsr_detail_rank: parseIntSafe(row['Detail Level Category (Rank)']), + bsr_detail_category: row['Detail Level Category (Name)'] || null, + avg_rating: parseCurrency(row['Average Rating']), // Uses existing parseCurrency which handles EU/US + num_reviews: parseIntSafe(row['Number of Reviews']), + buybox_owner: row['Buybox Seller Name'] || null, + buybox_price: parseCurrency(row['Buybox Price']), + amazon_has_buybox: row['Amazon Has Buybox'] === '1', + glance_views: parseIntSafe(row['Glance Views']), + })); + resolve(rows); + }, + error: (error: any) => { + reject(error); + } + }; + + if (typeof fileOrContent === 'string') { + Papa.parse(fileOrContent, config); + } else { + Papa.parse(fileOrContent, config); + } + }); +}; + +function parseIntSafe(val: string | undefined | null): number | null { + if (!val || typeof val !== 'string' || val.trim() === '') return null; + const cleaned = val.replace(/\./g, '').replace(',', '.').replace(/[^0-9.]/g, ''); + const num = parseInt(cleaned, 10); + return isNaN(num) ? null : num; +} + // Helper to parse currency values handling both EU (1.234,56) and US/Standard (1,234.56 or 1234.56) formats const parseCurrency = (value: string): number => { if (!value) return 0; diff --git a/services/supabase.ts b/services/supabase.ts index 21b51ab..df86e4e 100644 --- a/services/supabase.ts +++ b/services/supabase.ts @@ -1,5 +1,6 @@ import { createClient, SupabaseClient } from '@supabase/supabase-js'; +import { VendorDailyRow } from '../types'; const supabaseUrl = process.env.SUPABASE_URL || ''; const supabaseAnonKey = process.env.SUPABASE_ANON_KEY || ''; @@ -16,25 +17,6 @@ const getClient = (): SupabaseClient => { return _client; }; -export interface VendorDailyRow { - id?: number; - date: string; - market: string; - asin: string; - product_title: string | null; - tags: string | null; - bsr_top_rank: number | null; - bsr_top_category: string | null; - bsr_detail_rank: number | null; - bsr_detail_category: string | null; - avg_rating: number | null; - num_reviews: number | null; - buybox_owner: string | null; - buybox_price: number | null; - amazon_has_buybox: boolean | null; - glance_views: number | null; -} - export interface VendorFilters { markets?: string[]; tags?: string[]; diff --git a/types.ts b/types.ts index 21e986d..d9caf6c 100644 --- a/types.ts +++ b/types.ts @@ -242,3 +242,40 @@ export interface BuyBoxLostData { countries: string[]; // ['DE', 'FR', 'IT', 'UK', 'ES'] reasons: Record; // { DE: 'Amazon', FR: 'Unknown' } } + +export interface VendorDailyRow { + id?: number; + date: string; + market: string; + asin: string; + product_title: string | null; + tags: string | null; + bsr_top_rank: number | null; + bsr_top_category: string | null; + bsr_detail_rank: number | null; + bsr_detail_category: string | null; + avg_rating: number | null; + num_reviews: number | null; + buybox_owner: string | null; + buybox_price: number | null; + amazon_has_buybox: boolean | null; + glance_views: number | null; +} + +export interface VendorCSVRow { + Date: string; + Market: string; + ASIN: string; + 'Product Title': string; + Tags: string; + 'Top Level Category (Rank)': string; + 'Top Level Category (Name)': string; + 'Detail Level Category (Rank)': string; + 'Detail Level Category (Name)': string; + 'Average Rating': string; + 'Number of Reviews': string; + 'Buybox Seller Name': string; + 'Buybox Price': string; + 'Amazon Has Buybox': string; + 'Glance Views': string; +}