diff --git a/nuxt4-app/app/components/MetadatosCard.vue b/nuxt4-app/app/components/MetadatosCard.vue index b8447c9..2faea79 100644 --- a/nuxt4-app/app/components/MetadatosCard.vue +++ b/nuxt4-app/app/components/MetadatosCard.vue @@ -39,27 +39,54 @@ Columnas detectadas ({{ metadata.columns?.length || 0 }}): {{ (metadata.columns || []).join(', ') || 'Ninguna' }} -
-
- - {{ tableStore ? 'Última carga: ' + tableStore.formattedLastUpdated : 'No hay datos cargados' }} - - - ⚠️ Los datos pueden estar desactualizados - +
+
+
+ + {{ tableStore ? 'Última carga: ' + tableStore.formattedLastUpdated : 'No hay datos cargados' }} + + + ⚠️ Los datos pueden estar desactualizados + +
+ +
+ + + Últimos datos + + + + + Obtener todos + +
- + - - Cargar datos - + />
@@ -91,6 +118,11 @@ const props = defineProps() const { $getTableStore } = useNuxtApp() +// Loading states +const isLoadingLatest = ref(false) +const isLoadingAll = ref(false) +const loadingProgress = ref(0) + // Get the table store for this specific datasource (using name, not table) const tableStore = computed(() => { if (typeof $getTableStore === 'function') { @@ -104,16 +136,49 @@ const recordCount = computed(() => { return tableStore.value?.recordCount || 0 }) -async function loadData() { - const store = tableStore.value - if (store) { - if (!store.hasData) { - // If no data, initialize (which loads from cache or fetches) - await store.initialize() - } else { - // If already has data, refresh it - await store.refreshData() - } +async function loadLatestData() { + isLoadingLatest.value = true + loadingProgress.value = 0 + + try { + const store = tableStore.value + if (!store) return + + await store.loadLatestDataInBatches((progress) => { + loadingProgress.value = progress + }) + + loadingProgress.value = 100 + } catch (error) { + console.error('Error loading latest data:', error) + } finally { + setTimeout(() => { + isLoadingLatest.value = false + loadingProgress.value = 0 + }, 500) + } +} + +async function loadAllData() { + isLoadingAll.value = true + loadingProgress.value = 0 + + try { + const store = tableStore.value + if (!store) return + + await store.loadAllDataInBatches((progress) => { + loadingProgress.value = progress + }) + + loadingProgress.value = 100 + } catch (error) { + console.error('Error loading all data:', error) + } finally { + setTimeout(() => { + isLoadingAll.value = false + loadingProgress.value = 0 + }, 500) } } diff --git a/nuxt4-app/app/pages/explorer.vue b/nuxt4-app/app/pages/explorer.vue index 5ad0f49..5fb1823 100644 --- a/nuxt4-app/app/pages/explorer.vue +++ b/nuxt4-app/app/pages/explorer.vue @@ -32,29 +32,11 @@ - -
-
- - Última actualización: {{ currentTableStore.formattedLastUpdated }} - - - ⚠️ Los datos pueden estar desactualizados - -
- - - - Actualizar datos - -
+ +
@@ -214,12 +196,8 @@ const selectedTable = computed(() => { if (!metadata) return null return { - name: metadata.table, - label: `${metadata.table} (${metadata.rowCount || 0} registros)`, - rowCount: metadata.rowCount || 0, - primaryKey: metadata.primaryKey, - columns: metadata.columns || [], - ...metadata + ...metadata, + label: `${metadata.table} (${metadata.rowCount || 0} registros)` } }) @@ -241,7 +219,7 @@ const tableDropdownItems = computed((): DropdownMenuItem[] => { return metadataStore.allTables.map(metadata => ({ label: `${metadata.table} (${metadata.rowCount || 0})`, icon: 'i-lucide-table', - onSelect: () => selectTable(metadata.table) + onSelect: () => selectTable(metadata.name) })) }) diff --git a/nuxt4-app/app/stores/tableDataFactory.ts b/nuxt4-app/app/stores/tableDataFactory.ts index f0d454b..76889ac 100644 --- a/nuxt4-app/app/stores/tableDataFactory.ts +++ b/nuxt4-app/app/stores/tableDataFactory.ts @@ -29,6 +29,8 @@ export interface TableDataActions> { initialize(): Promise getRecord(id: string | number): T | undefined filterRecords(predicate: (record: T) => boolean): T[] + loadAllDataInBatches(onProgress?: (progress: number) => void): Promise + loadLatestDataInBatches(onProgress?: (progress: number) => void): Promise } /** @@ -57,36 +59,46 @@ export function createTableDataStore>( /** * Get all records */ - allRecords: (state): T[] => state.data, + allRecords(): T[] { + return this.data as T[] + }, /** * Check if data is available */ - hasData: (state): boolean => state.data.length > 0, + hasData(): boolean { + return this.data.length > 0 + }, /** * Check if data is currently loading */ - isLoading: (state): boolean => state.loading, + isLoading(): boolean { + return this.loading + }, /** * Check if there's an error */ - hasError: (state): boolean => !!state.error, + hasError(): boolean { + return !!this.error + }, /** * Get total number of records */ - recordCount: (state): number => state.data.length, + recordCount(): number { + return this.data.length + }, /** * Get formatted last updated time */ - formattedLastUpdated: (state): string => { - if (!state.lastUpdated) return 'Nunca' + formattedLastUpdated(): string { + if (!this.lastUpdated) return 'Nunca' try { - return new Date(state.lastUpdated).toLocaleString('es-ES', { + return new Date(this.lastUpdated).toLocaleString('es-ES', { year: 'numeric', month: 'long', day: 'numeric', @@ -101,10 +113,10 @@ export function createTableDataStore>( /** * Check if data is stale (older than 5 minutes) */ - isStale: (state): boolean => { - if (!state.lastUpdated) return true + isStale(): boolean { + if (!this.lastUpdated) return true - const lastUpdate = new Date(state.lastUpdated) + const lastUpdate = new Date(this.lastUpdated) const now = new Date() const fiveMinutes = 5 * 60 * 1000 @@ -186,19 +198,27 @@ export function createTableDataStore>( * Load data from localStorage cache */ loadFromCache(): void { - if (!process.client) return + if (!process.client) { + console.log(`[${tableName}] loadFromCache: Not on client, skipping`) + return + } try { + console.log(`[${tableName}] loadFromCache: Attempting to load from key: ${cacheKey}`) const cached = localStorage.getItem(cacheKey) if (cached) { const parsedCache = JSON.parse(cached) + console.log(`[${tableName}] loadFromCache: Found ${parsedCache.data?.length || 0} records in cache`) this.data = parsedCache.data || [] this.lastUpdated = parsedCache.lastUpdated || null this.limit = parsedCache.limit || defaultLimit this.initialized = true + console.log(`[${tableName}] loadFromCache: Successfully loaded, data.length: ${this.data.length}`) + } else { + console.log(`[${tableName}] loadFromCache: No cached data found`) } } catch (error) { - console.warn(`Failed to load ${tableName} data from cache:`, error) + console.error(`[${tableName}] loadFromCache: Failed to load data from cache:`, error) } }, @@ -259,7 +279,230 @@ export function createTableDataStore>( * Filter records by a predicate function */ filterRecords(predicate: (record: T) => boolean): T[] { - return this.data.filter(predicate) + return (this.data as T[]).filter(predicate) + }, + + /** + * Load all data in batches with progress callback + */ + async loadAllDataInBatches(onProgress?: (progress: number) => void): Promise { + console.log(`[${tableName}] loadAllDataInBatches: Starting...`) + this.loading = true + this.error = null + + try { + // Clear existing data + console.log(`[${tableName}] loadAllDataInBatches: Clearing ${this.data.length} existing records`) + this.data.splice(0, this.data.length) + console.log(`[${tableName}] loadAllDataInBatches: Data cleared, length now: ${this.data.length}`) + + let offset = 0 + const limit = 500 + let hasMore = true + let totalFetched = 0 + let estimatedTotal = 0 + + while (hasMore) { + console.log(`[${tableName}] loadAllDataInBatches: Fetching batch at offset ${offset}`) + const response = await $fetch(`/api/data/${tableName}`, { + params: { limit, offset } + }) + + if (!response) { + console.log(`[${tableName}] loadAllDataInBatches: No response, stopping`) + hasMore = false + break + } + + // Get total count from first response + if (offset === 0 && response.count) { + estimatedTotal = response.count + console.log(`[${tableName}] loadAllDataInBatches: Estimated total: ${estimatedTotal}`) + } + + if (response.records && response.records.length > 0) { + console.log(`[${tableName}] loadAllDataInBatches: Got ${response.records.length} records`) + this.data.push(...(response.records as T[])) + totalFetched += response.records.length + console.log(`[${tableName}] loadAllDataInBatches: Total fetched so far: ${totalFetched}, data.length: ${this.data.length}`) + + // Update progress + if (onProgress) { + if (estimatedTotal > 0) { + onProgress(Math.min(95, (totalFetched / estimatedTotal) * 100)) + } else { + onProgress(Math.min(95, (totalFetched / 10000) * 100)) + } + } + + // Check if we got fewer records than requested (means we're at the end) + hasMore = response.records.length === limit + offset += limit + } else { + console.log(`[${tableName}] loadAllDataInBatches: No records in response, stopping`) + hasMore = false + } + } + + this.lastUpdated = new Date().toISOString() + this.initialized = true + + console.log(`[${tableName}] loadAllDataInBatches: Finished loading ${this.data.length} records`) + + // Persist to localStorage + if (process.client) { + try { + console.log(`[${tableName}] loadAllDataInBatches: Persisting to localStorage with key: ${cacheKey}`) + const dataToSave = { + data: this.data, + lastUpdated: this.lastUpdated, + limit: this.limit + } + console.log(`[${tableName}] loadAllDataInBatches: Saving ${dataToSave.data.length} records`) + localStorage.setItem(cacheKey, JSON.stringify(dataToSave)) + console.log(`[${tableName}] loadAllDataInBatches: Successfully saved to localStorage`) + + // Verify it was saved + const saved = localStorage.getItem(cacheKey) + if (saved) { + const parsed = JSON.parse(saved) + console.log(`[${tableName}] loadAllDataInBatches: Verification - localStorage contains ${parsed.data?.length || 0} records`) + } + } catch (error) { + console.error(`[${tableName}] loadAllDataInBatches: Failed to persist data to localStorage:`, error) + } + } else { + console.log(`[${tableName}] loadAllDataInBatches: Not on client, skipping localStorage`) + } + + if (onProgress) { + onProgress(100) + } + } catch (error: any) { + this.error = this.extractErrorMessage(error) + console.error(`[${tableName}] loadAllDataInBatches: Error:`, error) + throw error + } finally { + this.loading = false + console.log(`[${tableName}] loadAllDataInBatches: Completed`) + } + }, + + /** + * Load only latest data (incremental) in batches with progress callback + */ + async loadLatestDataInBatches(onProgress?: (progress: number) => void): Promise { + console.log(`[${tableName}] loadLatestDataInBatches: Starting...`) + this.loading = true + this.error = null + + try { + // Find the most recent created_at in memory + let lastCreatedAt: string | null = null + if (this.data.length > 0) { + const sortedRecords = [...(this.data as any[])].sort((a, b) => { + const dateA = new Date(a.created_at || 0).getTime() + const dateB = new Date(b.created_at || 0).getTime() + return dateB - dateA + }) + lastCreatedAt = sortedRecords[0]?.created_at || null + console.log(`[${tableName}] loadLatestDataInBatches: Most recent created_at in memory: ${lastCreatedAt}`) + } else { + console.log(`[${tableName}] loadLatestDataInBatches: No data in memory, will fetch all`) + } + + let offset = 0 + const limit = 500 + let hasMore = true + let newRecordsCount = 0 + + while (hasMore) { + console.log(`[${tableName}] loadLatestDataInBatches: Fetching batch at offset ${offset}`) + const response = await $fetch(`/api/data/${tableName}`, { + params: { + limit, + offset, + orderBy: 'created_at', + orderDirection: 'desc' + } + }) + + if (!response || !response.records || response.records.length === 0) { + console.log(`[${tableName}] loadLatestDataInBatches: No response or records, stopping`) + hasMore = false + break + } + + console.log(`[${tableName}] loadLatestDataInBatches: Got ${response.records.length} records`) + + // Filter only records newer than what we have + const newRecords = lastCreatedAt + ? response.records.filter((r: any) => r.created_at > lastCreatedAt) + : response.records + + console.log(`[${tableName}] loadLatestDataInBatches: ${newRecords.length} new records after filtering`) + + if (newRecords.length > 0) { + this.data.unshift(...(newRecords as T[])) + newRecordsCount += newRecords.length + console.log(`[${tableName}] loadLatestDataInBatches: Total new records: ${newRecordsCount}, data.length: ${this.data.length}`) + } + + // Update progress + if (onProgress) { + onProgress(Math.min(95, (newRecordsCount / 500) * 100)) + } + + // Stop if we found records older than our last one + if (lastCreatedAt && response.records.some((r: any) => r.created_at <= lastCreatedAt)) { + console.log(`[${tableName}] loadLatestDataInBatches: Found older records, stopping`) + hasMore = false + } else { + hasMore = response.records.length === limit + offset += limit + } + } + + this.lastUpdated = new Date().toISOString() + console.log(`[${tableName}] loadLatestDataInBatches: Added ${newRecordsCount} new records, total: ${this.data.length}`) + + // Persist to localStorage + if (process.client) { + try { + console.log(`[${tableName}] loadLatestDataInBatches: Persisting to localStorage with key: ${cacheKey}`) + const dataToSave = { + data: this.data, + lastUpdated: this.lastUpdated, + limit: this.limit + } + console.log(`[${tableName}] loadLatestDataInBatches: Saving ${dataToSave.data.length} records`) + localStorage.setItem(cacheKey, JSON.stringify(dataToSave)) + console.log(`[${tableName}] loadLatestDataInBatches: Successfully saved to localStorage`) + + // Verify it was saved + const saved = localStorage.getItem(cacheKey) + if (saved) { + const parsed = JSON.parse(saved) + console.log(`[${tableName}] loadLatestDataInBatches: Verification - localStorage contains ${parsed.data?.length || 0} records`) + } + } catch (error) { + console.error(`[${tableName}] loadLatestDataInBatches: Failed to persist data to localStorage:`, error) + } + } else { + console.log(`[${tableName}] loadLatestDataInBatches: Not on client, skipping localStorage`) + } + + if (onProgress) { + onProgress(100) + } + } catch (error: any) { + this.error = this.extractErrorMessage(error) + console.error(`[${tableName}] loadLatestDataInBatches: Error:`, error) + throw error + } finally { + this.loading = false + console.log(`[${tableName}] loadLatestDataInBatches: Completed`) + } } } }) diff --git a/nuxt4-app/server/api/data/[table]/index.get.ts b/nuxt4-app/server/api/data/[table]/index.get.ts index 6fa16af..41ec602 100644 --- a/nuxt4-app/server/api/data/[table]/index.get.ts +++ b/nuxt4-app/server/api/data/[table]/index.get.ts @@ -12,11 +12,15 @@ export default defineEventHandler(async (event) => { const limitValue = Number.parseInt((query.limit as string) ?? '', 10) const limit = Number.isFinite(limitValue) ? Math.min(Math.max(limitValue, 1), 500) : undefined + const offsetValue = Number.parseInt((query.offset as string) ?? '', 10) + const offset = Number.isFinite(offsetValue) ? Math.max(offsetValue, 0) : undefined + const parsedQuery = parseQuerySegment(query.query as string | undefined) return await fetchTableData(table, { parsedQuery, limit, + offset, filters: { id: (query.id as string) || undefined, createdFrom: (query.created_from as string) || undefined, diff --git a/nuxt4-app/server/data-sources/vista_resumen_ingresos/config.ts b/nuxt4-app/server/data-sources/vista_resumen_ingresos/config.ts index d601ff1..53203b8 100644 --- a/nuxt4-app/server/data-sources/vista_resumen_ingresos/config.ts +++ b/nuxt4-app/server/data-sources/vista_resumen_ingresos/config.ts @@ -3,5 +3,6 @@ import type { TableConfig } from '../types' export const vistaResumenIngresosConfig: TableConfig = { name: 'vista_resumen_ingresos', table: 'vista_resumen_ingresos', - primaryKey: 'id' + primaryKey: 'id', + defaultSelect: '*, fecha as created_at' } \ No newline at end of file diff --git a/nuxt4-app/server/data-sources/vista_resumen_ingresos_por_comercio/config.ts b/nuxt4-app/server/data-sources/vista_resumen_ingresos_por_comercio/config.ts index 418f68d..2b4decf 100644 --- a/nuxt4-app/server/data-sources/vista_resumen_ingresos_por_comercio/config.ts +++ b/nuxt4-app/server/data-sources/vista_resumen_ingresos_por_comercio/config.ts @@ -3,5 +3,6 @@ import type { TableConfig } from '../types' export const vistaResumenIngresosPorComercioConfig: TableConfig = { name: 'vista_resumen_ingresos_por_comercio', table: 'vista_resumen_ingresos_por_comercio', - primaryKey: 'id' + primaryKey: 'id', + defaultSelect: '*, fecha as created_at' } \ No newline at end of file diff --git a/nuxt4-app/server/services/table-service.ts b/nuxt4-app/server/services/table-service.ts index ddc6401..39a7920 100644 --- a/nuxt4-app/server/services/table-service.ts +++ b/nuxt4-app/server/services/table-service.ts @@ -23,6 +23,7 @@ interface DataOptions { parsedQuery?: ParsedQuery | null filters?: DataFilters limit?: number + offset?: number } function serializeRow(row: unknown, config: TableConfig): GenericObject | null { @@ -261,6 +262,10 @@ export async function fetchTableData(tableName: string, options?: DataOptions) { query = query.limit(limit) } + if (options?.offset !== undefined && options.offset > 0) { + query = query.range(options.offset, options.offset + limit - 1) + } + const { data, error, count } = await query if (error) {