From 22800b08a149285be53fad9ea7730e9cb4f41dc5 Mon Sep 17 00:00:00 2001 From: aaaaaajie Date: Thu, 24 Apr 2025 12:40:41 +0800 Subject: [PATCH] perf: export xlsx --- packages/core/database/src/repository.ts | 46 ++++++++++ .../src/server/actions/export-xlsx.ts | 7 +- .../src/server/services/base-exporter.ts | 25 +++-- .../src/server/services/xlsx-exporter.ts | 92 +++++++++---------- 4 files changed, 104 insertions(+), 66 deletions(-) diff --git a/packages/core/database/src/repository.ts b/packages/core/database/src/repository.ts index 34873e96ad..479b0aefa7 100644 --- a/packages/core/database/src/repository.ts +++ b/packages/core/database/src/repository.ts @@ -47,6 +47,7 @@ import { RelationRepository } from './relation-repository/relation-repository'; import { updateAssociations, updateModelByValues } from './update-associations'; import { UpdateGuard } from './update-guard'; import { valuesToFilter } from './utils/filter-utils'; +import _ from 'lodash'; const debug = require('debug')('noco-database'); @@ -397,6 +398,51 @@ export class Repository Promise; + beforeFind?: (options: FindOptions) => Promise; + afterFind?: (rows: Model[], options: FindOptions) => Promise; + }, + ) { + const index = this.collection.model.primaryKeyAttribute || this.collection.model['_indexes'][0]; + let cursor = null; + let hasMoreData = true; + let isFirst = true; + while (hasMoreData) { + if (!isFirst) { + options.where = { ...options.where, [index]: { [Op.gt]: cursor } }; + } + if (isFirst) { + isFirst = false; + } + options.limit = options.chunkSize || 1000; + if (options.beforeFind) { + await options.beforeFind(options); + } + const records = await this.find(options); + if (options.afterFind) { + await options.afterFind(records, options); + } + if (records.length === 0) { + hasMoreData = false; + continue; + } + await options.callback(records, options); + cursor = records[records.length - 1][index]; + } + } + /** * find * @param options diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts index 23c86b49a1..e52fcc5974 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts @@ -11,7 +11,6 @@ import { Context, Next } from '@nocobase/actions'; import { Repository } from '@nocobase/database'; import { XlsxExporter } from '../services/xlsx-exporter'; -import XLSX from 'xlsx'; import { Mutex } from 'async-mutex'; import { DataSource } from '@nocobase/data-source-manager'; import { Logger } from '@nocobase/logger'; @@ -45,9 +44,11 @@ async function exportXlsxAction(ctx: Context, next: Next, logger: Logger) { }, }); - const wb = await xlsxExporter.run(ctx); try { - ctx.body = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }); + await xlsxExporter.run(ctx); + const buffer = xlsxExporter.getXlsxBuffer(); + xlsxExporter.cleanOutputFile(); + ctx.body = buffer; } catch (error) { logger.error('Error writing XLSX file:', error); throw error; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts index bfd1e4e698..a3db67724c 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts @@ -67,20 +67,21 @@ abstract class BaseExporter extends Eve await this.init(ctx); const { collection, chunkSize, repository } = this.options; - - const total = (await (repository || collection.repository).count(this.getFindOptions())) as number; + const repo = repository || collection.repository; + const total = (await repo.count(this.getFindOptions())) as number; this.logger?.info(`Found ${total} records to export from collection [${collection.name}]`); const totalCountStartTime = process.hrtime(); let current = 0; - this.logger?.debug(`findOptions: ${JSON.stringify(this.getFindOptions())}`); - await (repository || collection.repository).chunk({ + + // gt 200000, offset + limit will be slow,so use cursor + const chunkHandle = (total > 200000 ? repo.chunkWithCursor : repo.chunk).bind(repo); + const findOptions = { ...this.getFindOptions(), chunkSize: chunkSize || 200, beforeFind: async (options) => { this._batchQueryStartTime = process.hrtime(); }, afterFind: async (rows, options) => { - this.logger?.debug(`findOptions123: ${JSON.stringify(options)}`); if (this._batchQueryStartTime) { const diff = process.hrtime(this._batchQueryStartTime); const executionTime = (diff[0] * 1000 + diff[1] / 1000000).toFixed(2); @@ -105,13 +106,11 @@ abstract class BaseExporter extends Eve if (Number(executionTime) > 500) { this.logger?.debug(`HandleRow took too long, completed in ${executionTime}ms`); } - current += 1; - - this.emit('progress', { - total, - current, - }); } + this.emit('progress', { + total, + current: (current += rows.length), + }); const diff = process.hrtime(totalCountStartTime); const currentTotalCountTime = (diff[0] * 1000 + diff[1] / 1000000).toFixed(2); this.logger?.info( @@ -120,8 +119,8 @@ abstract class BaseExporter extends Eve )}%), totalCountTime: ${currentTotalCountTime}ms`, ); }, - }); - + }; + await chunkHandle(findOptions); this.logger?.info(`Export completed...... processed ${current} records in total`); return this.finalize(); } catch (error) { diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts index 900837d3fc..d81bafe4a9 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts @@ -1,6 +1,15 @@ -import XLSX from 'xlsx'; +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import * as Excel from 'exceljs'; import { BaseExporter, ExportOptions } from './base-exporter'; -import { NumberField } from '@nocobase/database'; +import fs from 'fs'; type ExportColumn = { dataIndex: Array; @@ -25,9 +34,10 @@ export class XlsxExporter extends BaseExporter col.dataIndex); @@ -35,61 +45,43 @@ export class XlsxExporter extends BaseExporter { - this.workbook = XLSX.utils.book_new(); - this.worksheet = XLSX.utils.sheet_new(); - - // write headers - XLSX.utils.sheet_add_aoa(this.worksheet, [this.renderHeaders(this.options.columns)], { - origin: 'A1', + this.outputPath = this.generateOutputPath('xlsx', '.xlsx'); + this.workbook = new Excel.stream.xlsx.WorkbookWriter({ + filename: this.outputPath, + useStyles: true, + useSharedStrings: false, // 减少内存使用 }); - - this.startRowNumber = 2; + this.worksheet = this.workbook.addWorksheet('Data', { + properties: { defaultRowHeight: 20 }, + }); + this.worksheet.columns = this.options.columns.map((x) => ({ + key: x.dataIndex[0], + header: x.title || x.defaultTitle, + })); + this.worksheet.getRow(1).font = { bold: true }; + this.worksheet.getRow(1).commit(); } async handleRow(row: any, ctx?): Promise { - const rowData = [ - this.options.columns.map((col) => { - return this.formatValue(row, col.dataIndex, ctx); - }), - ]; - - XLSX.utils.sheet_add_aoa(this.worksheet, rowData, { - origin: `A${this.startRowNumber}`, + const rowData = this.options.columns.map((col) => { + return this.formatValue(row, col.dataIndex, ctx); }); - this.startRowNumber += 1; + this.worksheet.addRow(rowData).commit(); } - async finalize(): Promise { - for (const col of this.options.columns) { - const fieldInstance = this.findFieldByDataIndex(col.dataIndex); - if (fieldInstance instanceof NumberField) { - // set column cell type to number - const colIndex = this.options.columns.indexOf(col); - const cellRange = XLSX.utils.decode_range(this.worksheet['!ref']); - - for (let r = 1; r <= cellRange.e.r; r++) { - const cell = this.worksheet[XLSX.utils.encode_cell({ c: colIndex, r })]; - // if cell and cell.v is a number, set cell.t to 'n' - if (cell && isNumeric(cell.v)) { - cell.t = 'n'; - } - } - } - } - - XLSX.utils.book_append_sheet(this.workbook, this.worksheet, 'Data'); + async finalize(): Promise { + await this.worksheet.commit(); + await this.workbook.commit(); return this.workbook; } - private renderHeaders(columns: Array) { - return columns.map((col) => { - const fieldInstance = this.findFieldByDataIndex(col.dataIndex); - return col.title || fieldInstance?.options.title || col.defaultTitle; - }); + cleanOutputFile() { + fs.unlink(this.outputPath, (err) => {}); + } + + getXlsxBuffer() { + const buffer = fs.readFileSync(this.outputPath); + return buffer; } } - -function isNumeric(n) { - return !isNaN(parseFloat(n)) && isFinite(n); -}