perf: export xlsx

packages/core/database/src/repository.ts

@@ -47,6 +47,7 @@ import { RelationRepository } from './relation-repository/relation-repository';
 import { updateAssociations, updateModelByValues } from './update-associations';
 import { UpdateGuard } from './update-guard';
 import { valuesToFilter } from './utils/filter-utils';
+import _ from 'lodash';
 
 const debug = require('debug')('noco-database');
 
@@ -397,6 +398,51 @@ export class Repository<TModelAttributes extends {} = any, TCreationAttributes e
     }
   }
 
+  /**
+   * Cursor-based pagination query function.
+   * Ideal for large datasets (e.g., millions of rows)
+   * Note:
+   *  1. does not support jumping to arbitrary pages (e.g., "Page 5")
+   *  2. Requires a stable, indexed sort field (e.g. ID, createdAt)
+   *  3. If custom orderBy is used, it must match the cursor field(s) and direction, otherwise results may be incorrect or unstable.
+   * @param options
+   */
+  async chunkWithCursor(
+    options: FindOptions & {
+      chunkSize: number;
+      callback: (rows: Model[], options: FindOptions) => Promise<void>;
+      beforeFind?: (options: FindOptions) => Promise<void>;
+      afterFind?: (rows: Model[], options: FindOptions) => Promise<void>;
+    },
+  ) {
+    const index = this.collection.model.primaryKeyAttribute || this.collection.model['_indexes'][0];
+    let cursor = null;
+    let hasMoreData = true;
+    let isFirst = true;
+    while (hasMoreData) {
+      if (!isFirst) {
+        options.where = { ...options.where, [index]: { [Op.gt]: cursor } };
+      }
+      if (isFirst) {
+        isFirst = false;
+      }
+      options.limit = options.chunkSize || 1000;
+      if (options.beforeFind) {
+        await options.beforeFind(options);
+      }
+      const records = await this.find(options);
+      if (options.afterFind) {
+        await options.afterFind(records, options);
+      }
+      if (records.length === 0) {
+        hasMoreData = false;
+        continue;
+      }
+      await options.callback(records, options);
+      cursor = records[records.length - 1][index];
+    }
+  }
+
   /**
    * find
    * @param options

packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts

@@ -11,7 +11,6 @@ import { Context, Next } from '@nocobase/actions';
 import { Repository } from '@nocobase/database';
 
 import { XlsxExporter } from '../services/xlsx-exporter';
-import XLSX from 'xlsx';
 import { Mutex } from 'async-mutex';
 import { DataSource } from '@nocobase/data-source-manager';
 import { Logger } from '@nocobase/logger';
@@ -45,9 +44,11 @@ async function exportXlsxAction(ctx: Context, next: Next, logger: Logger) {
     },
   });
 
-  const wb = await xlsxExporter.run(ctx);
   try {
-    ctx.body = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' });
+    await xlsxExporter.run(ctx);
+    const buffer = xlsxExporter.getXlsxBuffer();
+    xlsxExporter.cleanOutputFile();
+    ctx.body = buffer;
   } catch (error) {
     logger.error('Error writing XLSX file:', error);
     throw error;

packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts

@@ -67,20 +67,21 @@ abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends Eve
       await this.init(ctx);
 
       const { collection, chunkSize, repository } = this.options;
-
-      const total = (await (repository || collection.repository).count(this.getFindOptions())) as number;
+      const repo = repository || collection.repository;
+      const total = (await repo.count(this.getFindOptions())) as number;
       this.logger?.info(`Found ${total} records to export from collection [${collection.name}]`);
       const totalCountStartTime = process.hrtime();
       let current = 0;
-      this.logger?.debug(`findOptions: ${JSON.stringify(this.getFindOptions())}`);
-      await (repository || collection.repository).chunk({
+
+      // gt 200000, offset + limit will be slow,so use cursor
+      const chunkHandle = (total > 200000 ? repo.chunkWithCursor : repo.chunk).bind(repo);
+      const findOptions = {
         ...this.getFindOptions(),
         chunkSize: chunkSize || 200,
         beforeFind: async (options) => {
           this._batchQueryStartTime = process.hrtime();
         },
         afterFind: async (rows, options) => {
-          this.logger?.debug(`findOptions123: ${JSON.stringify(options)}`);
           if (this._batchQueryStartTime) {
             const diff = process.hrtime(this._batchQueryStartTime);
             const executionTime = (diff[0] * 1000 + diff[1] / 1000000).toFixed(2);
@@ -105,13 +106,11 @@ abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends Eve
             if (Number(executionTime) > 500) {
               this.logger?.debug(`HandleRow took too long, completed in ${executionTime}ms`);
             }
-            current += 1;
-
+          }
           this.emit('progress', {
             total,
-              current,
+            current: (current += rows.length),
           });
-          }
           const diff = process.hrtime(totalCountStartTime);
           const currentTotalCountTime = (diff[0] * 1000 + diff[1] / 1000000).toFixed(2);
           this.logger?.info(
@@ -120,8 +119,8 @@ abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends Eve
             )}%), totalCountTime: ${currentTotalCountTime}ms`,
           );
         },
-      });
-
+      };
+      await chunkHandle(findOptions);
       this.logger?.info(`Export completed...... processed ${current} records in total`);
       return this.finalize();
     } catch (error) {

packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts

@@ -1,6 +1,15 @@
-import XLSX from 'xlsx';
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import * as Excel from 'exceljs';
 import { BaseExporter, ExportOptions } from './base-exporter';
-import { NumberField } from '@nocobase/database';
+import fs from 'fs';
 
 type ExportColumn = {
   dataIndex: Array<string>;
@@ -25,9 +34,10 @@ export class XlsxExporter extends BaseExporter<XlsxExportOptions & { fields: Arr
    * 服务端进程被操作系统回收等问题。
    */
 
-  private workbook: XLSX.WorkBook;
-  private worksheet: XLSX.WorkSheet;
-  private startRowNumber: number;
+  private workbook: Excel.stream.xlsx.WorkbookWriter;
+  private worksheet: Excel.Worksheet;
+
+  public outputPath: string;
 
   constructor(options: XlsxExportOptions) {
     const fields = options.columns.map((col) => col.dataIndex);
@@ -35,61 +45,43 @@ export class XlsxExporter extends BaseExporter<XlsxExportOptions & { fields: Arr
   }
 
   async init(ctx?): Promise<void> {
-    this.workbook = XLSX.utils.book_new();
-    this.worksheet = XLSX.utils.sheet_new();
-
-    // write headers
-    XLSX.utils.sheet_add_aoa(this.worksheet, [this.renderHeaders(this.options.columns)], {
-      origin: 'A1',
+    this.outputPath = this.generateOutputPath('xlsx', '.xlsx');
+    this.workbook = new Excel.stream.xlsx.WorkbookWriter({
+      filename: this.outputPath,
+      useStyles: true,
+      useSharedStrings: false, // 减少内存使用
     });
-
-    this.startRowNumber = 2;
+    this.worksheet = this.workbook.addWorksheet('Data', {
+      properties: { defaultRowHeight: 20 },
+    });
+    this.worksheet.columns = this.options.columns.map((x) => ({
+      key: x.dataIndex[0],
+      header: x.title || x.defaultTitle,
+    }));
+    this.worksheet.getRow(1).font = { bold: true };
+    this.worksheet.getRow(1).commit();
   }
 
   async handleRow(row: any, ctx?): Promise<void> {
-    const rowData = [
-      this.options.columns.map((col) => {
+    const rowData = this.options.columns.map((col) => {
       return this.formatValue(row, col.dataIndex, ctx);
-      }),
-    ];
-
-    XLSX.utils.sheet_add_aoa(this.worksheet, rowData, {
-      origin: `A${this.startRowNumber}`,
     });
 
-    this.startRowNumber += 1;
+    this.worksheet.addRow(rowData).commit();
   }
 
-  async finalize(): Promise<XLSX.WorkBook> {
-    for (const col of this.options.columns) {
-      const fieldInstance = this.findFieldByDataIndex(col.dataIndex);
-      if (fieldInstance instanceof NumberField) {
-        // set column cell type to number
-        const colIndex = this.options.columns.indexOf(col);
-        const cellRange = XLSX.utils.decode_range(this.worksheet['!ref']);
-
-        for (let r = 1; r <= cellRange.e.r; r++) {
-          const cell = this.worksheet[XLSX.utils.encode_cell({ c: colIndex, r })];
-          // if cell and cell.v is a number, set cell.t to 'n'
-          if (cell && isNumeric(cell.v)) {
-            cell.t = 'n';
-          }
-        }
-      }
-    }
-
-    XLSX.utils.book_append_sheet(this.workbook, this.worksheet, 'Data');
+  async finalize(): Promise<any> {
+    await this.worksheet.commit();
+    await this.workbook.commit();
     return this.workbook;
   }
 
-  private renderHeaders(columns: Array<ExportColumn>) {
-    return columns.map((col) => {
-      const fieldInstance = this.findFieldByDataIndex(col.dataIndex);
-      return col.title || fieldInstance?.options.title || col.defaultTitle;
-    });
-  }
+  cleanOutputFile() {
+    fs.unlink(this.outputPath, (err) => {});
   }
 
-function isNumeric(n) {
-  return !isNaN(parseFloat(n)) && isFinite(n);
+  getXlsxBuffer() {
+    const buffer = fs.readFileSync(this.outputPath);
+    return buffer;
+  }
 }