Merge branch 'main' into next

packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts

@@ -1,3 +1,12 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
 import {
   FindOptions,
   ICollection,
@@ -10,6 +19,7 @@ import EventEmitter from 'events';
 import { deepGet } from '../utils/deep-get';
 import path from 'path';
 import os from 'os';
+import _ from 'lodash';
 
 export type ExportOptions = {
   collectionManager: ICollectionManager;
@@ -49,11 +59,11 @@ abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends Eve
 
     const { collection, chunkSize, repository } = this.options;
 
-    const total = await (repository || collection.repository).count(this.getFindOptions());
+    const total = await (repository || collection.repository).count(this.getFindOptions(ctx));
     let current = 0;
 
     await (repository || collection.repository).chunk({
-      ...this.getFindOptions(),
+      ...this.getFindOptions(ctx),
       chunkSize: chunkSize || 200,
       callback: async (rows, options) => {
         for (const row of rows) {
@@ -71,31 +81,34 @@ abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends Eve
     return this.finalize();
   }
 
-  protected getAppendOptionsFromFields() {
+  protected getAppendOptionsFromFields(ctx?) {
-    return this.options.fields
+    const fields = this.options.fields.map((x) => x[0]);
+    const hasPermissionFields = _.isEmpty(ctx?.permission?.can?.params)
+      ? fields
+      : _.intersection(ctx?.permission?.can?.params?.appends || [], fields);
+    return hasPermissionFields
       .map((field) => {
-        const fieldInstance = this.options.collection.getField(field[0]);
+        const fieldInstance = this.options.collection.getField(field);
         if (!fieldInstance) {
-          throw new Error(`Field "${field[0]}" not found: , please check the fields configuration.`);
+          throw new Error(`Field "${field}" not found: , please check the fields configuration.`);
         }
 
         if (fieldInstance.isRelationField()) {
-          return field.join('.');
+          return field;
         }
 
         return null;
       })
       .filter(Boolean);
   }
-
+  protected getFindOptions(ctx?) {
-  protected getFindOptions() {
     const { findOptions = {} } = this.options;
 
     if (this.limit) {
       findOptions.limit = this.limit;
     }
 
-    const appendOptions = this.getAppendOptionsFromFields();
+    const appendOptions = this.getAppendOptionsFromFields(ctx);
 
     if (appendOptions.length) {
       return {

packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts

@@ -2156,6 +2156,76 @@ describe('xlsx importer', () => {
     expect(await Post.repository.count()).toBe(1);
   });
 
+  it('should filter no permission columns', async () => {
+    const User = app.db.collection({
+      name: 'users',
+      fields: [
+        {
+          type: 'string',
+          name: 'name',
+        },
+        {
+          type: 'string',
+          name: 'email',
+        },
+      ],
+    });
+
+    await app.db.sync();
+
+    const templateCreator = new TemplateCreator({
+      collection: User,
+      explain: 'test',
+      columns: [
+        {
+          dataIndex: ['name'],
+          defaultTitle: '姓名',
+        },
+        {
+          dataIndex: ['email'],
+          defaultTitle: '邮箱',
+        },
+      ],
+    });
+
+    const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
+
+    const worksheet = template.Sheets[template.SheetNames[0]];
+
+    XLSX.utils.sheet_add_aoa(worksheet, [['User1', 'test@test.com']], {
+      origin: 'A3',
+    });
+
+    const importer = new XlsxImporter({
+      collectionManager: app.mainDataSource.collectionManager,
+      collection: User,
+      explain: 'test',
+      columns: [
+        {
+          dataIndex: ['name'],
+          defaultTitle: '姓名',
+        },
+        {
+          dataIndex: ['email'],
+          defaultTitle: '邮箱',
+        },
+      ],
+      workbook: template,
+    });
+    await importer.run({
+      context: {
+        permission: {
+          can: { params: { fields: ['name'] } },
+        },
+      },
+    });
+
+    expect(await User.repository.count()).toBe(1);
+    const user = await User.repository.findOne();
+    expect(user.get('name')).toBe('User1');
+    expect(user.get('email')).not.exist;
+  });
+
   it('should import time field successfully', async () => {
     const TimeCollection = app.db.collection({
       name: 'time_tests',

packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts

@@ -14,6 +14,8 @@ import { Collection as DBCollection, Database } from '@nocobase/database';
 import { Transaction } from 'sequelize';
 import EventEmitter from 'events';
 import { ImportValidationError, ImportError } from '../errors';
+import { Context } from '@nocobase/actions';
+import _ from 'lodash';
 
 export type ImportColumn = {
   dataIndex: Array<string>;
@@ -54,8 +56,9 @@ export class XlsxImporter extends EventEmitter {
     this.repository = options.repository ? options.repository : options.collection.repository;
   }
 
-  async validate() {
+  async validate(ctx?: Context) {
-    if (this.options.columns.length == 0) {
+    const columns = this.getColumnsByPermission(ctx);
+    if (columns.length == 0) {
       throw new ImportValidationError('Columns configuration is empty');
     }
 
@@ -66,7 +69,7 @@ export class XlsxImporter extends EventEmitter {
       }
     }
 
-    const data = await this.getData();
+    const data = await this.getData(ctx);
     return data;
   }
 
@@ -80,7 +83,7 @@ export class XlsxImporter extends EventEmitter {
     }
 
     try {
-      await this.validate();
+      await this.validate(options.context);
       const imported = await this.performImport(options);
 
       // @ts-ignore
@@ -111,7 +114,7 @@ export class XlsxImporter extends EventEmitter {
     }
 
     let hasImportedAutoIncrementPrimary = false;
-    for (const importedDataIndex of this.options.columns) {
+    for (const importedDataIndex of this.getColumnsByPermission(options?.context)) {
       if (importedDataIndex.dataIndex[0] === autoIncrementAttribute) {
         hasImportedAutoIncrementPrimary = true;
         break;
@@ -150,9 +153,18 @@ export class XlsxImporter extends EventEmitter {
     this.emit('seqReset', { maxVal, seqName: autoIncrInfo.seqName });
   }
 
+  private getColumnsByPermission(ctx: Context): ImportColumn[] {
+    const columns = this.options.columns;
+    return columns.filter((x) =>
+      _.isEmpty(ctx?.permission?.can?.params)
+        ? true
+        : _.includes(ctx?.permission?.can?.params?.fields || [], x.dataIndex[0]),
+    );
+  }
+
   async performImport(options?: RunOptions): Promise<any> {
     const transaction = options?.transaction;
-    const data = await this.getData();
+    const data = await this.getData(options?.context);
     const chunks = lodash.chunk(data.slice(1), this.options.chunkSize || 200);
 
     let handingRowIndex = 1;
@@ -271,25 +283,26 @@ export class XlsxImporter extends EventEmitter {
     return str;
   }
 
-  private getExpectedHeaders(): string[] {
+  private getExpectedHeaders(ctx?: Context): string[] {
-    return this.options.columns.map((col) => col.title || col.defaultTitle);
+    const columns = this.getColumnsByPermission(ctx);
+    return columns.map((col) => col.title || col.defaultTitle);
   }
 
-  async getData() {
+  async getData(ctx?: Context) {
     const workbook = this.options.workbook;
     const worksheet = workbook.Sheets[workbook.SheetNames[0]];
 
-    const data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][];
+    let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][];
 
     // Find and validate header row
-    const expectedHeaders = this.getExpectedHeaders();
+    const expectedHeaders = this.getExpectedHeaders(ctx);
-    const { headerRowIndex, headers } = this.findAndValidateHeaders(data);
+    const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders });
     if (headerRowIndex === -1) {
       throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', {
         headers: expectedHeaders.join(', '),
       });
     }
-
+    data = this.alignWithHeaders({ data, expectedHeaders, headers });
     // Extract data rows
     const rows = data.slice(headerRowIndex + 1);
 
@@ -301,8 +314,18 @@ export class XlsxImporter extends EventEmitter {
     return [headers, ...rows];
   }
 
-  private findAndValidateHeaders(data: string[][]): { headerRowIndex: number; headers: string[] } {
+  private alignWithHeaders(params: { headers: string[]; expectedHeaders: string[]; data: string[][] }): string[][] {
-    const expectedHeaders = this.getExpectedHeaders();
+    const { expectedHeaders, headers, data } = params;
+    const keepCols = headers.map((x, i) => (expectedHeaders.includes(x) ? i : -1)).filter((i) => i > -1);
+
+    return data.map((row) => keepCols.map((i) => row[i]));
+  }
+
+  private findAndValidateHeaders(options: { expectedHeaders: string[]; data: string[][] }): {
+    headerRowIndex: number;
+    headers: string[];
+  } {
+    const { data, expectedHeaders } = options;
 
     // Find header row and validate
     for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
@@ -310,31 +333,11 @@ export class XlsxImporter extends EventEmitter {
       const actualHeaders = row.filter((cell) => cell !== null && cell !== '');
 
       const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header));
-      const noExtraHeaders = actualHeaders.length === expectedHeaders.length;
 
-      if (allHeadersFound && noExtraHeaders) {
+      if (allHeadersFound) {
-        const mismatchIndex = expectedHeaders.findIndex((title, index) => actualHeaders[index] !== title);
+        const orderedHeaders = expectedHeaders.filter((h) => actualHeaders.includes(h));
-
+        return { headerRowIndex: rowIndex, headers: orderedHeaders };
-        if (mismatchIndex === -1) {
-          // All headers match
-          return { headerRowIndex: rowIndex, headers: actualHeaders };
-        } else {
-          // Found potential header row but with mismatch
-          throw new ImportValidationError(
-            'Header mismatch at column {{column}}: expected "{{expected}}", but got "{{actual}}"',
-            {
-              column: mismatchIndex + 1,
-              expected: expectedHeaders[mismatchIndex],
-              actual: actualHeaders[mismatchIndex] || 'empty',
-            },
-          );
-        }
       }
     }
-
-    // No row with matching headers found
-    throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', {
-      headers: expectedHeaders.join(', '),
-    });
   }
 }