From ced8af89ef8a3f20f86ecb6ab0390ebede2a6155 Mon Sep 17 00:00:00 2001 From: ajie Date: Wed, 23 Apr 2025 16:23:49 +0800 Subject: [PATCH] fix: import and export invalid when set field permissions (#6677) * fix: import and export invalid when set field permissions --- .../src/server/services/base-exporter.ts | 33 +++++--- .../server/__tests__/xlsx-importer.test.ts | 70 ++++++++++++++++ .../src/server/services/xlsx-importer.ts | 79 ++++++++++--------- 3 files changed, 134 insertions(+), 48 deletions(-) diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts index a86eea3c3d..d699a96b5d 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts @@ -1,3 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + import { FindOptions, ICollection, @@ -10,6 +19,7 @@ import EventEmitter from 'events'; import { deepGet } from '../utils/deep-get'; import path from 'path'; import os from 'os'; +import _ from 'lodash'; export type ExportOptions = { collectionManager: ICollectionManager; @@ -49,11 +59,11 @@ abstract class BaseExporter extends Eve const { collection, chunkSize, repository } = this.options; - const total = await (repository || collection.repository).count(this.getFindOptions()); + const total = await (repository || collection.repository).count(this.getFindOptions(ctx)); let current = 0; await (repository || collection.repository).chunk({ - ...this.getFindOptions(), + ...this.getFindOptions(ctx), chunkSize: chunkSize || 200, callback: async (rows, options) => { for (const row of rows) { @@ -71,31 +81,34 @@ abstract class BaseExporter extends Eve return this.finalize(); } - protected getAppendOptionsFromFields() { - return this.options.fields + protected getAppendOptionsFromFields(ctx?) { + const fields = this.options.fields.map((x) => x[0]); + const hasPermissionFields = _.isEmpty(ctx?.permission?.can?.params) + ? fields + : _.intersection(ctx?.permission?.can?.params?.appends || [], fields); + return hasPermissionFields .map((field) => { - const fieldInstance = this.options.collection.getField(field[0]); + const fieldInstance = this.options.collection.getField(field); if (!fieldInstance) { - throw new Error(`Field "${field[0]}" not found: , please check the fields configuration.`); + throw new Error(`Field "${field}" not found: , please check the fields configuration.`); } if (fieldInstance.isRelationField()) { - return field.join('.'); + return field; } return null; }) .filter(Boolean); } - - protected getFindOptions() { + protected getFindOptions(ctx?) { const { findOptions = {} } = this.options; if (this.limit) { findOptions.limit = this.limit; } - const appendOptions = this.getAppendOptionsFromFields(); + const appendOptions = this.getAppendOptionsFromFields(ctx); if (appendOptions.length) { return { diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts index 21153d3760..5f1fff7559 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts @@ -2156,6 +2156,76 @@ describe('xlsx importer', () => { expect(await Post.repository.count()).toBe(1); }); + it('should filter no permission columns', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + explain: 'test', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + }); + + const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook; + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa(worksheet, [['User1', 'test@test.com']], { + origin: 'A3', + }); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + explain: 'test', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook: template, + }); + await importer.run({ + context: { + permission: { + can: { params: { fields: ['name'] } }, + }, + }, + }); + + expect(await User.repository.count()).toBe(1); + const user = await User.repository.findOne(); + expect(user.get('name')).toBe('User1'); + expect(user.get('email')).not.exist; + }); + it('should import time field successfully', async () => { const TimeCollection = app.db.collection({ name: 'time_tests', diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts index 6641efcee6..1e850ab8cd 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts @@ -14,6 +14,8 @@ import { Collection as DBCollection, Database } from '@nocobase/database'; import { Transaction } from 'sequelize'; import EventEmitter from 'events'; import { ImportValidationError, ImportError } from '../errors'; +import { Context } from '@nocobase/actions'; +import _ from 'lodash'; export type ImportColumn = { dataIndex: Array; @@ -54,8 +56,9 @@ export class XlsxImporter extends EventEmitter { this.repository = options.repository ? options.repository : options.collection.repository; } - async validate() { - if (this.options.columns.length == 0) { + async validate(ctx?: Context) { + const columns = this.getColumnsByPermission(ctx); + if (columns.length == 0) { throw new ImportValidationError('Columns configuration is empty'); } @@ -66,7 +69,7 @@ export class XlsxImporter extends EventEmitter { } } - const data = await this.getData(); + const data = await this.getData(ctx); return data; } @@ -80,7 +83,7 @@ export class XlsxImporter extends EventEmitter { } try { - await this.validate(); + await this.validate(options.context); const imported = await this.performImport(options); // @ts-ignore @@ -111,7 +114,7 @@ export class XlsxImporter extends EventEmitter { } let hasImportedAutoIncrementPrimary = false; - for (const importedDataIndex of this.options.columns) { + for (const importedDataIndex of this.getColumnsByPermission(options?.context)) { if (importedDataIndex.dataIndex[0] === autoIncrementAttribute) { hasImportedAutoIncrementPrimary = true; break; @@ -150,9 +153,18 @@ export class XlsxImporter extends EventEmitter { this.emit('seqReset', { maxVal, seqName: autoIncrInfo.seqName }); } + private getColumnsByPermission(ctx: Context): ImportColumn[] { + const columns = this.options.columns; + return columns.filter((x) => + _.isEmpty(ctx?.permission?.can?.params) + ? true + : _.includes(ctx?.permission?.can?.params?.fields || [], x.dataIndex[0]), + ); + } + async performImport(options?: RunOptions): Promise { const transaction = options?.transaction; - const data = await this.getData(); + const data = await this.getData(options?.context); const chunks = lodash.chunk(data.slice(1), this.options.chunkSize || 200); let handingRowIndex = 1; @@ -271,25 +283,26 @@ export class XlsxImporter extends EventEmitter { return str; } - private getExpectedHeaders(): string[] { - return this.options.columns.map((col) => col.title || col.defaultTitle); + private getExpectedHeaders(ctx?: Context): string[] { + const columns = this.getColumnsByPermission(ctx); + return columns.map((col) => col.title || col.defaultTitle); } - async getData() { + async getData(ctx?: Context) { const workbook = this.options.workbook; const worksheet = workbook.Sheets[workbook.SheetNames[0]]; - const data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][]; + let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][]; // Find and validate header row - const expectedHeaders = this.getExpectedHeaders(); - const { headerRowIndex, headers } = this.findAndValidateHeaders(data); + const expectedHeaders = this.getExpectedHeaders(ctx); + const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders }); if (headerRowIndex === -1) { throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', { headers: expectedHeaders.join(', '), }); } - + data = this.alignWithHeaders({ data, expectedHeaders, headers }); // Extract data rows const rows = data.slice(headerRowIndex + 1); @@ -301,8 +314,18 @@ export class XlsxImporter extends EventEmitter { return [headers, ...rows]; } - private findAndValidateHeaders(data: string[][]): { headerRowIndex: number; headers: string[] } { - const expectedHeaders = this.getExpectedHeaders(); + private alignWithHeaders(params: { headers: string[]; expectedHeaders: string[]; data: string[][] }): string[][] { + const { expectedHeaders, headers, data } = params; + const keepCols = headers.map((x, i) => (expectedHeaders.includes(x) ? i : -1)).filter((i) => i > -1); + + return data.map((row) => keepCols.map((i) => row[i])); + } + + private findAndValidateHeaders(options: { expectedHeaders: string[]; data: string[][] }): { + headerRowIndex: number; + headers: string[]; + } { + const { data, expectedHeaders } = options; // Find header row and validate for (let rowIndex = 0; rowIndex < data.length; rowIndex++) { @@ -310,31 +333,11 @@ export class XlsxImporter extends EventEmitter { const actualHeaders = row.filter((cell) => cell !== null && cell !== ''); const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header)); - const noExtraHeaders = actualHeaders.length === expectedHeaders.length; - if (allHeadersFound && noExtraHeaders) { - const mismatchIndex = expectedHeaders.findIndex((title, index) => actualHeaders[index] !== title); - - if (mismatchIndex === -1) { - // All headers match - return { headerRowIndex: rowIndex, headers: actualHeaders }; - } else { - // Found potential header row but with mismatch - throw new ImportValidationError( - 'Header mismatch at column {{column}}: expected "{{expected}}", but got "{{actual}}"', - { - column: mismatchIndex + 1, - expected: expectedHeaders[mismatchIndex], - actual: actualHeaders[mismatchIndex] || 'empty', - }, - ); - } + if (allHeadersFound) { + const orderedHeaders = expectedHeaders.filter((h) => actualHeaders.includes(h)); + return { headerRowIndex: rowIndex, headers: orderedHeaders }; } } - - // No row with matching headers found - throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', { - headers: expectedHeaders.join(', '), - }); } }