fm453/nocobase
1
0
Fork 0
mirror of https://gitee.com/nocobase/nocobase.git synced 2025-05-05 05:29:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: export action (#5665)

Browse Source 
* chore: export xlsx command

* chore: container service

* chore: report export progress

* chore: export

* chore: performInsert

* chore: export command

* chore: base exporter

* chore: limit option

* chore: export

* chore: export

* chore: types

* chore: ws socket with auth

* refactor: refactor exporter, change addRows method to handleRow method to support row-by-row processing

* chore: xlsx exporter

* chore: base exporter

* chore: export limit

* chore: import action

* chore: load websocket

* chore: import action

* chore: object to cli args

* chore: import options

* chore: import action

* chore: import runner

* chore: import action

* chore: import options

* chore: import options

* chore: plugin load event

* chore: test

* chore: i18n

* chore: i18n

* chore: i18n

* fix: ws auth status

* chore: cache in data source

* chore: load datasource

* chore: import with field alias

* fix: build

* fix: test

* fix: build

* fix: import schema settings

* fix: import action

* fix: import progress

* chore: template creator

* fix: import error message

* fix: import error message

* chore: test

* chore: workflow dispatch event

* chore: export alias

* fix: typo

* chore: error render

* fix: event error

* chore: send message to tags

* fix: test

* fix: import action setting
This commit is contained in:
ChengLei Shao 2024-12-31 20:16:03 +08:00 committed by GitHub
parent 98103f7ad5
commit 61e7a89067
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 1740 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -32,6 +32,7 @@ storage/plugins
storage/tar
storage/tmp
storage/print-templates
storage/cache
storage/app.watch.ts
storage/.upgrading
storage/logs-e2e

6
packages/core/app/src/index.ts
View File

@ -13,12 +13,10 @@ import { getConfig } from './config';
async function initializeGateway() {
await runPluginStaticImports();
const config = await getConfig();
await Gateway.getInstance().run({
mainAppOptions: config,
});
}
initializeGateway().catch((e) => {
console.error(e);
process.exit(1);
});
initializeGateway();

140
packages/core/client/src/application/Application.tsx
View File

@ -76,6 +76,8 @@ export interface ApplicationOptions {
}
export class Application {
public eventBus = new EventTarget();
public providers: ComponentAndProps[] = [];
public router: RouterManager;
public scopes: Record<string, any> = {};
@ -96,13 +98,15 @@ export class Application {
public schemaInitializerManager: SchemaInitializerManager;
public schemaSettingsManager: SchemaSettingsManager;
public dataSourceManager: DataSourceManager;
public name: string;
loading = true;
maintained = false;
maintaining = false;
error = null;
private wsAuthorized = false;
get pm() {
return this.pluginManager;
}
@ -110,6 +114,14 @@ export class Application {
return this.options.disableAcl;
}
get isWsAuthorized() {
return this.wsAuthorized;
}
setWsAuthorized(authorized: boolean) {
this.wsAuthorized = authorized;
}
constructor(protected options: ApplicationOptions = {}) {
this.initRequireJs();
define(this, {
@ -140,6 +152,46 @@ export class Application {
this.i18n.on('languageChanged', (lng) => {
this.apiClient.auth.locale = lng;
});
this.initListeners();
}
private initListeners() {
this.eventBus.addEventListener('auth:tokenChanged', (event: CustomEvent) => {
this.setTokenInWebSocket(event.detail.token);
});
this.eventBus.addEventListener('maintaining:end', () => {
if (this.apiClient.auth.token) {
this.setTokenInWebSocket(this.apiClient.auth.token);
}
});
}
protected setTokenInWebSocket(token: string) {
if (this.maintaining) {
return;
}
this.ws.send(
JSON.stringify({
type: 'auth:token',
payload: {
token,
},
}),
);
}
setMaintaining(maintaining: boolean) {
// if maintaining is the same, do nothing
if (this.maintaining === maintaining) {
return;
}
this.maintaining = maintaining;
if (!maintaining) {
this.eventBus.dispatchEvent(new Event('maintaining:end'));
}
}
private initRequireJs() {
@ -240,40 +292,9 @@ export class Application {
}
async load() {
let loadFailed = false;
this.ws.on('message', (event) => {
const data = JSON.parse(event.data);
console.log(data.payload);
if (data?.payload?.refresh) {
window.location.reload();
return;
}
if (data.type === 'notification') {
this.notification[data.payload?.type || 'info']({ message: data.payload?.message });
return;
}
const maintaining = data.type === 'maintaining' && data.payload.code !== 'APP_RUNNING';
if (maintaining) {
this.maintaining = true;
this.error = data.payload;
} else {
// console.log('loadFailed', loadFailed);
if (loadFailed) {
window.location.reload();
return;
}
this.maintaining = false;
this.maintained = true;
this.error = null;
}
});
this.ws.on('serverDown', () => {
this.maintaining = true;
this.maintained = false;
});
this.ws.connect();
try {
this.loading = true;
await this.loadWebSocket();
await this.pm.load();
} catch (error) {
if (this.ws.enabled) {
@ -281,7 +302,6 @@ export class Application {
setTimeout(() => resolve(null), 1000);
});
}
loadFailed = true;
const toError = (error) => {
if (typeof error?.response?.data === 'string') {
const tempElement = document.createElement('div');
@ -305,6 +325,58 @@ export class Application {
this.loading = false;
}
async loadWebSocket() {
this.eventBus.addEventListener('ws:message:authorized', () => {
this.setWsAuthorized(true);
});
this.ws.on('message', (event) => {
const data = JSON.parse(event.data);
if (data?.payload?.refresh) {
window.location.reload();
return;
}
if (data.type === 'notification') {
this.notification[data.payload?.type || 'info']({ message: data.payload?.message });
return;
}
const maintaining = data.type === 'maintaining' && data.payload.code !== 'APP_RUNNING';
if (maintaining) {
this.setMaintaining(true);
this.error = data.payload;
} else {
this.setMaintaining(false);
this.maintained = true;
this.error = null;
const type = data.type;
if (!type) {
return;
}
const eventName = `ws:message:${type}`;
this.eventBus.dispatchEvent(new CustomEvent(eventName, { detail: data.payload }));
}
});
this.ws.on('serverDown', () => {
this.maintaining = true;
this.maintained = false;
});
this.ws.on('open', () => {
const token = this.apiClient.auth.token;
if (token) {
this.setTokenInWebSocket(token);
}
});
this.ws.connect();
}
getComponent<T = any>(Component: ComponentTypeAndString<T>, isShowError = true): ComponentType<T> | undefined {
const showError = (msg: string) => isShowError && console.error(msg);
if (!Component) {

1
packages/core/client/src/application/PluginManager.ts
View File

@ -108,6 +108,7 @@ export class PluginManager {
for (const plugin of this.pluginInstances.values()) {
await plugin.load();
this.app.eventBus.dispatchEvent(new CustomEvent(`plugin:${plugin.options.name}:loaded`, { detail: plugin }));
}
}
}

2
packages/core/client/src/application/WebSocketClient.ts
View File

@ -108,6 +108,7 @@ export class WebSocketClient {
this._reconnectTimes++;
const ws = new WebSocket(this.getURL(), this.options.protocols);
let pingIntervalTimer: any;
ws.onopen = () => {
console.log('[nocobase-ws]: connected.');
this.serverDown = false;
@ -121,6 +122,7 @@ export class WebSocketClient {
}
pingIntervalTimer = setInterval(() => this.send('ping'), this.pingInterval);
this.connected = true;
this.emit('open', {});
};
ws.onerror = async () => {
// setTimeout(() => this.connect(), this.reconnectInterval);

4
packages/core/client/src/locale/zh-CN.json
View File

@ -369,7 +369,7 @@
"Hide": "隐藏",
"Enable actions": "启用操作",
"Import": "导入",
"Export": "导出",
"Export": "导出记录",
"Customize": "自定义",
"Custom": "自定义",
"Function": "Function",
@ -740,7 +740,7 @@
"Plugin starting...": "插件启动中...",
"Plugin stopping...": "插件停止中...",
"Are you sure to delete this plugin?": "确定要删除此插件吗？",
"Are you sure to disable this plugin?": "确定要禁用此插件吗?",
"Are you sure to disable this plugin?": "确定要禁用此插件吗？",
"re-download file": "重新下载文件",
"Not enabled": "未启用",
"Search plugin": "搜索插件",

4
packages/core/sdk/src/APIClient.ts
View File

@ -166,6 +166,10 @@ export class Auth {
*/
setToken(token: string) {
this.setOption('token', token);
if (this.api['app']) {
this.api['app'].eventBus.dispatchEvent(new CustomEvent('auth:tokenChanged', { detail: token }));
}
}
/**

4
packages/core/server/src/application.ts
View File

@ -73,6 +73,7 @@ import { createPubSubManager, PubSubManager, PubSubManagerOptions } from './pub-
import { SyncMessageManager } from './sync-message-manager';
import packageJson from '../package.json';
import { ServiceContainer } from './service-container';
import { availableActions } from './acl/available-action';
import { AuditManager } from './audit-manager';
@ -245,6 +246,7 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
private _maintainingStatusBeforeCommand: MaintainingCommandStatus | null;
private _actionCommand: Command;
public container = new ServiceContainer();
public lockManager: LockManager;
constructor(public options: ApplicationOptions) {
@ -774,9 +776,11 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
try {
const commandName = options?.from === 'user' ? argv[0] : argv[2];
if (!this.cli.hasCommand(commandName)) {
await this.pm.loadCommands();
}
const command = await this.cli.parseAsync(argv, options);
this.setMaintaining({

67
packages/core/server/src/gateway/index.ts
View File

@ -29,6 +29,8 @@ import { applyErrorWithArgs, getErrorWithCode } from './errors';
import { IPCSocketClient } from './ipc-socket-client';
import { IPCSocketServer } from './ipc-socket-server';
import { WSServer } from './ws-server';
import { isMainThread, workerData } from 'node:worker_threads';
import process from 'node:process';
const compress = promisify(compression());
@ -346,11 +348,14 @@ export class Gateway extends EventEmitter {
const mainApp = AppSupervisor.getInstance().bootMainApp(options.mainAppOptions);
let runArgs: any = [process.argv, { throwError: true, from: 'node' }];
if (!isMainThread) {
runArgs = [workerData.argv, { throwError: true, from: 'user' }];
}
mainApp
.runAsCLI(process.argv, {
throwError: true,
from: 'node',
})
.runAsCLI(...runArgs)
.then(async () => {
if (!isStart && !(await mainApp.isStarted())) {
await mainApp.stop({ logging: false });
@ -358,8 +363,13 @@ export class Gateway extends EventEmitter {
})
.catch(async (e) => {
if (e.code !== 'commander.helpDisplayed') {
if (!isMainThread) {
throw e;
}
mainApp.log.error(e);
}
if (!isStart && !(await mainApp.isStarted())) {
await mainApp.stop({ logging: false });
}
@ -416,6 +426,55 @@ export class Gateway extends EventEmitter {
this.wsServer = new WSServer();
this.wsServer.on('message', async ({ client, message }) => {
const app = await AppSupervisor.getInstance().getApp(client.app);
const parsedMessage = JSON.parse(message.toString());
if (!parsedMessage.type) {
return;
}
// check app has event listener
if (!app.listenerCount(`ws:setTag`)) {
app.on('ws:setTag', ({ clientId, tagKey, tagValue }) => {
this.wsServer.setClientTag(clientId, tagKey, tagValue);
});
app.on('ws:sendToTag', ({ tagKey, tagValue, message }) => {
this.wsServer.sendToConnectionsByTags(
[
{ tagName: tagKey, tagValue },
{ tagName: 'app', tagValue: app.name },
],
message,
);
});
app.on('ws:sendToTags', ({ tags, message }) => {
this.wsServer.sendToConnectionsByTags(tags, message);
});
app.on('ws:authorized', ({ clientId, userId }) => {
this.wsServer.sendToConnectionsByTags(
[
{ tagName: 'userId', tagValue: userId },
{ tagName: 'app', tagValue: app.name },
],
{ type: 'authorized' },
);
});
}
const eventName = `ws:message:${parsedMessage.type}`;
app.emit(eventName, {
clientId: client.id,
tags: [...client.tags],
payload: parsedMessage.payload,
});
});
this.server.on('upgrade', (request, socket, head) => {
const { pathname } = parse(request.url);

58
packages/core/server/src/gateway/ws-server.ts
View File

@ -8,13 +8,14 @@
*/
import { Gateway, IncomingRequest } from '../gateway';
import WebSocket, { WebSocketServer } from 'ws';
import WebSocket, { WebSocketServer as WSS } from 'ws';
import { nanoid } from 'nanoid';
import { IncomingMessage } from 'http';
import { AppSupervisor } from '../app-supervisor';
import { applyErrorWithArgs, getErrorWithCode } from './errors';
import lodash from 'lodash';
import { Logger } from '@nocobase/logger';
import EventEmitter from 'events';
declare class WebSocketWithId extends WebSocket {
id: string;
@ -22,10 +23,11 @@ declare class WebSocketWithId extends WebSocket {
interface WebSocketClient {
ws: WebSocketWithId;
tags: string[];
tags: Set<string>;
url: string;
headers: any;
app?: string;
id: string;
}
function getPayloadByErrorCode(code, options) {
@ -33,13 +35,14 @@ function getPayloadByErrorCode(code, options) {
return lodash.omit(applyErrorWithArgs(error, options), ['status', 'maintaining']);
}
export class WSServer {
export class WSServer extends EventEmitter {
wss: WebSocket.Server;
webSocketClients = new Map<string, WebSocketClient>();
logger: Logger;
constructor() {
this.wss = new WebSocketServer({ noServer: true });
super();
this.wss = new WSS({ noServer: true });
this.wss.on('connection', (ws: WebSocketWithId, request: IncomingMessage) => {
const client = this.addNewConnection(ws, request);
@ -53,18 +56,33 @@ export class WSServer {
ws.on('close', () => {
this.removeConnection(ws.id);
});
ws.on('message', (message) => {
if (message.toString() === 'ping') {
return;
}
this.emit('message', {
client,
message,
});
});
});
Gateway.getInstance().on('appSelectorChanged', () => {
// reset connection app tags
this.loopThroughConnections(async (client) => {
const handleAppName = await Gateway.getInstance().getRequestHandleAppName({
url: client.url,
headers: client.headers,
});
client.tags = client.tags.filter((tag) => !tag.startsWith('app#'));
client.tags.push(`app#${handleAppName}`);
for (const tag of client.tags) {
if (tag.startsWith('app#')) {
client.tags.delete(tag);
}
}
client.tags.add(`app#${handleAppName}`);
AppSupervisor.getInstance().bootStrapApp(handleAppName);
});
@ -121,21 +139,26 @@ export class WSServer {
addNewConnection(ws: WebSocketWithId, request: IncomingMessage) {
const id = nanoid();
ws.id = id;
this.webSocketClients.set(id, {
ws,
tags: [],
tags: new Set(),
url: request.url,
headers: request.headers,
id,
});
this.setClientApp(this.webSocketClients.get(id));
return this.webSocketClients.get(id);
}
setClientTag(clientId: string, tagKey: string, tagValue: string) {
const client = this.webSocketClients.get(clientId);
client.tags.add(`${tagKey}#${tagValue}`);
console.log(`client tags: ${Array.from(client.tags)}`);
}
async setClientApp(client: WebSocketClient) {
const req: IncomingRequest = {
url: client.url,
@ -146,7 +169,7 @@ export class WSServer {
client.app = handleAppName;
console.log(`client tags: app#${handleAppName}`);
client.tags.push(`app#${handleAppName}`);
client.tags.add(`app#${handleAppName}`);
const hasApp = AppSupervisor.getInstance().hasApp(handleAppName);
@ -191,8 +214,19 @@ export class WSServer {
}
sendToConnectionsByTag(tagName: string, tagValue: string, sendMessage: object) {
this.sendToConnectionsByTags([{ tagName, tagValue }], sendMessage);
}
/**
* Send message to clients that match all the given tag conditions
* @param tags Array of tag conditions, each condition is an object with tagName and tagValue
* @param sendMessage Message to be sent
*/
sendToConnectionsByTags(tags: Array<{ tagName: string; tagValue: string }>, sendMessage: object) {
this.loopThroughConnections((client: WebSocketClient) => {
if (client.tags.includes(`${tagName}#${tagValue}`)) {
const allTagsMatch = tags.every(({ tagName, tagValue }) => client.tags.has(`${tagName}#${tagValue}`));
if (allTagsMatch) {
this.sendMessageToConnection(client, sendMessage);
}
});

5
packages/core/server/src/plugin-manager/plugin-manager.ts
View File

@ -395,7 +395,7 @@ export class PluginManager {
const source = [];
for (const packageName of packageNames) {
const dirname = await getPluginBasePath(packageName);
const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? 'ts' : 'js'));
const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? '{ts,js}' : 'js'));
source.push(directory.replaceAll(sep, '/'));
}
@ -403,7 +403,7 @@ export class PluginManager {
if (typeof plugin === 'string') {
const { packageName } = await PluginManager.parseName(plugin);
const dirname = await getPluginBasePath(packageName);
const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? 'ts' : 'js'));
const directory = join(dirname, 'server/commands/*.' + (basename(dirname) === 'src' ? '{ts,js}' : 'js'));
source.push(directory.replaceAll(sep, '/'));
}
}
@ -411,6 +411,7 @@ export class PluginManager {
ignore: ['**/*.d.ts'],
cwd: process.env.NODE_MODULES_PATH,
});
for (const file of files) {
const callback = await importModule(file);
callback(this.app);

15
packages/core/server/src/service-container.ts Normal file
View File

@ -0,0 +1,15 @@
export class ServiceContainer {
private services: Map<string, any> = new Map();
public register<T>(name: string, service: T) {
if (typeof service === 'function') {
service = service();
}
this.services.set(name, service);
}
public get<T>(name: string): T {
return this.services.get(name);
}
}

2
packages/core/utils/src/index.ts
View File

@ -36,6 +36,6 @@ export * from './uid';
export * from './url';
export * from './i18n';
export * from './wrap-middleware';
export * from './object-to-cli-args';
export { dayjs, lodash };
export { Schema } from '@formily/json-schema';

13
packages/core/utils/src/object-to-cli-args.ts Normal file
View File

@ -0,0 +1,13 @@
/**
* Convert object to CLI arguments array
* @param obj Object to convert
* @returns CLI arguments array
*/
export function objectToCliArgs(obj: Record<string, any>): string[] {
return Object.entries(obj)
.filter(([_, value]) => value !== null && value !== undefined)
.map(([key, value]) => {
const stringValue = typeof value === 'object' ? JSON.stringify(value) : value;
return `--${key}=${stringValue}`;
});
}

3
packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx
View File

@ -54,11 +54,14 @@ export const useExportAction = () => {
content: t('Export warning', { limit: exportLimit }),
okText: t('Start export'),
});
if (!confirmed) {
return;
}
field.data.loading = true;
const { exportSettings } = lodash.cloneDeep(actionSchema?.['x-action-settings'] ?? {});
exportSettings.forEach((es) => {
const { uiSchema, interface: fieldInterface } =
getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {};

6
packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json
View File

@ -1,7 +1,7 @@
{
"Export warning": "每次最多导出 {{limit}} 行数据，超出的将被忽略。",
"Start export": "开始导出",
"another export action is running, please try again later.": "另一导出任务正在运行，请稍后重试。",
"Export warning": "每次最多导出记录 {{limit}} 行数据，超出的将被忽略。",
"Start export": "开始导出记录",
"another export action is running, please try again later.": "另一导出记录任务正在运行，请稍后重试。",
"True": "是",
"False": "否"
}

126
packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts
View File

@ -9,7 +9,7 @@
import { createMockServer, MockServer } from '@nocobase/test';
import { uid } from '@nocobase/utils';
import XlsxExporter from '../xlsx-exporter';
import { XlsxExporter } from '../services/xlsx-exporter';
import XLSX from 'xlsx';
import fs from 'fs';
import path from 'path';
@ -510,6 +510,7 @@ describe('export to xlsx', () => {
afterEach(async () => {
await app.destroy();
vi.unstubAllEnvs();
});
it('should throw error when export field not exists', async () => {
@ -1374,4 +1375,127 @@ describe('export to xlsx', () => {
fs.unlinkSync(xlsxFilePath);
}
});
it('should respect the EXPORT_LIMIT env variable', async () => {
vi.stubEnv('EXPORT_LIMIT', '30'); // Set a small limit
const User = app.db.collection({
name: 'users',
fields: [
{ type: 'string', name: 'name' },
{ type: 'integer', name: 'age' },
],
});
await app.db.sync();
const values = Array.from({ length: 100 }).map((_, index) => {
return {
name: `user${index}`,
age: index % 100,
};
});
await User.model.bulkCreate(values);
const exporter = new XlsxExporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
chunkSize: 10,
columns: [
{ dataIndex: ['name'], defaultTitle: 'Name' },
{ dataIndex: ['age'], defaultTitle: 'Age' },
],
});
const wb = await exporter.run();
const firstSheet = wb.Sheets[wb.SheetNames[0]];
const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 });
expect(sheetData.length).toBe(31); // 30 users + 1 header
expect(sheetData[0]).toEqual(['Name', 'Age']); // header
expect(sheetData[1]).toEqual(['user0', 0]); // first user
expect(sheetData[30]).toEqual(['user29', 29]); // last user
});
it('should use default EXPORT_LIMIT (2000) when env not set', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{ type: 'string', name: 'name' },
{ type: 'integer', name: 'age' },
],
});
await app.db.sync();
const values = Array.from({ length: 2500 }).map((_, index) => {
return {
name: `user${index}`,
age: index % 100,
};
});
await User.model.bulkCreate(values);
const exporter = new XlsxExporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
chunkSize: 10,
columns: [
{ dataIndex: ['name'], defaultTitle: 'Name' },
{ dataIndex: ['age'], defaultTitle: 'Age' },
],
});
const wb = await exporter.run();
const firstSheet = wb.Sheets[wb.SheetNames[0]];
const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 });
expect(sheetData.length).toBe(2001); // 2000 users + 1 header
expect(sheetData[0]).toEqual(['Name', 'Age']); // header
expect(sheetData[1]).toEqual(['user0', 0]); // first user
expect(sheetData[2000]).toEqual(['user1999', 99]); // last user
});
it('should respect the limit option when exporting data', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{ type: 'string', name: 'name' },
{ type: 'integer', name: 'age' },
],
});
await app.db.sync();
const values = Array.from({ length: 100 }).map((_, index) => {
return {
name: `user${index}`,
age: index % 100,
};
});
await User.model.bulkCreate(values);
const exporter = new XlsxExporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
chunkSize: 10,
limit: 10,
columns: [
{ dataIndex: ['name'], defaultTitle: 'Name' },
{ dataIndex: ['age'], defaultTitle: 'Age' },
],
});
const wb = await exporter.run();
const firstSheet = wb.Sheets[wb.SheetNames[0]];
const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 });
expect(sheetData.length).toBe(11); // 10 users + 1 header
expect(sheetData[0]).toEqual(['Name', 'Age']); // header
expect(sheetData[1]).toEqual(['user0', 0]); // first user
expect(sheetData[10]).toEqual(['user9', 9]); // last user
});
});

6
packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts
View File

@ -10,7 +10,7 @@
import { Context, Next } from '@nocobase/actions';
import { Repository } from '@nocobase/database';
import XlsxExporter from '../xlsx-exporter';
import { XlsxExporter } from '../services/xlsx-exporter';
import XLSX from 'xlsx';
import { Mutex } from 'async-mutex';
import { DataSource } from '@nocobase/data-source-manager';
@ -54,6 +54,10 @@ async function exportXlsxAction(ctx: Context, next: Next) {
}
export async function exportXlsx(ctx: Context, next: Next) {
if (ctx.exportHandled) {
return await next();
}
if (mutex.isLocked()) {
throw new Error(
ctx.t(`another export action is running, please try again later.`, {

4
packages/plugins/@nocobase/plugin-action-export/src/server/index.ts
View File

@ -46,9 +46,13 @@ export class PluginActionExportServer extends Plugin {
dataSource.acl.setAvailableAction('export', {
displayName: '{{t("Export")}}',
allowConfigureFields: true,
aliases: ['export', 'exportAttachments'],
});
});
}
}
export default PluginActionExportServer;
export * from './services/base-exporter';
export * from './services/xlsx-exporter';

172
packages/plugins/@nocobase/plugin-action-export/src/server/services/base-exporter.ts Normal file
View File

@ -0,0 +1,172 @@
import {
FindOptions,
ICollection,
ICollectionManager,
IField,
IModel,
IRelationField,
} from '@nocobase/data-source-manager';
import EventEmitter from 'events';
import { deepGet } from '../utils/deep-get';
import path from 'path';
import os from 'os';
export type ExportOptions = {
collectionManager: ICollectionManager;
collection: ICollection;
repository?: any;
fields: Array<Array<string>>;
findOptions?: FindOptions;
chunkSize?: number;
limit?: number;
};
abstract class BaseExporter<T extends ExportOptions = ExportOptions> extends EventEmitter {
/**
* You can adjust the maximum number of exported rows based on business needs and system
* available resources. However, please note that you need to fully understand the risks
* after the modification. Increasing the maximum number of rows that can be exported may
* increase system resource usage, leading to increased processing delays for other
* requests, or even server processes being recycled by the operating system.
*
* 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险，
* 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至
* 服务端进程被操作系统回收等问题。
*/
protected limit: number;
protected constructor(protected options: T) {
super();
this.limit = options.limit ?? (process.env['EXPORT_LIMIT'] ? parseInt(process.env['EXPORT_LIMIT']) : 2000);
}
abstract init(ctx?): Promise<void>;
abstract finalize(): Promise<any>;
abstract handleRow(row: any, ctx?): Promise<void>;
async run(ctx?): Promise<any> {
await this.init(ctx);
const { collection, chunkSize, repository } = this.options;
const total = await (repository || collection.repository).count(this.getFindOptions());
let current = 0;
await (repository || collection.repository).chunk({
...this.getFindOptions(),
chunkSize: chunkSize || 200,
callback: async (rows, options) => {
for (const row of rows) {
await this.handleRow(row, ctx);
current += 1;
this.emit('progress', {
total,
current,
});
}
},
});
return this.finalize();
}
protected getAppendOptionsFromFields() {
return this.options.fields
.map((field) => {
const fieldInstance = this.options.collection.getField(field[0]);
if (!fieldInstance) {
throw new Error(`Field "${field[0]}" not found: , please check the fields configuration.`);
}
if (fieldInstance.isRelationField()) {
return field.join('.');
}
return null;
})
.filter(Boolean);
}
protected getFindOptions() {
const { findOptions = {} } = this.options;
if (this.limit) {
findOptions.limit = this.limit;
}
const appendOptions = this.getAppendOptionsFromFields();
if (appendOptions.length) {
return {
...findOptions,
appends: appendOptions,
};
}
return findOptions;
}
protected findFieldByDataIndex(dataIndex: Array<string>): IField {
const { collection } = this.options;
const currentField = collection.getField(dataIndex[0]);
if (dataIndex.length > 1) {
let targetCollection: ICollection;
for (let i = 0; i < dataIndex.length; i++) {
const isLast = i === dataIndex.length - 1;
if (isLast) {
return targetCollection.getField(dataIndex[i]);
}
targetCollection = (currentField as IRelationField).targetCollection();
}
}
return currentField;
}
protected renderRawValue(value) {
if (typeof value === 'object' && value !== null) {
return JSON.stringify(value);
}
return value;
}
protected getFieldRenderer(field?: IField, ctx?): (value) => any {
const InterfaceClass = this.options.collectionManager.getFieldInterface(field?.options?.interface);
if (!InterfaceClass) {
return this.renderRawValue;
}
const fieldInterface = new InterfaceClass(field?.options);
return (value) => fieldInterface.toString(value, ctx);
}
protected formatValue(rowData: IModel, dataIndex: Array<string>, ctx?) {
rowData = rowData.toJSON();
const value = rowData[dataIndex[0]];
const field = this.findFieldByDataIndex(dataIndex);
const render = this.getFieldRenderer(field, ctx);
if (dataIndex.length > 1) {
const deepValue = deepGet(rowData, dataIndex);
if (Array.isArray(deepValue)) {
return deepValue.map(render).join(',');
}
return render(deepValue);
}
return render(value);
}
public generateOutputPath(prefix = 'export', ext = '', destination = os.tmpdir()): string {
const fileName = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`;
return path.join(destination, fileName);
}
}
export { BaseExporter };

95
packages/plugins/@nocobase/plugin-action-export/src/server/services/xlsx-exporter.ts Normal file
View File

@ -0,0 +1,95 @@
import XLSX from 'xlsx';
import { BaseExporter, ExportOptions } from './base-exporter';
import { NumberField } from '@nocobase/database';
type ExportColumn = {
dataIndex: Array<string>;
title?: string;
defaultTitle: string;
};
type XlsxExportOptions = Omit<ExportOptions, 'fields'> & {
columns: Array<ExportColumn>;
};
export class XlsxExporter extends BaseExporter<XlsxExportOptions & { fields: Array<Array<string>> }> {
/**
* You can adjust the maximum number of exported rows based on business needs and system
* available resources. However, please note that you need to fully understand the risks
* after the modification. Increasing the maximum number of rows that can be exported may
* increase system resource usage, leading to increased processing delays for other
* requests, or even server processes being recycled by the operating system.
*
* 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险，
* 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至
* 服务端进程被操作系统回收等问题。
*/
private workbook: XLSX.WorkBook;
private worksheet: XLSX.WorkSheet;
private startRowNumber: number;
constructor(options: XlsxExportOptions) {
const fields = options.columns.map((col) => col.dataIndex);
super({ ...options, fields });
}
async init(ctx?): Promise<void> {
this.workbook = XLSX.utils.book_new();
this.worksheet = XLSX.utils.sheet_new();
// write headers
XLSX.utils.sheet_add_aoa(this.worksheet, [this.renderHeaders(this.options.columns)], {
origin: 'A1',
});
this.startRowNumber = 2;
}
async handleRow(row: any, ctx?): Promise<void> {
const rowData = [
this.options.columns.map((col) => {
return this.formatValue(row, col.dataIndex, ctx);
}),
];
XLSX.utils.sheet_add_aoa(this.worksheet, rowData, {
origin: `A${this.startRowNumber}`,
});
this.startRowNumber += 1;
}
async finalize(): Promise<XLSX.WorkBook> {
for (const col of this.options.columns) {
const fieldInstance = this.findFieldByDataIndex(col.dataIndex);
if (fieldInstance instanceof NumberField) {
// set column cell type to number
const colIndex = this.options.columns.indexOf(col);
const cellRange = XLSX.utils.decode_range(this.worksheet['!ref']);
for (let r = 1; r <= cellRange.e.r; r++) {
const cell = this.worksheet[XLSX.utils.encode_cell({ c: colIndex, r })];
// if cell and cell.v is a number, set cell.t to 'n'
if (cell && isNumeric(cell.v)) {
cell.t = 'n';
}
}
}
}
XLSX.utils.book_append_sheet(this.workbook, this.worksheet, 'Data');
return this.workbook;
}
private renderHeaders(columns: Array<ExportColumn>) {
return columns.map((col) => {
const fieldInstance = this.findFieldByDataIndex(col.dataIndex);
return col.title || fieldInstance?.options.title || col.defaultTitle;
});
}
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}

221
packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts
View File

@ -1,221 +0,0 @@
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import {
FindOptions,
ICollection,
ICollectionManager,
IField,
IModel,
IRelationField,
} from '@nocobase/data-source-manager';
import XLSX from 'xlsx';
import { deepGet } from './utils/deep-get';
import { NumberField } from '@nocobase/database';
type ExportColumn = {
dataIndex: Array<string>;
title?: string;
defaultTitle: string;
};
type ExportOptions = {
collectionManager: ICollectionManager;
collection: ICollection;
repository?: any;
columns: Array<ExportColumn>;
findOptions?: FindOptions;
chunkSize?: number;
};
class XlsxExporter {
/**
* You can adjust the maximum number of exported rows based on business needs and system
* available resources. However, please note that you need to fully understand the risks
* after the modification. Increasing the maximum number of rows that can be exported may
* increase system resource usage, leading to increased processing delays for other
* requests, or even server processes being recycled by the operating system.
*
* 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险，
* 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至
* 服务端进程被操作系统回收等问题。
*/
limit = process.env['EXPORT_LIMIT'] ? parseInt(process.env['EXPORT_LIMIT']) : 2000;
constructor(private options: ExportOptions) {}
async run(ctx?): Promise<XLSX.WorkBook> {
const { collection, columns, chunkSize, repository } = this.options;
const workbook = XLSX.utils.book_new();
const worksheet = XLSX.utils.sheet_new();
// write headers
XLSX.utils.sheet_add_aoa(worksheet, [this.renderHeaders()], {
origin: 'A1',
});
let startRowNumber = 2;
await (repository || collection.repository).chunk({
...this.getFindOptions(),
chunkSize: chunkSize || 200,
callback: async (rows, options) => {
const chunkData = rows.map((r) => {
return columns.map((col) => {
return this.renderCellValue(r, col, ctx);
});
});
XLSX.utils.sheet_add_aoa(worksheet, chunkData, {
origin: `A${startRowNumber}`,
});
startRowNumber += rows.length;
await new Promise((resolve) => {
setTimeout(resolve, 50);
});
},
});
for (const col of columns) {
const field = this.findFieldByDataIndex(col.dataIndex);
if (field instanceof NumberField) {
// set column cell type to number
const colIndex = columns.indexOf(col);
const cellRange = XLSX.utils.decode_range(worksheet['!ref']);
for (let r = 1; r <= cellRange.e.r; r++) {
const cell = worksheet[XLSX.utils.encode_cell({ c: colIndex, r })];
// if cell and cell.v is a number, set cell.t to 'n'
if (cell && isNumeric(cell.v)) {
cell.t = 'n';
}
}
}
}
XLSX.utils.book_append_sheet(workbook, worksheet, 'Data');
return workbook;
}
private getAppendOptionsFromColumns() {
return this.options.columns
.map((col) => {
if (col.dataIndex.length > 1) {
return col.dataIndex.join('.');
}
const field = this.options.collection.getField(col.dataIndex[0]);
if (!field) {
throw new Error(`Field "${col.dataIndex[0]}" not found: , please check the columns configuration.`);
}
if (field.isRelationField()) {
return col.dataIndex[0];
}
return null;
})
.filter(Boolean);
}
private getFindOptions() {
const { findOptions = {} } = this.options;
findOptions.limit = this.limit;
const appendOptions = this.getAppendOptionsFromColumns();
if (appendOptions.length) {
return {
...findOptions,
appends: appendOptions,
};
}
return findOptions;
}
private findFieldByDataIndex(dataIndex: Array<string>): IField {
const { collection } = this.options;
const currentField = collection.getField(dataIndex[0]);
if (dataIndex.length > 1) {
let targetCollection: ICollection;
for (let i = 0; i < dataIndex.length; i++) {
const isLast = i === dataIndex.length - 1;
if (isLast) {
return targetCollection.getField(dataIndex[i]);
}
targetCollection = (currentField as IRelationField).targetCollection();
}
}
return currentField;
}
private renderHeaders() {
return this.options.columns.map((col) => {
const field = this.findFieldByDataIndex(col.dataIndex);
if (col.title) {
return col.title;
}
return field?.options.title || col.defaultTitle;
});
}
private renderRawValue(value) {
if (typeof value === 'object' && value !== null) {
return JSON.stringify(value);
}
return value;
}
private getFieldRenderer(field?: IField, ctx?): (value) => any {
const InterfaceClass = this.options.collectionManager.getFieldInterface(field?.options?.interface);
if (!InterfaceClass) {
return this.renderRawValue;
}
const fieldInternface = new InterfaceClass(field?.options);
return (value) => fieldInternface.toString(value, ctx);
}
private renderCellValue(rowData: IModel, column: ExportColumn, ctx?) {
const { dataIndex } = column;
rowData = rowData.toJSON();
const value = rowData[dataIndex[0]];
const field = this.findFieldByDataIndex(dataIndex);
const render = this.getFieldRenderer(field, ctx);
if (dataIndex.length > 1) {
const deepValue = deepGet(rowData, dataIndex);
if (Array.isArray(deepValue)) {
return deepValue.map(render).join(',');
}
return render(deepValue);
}
return render(value);
}
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
export default XlsxExporter;

3
packages/plugins/@nocobase/plugin-action-import/package.json
View File

@ -25,7 +25,8 @@
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-i18next": "^11.15.1",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
"exceljs": "^4.4.0"
},
"peerDependencies": {
"@nocobase/actions": "1.x",

130
packages/plugins/@nocobase/plugin-action-import/src/client/ImportAction.tsx Normal file
View File

@ -0,0 +1,130 @@
import { css } from '@emotion/css';
import { ISchema, useFieldSchema } from '@formily/react';
import { Action, ActionContextProvider, SchemaComponent } from '@nocobase/client';
import React, { useState } from 'react';
import { NAMESPACE } from './constants';
import { useTranslation } from 'react-i18next';
import { UploadOutlined } from '@ant-design/icons';
const importFormSchema: ISchema = {
type: 'void',
name: 'import-modal',
title: `{{ t("Import Data", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action.Modal',
'x-decorator': 'Form',
'x-component-props': {
width: '100%',
style: {
maxWidth: '750px',
},
className: css`
.ant-formily-item-label {
height: var(--controlHeightLG);
}
`,
},
properties: {
formLayout: {
type: 'void',
'x-component': 'FormLayout',
properties: {
warning: {
type: 'void',
'x-component': 'ImportWarning',
},
download: {
type: 'void',
title: `{{ t("Step 1: Download template", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'FormItem',
'x-acl-ignore': true,
properties: {
tip: {
type: 'void',
'x-component': 'DownloadTips',
},
downloadAction: {
type: 'void',
title: `{{ t("Download template", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action',
'x-component-props': {
className: css`
margin-top: 5px;
`,
useAction: '{{ useDownloadXlsxTemplateAction }}',
},
},
},
},
upload: {
type: 'array',
title: `{{ t("Step 2: Upload Excel", {ns: "${NAMESPACE}" }) }}`,
'x-decorator': 'FormItem',
'x-acl-ignore': true,
'x-component': 'Upload.Dragger',
'x-validator': '{{ uploadValidator }}',
'x-component-props': {
action: '',
height: '150px',
tipContent: `{{ t("Upload placeholder", {ns: "${NAMESPACE}" }) }}`,
beforeUpload: '{{ beforeUploadHandler }}',
},
},
},
},
footer: {
'x-component': 'Action.Modal.Footer',
'x-component-props': {},
properties: {
actions: {
type: 'void',
'x-component': 'ActionBar',
'x-component-props': {},
properties: {
cancel: {
type: 'void',
title: '{{ t("Cancel") }}',
'x-component': 'Action',
'x-component-props': {
useAction: '{{ cm.useCancelAction }}',
},
},
startImport: {
type: 'void',
title: `{{ t("Start import", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action',
'x-component-props': {
type: 'primary',
htmlType: 'submit',
useAction: '{{ useImportStartAction }}',
},
'x-reactions': {
dependencies: ['upload'],
fulfill: {
run: 'validateUpload($form, $self, $deps)',
},
},
},
},
},
},
},
},
};
export const ImportAction = (props) => {
const [visible, setVisible] = useState(false);
const { t } = useTranslation(NAMESPACE);
const fieldSchema = useFieldSchema();
return (
<ActionContextProvider value={{ visible, setVisible, fieldSchema }}>
<Action
icon={props.icon || <UploadOutlined />}
title={fieldSchema?.title || t('Import', { ns: 'action-import' })}
{...props}
onClick={() => setVisible(true)}
/>
<SchemaComponent schema={importFormSchema} />
</ActionContextProvider>
);
};

8
packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionContext.tsx Normal file
View File

@ -0,0 +1,8 @@
import React from 'react';
import { Schema } from '@formily/react';
export const ImportActionContext = React.createContext<any>({});
export const useImportActionContext = () => {
return React.useContext(ImportActionContext);
};

172
packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx
View File

@ -11,17 +11,17 @@ import type { ISchema } from '@formily/react';
import { Schema } from '@formily/react';
import { merge } from '@formily/shared';
import {
css,
SchemaInitializerItem,
useCollection_deprecated,
useSchemaInitializer,
useSchemaInitializerItem,
} from '@nocobase/client';
import { Alert } from 'antd';
import React from 'react';
import { NAMESPACE } from './constants';
import { useImportTranslation } from './locale';
import { useFields } from './useFields';
import { Alert } from 'antd';
import { lodash } from '@nocobase/utils/client';
const findSchema = (schema: Schema, key: string, action: string) => {
return schema.reduceProperties((buf, s) => {
@ -36,11 +36,38 @@ const findSchema = (schema: Schema, key: string, action: string) => {
});
};
const initImportSettings = (fields) => {
export const initImportSettings = (fields) => {
const importColumns = fields?.filter((f) => !f.children).map((f) => ({ dataIndex: [f.name] }));
return { importColumns, explain: '' };
};
export const ImportActionInitializer = () => {
const itemConfig = useSchemaInitializerItem();
const { insert } = useSchemaInitializer();
const { name } = useCollection_deprecated();
const fields = useFields(name);
const schema: ISchema = {
type: 'void',
title: '{{ t("Import") }}',
'x-component': 'ImportAction',
'x-action': 'importXlsx',
'x-settings': 'actionSettings:import',
};
return (
<SchemaInitializerItem
title={itemConfig.title}
onClick={() => {
lodash.set(schema, 'x-action-settings.importSettings', initImportSettings(fields));
const s = merge(schema || {}, itemConfig.schema || {});
itemConfig?.schemaInitialize?.(s);
insert(s);
}}
/>
);
};
export const ImportWarning = () => {
const { t } = useImportTranslation();
return <Alert type="warning" style={{ marginBottom: '10px' }} message={t('Import warnings', { limit: 2000 })} />;
@ -50,142 +77,3 @@ export const DownloadTips = () => {
const { t } = useImportTranslation();
return <Alert type="info" style={{ marginBottom: '10px', whiteSpace: 'pre-line' }} message={t('Download tips')} />;
};
export const ImportActionInitializer = () => {
const itemConfig = useSchemaInitializerItem();
const { insert } = useSchemaInitializer();
const { name } = useCollection_deprecated();
const fields = useFields(name);
const schema: ISchema = {
type: 'void',
title: '{{ t("Import") }}',
'x-action': 'importXlsx',
'x-action-settings': {
importSettings: { importColumns: [], explain: '' },
},
'x-toolbar': 'ActionSchemaToolbar',
'x-settings': 'actionSettings:import',
'x-component': 'Action',
'x-component-props': {
icon: 'CloudUploadOutlined',
openMode: 'modal',
},
properties: {
modal: {
type: 'void',
title: `{{ t("Import Data", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action.Modal',
'x-decorator': 'Form',
'x-component-props': {
width: '100%',
style: {
maxWidth: '750px',
},
className: css`
.ant-formily-item-label {
height: var(--controlHeightLG);
}
`,
},
properties: {
formLayout: {
type: 'void',
'x-component': 'FormLayout',
properties: {
warning: {
type: 'void',
'x-component': 'ImportWarning',
},
download: {
type: 'void',
title: `{{ t("Step 1: Download template", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'FormItem',
'x-acl-ignore': true,
properties: {
tip: {
type: 'void',
'x-component': 'DownloadTips',
},
downloadAction: {
type: 'void',
title: `{{ t("Download template", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action',
'x-component-props': {
className: css`
margin-top: 5px;
`,
useAction: '{{ useDownloadXlsxTemplateAction }}',
},
},
},
},
upload: {
type: 'array',
title: `{{ t("Step 2: Upload Excel", {ns: "${NAMESPACE}" }) }}`,
'x-decorator': 'FormItem',
'x-acl-ignore': true,
'x-component': 'Upload.Dragger',
'x-validator': '{{ uploadValidator }}',
'x-component-props': {
action: '',
height: '150px',
tipContent: `{{ t("Upload placeholder", {ns: "${NAMESPACE}" }) }}`,
beforeUpload: '{{ beforeUploadHandler }}',
},
},
},
},
footer: {
'x-component': 'Action.Modal.Footer',
'x-component-props': {},
properties: {
actions: {
type: 'void',
'x-component': 'ActionBar',
'x-component-props': {},
properties: {
cancel: {
type: 'void',
title: '{{ t("Cancel") }}',
'x-component': 'Action',
'x-component-props': {
useAction: '{{ cm.useCancelAction }}',
},
},
startImport: {
type: 'void',
title: `{{ t("Start import", {ns: "${NAMESPACE}" }) }}`,
'x-component': 'Action',
'x-component-props': {
type: 'primary',
htmlType: 'submit',
useAction: '{{ useImportStartAction }}',
},
'x-reactions': {
dependencies: ['upload'],
fulfill: {
run: 'validateUpload($form, $self, $deps)',
},
},
},
},
},
},
},
},
},
},
};
return (
<SchemaInitializerItem
title={itemConfig.title}
onClick={() => {
schema['x-action-settings']['importSettings'] = initImportSettings(fields);
const s = merge(schema || {}, itemConfig.schema || {});
itemConfig?.schemaInitialize?.(s);
insert(s);
}}
/>
);
};

3
packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx
View File

@ -15,12 +15,13 @@ import { ImportContext } from './context';
import { ImportModal, ImportStatus } from './ImportModal';
import { useDownloadXlsxTemplateAction, useImportStartAction } from './useImportAction';
import { useShared } from './useShared';
import { ImportAction } from './ImportAction';
export const ImportPluginProvider = (props: any) => {
const { uploadValidator, beforeUploadHandler, validateUpload } = useShared();
return (
<SchemaComponentOptions
components={{ ImportActionInitializer, ImportDesigner, ImportWarning, DownloadTips }}
components={{ ImportActionInitializer, ImportDesigner, ImportWarning, DownloadTips, ImportAction }}
scope={{
uploadValidator,
validateUpload,

11
packages/plugins/@nocobase/plugin-action-import/src/client/schemaSettings.ts → packages/plugins/@nocobase/plugin-action-import/src/client/schemaSettings.tsx
View File

@ -9,10 +9,14 @@
import { ArrayItems } from '@formily/antd-v5';
import { ISchema, useField, useFieldSchema } from '@formily/react';
import { ButtonEditor, SchemaSettings, useDesignable, useSchemaToolbar } from '@nocobase/client';
import { ButtonEditor, SchemaSettings, type, useDesignable, useSchemaToolbar } from '@nocobase/client';
import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useShared } from './useShared';
import { Button, Space } from 'antd';
import { Action } from '@nocobase/client';
import React from 'react';
import { useDownloadXlsxTemplateAction } from './useImportAction';
export const importActionSchemaSettings = new SchemaSettings({
name: 'actionSettings:import',
@ -42,7 +46,9 @@ export const importActionSchemaSettings = new SchemaSettings({
return {
title: t('Importable fields'),
schema: schema,
schema: {
...schema,
},
initialValues: { ...(fieldSchema?.['x-action-settings']?.importSettings ?? {}) },
components: { ArrayItems },
onSubmit: ({ importColumns, explain }: any) => {
@ -51,6 +57,7 @@ export const importActionSchemaSettings = new SchemaSettings({
.map((item) => ({
dataIndex: item.dataIndex.map((di) => di.name ?? di),
title: item.title,
description: item.description,
}));
fieldSchema['x-action-settings']['importSettings'] = { importColumns: columns, explain };

46
packages/plugins/@nocobase/plugin-action-import/src/client/useDownloadXlsxTemplateAction.ts Normal file
View File

@ -0,0 +1,46 @@
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import {
useAPIClient,
useRecord,
useBlockRequestContext,
useCollection_deprecated,
useCompile,
} from '@nocobase/client';
import { saveAs } from 'file-saver';
export const useDownloadXlsxTemplateAction = () => {
const { resource } = useBlockRequestContext();
const compile = useCompile();
const record = useRecord();
const { title } = useCollection_deprecated();
return {
async run() {
const { explain, importColumns } = record;
const { data } = await resource.downloadXlsxTemplate(
{
values: {
title: compile(title),
explain,
columns: compile(importColumns),
},
},
{
method: 'post',
responseType: 'blob',
},
);
const blob = new Blob([data], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' });
saveAs(blob, `${compile(title)}.xlsx`);
},
};
};

45
packages/plugins/@nocobase/plugin-action-import/src/client/useImportAction.ts
View File

@ -26,11 +26,18 @@ import { NAMESPACE } from './constants';
import { useImportContext } from './context';
import { ImportStatus } from './ImportModal';
import { useEffect } from 'react';
import { useFields } from './useFields';
import { initImportSettings } from './ImportActionInitializer';
import { useImportActionContext } from './ImportActionContext';
const useImportSchema = (s: Schema) => {
let schema = s;
const useImportSchema = () => {
const { fieldSchema: actionSchema } = useActionContext();
const fieldSchema = useFieldSchema();
let schema = actionSchema || fieldSchema;
while (schema && schema['x-action'] !== 'importXlsx') {
schema = schema.parent;
console.log('schema', schema);
}
return { schema };
};
@ -50,7 +57,8 @@ export const useDownloadXlsxTemplateAction = () => {
const { getCollectionJoinField, getCollectionField } = useCollectionManager_deprecated();
const { name, title, getField } = useCollection_deprecated();
const { t } = useTranslation(NAMESPACE);
const { schema: importSchema } = useImportSchema(actionSchema);
const { schema: importSchema } = useImportSchema();
return {
async run() {
const { importColumns, explain } = lodash.cloneDeep(
@ -103,18 +111,19 @@ export const useImportStartAction = () => {
const { getCollectionJoinField, getCollectionField } = useCollectionManager_deprecated();
const { name, title, getField } = useCollection_deprecated();
const { t } = useTranslation(NAMESPACE);
const { schema: importSchema } = useImportSchema(actionSchema);
const { schema: importSchema } = useImportSchema();
const form = useForm();
const { setVisible, fieldSchema } = useActionContext();
const { setImportModalVisible, setImportStatus, setImportResult } = useImportContext();
const { upload } = form.values;
const dataBlockProps = useDataBlockProps() || ({} as any);
const dataBlockProps = useDataBlockProps();
const headers = useDataSourceHeaders(dataBlockProps.dataSource);
const newResource = useDataBlockResource();
useEffect(() => {
form.reset();
}, []);
return {
async run() {
const { importColumns, explain } = lodash.cloneDeep(
@ -140,11 +149,30 @@ export const useImportStartAction = () => {
return column;
})
.filter(Boolean);
const formData = new FormData();
const uploadFiles = form.values.upload.map((f) => f.originFileObj);
formData.append('file', uploadFiles[0]);
formData.append('columns', JSON.stringify(columns));
formData.append('explain', explain);
const { triggerWorkflow, identifyDuplicates, uniqueField, duplicateStrategy } = form.values;
if (triggerWorkflow !== undefined) {
formData.append('triggerWorkflow', JSON.stringify(triggerWorkflow));
}
if (identifyDuplicates) {
formData.append(
'duplicateOption',
JSON.stringify({
uniqueField,
mode: duplicateStrategy,
}),
);
}
setVisible(false);
setImportModalVisible(true);
setImportStatus(ImportStatus.IMPORTING);
@ -161,8 +189,15 @@ export const useImportStartAction = () => {
setImportResult(data);
form.reset();
if (!data.data.taskId) {
setImportResult(data);
await service?.refresh?.();
setImportStatus(ImportStatus.IMPORTED);
} else {
setImportModalVisible(false);
setVisible(false);
}
} catch (error) {
setImportModalVisible(false);
setVisible(true);

16
packages/plugins/@nocobase/plugin-action-import/src/client/useShared.ts
View File

@ -73,6 +73,22 @@ export const useShared = () => {
changeOnSelect: false,
},
},
title: {
type: 'string',
'x-decorator': 'FormItem',
'x-component': 'Input',
'x-component-props': {
placeholder: '{{ t("Custom column title") }}',
},
},
description: {
type: 'string',
'x-decorator': 'FormItem',
'x-component': 'Input',
'x-component-props': {
placeholder: `{{ t("Field description placeholder", {ns: "${NAMESPACE}"}) }}`,
},
},
remove: {
type: 'void',
'x-decorator': 'FormItem',

1
packages/plugins/@nocobase/plugin-action-import/src/index.ts
View File

@ -9,6 +9,7 @@
// @ts-ignore
import { name } from '../package.json';
export * from './server';
export { default } from './server';
export const namespace = name;

16
packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json
View File

@ -8,6 +8,7 @@
"Download template": "Download template",
"Step 1: Download template": "Step 1: Download template",
"Step 2: Upload Excel": "Step 2: Upload Excel",
"Step 3: Import options": "Step 3: Import options",
"Download tips": "- Download the template and fill in the data according to the format \r\n - Import only the first worksheet \r\n - Do not change the header of the template to prevent import failure",
"Import warnings": "You can import up to {{limit}} rows of data at a time, any excess will be ignored.",
"Upload placeholder": "Drag and drop the file here or click to upload, file size should not exceed 30M",
@ -24,5 +25,18 @@
"Incorrect time format": "Incorrect time format",
"Incorrect date format": "Incorrect date format",
"Incorrect email format": "Incorrect email format",
"Illegal percentage format": "Illegal percentage format"
"Illegal percentage format": "Illegal percentage format",
"Imported template does not match, please download again.": "Imported template does not match, please download again.",
"another import action is running, please try again later.": "another import action is running, please try again later.",
"Please select": "Please select",
"Custom column title": "Custom column title",
"Field description": "Field description",
"Field description placeholder": "Enter field description",
"Columns configuration is empty": "Columns configuration is empty",
"Field not found: {{field}}": "Field not found: {{field}}",
"Headers not found. Expected headers: {{headers}}": "Headers not found. Expected headers: {{headers}}",
"Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"": "Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"",
"No data to import": "No data to import",
"Failed to import row {{row}}, {{message}}, row data: {{data}}": "Failed to import row {{row}}, {{message}}, row data: {{data}}",
"import-error": "Failed to import row {{rowIndex}}, row data: {{rowData}}, cause: {{causeMessage}}"
}

15
packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json
View File

@ -3,11 +3,13 @@
"File size cannot exceed 10M": "文件大小不能超过10M",
"Please upload the file of Excel": "请上传Excel的文件",
"Import Data": "导入数据",
"Import": "导入",
"Start import": "开始导入",
"Import explain": "说明",
"Download template": "下载模板",
"Step 1: Download template": "1.下载模板",
"Step 2: Upload Excel": "2.上传完善后的表格",
"Step 3: Import options": "3.导入选项",
"Download tips": "- 下载模板后，按格式填写数据\r\n - 只导入第一张工作表\r\n - 请勿改模板表头，防止导入失败",
"Import warnings": "每次最多导入 {{limit}} 行数据，超出的将被忽略。",
"Upload placeholder": "将文件拖曳到此处或点击上传，文件大小不超过10M",
@ -18,6 +20,7 @@
"Done": "完成",
"Yes": "是",
"No": "否",
"Please select": "请选择",
"Field {{fieldName}} does not exist": "字段 {{fieldName}} 不存在",
"can not find value": "找不到对应值",
"password is empty": "密码为空",
@ -26,5 +29,15 @@
"Incorrect email format": "邮箱格式不正确",
"Illegal percentage format": "百分比格式有误",
"Imported template does not match, please download again.": "导入模板不匹配，请检查导入文件标题行或重新下载导入模板",
"another import action is running, please try again later.": "另一导入任务正在运行，请稍后重试。"
"another import action is running, please try again later.": "另一导入任务正在运行，请稍后重试。",
"Custom column title": "自定义列标题",
"Field description": "字段说明",
"Field description placeholder": "请输入字段说明",
"Columns configuration is empty": "列配置为空",
"Field not found: {{field}}": "字段未找到：{{field}}",
"Headers not found. Expected headers: {{headers}}": "未找到表头。预期的表头：{{headers}}",
"Header mismatch at column {{column}}: expected \"{{expected}}\", but got \"{{actual}}\"": "第 {{column}} 列的表头不匹配：预期 \"{{expected}}\"，实际是 \"{{actual}}\"",
"No data to import": "没有数据可导入",
"Failed to import row {{row}}, {{message}}, row data: {{data}}": "导入第 {{row}} 行失败，{{message}}，行数据：{{data}}",
"import-error": "导入第 {{rowIndex}} 行失败，行数据：{{rowData}}, 原因：{{causeMessage}}"
}

7
packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts
View File

@ -56,7 +56,7 @@ describe('download template', () => {
],
});
const workbook = await templateCreator.run();
const workbook = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const sheet0 = workbook.Sheets[workbook.SheetNames[0]];
const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false });
@ -64,7 +64,8 @@ describe('download template', () => {
expect(explainData[0]).toEqual(explain);
const headerData = sheetData[1];
expect(headerData).toEqual(['Name', 'Email']);
expect(headerData[0]).toEqual('Name');
expect(headerData[1]).toEqual('Email');
});
it('should render template', async () => {
@ -99,7 +100,7 @@ describe('download template', () => {
],
});
const workbook = await templateCreator.run();
const workbook = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const sheet0 = workbook.Sheets[workbook.SheetNames[0]];
const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false });
const headerData = sheetData[0];

380
packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts
View File

@ -26,6 +26,85 @@ describe('xlsx importer', () => {
await app.destroy();
});
describe('validate empty data', () => {
let User;
beforeEach(async () => {
User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
],
});
await app.db.sync();
});
it('should throw error when file only has header row', async () => {
const templateCreator = new TemplateCreator({
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: 'Name',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
// template already has header row, no need to add data
const importer = new XlsxImporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: 'Name',
},
],
workbook: template,
});
await expect(importer.validate()).rejects.toThrow('No data to import');
});
it('should pass validation when file has header and data rows', async () => {
const templateCreator = new TemplateCreator({
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: 'Name',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
XLSX.utils.sheet_add_aoa(worksheet, [['Test Data 1'], ['Test Data 2']], {
origin: 'A2',
});
const importer = new XlsxImporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: 'Name',
},
],
workbook: template,
});
await expect(importer.validate()).resolves.toBeUndefined();
});
});
describe('import with date field', () => {
let User;
@ -96,7 +175,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -142,7 +221,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -188,7 +267,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -234,7 +313,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -336,7 +415,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -378,7 +457,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -420,7 +499,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -524,7 +603,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -566,7 +645,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -686,7 +765,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -737,7 +816,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -780,7 +859,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -839,7 +918,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -872,7 +951,6 @@ describe('xlsx importer', () => {
}
expect(error).toBeTruthy();
console.log(error.message);
});
it('should report validation error message on unique validation', async () => {
@ -907,7 +985,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
XLSX.utils.sheet_add_aoa(
@ -946,7 +1024,6 @@ describe('xlsx importer', () => {
}
expect(error).toBeTruthy();
console.log(error.message);
});
it('should import china region field', async () => {
@ -980,7 +1057,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1078,7 +1155,7 @@ describe('xlsx importer', () => {
columns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1168,7 +1245,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1269,7 +1346,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1372,7 +1449,7 @@ describe('xlsx importer', () => {
let error;
try {
importer.getData();
await importer.validate();
} catch (e) {
error = e;
}
@ -1409,7 +1486,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const importer = new XlsxImporter({
collectionManager: app.mainDataSource.collectionManager,
@ -1427,9 +1504,15 @@ describe('xlsx importer', () => {
workbook: template,
});
// insert a row for test
const worksheet = template.Sheets[template.SheetNames[0]];
XLSX.utils.sheet_add_aoa(worksheet, [['User1', 'test@test.com']], {
origin: 'A2',
});
let error;
try {
importer.getData();
await importer.getData();
} catch (e) {
error = e;
}
@ -1567,7 +1650,7 @@ describe('xlsx importer', () => {
columns: importColumns,
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1629,7 +1712,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1698,7 +1781,7 @@ describe('xlsx importer', () => {
],
});
const template = await templateCreator.run();
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
@ -1739,4 +1822,249 @@ describe('xlsx importer', () => {
expect(await User.repository.count()).toBe(0);
expect(error).toBeTruthy();
});
it('should import data with multiline explain and field descriptions', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
{
type: 'string',
name: 'email',
},
],
});
await app.db.sync();
const templateCreator = new TemplateCreator({
collection: User,
explain: '这是第一行说明\n这是第二行说明',
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
description: '请输入用户姓名',
},
{
dataIndex: ['email'],
defaultTitle: '邮箱',
description: '请输入有效的邮箱地址',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const headerRowIndex = templateCreator.getHeaderRowIndex();
console.log({ headerRowIndex });
const worksheet = template.Sheets[template.SheetNames[0]];
XLSX.utils.sheet_add_aoa(
worksheet,
[
['User1', 'test@test.com'],
['User2', 'test2@test.com'],
],
{
origin: `A${headerRowIndex + 1}`,
},
);
const importer = new XlsxImporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
description: '请输入用户姓名',
},
{
dataIndex: ['email'],
defaultTitle: '邮箱',
description: '请输入有效的邮箱地址',
},
],
workbook: template,
});
await importer.run();
expect(await User.repository.count()).toBe(2);
const users = await User.repository.find();
expect(users[0].get('name')).toBe('User1');
expect(users[0].get('email')).toBe('test@test.com');
expect(users[1].get('name')).toBe('User2');
expect(users[1].get('email')).toBe('test2@test.com');
});
describe('template creator', () => {
it('should create template with explain and field descriptions', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
{
type: 'string',
name: 'email',
},
],
});
const templateCreator = new TemplateCreator({
collection: User,
explain: '这是导入说明\n第二行说明',
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
description: '请输入用户姓名',
},
{
dataIndex: ['email'],
defaultTitle: '邮箱',
description: '请输入有效的邮箱地址',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][];
// 验证说明文本
expect(data[0][0]).toBe('这是导入说明');
expect(data[1][0]).toBe('第二行说明');
// 验证空行
expect(data[2][0]).toBe('');
// 验证字段描述
expect(data[3][0]).toBe('姓名：请输入用户姓名');
expect(data[4][0]).toBe('邮箱：请输入有效的邮箱地址');
// 验证表头
expect(data[5]).toEqual(['姓名', '邮箱']);
});
it('should create template with only field descriptions', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
],
});
const templateCreator = new TemplateCreator({
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
description: '请输入用户姓名',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][];
// 验证字段描述
expect(data[0][0]).toBe('姓名：请输入用户姓名');
// 验证表头
expect(data[1]).toEqual(['姓名']);
});
it('should create template with only explain', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
],
});
const templateCreator = new TemplateCreator({
collection: User,
explain: '这是导入说明',
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
const data = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as string[][];
// 验证说明文本
expect(data[0][0]).toBe('这是导入说明');
// 验证表头
expect(data[1]).toEqual(['姓名']);
});
});
it('should import data with single column', async () => {
const User = app.db.collection({
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
],
});
await app.db.sync();
const templateCreator = new TemplateCreator({
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
},
],
});
const template = (await templateCreator.run({ returnXLSXWorkbook: true })) as XLSX.WorkBook;
const worksheet = template.Sheets[template.SheetNames[0]];
XLSX.utils.sheet_add_aoa(worksheet, [['User1'], ['User2']], {
origin: 'A2',
});
const importer = new XlsxImporter({
collectionManager: app.mainDataSource.collectionManager,
collection: User,
columns: [
{
dataIndex: ['name'],
defaultTitle: '姓名',
},
],
workbook: template,
});
await importer.run();
expect(await User.repository.count()).toBe(2);
const users = await User.repository.find();
expect(users[0].get('name')).toBe('User1');
expect(users[1].get('name')).toBe('User2');
});
});

23
packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts
View File

@ -9,31 +9,24 @@
import { Context, Next } from '@nocobase/actions';
import { TemplateCreator } from '../services/template-creator';
import XLSX from 'xlsx';
import { Workbook } from 'exceljs';
export async function downloadXlsxTemplate(ctx: Context, next: Next) {
let { columns } = ctx.request.body as any;
const { explain, title } = ctx.request.body as any;
if (typeof columns === 'string') {
columns = JSON.parse(columns);
}
const { resourceName, values = {} } = ctx.action.params;
const { collection } = ctx.db;
const templateCreator = new TemplateCreator({
explain,
title,
columns,
collection,
...values,
});
const workbook = await templateCreator.run();
const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' });
const buffer = await (workbook as Workbook).xlsx.writeBuffer();
ctx.set('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
ctx.set('Content-Disposition', `attachment; filename="${resourceName}-import-template.xlsx"`);
ctx.body = buffer;
ctx.set({
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename=${encodeURIComponent(title)}.xlsx`,
});
await next();
}

31
packages/plugins/@nocobase/plugin-action-import/src/server/errors.ts Normal file
View File

@ -0,0 +1,31 @@
export class ImportValidationError extends Error {
code: string;
params?: Record<string, any>;
constructor(code: string, params?: Record<string, any>) {
super(code);
this.code = code;
this.params = params;
this.name = 'ImportValidationError';
}
}
export interface ImportErrorOptions {
rowIndex: number;
rowData: any;
cause?: Error;
}
export class ImportError extends Error {
rowIndex: number;
rowData: any;
cause?: Error;
constructor(message: string, options: ImportErrorOptions) {
super(message);
this.name = 'ImportError';
this.rowIndex = options.rowIndex;
this.rowData = options.rowData;
this.cause = options.cause;
}
}

45
packages/plugins/@nocobase/plugin-action-import/src/server/index.ts
View File

@ -10,7 +10,7 @@
import { Plugin } from '@nocobase/server';
import { downloadXlsxTemplate, importXlsx } from './actions';
import { importMiddleware } from './middleware';
import { ImportError, ImportValidationError } from './errors';
export class PluginActionImportServer extends Plugin {
beforeLoad() {
this.app.on('afterInstall', async () => {
@ -55,7 +55,50 @@ export class PluginActionImportServer extends Plugin {
dataSource.acl.allow('*', 'downloadXlsxTemplate', 'loggedIn');
});
const errorHandlerPlugin = this.app.getPlugin<any>('error-handler');
errorHandlerPlugin.errorHandler.register(
(err) => err instanceof ImportValidationError,
(err: ImportValidationError, ctx) => {
ctx.status = 400;
ctx.body = {
errors: [
{
message: ctx.i18n.t(err.code, {
...err.params,
ns: 'action-import',
}),
},
],
};
},
);
errorHandlerPlugin.errorHandler.register(
(err) => err.name === 'ImportError',
(err: ImportError, ctx) => {
ctx.status = 400;
const causeError = err.cause;
errorHandlerPlugin.errorHandler.renderError(causeError, ctx);
ctx.body = {
errors: [
{
message: ctx.i18n.t('import-error', {
ns: 'action-import',
rowData: err.rowData,
rowIndex: err.rowIndex,
causeMessage: ctx.body.errors[0].message,
}),
},
],
};
},
);
}
}
export default PluginActionImportServer;
export * from './services/xlsx-importer';
export * from './services/template-creator';

143
packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts
View File

@ -9,40 +9,153 @@
import { ICollection } from '@nocobase/data-source-manager';
import { ImportColumn } from './xlsx-importer';
import XLSX, { WorkBook } from 'xlsx';
import { WorkbookConverter } from '../utils/workbook-converter';
import * as XLSX from 'xlsx';
import { Workbook as ExcelJSWorkbook } from 'exceljs';
type TemplateCreatorOptions = {
export type TemplateCreatorOptions = {
collection?: ICollection;
title?: string;
explain?: string;
columns: Array<ImportColumn>;
};
export type TemplateResult = {
workbook: XLSX.WorkBook | ExcelJSWorkbook;
headerRowIndex: number;
};
export class TemplateCreator {
private headerRowIndex: number;
constructor(private options: TemplateCreatorOptions) {}
async run(): Promise<WorkBook> {
const workbook = XLSX.utils.book_new();
const worksheet = XLSX.utils.sheet_new();
const data = [this.renderHeaders()];
if (this.options.explain && this.options.explain?.trim() !== '') {
data.unshift([this.options.explain]);
getHeaderRowIndex() {
return this.headerRowIndex;
}
// write headers
XLSX.utils.sheet_add_aoa(worksheet, data, {
origin: 'A1',
async run(options?: any): Promise<XLSX.WorkBook | ExcelJSWorkbook> {
const workbook = new ExcelJSWorkbook();
const worksheet = workbook.addWorksheet('Sheet 1');
const headers = this.renderHeaders();
let currentRow = 1;
let explainText = '';
if (this.options.explain && this.options.explain?.trim() !== '') {
explainText = this.options.explain;
}
const fieldDescriptions = this.options.columns
.filter((col) => col.description)
.map((col) => `${col.title || col.defaultTitle}：${col.description}`);
if (fieldDescriptions.length > 0) {
if (explainText) {
explainText += '\n\n';
}
explainText += fieldDescriptions.join('\n');
}
if (explainText.trim() !== '') {
const lines = explainText.split('\n');
const EXPLAIN_MERGE_COLUMNS = 5; // Default merge 5 columns
// Pre-set the required number of columns and their widths
const columnsNeeded = Math.max(EXPLAIN_MERGE_COLUMNS, headers.length);
for (let i = 1; i <= columnsNeeded; i++) {
const col = worksheet.getColumn(i);
// Set first column wider
col.width = i === 1 ? 60 : 30;
}
lines.forEach((line, index) => {
const row = worksheet.getRow(index + 1);
// Merge first 5 cells for explanation text
worksheet.mergeCells(index + 1, 1, index + 1, EXPLAIN_MERGE_COLUMNS);
const cell = row.getCell(1);
cell.value = line;
cell.alignment = {
vertical: 'middle',
horizontal: 'left',
indent: 1,
wrapText: true,
};
});
XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet 1');
currentRow = lines.length + 1;
}
// Write headers and set styles
const headerRow = worksheet.getRow(currentRow);
headerRow.height = 25;
headers.forEach((header, index) => {
const cell = headerRow.getCell(index + 1);
cell.value = header;
cell.font = {
bold: true,
size: 10,
name: 'Arial',
};
cell.border = {
top: { style: 'thin' },
bottom: { style: 'thin' },
left: { style: 'thin' },
right: { style: 'thin' },
};
cell.fill = {
type: 'pattern',
pattern: 'solid',
fgColor: { argb: 'FFE9ECEF' }, // Darker gray background for headers
};
cell.alignment = {
vertical: 'middle',
horizontal: 'left',
indent: 1,
wrapText: true,
};
// 如果是第一列，恢复到正常宽度
if (index === 0) {
worksheet.getColumn(1).width = 30;
}
});
// Set column widths
headers.forEach((_, index) => {
const col = worksheet.getColumn(index + 1);
col.width = 30;
// Set style for the first column
if (index === 0) {
col.eachCell({ includeEmpty: false }, (cell, rowNumber) => {
// Only set background color for headers, not for other rows
if (rowNumber === currentRow) {
cell.fill = {
type: 'pattern',
pattern: 'solid',
fgColor: { argb: 'FFE9ECEF' },
};
}
});
}
});
this.headerRowIndex = currentRow;
if (options?.returnXLSXWorkbook) {
return await WorkbookConverter.excelJSToXLSX(workbook);
}
return workbook;
}
renderHeaders() {
return this.options.columns.map((col) => {
return col.defaultTitle;
return col.title || col.defaultTitle;
});
}
}

164
packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts
View File

@ -7,29 +7,32 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import XLSX, { WorkBook } from 'xlsx';
import * as XLSX from 'xlsx';
import lodash from 'lodash';
import { ICollection, ICollectionManager, IRelationField } from '@nocobase/data-source-manager';
import { Collection as DBCollection, Database } from '@nocobase/database';
import { Transaction } from 'sequelize';
import EventEmitter from 'events';
import { ImportValidationError, ImportError } from '../errors';
export type ImportColumn = {
dataIndex: Array<string>;
defaultTitle: string;
title?: string;
description?: string;
};
type ImporterOptions = {
export type ImporterOptions = {
collectionManager: ICollectionManager;
collection: ICollection;
columns: Array<ImportColumn>;
workbook: WorkBook;
workbook: any;
chunkSize?: number;
explain?: string;
repository?: any;
};
type RunOptions = {
export type RunOptions = {
transaction?: Transaction;
context?: any;
};
@ -37,9 +40,13 @@ type RunOptions = {
export class XlsxImporter extends EventEmitter {
private repository;
constructor(private options: ImporterOptions) {
constructor(protected options: ImporterOptions) {
super();
if (typeof options.columns === 'string') {
options.columns = JSON.parse(options.columns);
}
if (options.columns.length == 0) {
throw new Error(`columns is empty`);
}
@ -47,6 +54,21 @@ export class XlsxImporter extends EventEmitter {
this.repository = options.repository ? options.repository : options.collection.repository;
}
async validate() {
if (this.options.columns.length == 0) {
throw new ImportValidationError('Columns configuration is empty');
}
for (const column of this.options.columns) {
const field = this.options.collection.getField(column.dataIndex[0]);
if (!field) {
throw new ImportValidationError('Field not found: {{field}}', { field: column.dataIndex[0] });
}
}
await this.getData();
}
async run(options: RunOptions = {}) {
let transaction = options.transaction;
@ -57,6 +79,7 @@ export class XlsxImporter extends EventEmitter {
}
try {
await this.validate();
const imported = await this.performImport(options);
// @ts-ignore
@ -69,7 +92,6 @@ export class XlsxImporter extends EventEmitter {
return imported;
} catch (error) {
transaction && (await transaction.rollback());
throw error;
}
}
@ -127,19 +149,21 @@ export class XlsxImporter extends EventEmitter {
this.emit('seqReset', { maxVal, seqName: autoIncrInfo.seqName });
}
async performImport(options?: RunOptions) {
async performImport(options?: RunOptions): Promise<any> {
const transaction = options?.transaction;
const rows = this.getData();
const chunks = lodash.chunk(rows, this.options.chunkSize || 200);
const data = await this.getData();
const chunks = lodash.chunk(data.slice(1), this.options.chunkSize || 200);
let handingRowIndex = 1;
let imported = 0;
// Calculate total rows to be imported
const total = data.length - 1; // Subtract header row
if (this.options.explain) {
handingRowIndex += 1;
}
let imported = 0;
for (const chunkRows of chunks) {
for (const row of chunkRows) {
const rowValues = {};
@ -151,7 +175,9 @@ export class XlsxImporter extends EventEmitter {
const field = this.options.collection.getField(column.dataIndex[0]);
if (!field) {
throw new Error(`Field not found: ${column.dataIndex[0]}`);
throw new ImportValidationError('Import validation.Field not found', {
field: column.dataIndex[0],
});
}
const str = row[index];
@ -185,32 +211,49 @@ export class XlsxImporter extends EventEmitter {
rowValues[dataKey] = await interfaceInstance.toValue(this.trimString(str), ctx);
}
await this.repository.create({
await this.performInsert({
values: rowValues,
context: options?.context,
transaction,
context: options?.context,
});
imported += 1;
// Emit progress event
this.emit('progress', {
total,
current: imported,
});
await new Promise((resolve) => setTimeout(resolve, 5));
} catch (error) {
throw new Error(
`failed to import row ${handingRowIndex}, ${this.renderErrorMessage(error)}, rowData: ${JSON.stringify(
rowValues,
)}`,
{ cause: error },
);
throw new ImportError(`Import failed at row ${handingRowIndex}`, {
rowIndex: handingRowIndex,
rowData: Object.entries(rowValues)
.map(([key, value]) => `${key}: ${value}`)
.join(', '),
cause: error,
});
}
}
// await to prevent high cpu usage
await new Promise((resolve) => setTimeout(resolve, 50));
}
return imported;
}
async performInsert(insertOptions: { values: any; transaction: Transaction; context: any; hooks?: boolean }) {
const { values, transaction, context } = insertOptions;
return this.repository.create({
values,
context,
transaction,
hooks: insertOptions.hooks == undefined ? true : insertOptions.hooks,
});
}
renderErrorMessage(error) {
let message = error.message;
if (error.parent) {
@ -227,33 +270,70 @@ export class XlsxImporter extends EventEmitter {
return str;
}
getData() {
const firstSheet = this.firstSheet();
const rows = XLSX.utils.sheet_to_json(firstSheet, { header: 1, defval: null });
if (this.options.explain) {
rows.shift();
private getExpectedHeaders(): string[] {
return this.options.columns.map((col) => col.title || col.defaultTitle);
}
const headers = rows[0];
async getData() {
const workbook = this.options.workbook;
const worksheet = workbook.Sheets[workbook.SheetNames[0]];
const columns = this.options.columns;
const data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null }) as string[][];
// validate headers
for (let i = 0; i < columns.length; i++) {
const column = columns[i];
if (column.defaultTitle !== headers[i]) {
throw new Error(`Invalid header: ${column.defaultTitle} !== ${headers[i]}`);
// Find and validate header row
const expectedHeaders = this.getExpectedHeaders();
const { headerRowIndex, headers } = this.findAndValidateHeaders(data);
if (headerRowIndex === -1) {
throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', {
headers: expectedHeaders.join(', '),
});
}
// Extract data rows
const rows = data.slice(headerRowIndex + 1);
// if no data rows, throw error
if (rows.length === 0) {
throw new ImportValidationError('No data to import');
}
return [headers, ...rows];
}
private findAndValidateHeaders(data: string[][]): { headerRowIndex: number; headers: string[] } {
const expectedHeaders = this.getExpectedHeaders();
// Find header row and validate
for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
const row = data[rowIndex];
const actualHeaders = row.filter((cell) => cell !== null && cell !== '');
const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header));
const noExtraHeaders = actualHeaders.length === expectedHeaders.length;
if (allHeadersFound && noExtraHeaders) {
const mismatchIndex = expectedHeaders.findIndex((title, index) => actualHeaders[index] !== title);
if (mismatchIndex === -1) {
// All headers match
return { headerRowIndex: rowIndex, headers: actualHeaders };
} else {
// Found potential header row but with mismatch
throw new ImportValidationError(
'Header mismatch at column {{column}}: expected "{{expected}}", but got "{{actual}}"',
{
column: mismatchIndex + 1,
expected: expectedHeaders[mismatchIndex],
actual: actualHeaders[mismatchIndex] || 'empty',
},
);
}
}
}
// remove header
rows.shift();
return rows;
}
firstSheet() {
return this.options.workbook.Sheets[this.options.workbook.SheetNames[0]];
// No row with matching headers found
throw new ImportValidationError('Headers not found. Expected headers: {{headers}}', {
headers: expectedHeaders.join(', '),
});
}
}

18
packages/plugins/@nocobase/plugin-action-import/src/server/utils/workbook-converter.ts Normal file
View File

@ -0,0 +1,18 @@
import * as XLSX from 'xlsx';
import { Workbook as ExcelJSWorkbook } from 'exceljs';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { promisify } from 'util';
export class WorkbookConverter {
/**
* Convert ExcelJS Workbook to XLSX Workbook
*/
static async excelJSToXLSX(workbook: ExcelJSWorkbook): Promise<XLSX.WorkBook> {
// Convert ExcelJS workbook to buffer in memory
const buffer = await workbook.xlsx.writeBuffer();
// Convert buffer to XLSX workbook
return XLSX.read(buffer, { type: 'buffer' });
}
}

27
packages/plugins/@nocobase/plugin-auth/src/server/plugin.ts
View File

@ -112,6 +112,33 @@ export class PluginAuthServer extends Plugin {
await this.cache.del(`auth:${userId}`);
});
this.app.on('ws:message:auth:token', async ({ clientId, payload }) => {
const auth = await this.app.authManager.get('basic', {
getBearerToken: () => payload.token,
app: this.app,
db: this.app.db,
cache: this.app.cache,
logger: this.app.logger,
} as any);
const user = await auth.check();
if (!user) {
this.app.logger.error(`Invalid token: ${payload.token}`);
return;
}
this.app.emit(`ws:setTag`, {
clientId,
tagKey: 'userId',
tagValue: user.id,
});
this.app.emit(`ws:authorized`, {
clientId,
userId: user.id,
});
});
this.app.auditManager.registerActions([
{
name: 'auth:signIn',

3
packages/plugins/@nocobase/plugin-data-source-manager/src/server/models/data-source.ts
View File

@ -13,6 +13,7 @@ import { setCurrentRole } from '@nocobase/plugin-acl';
import { ACL, AvailableActionOptions } from '@nocobase/acl';
import { DataSourcesRolesModel } from './data-sources-roles-model';
import PluginDataSourceManagerServer from '../plugin';
import * as path from 'path';
const availableActions: {
[key: string]: AvailableActionOptions;
@ -98,6 +99,8 @@ export class DataSourceModel extends Model {
name: dataSourceKey,
logger: app.logger.child({ dataSourceKey }),
sqlLogger: app.sqlLogger.child({ dataSourceKey }),
cache: app.cache,
storagePath: path.join(process.cwd(), 'storage', 'cache', 'apps', app.name),
});
dataSource.on('loadingProgress', (progress) => {

18
packages/plugins/@nocobase/plugin-error-handler/src/server/error-handler.ts
View File

@ -36,6 +36,16 @@ export class ErrorHandler {
};
}
renderError(err, ctx) {
for (const handler of this.handlers) {
if (handler.guard(err)) {
return handler.render(err, ctx);
}
}
this.defaultHandler(err, ctx);
}
middleware() {
const self = this;
return async function errorHandler(ctx, next) {
@ -48,13 +58,7 @@ export class ErrorHandler {
ctx.status = err.statusCode;
}
for (const handler of self.handlers) {
if (handler.guard(err)) {
return handler.render(err, ctx);
}
}
self.defaultHandler(err, ctx);
self.renderError(err, ctx);
}
};
}

1
packages/plugins/@nocobase/plugin-error-handler/src/server/index.ts
View File

@ -8,3 +8,4 @@
*/
export { PluginErrorHandlerServer as default } from './server';
export { ErrorHandler } from './error-handler';

4
packages/plugins/@nocobase/plugin-error-handler/src/server/server.ts
View File

@ -45,7 +45,9 @@ export class PluginErrorHandlerServer extends Plugin {
};
this.errorHandler.register(
(err) => err?.errors?.length && err instanceof BaseError,
(err) =>
err?.errors?.length &&
(err.name === 'SequelizeValidationError' || err.name === 'SequelizeUniqueConstraintError'),
(err, ctx) => {
ctx.body = {
errors: err.errors.map((err) => {

5
packages/plugins/@nocobase/plugin-workflow/src/server/Plugin.ts
View File

@ -301,6 +301,11 @@ export default class PluginWorkflowServer extends Plugin {
this.dispatch();
}, 300_000);
this.app.on('workflow:dispatch', () => {
this.app.logger.info('workflow:dispatch');
this.dispatch();
});
// check for not started executions
this.dispatch();
});