From 2063227f4a80861bf7d006db849fdff0ae8ea53e Mon Sep 17 00:00:00 2001 From: ChengLei Shao Date: Wed, 5 Jun 2024 17:52:43 +0800 Subject: [PATCH] refactor: export & import plugin (#4468) * feat: chunk method in repository * chore: xlsx export test * chore: xlsx export * chore: export action * chore: export action * chore: code * feat: database interface manager * feat: export with ui schema * chore: console.log * chore: export with china region field * chore: export with attachments * chore: export with multiple select * chore: export with interface * chore: export action * fix: export with datetime file * chore: limit export action running in same time * chore: yarn.lock * fix: render json value * chore: chunk with limit * feat: add EXPORT_LIMIT env config * fix: typo * fix: type * chore: asyn mutex version * chore: test * chore: test * fix: export null value * chore: test * chore: createdAt test * fix: export with createdAt * chore: import template * chore: xlsx importer * chore: import run * chore: export with data source api * chore: toValue api in interface * fix: build * chore: import with transaction * fix: build database * chore: many to one interface * chore: code * chore: import with associations * chore: default toValue * chore: import template with explain * chore: import with explain template * chore: reset id seq after import * chore: download template action * fix: database build * fix: build * fix: build * fix: test * chore: import with number field * chore: import with boolean field * chore: json interface * chore: import action * chore: typo * chore: i18n * chore: select interface * chore: china region interface * chore: datetiem field * chore: cast to array * fix: import tips * chore: import await * fix: test * fix: test in mariadb * chore: comments * chore: comments * fix: parse date with empty string * fix: read import limit * fix: type * fix: test in mariadb * chore: skip bigint test in sqlite * chore: skip bigint test in sqlite * chore: download tip i18n keys * fix: download tips * feat(client): add new variable named 'URL search params' and support link action (#4506) * feat: support link action * feat(client): add new variable named 'URL search params' * chore: add translation * fix: avoid crashing * chore: fix failed test * feat: link action * feat: link action * fix: remove filter parameters with undefined values * feat: link action * feat: add support for default values in filter form fields * refactor: code improve * refactor: locale improve * refactor: locale improve * test: add e2e test * refactor: locale improve * refactor: locale improve * fix: resolve operation issues with variables * refactor: code improve * chore: enable direct selection of variables as default value * chore: use qs to parse query string * fix: menu selectKeys (T-4373) * refactor: use qs to stringify search params * refactor: locale improve * refactor: locale improve * chore: fix failed tests * fix: resolve issue where setting Data scope is not work * chore: fix failed e2e tests * chore: make e2e tests more stable * chore: add translation * chore: make e2e tests more stable * fix: resolve the issue of error when saving data scope * feat: trigger variable parsing after context change * test: add unit tests * test: add e2e test * refactor: extract template * chore: fix failed unit tests * chore: fix failed e2e test * fix(Link): hide linkage rules in top link (T-4410) * fix(permission): remove URL search params variable from data scope * chore: make more stable * chore: make e2e test more stable * fix(Link): reduce size for variable * fix: clear previous context (T-4449) * fix(calendar, map): resolve initial data scope setting error (T-4450) * fix: correct concatenation of query string (T-4453) --------- Co-authored-by: katherinehhh Co-authored-by: jack zhang <1098626505@qq.com> * refactor(FormV2): add FormWithDataTemplates component (#4551) * Revert "fix(client): fix data template style (#4536)" This reverts commit db66090ab279508473e74803dbb8637341fa6f3f. * refactor(FormV2): add FormWithDataTemplates component * chore: fix failed e2e tests * chore: make e2e test more stable * chore: import warning i18n * chore: import warning i18n * fix: bug * fix: export action loading * fix: bug * chore: map field interface * fix: merge bug --------- Co-authored-by: xilesun <2013xile@gmail.com> Co-authored-by: Zeke Zhang <958414905@qq.com> Co-authored-by: katherinehhh Co-authored-by: jack zhang <1098626505@qq.com> --- .../src/appInfo/CurrentAppInfoProvider.tsx | 1 + .../client/src/block-provider/hooks/index.ts | 1 + .../link/customizeLinkActionSettings.tsx | 3 +- .../schema-component/antd/action/Action.tsx | 4 +- .../src/collection-field.ts | 4 + .../src/collection-manager.ts | 25 +- .../src/sequelize-collection-manager.ts | 19 +- .../core/data-source-manager/src/types.ts | 22 +- .../database/src/__tests__/bigint.test.ts | 3 +- .../interfaces/datetime-interface.test.ts | 93 ++ .../interfaces/interface-manager.test.ts | 36 + .../multiple-select-interface.test.ts | 42 + .../interfaces/number-interface.test.ts | 31 + .../interfaces/percent-interface.test.ts | 31 + .../interfaces/select-interface.test.ts | 42 + .../src/__tests__/percent2float.test.ts | 1 + .../src/__tests__/repository/chunk.test.ts | 103 +++ packages/core/database/src/database.ts | 4 + packages/core/database/src/fields/field.ts | 4 + .../database/src/fields/relation-field.ts | 8 + packages/core/database/src/index.ts | 1 + .../core/database/src/interface-manager.ts | 25 + .../database/src/interfaces/base-interface.ts | 53 ++ .../src/interfaces/boolean-interface.ts | 46 + .../src/interfaces/checkboxes-interface.ts | 12 + .../src/interfaces/datetime-interface.ts | 71 ++ .../core/database/src/interfaces/index.ts | 15 + .../src/interfaces/integer-interface.ts | 16 + .../database/src/interfaces/json-interface.ts | 20 + .../src/interfaces/many-to-many-interface.ts | 12 + .../src/interfaces/many-to-one-interface.ts} | 4 +- .../interfaces/multiple-select-interface.ts | 33 + .../src/interfaces/number-interface.ts | 53 ++ .../one-belongs-to-one-interface.ts | 12 + .../src/interfaces/one-has-one-interface.ts | 12 + .../src/interfaces/one-to-many-interface.ts | 12 + .../src/interfaces/percent-interface.ts | 28 + .../src/interfaces/select-interface.ts | 24 + .../src/interfaces/to-many-interface.ts | 29 + .../src/interfaces/to-one-interface.ts | 35 + .../core/database/src/interfaces/utils.ts | 53 ++ packages/core/database/src/model.ts | 19 + .../query-interface/mysql-query-interface.ts | 8 +- .../postgres-query-interface.ts | 8 + .../src/query-interface/query-interface.ts | 1 + .../query-interface/sqlite-query-interface.ts | 6 +- packages/core/database/src/repository.ts | 36 + .../plugin-acl/src/client/__e2e__/utils.ts | 3 +- .../plugin-action-export/package.json | 4 +- .../src/client/useExportAction.tsx | 24 +- .../src/locale/en-US.json | 2 +- .../src/locale/zh-CN.json | 2 +- .../server/__tests__/export-to-xlsx.test.ts | 870 ++++++++++++++++++ .../src/server/__tests__/utils/utils.test.ts | 60 -- .../src/server/actions/export-xlsx.ts | 67 +- .../src/server/collections/.gitkeep | 0 .../plugin-action-export/src/server/index.ts | 9 +- .../src/server/models/.gitkeep | 0 .../src/server/renders/index.ts | 182 ---- .../src/server/renders/renders.ts | 139 --- .../src/server/repositories/.gitkeep | 0 .../src/server/utils/columns2Appends.ts | 28 - .../src/server/utils/deep-get.ts | 25 + .../src/server/xlsx-exporter.ts | 200 ++++ .../plugin-action-import/package.json | 2 +- .../src/client/ImportActionInitializer.tsx | 22 +- .../src/client/ImportModal.tsx | 2 +- .../src/client/ImportPluginProvider.tsx | 4 +- .../src/client/useImportAction.ts | 6 + .../src/locale/en-US.json | 6 +- .../src/locale/es-ES.json | 4 +- .../src/locale/ko_KR.json | 6 +- .../src/locale/pt-BR.json | 4 +- .../src/locale/zh-CN.json | 6 +- .../__tests__/download-template.test.ts | 108 +++ .../server/__tests__/xlsx-importer.test.ts | 706 ++++++++++++++ ...xTemplate.ts => download-xlsx-template.ts} | 27 +- .../src/server/actions/import-xlsx.ts | 73 ++ .../src/server/actions/importXlsx.ts | 177 ---- .../src/server/actions/index.ts | 4 +- .../src/server/collections/.gitkeep | 0 .../plugin-action-import/src/server/index.ts | 11 +- .../src/server/models/.gitkeep | 0 .../src/server/repositories/.gitkeep | 0 .../src/server/services/template-creator.ts | 48 + .../src/server/services/xlsx-importer.ts | 222 +++++ .../plugin-client/src/server/server.ts | 9 +- .../src/server/index.ts | 3 + .../interfaces/china-region-interface.ts | 42 + .../server/__tests__/formula-field.test.ts | 2 +- .../server/interfaces/attachment-interface.ts | 32 + .../plugin-file-manager/src/server/server.ts | 3 + .../src/server/__tests__/interfaces.test.ts | 78 ++ .../plugin-map/src/server/interfaces/index.ts | 53 ++ .../@nocobase/plugin-map/src/server/plugin.ts | 6 + yarn.lock | 4 + 96 files changed, 3695 insertions(+), 711 deletions(-) create mode 100644 packages/core/database/src/__tests__/interfaces/datetime-interface.test.ts create mode 100644 packages/core/database/src/__tests__/interfaces/interface-manager.test.ts create mode 100644 packages/core/database/src/__tests__/interfaces/multiple-select-interface.test.ts create mode 100644 packages/core/database/src/__tests__/interfaces/number-interface.test.ts create mode 100644 packages/core/database/src/__tests__/interfaces/percent-interface.test.ts create mode 100644 packages/core/database/src/__tests__/interfaces/select-interface.test.ts create mode 100644 packages/core/database/src/__tests__/repository/chunk.test.ts create mode 100644 packages/core/database/src/interface-manager.ts create mode 100644 packages/core/database/src/interfaces/base-interface.ts create mode 100644 packages/core/database/src/interfaces/boolean-interface.ts create mode 100644 packages/core/database/src/interfaces/checkboxes-interface.ts create mode 100644 packages/core/database/src/interfaces/datetime-interface.ts create mode 100644 packages/core/database/src/interfaces/index.ts create mode 100644 packages/core/database/src/interfaces/integer-interface.ts create mode 100644 packages/core/database/src/interfaces/json-interface.ts create mode 100644 packages/core/database/src/interfaces/many-to-many-interface.ts rename packages/{plugins/@nocobase/plugin-action-export/src/server/utils/index.ts => core/database/src/interfaces/many-to-one-interface.ts} (72%) create mode 100644 packages/core/database/src/interfaces/multiple-select-interface.ts create mode 100644 packages/core/database/src/interfaces/number-interface.ts create mode 100644 packages/core/database/src/interfaces/one-belongs-to-one-interface.ts create mode 100644 packages/core/database/src/interfaces/one-has-one-interface.ts create mode 100644 packages/core/database/src/interfaces/one-to-many-interface.ts create mode 100644 packages/core/database/src/interfaces/percent-interface.ts create mode 100644 packages/core/database/src/interfaces/select-interface.ts create mode 100644 packages/core/database/src/interfaces/to-many-interface.ts create mode 100644 packages/core/database/src/interfaces/to-one-interface.ts create mode 100644 packages/core/database/src/interfaces/utils.ts create mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/utils/utils.test.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/collections/.gitkeep delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/models/.gitkeep delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/renders/index.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/renders/renders.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/repositories/.gitkeep delete mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/utils/columns2Appends.ts create mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/utils/deep-get.ts create mode 100644 packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts rename packages/plugins/@nocobase/plugin-action-import/src/server/actions/{downloadXlsxTemplate.ts => download-xlsx-template.ts} (69%) create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/actions/import-xlsx.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/actions/importXlsx.ts delete mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/collections/.gitkeep delete mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/models/.gitkeep delete mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/repositories/.gitkeep create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts create mode 100644 packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts create mode 100644 packages/plugins/@nocobase/plugin-field-china-region/src/server/interfaces/china-region-interface.ts create mode 100644 packages/plugins/@nocobase/plugin-file-manager/src/server/interfaces/attachment-interface.ts create mode 100644 packages/plugins/@nocobase/plugin-map/src/server/__tests__/interfaces.test.ts create mode 100644 packages/plugins/@nocobase/plugin-map/src/server/interfaces/index.ts diff --git a/packages/core/client/src/appInfo/CurrentAppInfoProvider.tsx b/packages/core/client/src/appInfo/CurrentAppInfoProvider.tsx index ea1ed3aea1..1d588a07b9 100644 --- a/packages/core/client/src/appInfo/CurrentAppInfoProvider.tsx +++ b/packages/core/client/src/appInfo/CurrentAppInfoProvider.tsx @@ -22,6 +22,7 @@ export const useCurrentAppInfo = () => { }; lang: string; version: string; + exportLimit?: number; }; }>(CurrentAppInfoContext); }; diff --git a/packages/core/client/src/block-provider/hooks/index.ts b/packages/core/client/src/block-provider/hooks/index.ts index dec8a38fa4..307756efc5 100644 --- a/packages/core/client/src/block-provider/hooks/index.ts +++ b/packages/core/client/src/block-provider/hooks/index.ts @@ -1490,6 +1490,7 @@ export function useLinkActionProps() { }; } + export async function replaceVariableValue( url: string, variables: VariablesContextType, diff --git a/packages/core/client/src/modules/actions/link/customizeLinkActionSettings.tsx b/packages/core/client/src/modules/actions/link/customizeLinkActionSettings.tsx index 43f6c193fb..bccbbbe7b7 100644 --- a/packages/core/client/src/modules/actions/link/customizeLinkActionSettings.tsx +++ b/packages/core/client/src/modules/actions/link/customizeLinkActionSettings.tsx @@ -12,10 +12,9 @@ import { useField, useFieldSchema } from '@formily/react'; import _ from 'lodash'; import React from 'react'; import { useTranslation } from 'react-i18next'; -import { useCollectionRecord, useDesignable, useRecord } from '../../../'; +import { useCollectionRecord, useDesignable, useFormBlockContext, useRecord } from '../../../'; import { useSchemaToolbar } from '../../../application'; import { SchemaSettings } from '../../../application/schema-settings/SchemaSettings'; -import { useFormBlockContext } from '../../../block-provider/FormBlockProvider'; import { useCollection_deprecated } from '../../../collection-manager'; import { ButtonEditor, RemoveButton } from '../../../schema-component/antd/action/Action.Designer'; import { SchemaSettingsLinkageRules, SchemaSettingsModalItem } from '../../../schema-settings'; diff --git a/packages/core/client/src/schema-component/antd/action/Action.tsx b/packages/core/client/src/schema-component/antd/action/Action.tsx index 8a78389278..f6aa0f010f 100644 --- a/packages/core/client/src/schema-component/antd/action/Action.tsx +++ b/packages/core/client/src/schema-component/antd/action/Action.tsx @@ -79,7 +79,7 @@ export const Action: ComposedAction = withDynamicSchemaProps( const [formValueChanged, setFormValueChanged] = useState(false); const Designer = useDesigner(); const field = useField(); - const { run, element } = useAction(actionCallback); + const { run, element, disabled: disableAction } = useAction(actionCallback); const fieldSchema = useFieldSchema(); const compile = useCompile(); const form = useForm(); @@ -90,7 +90,7 @@ export const Action: ComposedAction = withDynamicSchemaProps( const openMode = fieldSchema?.['x-component-props']?.['openMode']; const openSize = fieldSchema?.['x-component-props']?.['openSize']; - const disabled = form.disabled || field.disabled || field.data?.disabled || propsDisabled; + const disabled = form.disabled || field.disabled || field.data?.disabled || propsDisabled || disableAction; const linkageRules = useMemo(() => fieldSchema?.['x-linkage-rules'] || [], [fieldSchema?.['x-linkage-rules']]); const { designable } = useDesignable(); const tarComponent = useComponent(component) || component; diff --git a/packages/core/data-source-manager/src/collection-field.ts b/packages/core/data-source-manager/src/collection-field.ts index 889b13cf62..d8f844ffee 100644 --- a/packages/core/data-source-manager/src/collection-field.ts +++ b/packages/core/data-source-manager/src/collection-field.ts @@ -22,4 +22,8 @@ export class CollectionField implements IField { ...options, }; } + + isRelationField(): boolean { + return false; + } } diff --git a/packages/core/data-source-manager/src/collection-manager.ts b/packages/core/data-source-manager/src/collection-manager.ts index 640031787a..7f29a55178 100644 --- a/packages/core/data-source-manager/src/collection-manager.ts +++ b/packages/core/data-source-manager/src/collection-manager.ts @@ -8,7 +8,14 @@ */ import { Collection } from './collection'; -import { CollectionOptions, ICollection, ICollectionManager, IRepository, MergeOptions } from './types'; +import { + CollectionOptions, + ICollection, + ICollectionManager, + IFieldInterface, + IRepository, + MergeOptions, +} from './types'; export class CollectionManager implements ICollectionManager { protected collections = new Map(); @@ -38,8 +45,17 @@ export class CollectionManager implements ICollectionManager { /* istanbul ignore next -- @preserve */ registerFieldTypes() {} - /* istanbul ignore next -- @preserve */ - registerFieldInterfaces() {} + registerFieldInterfaces(interfaces: Record IFieldInterface>) { + Object.keys(interfaces).forEach((key) => { + this.registerFieldInterface(key, interfaces[key]); + }); + } + + registerFieldInterface(name: string, fieldInterface: new (options: any) => IFieldInterface): void {} + + getFieldInterface(name: string): { new (options: any): IFieldInterface | undefined } { + return; + } /* istanbul ignore next -- @preserve */ registerCollectionTemplates() {} @@ -87,7 +103,8 @@ export class CollectionManager implements ICollectionManager { async sync() {} - protected newCollection(options) { + protected newCollection(options): ICollection { + // @ts-ignore return new Collection(options, this); } } diff --git a/packages/core/data-source-manager/src/sequelize-collection-manager.ts b/packages/core/data-source-manager/src/sequelize-collection-manager.ts index 915c0afc9f..eff1e5390b 100644 --- a/packages/core/data-source-manager/src/sequelize-collection-manager.ts +++ b/packages/core/data-source-manager/src/sequelize-collection-manager.ts @@ -9,8 +9,15 @@ /* istanbul ignore file -- @preserve */ -import Database from '@nocobase/database'; -import { CollectionOptions, ICollection, ICollectionManager, IRepository, MergeOptions } from './types'; +import { Database } from '@nocobase/database'; +import { + CollectionOptions, + ICollection, + ICollectionManager, + IFieldInterface, + IRepository, + MergeOptions, +} from './types'; export class SequelizeCollectionManager implements ICollectionManager { db: Database; @@ -97,4 +104,12 @@ export class SequelizeCollectionManager implements ICollectionManager { async sync() { await this.db.sync(); } + + registerFieldInterface(name: string, fieldInterface: new (options: any) => IFieldInterface): void { + this.db.interfaceManager.registerInterfaceType(name, fieldInterface); + } + + getFieldInterface(name: string): { new (options: any): IFieldInterface | undefined } { + return this.db.interfaceManager.getInterfaceType(name); + } } diff --git a/packages/core/data-source-manager/src/types.ts b/packages/core/data-source-manager/src/types.ts index 567576a454..f9a6364fcc 100644 --- a/packages/core/data-source-manager/src/types.ts +++ b/packages/core/data-source-manager/src/types.ts @@ -33,8 +33,22 @@ export type FieldOptions = { export interface IField { options: FieldOptions; + isRelationField(): boolean; } +export interface IRelationField extends IField { + targetCollection(): ICollection; +} + +export interface IFieldInterface { + options: FieldOptions; + + toString(value: any, ctx?: any): string; + toValue(str: string, ctx?: any): any; +} + +export type FindOptions = any; + export interface ICollection { repository: IRepository; @@ -58,7 +72,7 @@ export interface IModel { } export interface IRepository { - find(options?: any): Promise; + find(options?: FindOptions): Promise; findOne(options?: any): Promise; @@ -82,7 +96,11 @@ export type MergeOptions = { export interface ICollectionManager { registerFieldTypes(types: Record): void; - registerFieldInterfaces(interfaces: Record): void; + registerFieldInterfaces(interfaces: Record IFieldInterface>): void; + + registerFieldInterface(name: string, fieldInterface: new (options: any) => IFieldInterface): void; + + getFieldInterface(name: string): new (options: any) => IFieldInterface | undefined; registerCollectionTemplates(templates: Record): void; diff --git a/packages/core/database/src/__tests__/bigint.test.ts b/packages/core/database/src/__tests__/bigint.test.ts index ecdfe71911..621c8700b7 100644 --- a/packages/core/database/src/__tests__/bigint.test.ts +++ b/packages/core/database/src/__tests__/bigint.test.ts @@ -10,7 +10,7 @@ import { Database } from '../database'; import { mockDatabase } from './index'; -describe.runIf(process.env.DB_DIALECT == 'postgres')('collection', () => { +describe.skipIf(process.env['DB_DIALECT'] === 'sqlite')('collection', () => { let db: Database; beforeEach(async () => { @@ -84,7 +84,6 @@ describe.runIf(process.env.DB_DIALECT == 'postgres')('collection', () => { const item = await Test.repository.findOne(); - console.log(item.toJSON()); expect(item.toJSON()['id']).toBe('35809622393264128'); }); diff --git a/packages/core/database/src/__tests__/interfaces/datetime-interface.test.ts b/packages/core/database/src/__tests__/interfaces/datetime-interface.test.ts new file mode 100644 index 0000000000..e5a0deab12 --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/datetime-interface.test.ts @@ -0,0 +1,93 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { Collection } from '../../collection'; +import { DatetimeInterface } from '../../interfaces/datetime-interface'; +import dayjs from 'dayjs'; + +describe('Date time interface', () => { + let db: Database; + let testCollection: Collection; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + testCollection = db.collection({ + name: 'tests', + fields: [ + { + name: 'date', + type: 'date', + }, + { + name: 'dateOnly', + type: 'date', + uiSchema: { + ['x-component-props']: { + showTime: false, + gmt: false, + }, + }, + }, + { + name: 'dateTime', + type: 'date', + uiSchema: { + ['x-component-props']: { + showTime: true, + }, + }, + }, + { + name: 'dateTimeGmt', + type: 'date', + uiSchema: { + ['x-component-props']: { + showTime: true, + gmt: true, + }, + }, + }, + ], + }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should to value', async () => { + const interfaceInstance = new DatetimeInterface(); + expect(await interfaceInstance.toValue('')).toBe(null); + + expect(await interfaceInstance.toValue('20231223')).toBe(dayjs('2023-12-23 00:00:00.000').toISOString()); + expect(await interfaceInstance.toValue('2023/12/23')).toBe(dayjs('2023-12-23 00:00:00.000').toISOString()); + expect(await interfaceInstance.toValue('2023-12-23')).toBe(dayjs('2023-12-23 00:00:00.000').toISOString()); + expect(await interfaceInstance.toValue(42510)).toBe('2016-05-20T00:00:00.000Z'); + expect(await interfaceInstance.toValue('42510')).toBe('2016-05-20T00:00:00.000Z'); + expect(await interfaceInstance.toValue('2016-05-20T00:00:00.000Z')).toBe('2016-05-20T00:00:00.000Z'); + expect( + await interfaceInstance.toValue('2016-05-20 04:22:22', { + field: testCollection.getField('dateOnly'), + }), + ).toBe('2016-05-20T00:00:00.000Z'); + expect( + await interfaceInstance.toValue('2016-05-20 01:00:00', { + field: testCollection.getField('dateTime'), + }), + ).toBe(dayjs('2016-05-20 01:00:00').toISOString()); + expect( + await interfaceInstance.toValue('2016-05-20 01:00:00', { + field: testCollection.getField('dateTimeGmt'), + }), + ).toBe('2016-05-20T01:00:00.000Z'); + }); +}); diff --git a/packages/core/database/src/__tests__/interfaces/interface-manager.test.ts b/packages/core/database/src/__tests__/interfaces/interface-manager.test.ts new file mode 100644 index 0000000000..efdbd192df --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/interface-manager.test.ts @@ -0,0 +1,36 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { BaseInterface } from '../../interfaces/base-interface'; + +describe('interface manager', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should register field interface', async () => { + class TestInterface extends BaseInterface { + toString(value: any) { + return `test-${value}`; + } + } + + db.interfaceManager.registerInterfaceType('test', TestInterface); + expect(db.interfaceManager.getInterfaceType('test')).toBe(TestInterface); + }); +}); diff --git a/packages/core/database/src/__tests__/interfaces/multiple-select-interface.test.ts b/packages/core/database/src/__tests__/interfaces/multiple-select-interface.test.ts new file mode 100644 index 0000000000..4dde27ef52 --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/multiple-select-interface.test.ts @@ -0,0 +1,42 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { MultipleSelectInterface } from '../../interfaces/multiple-select-interface'; + +describe('MultipleSelectInterface', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + describe('toValue', () => { + it('should return value', async () => { + const options = { + uiSchema: { + enum: [ + { value: '1', label: 'Label1' }, + { value: '2', label: 'Label2' }, + ], + }, + }; + + const interfaceInstance = new MultipleSelectInterface(options); + const value = await interfaceInstance.toValue('Label1,Label2'); + expect(value).toEqual(['1', '2']); + }); + }); +}); diff --git a/packages/core/database/src/__tests__/interfaces/number-interface.test.ts b/packages/core/database/src/__tests__/interfaces/number-interface.test.ts new file mode 100644 index 0000000000..340652952c --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/number-interface.test.ts @@ -0,0 +1,31 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { NumberInterface } from '../../interfaces/number-interface'; + +describe('number interface', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should handle bigint', async () => { + const numberInterface = new NumberInterface({}); + const value = await numberInterface.toValue('12312312321312321321'); + expect(value).toBe('12312312321312321321'); + }); +}); diff --git a/packages/core/database/src/__tests__/interfaces/percent-interface.test.ts b/packages/core/database/src/__tests__/interfaces/percent-interface.test.ts new file mode 100644 index 0000000000..fcf057f00e --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/percent-interface.test.ts @@ -0,0 +1,31 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { PercentInterface } from '../../interfaces/percent-interface'; + +describe('percent interface', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + it('should render percent interface', async () => { + const percentInterface = new PercentInterface({}); + const value = percentInterface.toString(0.5); + expect(value).toBe('50%'); + }); +}); diff --git a/packages/core/database/src/__tests__/interfaces/select-interface.test.ts b/packages/core/database/src/__tests__/interfaces/select-interface.test.ts new file mode 100644 index 0000000000..afe50abe47 --- /dev/null +++ b/packages/core/database/src/__tests__/interfaces/select-interface.test.ts @@ -0,0 +1,42 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '..'; +import { Database } from '../../database'; +import { SelectInterface } from '../../interfaces/select-interface'; + +describe('SelectInterface', () => { + let db: Database; + + beforeEach(async () => { + db = mockDatabase(); + await db.clean({ drop: true }); + }); + + afterEach(async () => { + await db.close(); + }); + + describe('toValue', () => { + it('should return value', async () => { + const options = { + uiSchema: { + enum: [ + { value: '1', label: 'Label1' }, + { value: '2', label: 'Label2' }, + ], + }, + }; + + const interfaceInstance = new SelectInterface(options); + const value = await interfaceInstance.toValue('Label1'); + expect(value).toEqual('1'); + }); + }); +}); diff --git a/packages/core/database/src/__tests__/percent2float.test.ts b/packages/core/database/src/__tests__/percent2float.test.ts index b216f78ea2..5d14a4ab38 100644 --- a/packages/core/database/src/__tests__/percent2float.test.ts +++ b/packages/core/database/src/__tests__/percent2float.test.ts @@ -19,5 +19,6 @@ describe('percent2float', () => { it('should be a floating point number', () => { expect(percent2float('123%')).toBe(1.23); expect(percent2float('22.5507%')).toBe(0.225507); // not 0.22550699999999999 + expect(percent2float('10%')).toBe(0.1); }); }); diff --git a/packages/core/database/src/__tests__/repository/chunk.test.ts b/packages/core/database/src/__tests__/repository/chunk.test.ts new file mode 100644 index 0000000000..e04d358353 --- /dev/null +++ b/packages/core/database/src/__tests__/repository/chunk.test.ts @@ -0,0 +1,103 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { mockDatabase } from '../index'; +import Database from '../../database'; +import { Collection } from '../../collection'; +import { vi } from 'vitest'; + +describe('repository chunk', () => { + let db: Database; + let Post: Collection; + let User: Collection; + + afterEach(async () => { + await db.close(); + }); + + beforeEach(async () => { + db = mockDatabase(); + + await db.clean({ drop: true }); + + User = db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'hasMany', + name: 'posts', + }, + ], + }); + + Post = db.collection({ + name: 'posts', + fields: [ + { + type: 'string', + name: 'title', + }, + { + type: 'belongsTo', + name: 'user', + }, + ], + }); + + await db.sync(); + }); + + it('should find items with chunk', async () => { + const values = Array.from({ length: 99 }, (_, i) => ({ name: `user-${i}` })); + + const repository = db.getRepository('users'); + + await repository.create({ + values, + }); + + const chunkCallback = vi.fn(); + + await repository.chunk({ + chunkSize: 10, + callback: chunkCallback, + }); + + expect(chunkCallback).toHaveBeenCalledTimes(10); + }); + + it('should chunk with limit', async () => { + const values = Array.from({ length: 99 }, (_, i) => ({ name: `user-${i}` })); + + const repository = db.getRepository('users'); + + await repository.create({ + values, + }); + + let totalCount = 0; + const chunkCallback = vi.fn(); + + await repository.chunk({ + chunkSize: 10, + limit: 11, + callback: async (rows) => { + chunkCallback(); + totalCount += rows.length; + }, + }); + + expect(chunkCallback).toHaveBeenCalledTimes(2); + expect(totalCount).toBe(11); + }); +}); diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 364c612245..0b621b2f41 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -84,6 +84,8 @@ import { import { patchSequelizeQueryInterface, snakeCase } from './utils'; import { BaseValueParser, registerFieldValueParsers } from './value-parsers'; import { ViewCollection } from './view-collection'; +import { InterfaceManager } from './interface-manager'; +import { registerInterfaces } from './interfaces/utils'; export type MergeOptions = merge.Options; @@ -180,6 +182,7 @@ export class Database extends EventEmitter implements AsyncEmitter { delayCollectionExtend = new Map(); logger: Logger; collectionGroupManager = new CollectionGroupManager(this); + interfaceManager = new InterfaceManager(this); collectionFactory: CollectionFactory = new CollectionFactory(this); declare emitAsync: (event: string | symbol, ...args: any[]) => Promise; @@ -276,6 +279,7 @@ export class Database extends EventEmitter implements AsyncEmitter { }); } + registerInterfaces(this); registerFieldValueParsers(this); this.initOperators(); diff --git a/packages/core/database/src/fields/field.ts b/packages/core/database/src/fields/field.ts index 2f529d002d..b9f230dd4f 100644 --- a/packages/core/database/src/fields/field.ts +++ b/packages/core/database/src/fields/field.ts @@ -58,6 +58,10 @@ export abstract class Field { abstract get dataType(); + isRelationField() { + return false; + } + async sync(syncOptions: SyncOptions) { await this.collection.sync({ ...syncOptions, diff --git a/packages/core/database/src/fields/relation-field.ts b/packages/core/database/src/fields/relation-field.ts index ae90e85895..66bf1aa4c5 100644 --- a/packages/core/database/src/fields/relation-field.ts +++ b/packages/core/database/src/fields/relation-field.ts @@ -44,6 +44,14 @@ export abstract class RelationField extends Field { return this.context.database.sequelize.models[this.target]; } + targetCollection() { + return this.context.database.getCollection(this.target); + } + + isRelationField(): boolean { + return true; + } + keyPairsTypeMatched(type1, type2) { type1 = type1.toLowerCase(); type2 = type2.toLowerCase(); diff --git a/packages/core/database/src/index.ts b/packages/core/database/src/index.ts index 810981a835..d54dd11ded 100644 --- a/packages/core/database/src/index.ts +++ b/packages/core/database/src/index.ts @@ -52,3 +52,4 @@ export * from './view-collection'; export * from './view/view-inference'; export * from './helpers'; export { default as sqlParser, SQLParserTypes } from './sql-parser'; +export * from './interfaces'; diff --git a/packages/core/database/src/interface-manager.ts b/packages/core/database/src/interface-manager.ts new file mode 100644 index 0000000000..3cf6c1b3a1 --- /dev/null +++ b/packages/core/database/src/interface-manager.ts @@ -0,0 +1,25 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import Database from './database'; +import { BaseInterface } from './interfaces/base-interface'; + +export class InterfaceManager { + interfaceTypes: Map BaseInterface> = new Map(); + + constructor(private db: Database) {} + + registerInterfaceType(name, iface) { + this.interfaceTypes.set(name, iface); + } + + getInterfaceType(name): new (options) => BaseInterface { + return this.interfaceTypes.get(name); + } +} diff --git a/packages/core/database/src/interfaces/base-interface.ts b/packages/core/database/src/interfaces/base-interface.ts new file mode 100644 index 0000000000..60cc081d7b --- /dev/null +++ b/packages/core/database/src/interfaces/base-interface.ts @@ -0,0 +1,53 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +export abstract class BaseInterface { + constructor(public options: any = {}) {} + + /** + * cast value to string + * @param value + * @param ctx + */ + toString(value: any, ctx?: any) { + return value; + } + + /** + * parse string to value + * @param str + * @param ctx + */ + async toValue(str: any, ctx?: any): Promise { + return str; + } + + /** + * cast value to array + * eg: 'a,b,c' => ['a', 'b', 'c'] + * eg: ['a', 'b', 'c'] => ['a', 'b', 'c'] + * @param value + * @param splitter + */ + castArray(value: any, splitter?: string) { + let values: string[] = []; + if (!value) { + values = []; + } else if (typeof value === 'string') { + values = value.split(splitter || /,|，|、/); + } else if (Array.isArray(value)) { + values = value; + } + return values.map((v) => this.trim(v)).filter(Boolean); + } + + trim(value: any) { + return typeof value === 'string' ? value.trim() : value; + } +} diff --git a/packages/core/database/src/interfaces/boolean-interface.ts b/packages/core/database/src/interfaces/boolean-interface.ts new file mode 100644 index 0000000000..b263a3641f --- /dev/null +++ b/packages/core/database/src/interfaces/boolean-interface.ts @@ -0,0 +1,46 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class BooleanInterface extends BaseInterface { + async toValue(value: string, ctx?: any): Promise { + if (typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + return !!value; + } + + if (typeof value === 'string') { + if (!value) { + return false; + } + + if (['1', 'y', 'yes', 'true', '是'].includes(value.toLowerCase())) { + return true; + } else if (['0', 'n', 'no', 'false', '否'].includes(value.toLowerCase())) { + return false; + } + } + + throw new Error(`Invalid value - ${JSON.stringify(value)}`); + } + + toString(value: any, ctx?: any) { + const enumConfig = this.options.uiSchema?.enum || []; + if (enumConfig?.length > 0) { + const option = enumConfig.find((item) => item.value === value); + return option?.label; + } else { + return value ? '是' : value === null || value === undefined ? '' : '否'; + } + } +} diff --git a/packages/core/database/src/interfaces/checkboxes-interface.ts b/packages/core/database/src/interfaces/checkboxes-interface.ts new file mode 100644 index 0000000000..e1e53468f4 --- /dev/null +++ b/packages/core/database/src/interfaces/checkboxes-interface.ts @@ -0,0 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { MultipleSelectInterface } from './multiple-select-interface'; + +export class CheckboxesInterface extends MultipleSelectInterface {} diff --git a/packages/core/database/src/interfaces/datetime-interface.ts b/packages/core/database/src/interfaces/datetime-interface.ts new file mode 100644 index 0000000000..fed3a38753 --- /dev/null +++ b/packages/core/database/src/interfaces/datetime-interface.ts @@ -0,0 +1,71 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; +import { getDefaultFormat, moment2str, str2moment } from '@nocobase/utils'; +import dayjs from 'dayjs'; +import { getJsDateFromExcel } from 'excel-date-to-js'; + +function isDate(v) { + return v instanceof Date; +} + +function isNumeric(str: any) { + if (typeof str === 'number') return true; + if (typeof str != 'string') return false; + return !isNaN(str as any) && !isNaN(parseFloat(str)); +} + +function resolveTimeZoneFromCtx(ctx) { + if (ctx?.get && ctx?.get('X-Timezone')) { + return ctx.get('X-Timezone'); + } + + return 0; +} + +export class DatetimeInterface extends BaseInterface { + async toValue(value: any, ctx: any = {}): Promise { + if (!value) { + return null; + } + + if (typeof value === 'string') { + const match = /^(\d{4})[-/]?(\d{2})[-/]?(\d{2})$/.exec(value); + if (match) { + const m = dayjs(`${match[1]}-${match[2]}-${match[3]} 00:00:00.000`); + return m.toISOString(); + } + } + + if (dayjs.isDayjs(value)) { + return value; + } else if (isDate(value)) { + return value; + } else if (isNumeric(value)) { + return getJsDateFromExcel(value).toISOString(); + } else if (typeof value === 'string') { + const props = ctx.field?.options?.uiSchema?.['x-component-props'] || {}; + const m = dayjs(value); + if (m.isValid()) { + return moment2str(m, props); + } + } + + throw new Error(`Invalid date - ${value}`); + } + + toString(value: any, ctx?: any) { + const utcOffset = resolveTimeZoneFromCtx(ctx); + const props = this.options?.uiSchema?.['x-component-props'] ?? {}; + const format = getDefaultFormat(props); + const m = str2moment(value, { ...props, utcOffset }); + return m ? m.format(format) : ''; + } +} diff --git a/packages/core/database/src/interfaces/index.ts b/packages/core/database/src/interfaces/index.ts new file mode 100644 index 0000000000..00eba5d9d8 --- /dev/null +++ b/packages/core/database/src/interfaces/index.ts @@ -0,0 +1,15 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +export * from './base-interface'; +export * from './percent-interface'; +export * from './multiple-select-interface'; +export * from './select-interface'; +export * from './datetime-interface'; +export * from './boolean-interface'; diff --git a/packages/core/database/src/interfaces/integer-interface.ts b/packages/core/database/src/interfaces/integer-interface.ts new file mode 100644 index 0000000000..9b124ba16d --- /dev/null +++ b/packages/core/database/src/interfaces/integer-interface.ts @@ -0,0 +1,16 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { NumberInterface } from './number-interface'; + +export class IntegerInterface extends NumberInterface { + validate(value): boolean { + return true; + } +} diff --git a/packages/core/database/src/interfaces/json-interface.ts b/packages/core/database/src/interfaces/json-interface.ts new file mode 100644 index 0000000000..d5310cefba --- /dev/null +++ b/packages/core/database/src/interfaces/json-interface.ts @@ -0,0 +1,20 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class JsonInterface extends BaseInterface { + async toValue(value: string, ctx?: any): Promise { + return JSON.parse(value); + } + + toString(value: any, ctx?: any) { + return JSON.stringify(value); + } +} diff --git a/packages/core/database/src/interfaces/many-to-many-interface.ts b/packages/core/database/src/interfaces/many-to-many-interface.ts new file mode 100644 index 0000000000..136d95b07f --- /dev/null +++ b/packages/core/database/src/interfaces/many-to-many-interface.ts @@ -0,0 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { ToManyInterface } from './to-many-interface'; + +export class ManyToManyInterface extends ToManyInterface {} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/utils/index.ts b/packages/core/database/src/interfaces/many-to-one-interface.ts similarity index 72% rename from packages/plugins/@nocobase/plugin-action-export/src/server/utils/index.ts rename to packages/core/database/src/interfaces/many-to-one-interface.ts index 3b36769386..5b6207b858 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/utils/index.ts +++ b/packages/core/database/src/interfaces/many-to-one-interface.ts @@ -7,4 +7,6 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -export * from './columns2Appends'; +import { ToOneInterface } from './to-one-interface'; + +export class ManyToOneInterface extends ToOneInterface {} diff --git a/packages/core/database/src/interfaces/multiple-select-interface.ts b/packages/core/database/src/interfaces/multiple-select-interface.ts new file mode 100644 index 0000000000..bec77363ef --- /dev/null +++ b/packages/core/database/src/interfaces/multiple-select-interface.ts @@ -0,0 +1,33 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; +import lodash from 'lodash'; + +export class MultipleSelectInterface extends BaseInterface { + async toValue(str: string, ctx?: any): Promise { + const items = this.castArray(str); + const enumConfig = this.options.uiSchema?.enum || []; + return items.map((item) => { + const option = enumConfig.find((option) => option.label === item); + return option ? option.value : item; + }); + } + + toString(value: any, ctx?: any) { + const enumConfig = this.options.uiSchema?.enum || []; + return lodash + .castArray(value) + .map((value) => { + const option = enumConfig.find((item) => item.value === value); + return option ? option.label : value; + }) + .join(','); + } +} diff --git a/packages/core/database/src/interfaces/number-interface.ts b/packages/core/database/src/interfaces/number-interface.ts new file mode 100644 index 0000000000..5c19a45175 --- /dev/null +++ b/packages/core/database/src/interfaces/number-interface.ts @@ -0,0 +1,53 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class NumberInterface extends BaseInterface { + sanitizeValue(value: any) { + if (typeof value === 'string') { + if (['n/a', '-'].includes(value.toLowerCase())) { + return null; + } + + if (value.includes(',')) { + value = value.replace(/,/g, ''); + } + } + + return value; + } + + async toValue(value: any) { + if (value === null || value === undefined || typeof value === 'number') { + return value; + } + + if (!value) { + return null; + } + + const sanitizedValue = this.sanitizeValue(value); + const numberValue = this.parseValue(sanitizedValue); + + if (!this.validate(numberValue)) { + throw new Error(`Invalid number value: "${value}"`); + } + + return numberValue; + } + + parseValue(value) { + return value; + } + + validate(value) { + return !isNaN(value); + } +} diff --git a/packages/core/database/src/interfaces/one-belongs-to-one-interface.ts b/packages/core/database/src/interfaces/one-belongs-to-one-interface.ts new file mode 100644 index 0000000000..0063c3d7e6 --- /dev/null +++ b/packages/core/database/src/interfaces/one-belongs-to-one-interface.ts @@ -0,0 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { ToOneInterface } from './to-one-interface'; + +export class OneBelongsToOneInterface extends ToOneInterface {} diff --git a/packages/core/database/src/interfaces/one-has-one-interface.ts b/packages/core/database/src/interfaces/one-has-one-interface.ts new file mode 100644 index 0000000000..189726fca7 --- /dev/null +++ b/packages/core/database/src/interfaces/one-has-one-interface.ts @@ -0,0 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { ToOneInterface } from './to-one-interface'; + +export class OneHasOneInterface extends ToOneInterface {} diff --git a/packages/core/database/src/interfaces/one-to-many-interface.ts b/packages/core/database/src/interfaces/one-to-many-interface.ts new file mode 100644 index 0000000000..de43e19395 --- /dev/null +++ b/packages/core/database/src/interfaces/one-to-many-interface.ts @@ -0,0 +1,12 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { ToManyInterface } from './to-many-interface'; + +export class OneToManyInterface extends ToManyInterface {} diff --git a/packages/core/database/src/interfaces/percent-interface.ts b/packages/core/database/src/interfaces/percent-interface.ts new file mode 100644 index 0000000000..baa276fcc2 --- /dev/null +++ b/packages/core/database/src/interfaces/percent-interface.ts @@ -0,0 +1,28 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { toFixedByStep } from '@nocobase/utils'; +import { NumberInterface } from './number-interface'; +import { percent2float } from '../utils'; + +export class PercentInterface extends NumberInterface { + parseValue(value) { + if (typeof value === 'string' && value.endsWith('%')) { + const parsedValue = percent2float(value); + return parsedValue; + } + + return value; + } + + toString(value) { + const step = this.options?.uiSchema?.['x-component-props']?.['step'] ?? 0; + return value && `${toFixedByStep(value * 100, step)}%`; + } +} diff --git a/packages/core/database/src/interfaces/select-interface.ts b/packages/core/database/src/interfaces/select-interface.ts new file mode 100644 index 0000000000..a904a150be --- /dev/null +++ b/packages/core/database/src/interfaces/select-interface.ts @@ -0,0 +1,24 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class SelectInterface extends BaseInterface { + async toValue(str: string, ctx?: any): Promise { + const enumConfig = this.options.uiSchema?.enum || []; + const option = enumConfig.find((item) => item.label === str); + return option?.value || str; + } + + toString(value: any, ctx?: any) { + const enumConfig = this.options.uiSchema?.enum || []; + const option = enumConfig.find((item) => item.value === value); + return option?.label || value; + } +} diff --git a/packages/core/database/src/interfaces/to-many-interface.ts b/packages/core/database/src/interfaces/to-many-interface.ts new file mode 100644 index 0000000000..185f84ef11 --- /dev/null +++ b/packages/core/database/src/interfaces/to-many-interface.ts @@ -0,0 +1,29 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class ToManyInterface extends BaseInterface { + async toValue(str: string, ctx?: any) { + const items = str.split(','); + + const { filterKey, targetCollection, transaction } = ctx; + + const targetInstances = await targetCollection.repository.find({ + filter: { + [filterKey]: items, + }, + transaction, + }); + + const primaryKeyAttribute = targetCollection.model.primaryKeyAttribute; + + return targetInstances.map((targetInstance) => targetInstance[primaryKeyAttribute]); + } +} diff --git a/packages/core/database/src/interfaces/to-one-interface.ts b/packages/core/database/src/interfaces/to-one-interface.ts new file mode 100644 index 0000000000..4dffbc0977 --- /dev/null +++ b/packages/core/database/src/interfaces/to-one-interface.ts @@ -0,0 +1,35 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from './base-interface'; + +export class ToOneInterface extends BaseInterface { + toString(value: any, ctx?: any): string { + return value; + } + + async toValue(str: string, ctx?: any) { + const { filterKey, targetCollection, transaction } = ctx; + + const targetInstance = await targetCollection.repository.findOne({ + filter: { + [filterKey]: str, + }, + transaction, + }); + + const primaryKeyAttribute = targetCollection.model.primaryKeyAttribute; + + if (targetInstance) { + return targetInstance[primaryKeyAttribute]; + } + + return null; + } +} diff --git a/packages/core/database/src/interfaces/utils.ts b/packages/core/database/src/interfaces/utils.ts new file mode 100644 index 0000000000..394cbf20b3 --- /dev/null +++ b/packages/core/database/src/interfaces/utils.ts @@ -0,0 +1,53 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import Database from '../database'; +import { + BooleanInterface, + DatetimeInterface, + MultipleSelectInterface, + PercentInterface, + SelectInterface, +} from './index'; +import { ManyToOneInterface } from './many-to-one-interface'; +import { ManyToManyInterface } from './many-to-many-interface'; +import { OneHasOneInterface } from './one-has-one-interface'; +import { OneBelongsToOneInterface } from './one-belongs-to-one-interface'; +import { OneToManyInterface } from './one-to-many-interface'; +import { IntegerInterface } from './integer-interface'; +import { NumberInterface } from './number-interface'; +import { JsonInterface } from './json-interface'; + +const interfaces = { + integer: IntegerInterface, + number: NumberInterface, + multipleSelect: MultipleSelectInterface, + checkboxes: MultipleSelectInterface, + checkboxGroup: MultipleSelectInterface, + select: SelectInterface, + radio: SelectInterface, + radioGroup: SelectInterface, + percent: PercentInterface, + datetime: DatetimeInterface, + createdAt: DatetimeInterface, + updatedAt: DatetimeInterface, + boolean: BooleanInterface, + json: JsonInterface, + oho: OneHasOneInterface, + obo: OneBelongsToOneInterface, + o2m: OneToManyInterface, + m2o: ManyToOneInterface, + m2m: ManyToManyInterface, +}; + +export function registerInterfaces(db: Database) { + for (const [interfaceName, type] of Object.entries(interfaces)) { + db.interfaceManager.registerInterfaceType(interfaceName, type); + } +} diff --git a/packages/core/database/src/model.ts b/packages/core/database/src/model.ts index c8e168ef52..0194250137 100644 --- a/packages/core/database/src/model.ts +++ b/packages/core/database/src/model.ts @@ -91,6 +91,7 @@ export class Model fn.apply(this, [carry, options]), obj); }; @@ -148,6 +149,24 @@ export class Model { + return options.model.rawAttributes[key].type.constructor.name === 'BIGINT'; + }); + + for (const key of bigIntKeys) { + if (obj[key] !== null && obj[key] !== undefined && typeof obj[key] !== 'string' && typeof obj[key] !== 'number') { + obj[key] = obj[key].toString(); + } + } + + return obj; + } + private sortAssociations(data, { field }: JSONTransformerOptions): any { const sortBy = field.options.sortBy; return sortBy ? this.sortArray(data, sortBy) : data; diff --git a/packages/core/database/src/query-interface/mysql-query-interface.ts b/packages/core/database/src/query-interface/mysql-query-interface.ts index 2b125d5acc..d62059e411 100644 --- a/packages/core/database/src/query-interface/mysql-query-interface.ts +++ b/packages/core/database/src/query-interface/mysql-query-interface.ts @@ -98,18 +98,18 @@ export default class MysqlQueryInterface extends QueryInterface { return results[0]['Create Table']; } - async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string }): Promise<{ + async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string; transaction: Transaction }): Promise<{ seqName?: string; currentVal: number; }> { - const { tableInfo, fieldName } = options; + const { tableInfo, fieldName, transaction } = options; const sql = `SELECT AUTO_INCREMENT as currentVal FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = '${tableInfo.tableName}';`; - const results = await this.db.sequelize.query(sql, { type: 'SELECT' }); + const results = await this.db.sequelize.query(sql, { type: 'SELECT', transaction }); let currentVal = results[0]['currentVal'] as number; @@ -117,7 +117,7 @@ export default class MysqlQueryInterface extends QueryInterface { // use max value of field instead const maxSql = `SELECT MAX(${fieldName}) as currentVal FROM ${tableInfo.tableName};`; - const maxResults = await this.db.sequelize.query(maxSql, { type: 'SELECT' }); + const maxResults = await this.db.sequelize.query(maxSql, { type: 'SELECT', transaction }); currentVal = maxResults[0]['currentVal'] as number; } diff --git a/packages/core/database/src/query-interface/postgres-query-interface.ts b/packages/core/database/src/query-interface/postgres-query-interface.ts index eda08b0cd7..2d251ddf51 100644 --- a/packages/core/database/src/query-interface/postgres-query-interface.ts +++ b/packages/core/database/src/query-interface/postgres-query-interface.ts @@ -52,9 +52,11 @@ export default class PostgresQueryInterface extends QueryInterface { async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string; + transaction: Transaction; }): Promise<{ seqName?: string; currentVal: number }> { const fieldName = options.fieldName || 'id'; const tableInfo = options.tableInfo; + const transaction = options.transaction; const sequenceNameResult = await this.db.sequelize.query( `SELECT column_default @@ -62,6 +64,9 @@ export default class PostgresQueryInterface extends QueryInterface { WHERE table_name = '${tableInfo.tableName}' and table_schema = '${tableInfo.schema || 'public'}' and "column_name" = '${fieldName}';`, + { + transaction, + }, ); const columnDefault = sequenceNameResult[0][0]['column_default']; @@ -74,6 +79,9 @@ export default class PostgresQueryInterface extends QueryInterface { const sequenceCurrentValResult = await this.db.sequelize.query( `select last_value from ${sequenceName}`, + { + transaction, + }, ); const sequenceCurrentVal = parseInt(sequenceCurrentValResult[0][0]['last_value']); diff --git a/packages/core/database/src/query-interface/query-interface.ts b/packages/core/database/src/query-interface/query-interface.ts index e0d32cd4b6..f042530fd4 100644 --- a/packages/core/database/src/query-interface/query-interface.ts +++ b/packages/core/database/src/query-interface/query-interface.ts @@ -64,6 +64,7 @@ export default abstract class QueryInterface { abstract getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string; + transaction?: Transaction; }): Promise<{ seqName?: string; currentVal: number }>; abstract setAutoIncrementVal(options: { diff --git a/packages/core/database/src/query-interface/sqlite-query-interface.ts b/packages/core/database/src/query-interface/sqlite-query-interface.ts index 8663aebda5..2f6b93c877 100644 --- a/packages/core/database/src/query-interface/sqlite-query-interface.ts +++ b/packages/core/database/src/query-interface/sqlite-query-interface.ts @@ -103,11 +103,11 @@ export default class SqliteQueryInterface extends QueryInterface { return Promise.resolve(undefined); } - async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string }): Promise<{ + async getAutoIncrementInfo(options: { tableInfo: TableInfo; fieldName: string; transaction: Transaction }): Promise<{ seqName?: string; currentVal: number; }> { - const { tableInfo } = options; + const { tableInfo, transaction } = options; const tableName = tableInfo.tableName; @@ -115,7 +115,7 @@ export default class SqliteQueryInterface extends QueryInterface { FROM sqlite_sequence WHERE name = '${tableName}';`; - const results = await this.db.sequelize.query(sql, { type: 'SELECT' }); + const results = await this.db.sequelize.query(sql, { type: 'SELECT', transaction }); const row = results[0]; diff --git a/packages/core/database/src/repository.ts b/packages/core/database/src/repository.ts index 353b83cb17..ca7a72147b 100644 --- a/packages/core/database/src/repository.ts +++ b/packages/core/database/src/repository.ts @@ -390,6 +390,42 @@ export class Repository Promise }, + ) { + const { chunkSize, callback, limit: overallLimit } = options; + const transaction = await this.getTransaction(options); + + let offset = 0; + let totalProcessed = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + // Calculate the limit for the current chunk + const currentLimit = overallLimit !== undefined ? Math.min(chunkSize, overallLimit - totalProcessed) : chunkSize; + + const rows = await this.find({ + ...options, + limit: currentLimit, + offset, + transaction, + }); + + if (rows.length === 0) { + break; + } + + await callback(rows, options); + + offset += currentLimit; + totalProcessed += rows.length; + + if (overallLimit !== undefined && totalProcessed >= overallLimit) { + break; + } + } + } + /** * find * @param options diff --git a/packages/plugins/@nocobase/plugin-acl/src/client/__e2e__/utils.ts b/packages/plugins/@nocobase/plugin-acl/src/client/__e2e__/utils.ts index 71518ada1e..625531e62c 100644 --- a/packages/plugins/@nocobase/plugin-acl/src/client/__e2e__/utils.ts +++ b/packages/plugins/@nocobase/plugin-acl/src/client/__e2e__/utils.ts @@ -8,6 +8,7 @@ */ import { general, PageConfig } from '@nocobase/test/e2e'; + /** * 页面中有一个空的 Table 区块，并且配有字段:普通字段和关系字段 */ @@ -650,7 +651,7 @@ export const oneTableBlock: PageConfig = { marginBottom: 'var(--marginSM)', backgroundColor: 'var(--colorInfoBg)', }, - content: '{{ t("Download tip", {ns: "import" }) }}', + content: '{{ t("Download tips", {ns: "import" }) }}', }, _isJSONSchemaObject: true, 'x-uid': 'p47ou5drhji', diff --git a/packages/plugins/@nocobase/plugin-action-export/package.json b/packages/plugins/@nocobase/plugin-action-export/package.json index eee70ab6c2..29ed701187 100644 --- a/packages/plugins/@nocobase/plugin-action-export/package.json +++ b/packages/plugins/@nocobase/plugin-action-export/package.json @@ -17,7 +17,9 @@ "file-saver": "^2.0.5", "node-xlsx": "^0.16.1", "react": "^18.2.0", - "react-i18next": "^11.15.1" + "react-i18next": "^11.15.1", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz", + "async-mutex": "^0.3.2" }, "peerDependencies": { "@nocobase/actions": "1.x", diff --git a/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx b/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx index c520a7a68c..4886094452 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx +++ b/packages/plugins/@nocobase/plugin-action-export/src/client/useExportAction.tsx @@ -7,21 +7,24 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -import { useFieldSchema } from '@formily/react'; +import { useField, useFieldSchema } from '@formily/react'; import { + mergeFilter, useBlockRequestContext, useCollection_deprecated, useCollectionManager_deprecated, useCompile, - mergeFilter, + useCurrentAppInfo, } from '@nocobase/client'; import lodash from 'lodash'; import { saveAs } from 'file-saver'; import { App } from 'antd'; import { useExportTranslation } from './locale'; +import { useMemo } from 'react'; export const useExportAction = () => { const { service, resource, props } = useBlockRequestContext(); + const appInfo = useCurrentAppInfo(); const defaultFilter = props?.params.filter; const actionSchema = useFieldSchema(); const compile = useCompile(); @@ -30,11 +33,22 @@ export const useExportAction = () => { const { t } = useExportTranslation(); const { modal } = App.useApp(); const filters = service.params?.[1]?.filters || {}; + const field = useField(); + const exportLimit = useMemo(() => { + if (appInfo?.data?.exportLimit) { + return appInfo.data.exportLimit; + } + + return 2000; + }, [appInfo]); + return { async onClick() { + field.data = field.data || {}; + field.data.loading = true; const confirmed = await modal.confirm({ title: t('Export'), - content: t('Export warning'), + content: t('Export warning', { limit: exportLimit }), okText: t('Start export'), }); if (!confirmed) { @@ -53,9 +67,6 @@ export const useExportAction = () => { ]; } es.defaultTitle = uiSchema?.title; - if (fieldInterface === 'chinaRegion') { - es.dataIndex.push('name'); - } }); const { data } = await resource.export( { @@ -73,6 +84,7 @@ export const useExportAction = () => { }, ); const blob = new Blob([data], { type: 'application/x-xls' }); + field.data.loading = false; saveAs(blob, `${compile(title)}.xlsx`); }, }; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/locale/en-US.json b/packages/plugins/@nocobase/plugin-action-export/src/locale/en-US.json index e1366faf9a..19f7aaa58e 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/locale/en-US.json +++ b/packages/plugins/@nocobase/plugin-action-export/src/locale/en-US.json @@ -1,4 +1,4 @@ { - "Export warning": "You can export up to 200 rows of data at a time, any excess will be ignored.", + "Export warning": "You can export up to {{limit}} rows of data at a time, any excess will be ignored.", "Start export": "Start export" } diff --git a/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json index b9187b2e79..dbe6fb2659 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json +++ b/packages/plugins/@nocobase/plugin-action-export/src/locale/zh-CN.json @@ -1,4 +1,4 @@ { - "Export warning": "每次最多导出 200 行数据，超出的将被忽略。", + "Export warning": "每次最多导出 {{limit}} 行数据，超出的将被忽略。", "Start export": "开始导出" } diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts new file mode 100644 index 0000000000..d1baa8ab22 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/export-to-xlsx.test.ts @@ -0,0 +1,870 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { createMockServer, MockServer } from '@nocobase/test'; +import { uid } from '@nocobase/utils'; +import XlsxExporter from '../xlsx-exporter'; +import XLSX from 'xlsx'; +import fs from 'fs'; +import path from 'path'; +import { BaseInterface } from '@nocobase/database'; +import moment from 'moment'; + +XLSX.set_fs(fs); + +describe('export to xlsx with preset', () => { + let app: MockServer; + + beforeEach(async () => { + app = await createMockServer({ + plugins: ['nocobase', 'map'], + }); + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should export with map field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { + name: 'circle', + type: 'circle', + interface: 'circle', + uiSchema: { + 'x-component-props': { mapType: 'amap' }, + type: 'void', + 'x-component': 'Map', + 'x-component-designer': 'Map.Designer', + title: 'circle', + }, + }, + ], + }); + + await app.db.sync(); + + await Post.repository.create({ + values: { + title: 'p1', + circle: [116.397428, 39.90923, 3241], + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + chunkSize: 10, + columns: [ + { dataIndex: ['title'], defaultTitle: 'Title' }, + { + dataIndex: ['circle'], + defaultTitle: 'circle', + }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const header = sheetData[0]; + expect(header).toEqual(['Title', 'circle']); + + const firstUser = sheetData[1]; + expect(firstUser[1]).toEqual('116.397428,39.90923,3241'); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with attachment field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { + name: 'attachment1', + type: 'belongsToMany', + interface: 'attachment', + uiSchema: { + 'x-component-props': { + accept: 'image/*', + multiple: true, + }, + type: 'array', + 'x-component': 'Upload.Attachment', + title: 'attachment1', + }, + target: 'attachments', + storage: 'local', + through: 'postsAttachments', + }, + ], + }); + + await app.db.sync(); + + await Post.repository.create({ + values: { + title: 'p1', + attachment1: [ + { + title: 'nocobase-logo1', + filename: '682e5ad037dd02a0fe4800a3e91c283b.png', + extname: '.png', + mimetype: 'image/png', + url: 'https://nocobase.oss-cn-beijing.aliyuncs.com/test1.png', + }, + { + title: 'nocobase-logo2', + filename: '682e5ad037dd02a0fe4800a3e91c283b.png', + extname: '.png', + mimetype: 'image/png', + url: 'https://nocobase.oss-cn-beijing.aliyuncs.com/test2.png', + }, + ], + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + chunkSize: 10, + columns: [ + { dataIndex: ['title'], defaultTitle: 'Title' }, + { + dataIndex: ['attachment1'], + defaultTitle: 'attachment', + }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const header = sheetData[0]; + expect(header).toEqual(['Title', 'attachment']); + + const firstUser = sheetData[1]; + expect(firstUser[1]).toEqual( + 'https://nocobase.oss-cn-beijing.aliyuncs.com/test1.png,https://nocobase.oss-cn-beijing.aliyuncs.com/test2.png', + ); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with china region field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { + type: 'belongsToMany', + target: 'chinaRegions', + through: 'userRegions', + targetKey: 'code', + interface: 'chinaRegion', + name: 'region', + }, + ], + }); + + await app.db.sync(); + + await Post.repository.create({ + values: { + title: 'post0', + region: [{ code: '14' }, { code: '1404' }, { code: '140406' }], + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + chunkSize: 10, + columns: [ + { dataIndex: ['title'], defaultTitle: 'Title' }, + { + dataIndex: ['region'], + defaultTitle: 'region', + }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const header = sheetData[0]; + expect(header).toEqual(['Title', 'region']); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['post0', '山西省/长治市/潞城区']); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); +}); + +describe('export to xlsx', () => { + let app: MockServer; + + beforeEach(async () => { + app = await createMockServer({ + plugins: ['action-export'], + }); + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should export with json field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { + name: 'title', + type: 'string', + }, + { + name: 'json', + type: 'json', + }, + ], + }); + + await app.db.sync(); + + await Post.repository.create({ + values: { + title: 'some_title', + json: { + a: { + b: 'c', + }, + }, + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + chunkSize: 10, + columns: [ + { dataIndex: ['title'], defaultTitle: '' }, + { + dataIndex: ['json'], + defaultTitle: '', + }, + ], + }); + + const wb = await exporter.run({}); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual([ + 'some_title', + JSON.stringify({ + a: { + b: 'c', + }, + }), + ]); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with datetime field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { + name: 'title', + type: 'string', + }, + { + uiSchema: { + 'x-component-props': { dateFormat: 'YYYY-MM-DD', gmt: false, showTime: true, timeFormat: 'HH:mm:ss' }, + type: 'string', + 'x-component': 'DatePicker', + title: 'test_date', + }, + name: 'test_date', + type: 'date', + interface: 'datetime', + }, + ], + }); + + await app.db.sync(); + + await Post.repository.create({ + values: { + title: 'some_title', + test_date: '2024-05-10T01:42:35.000Z', + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + chunkSize: 10, + columns: [ + { dataIndex: ['title'], defaultTitle: '' }, + { + dataIndex: ['test_date'], + defaultTitle: '', + }, + ], + }); + + const wb = await exporter.run({ + get() { + return '+08:00'; + }, + }); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['some_title', '2024-05-10 09:42:35']); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with multi select', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name', title: '姓名' }, + { + uiSchema: { + enum: [ + { + value: '123', + label: 'Label123', + color: 'orange', + }, + { + value: '223', + label: 'Label223', + color: 'lime', + }, + ], + type: 'array', + 'x-component': 'Select', + 'x-component-props': { + mode: 'multiple', + }, + title: 'multi-select', + }, + defaultValue: [], + name: 'multiSelect', + type: 'array', + interface: 'multipleSelect', + }, + ], + }); + + await app.db.sync(); + + await User.repository.create({ + values: { + name: 'u1', + multiSelect: ['123', '223'], + }, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { + dataIndex: ['multiSelect'], + defaultTitle: '', + }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + XLSX.writeFile(wb, xlsxFilePath); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['u1', 'Label123,Label223']); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with different ui schema', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name', title: '姓名' }, + { type: 'integer', name: 'age', title: '年龄' }, + { + type: 'integer', + name: 'testInterface', + interface: 'testInterface', + title: 'Interface 测试', + uiSchema: { test: 'testValue' }, + }, + ], + }); + + class TestInterface extends BaseInterface { + toString(value, ctx) { + return `${this.options.uiSchema.test}.${value}`; + } + } + + app.db.interfaceManager.registerInterfaceType('testInterface', TestInterface); + + await app.db.sync(); + const values = Array.from({ length: 20 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + testInterface: index, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { + dataIndex: ['testInterface'], + defaultTitle: '', + }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + await new Promise((resolve, reject) => { + XLSX.writeFileAsync( + xlsxFilePath, + wb, + { + type: 'array', + }, + () => { + resolve(123); + }, + ); + }); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const header = sheetData[0]; + expect(header).toEqual(['姓名', 'Interface 测试']); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['user0', 'testValue.0']); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with empty title', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name', title: '姓名' }, + { type: 'integer', name: 'age', title: '年龄' }, + ], + }); + + await app.db.sync(); + const values = Array.from({ length: 20 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + findOptions: { + filter: { + age: { + $gt: 9, + }, + }, + }, + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + await new Promise((resolve, reject) => { + XLSX.writeFileAsync( + xlsxFilePath, + wb, + { + type: 'array', + }, + () => { + resolve(123); + }, + ); + }); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + + const header = sheetData[0]; + expect(header).toEqual(['姓名', '年龄']); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with filter', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + ], + }); + + await app.db.sync(); + + const values = Array.from({ length: 20 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + findOptions: { + filter: { + age: { + $gt: 9, + }, + }, + }, + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + await new Promise((resolve, reject) => { + XLSX.writeFileAsync( + xlsxFilePath, + wb, + { + type: 'array', + }, + () => { + resolve(123); + }, + ); + }); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + expect(sheetData.length).toBe(11); // 10 users + 1 header + + const header = sheetData[0]; + expect(header).toEqual(['Name', 'Age']); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['user10', 10]); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export with associations', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + { + type: 'hasMany', + name: 'posts', + target: 'posts', + }, + { + type: 'belongsToMany', + name: 'groups', + target: 'groups', + through: 'usersGroups', + }, + { + name: 'createdAt', + type: 'date', + interface: 'createdAt', + field: 'createdAt', + uiSchema: { + type: 'datetime', + title: '{{t("Created at")}}', + 'x-component': 'DatePicker', + 'x-component-props': {}, + 'x-read-pretty': true, + }, + }, + ], + }); + + const Post = app.db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { type: 'belongsTo', name: 'user', target: 'users' }, + ], + }); + + const Group = app.db.collection({ + name: 'groups', + fields: [{ type: 'string', name: 'name' }], + }); + + await app.db.sync(); + + const [group1, group2, group3] = await Group.repository.create({ + values: [{ name: 'group1' }, { name: 'group2' }, { name: 'group3' }], + }); + + const values = Array.from({ length: 22 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + groups: [ + { + id: group1.get('id'), + }, + { + id: group2.get('id'), + }, + { + id: group3.get('id'), + }, + ], + posts: Array.from({ length: 3 }).map((_, postIndex) => { + return { + title: `post${postIndex}`, + }; + }), + }; + }); + + await User.repository.create({ + values, + }); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + { dataIndex: ['posts', 'title'], defaultTitle: 'Post Title' }, + { dataIndex: ['groups', 'name'], defaultTitle: 'Group Names' }, + { dataIndex: ['createdAt'], defaultTitle: 'Created at' }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + await new Promise((resolve, reject) => { + XLSX.writeFileAsync( + xlsxFilePath, + wb, + { + type: 'array', + }, + () => { + resolve(123); + }, + ); + }); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + expect(sheetData.length).toBe(23); // 22 users * 3 posts + 1 header + + const header = sheetData[0]; + expect(header).toEqual(['Name', 'Age', 'Post Title', 'Group Names', 'Created at']); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual([ + 'user0', + 0, + 'post0,post1,post2', + 'group1,group2,group3', + moment().format('YYYY-MM-DD'), + ]); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); + + it('should export data to xlsx', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { type: 'string', name: 'name' }, + { type: 'integer', name: 'age' }, + ], + }); + + await app.db.sync(); + + const values = Array.from({ length: 22 }).map((_, index) => { + return { + name: `user${index}`, + age: index % 100, + }; + }); + + await User.model.bulkCreate(values); + + const exporter = new XlsxExporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + chunkSize: 10, + columns: [ + { dataIndex: ['name'], defaultTitle: 'Name' }, + { dataIndex: ['age'], defaultTitle: 'Age' }, + ], + }); + + const wb = await exporter.run(); + + const xlsxFilePath = path.resolve(__dirname, `t_${uid()}.xlsx`); + try { + await new Promise((resolve, reject) => { + XLSX.writeFileAsync( + xlsxFilePath, + wb, + { + type: 'array', + }, + () => { + resolve(123); + }, + ); + }); + + // read xlsx file + const workbook = XLSX.readFile(xlsxFilePath); + const firstSheet = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(firstSheet, { header: 1 }); + expect(sheetData.length).toBe(23); // 22 users + 1 header + + const header = sheetData[0]; + expect(header).toEqual(['Name', 'Age']); + + const firstUser = sheetData[1]; + expect(firstUser).toEqual(['user0', 0]); + } finally { + fs.unlinkSync(xlsxFilePath); + } + }); +}); diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/utils/utils.test.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/utils/utils.test.ts deleted file mode 100644 index b0563a6bd5..0000000000 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/__tests__/utils/utils.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import Database from '@nocobase/database'; -import { mockServer, MockServer } from '@nocobase/test'; - -describe('utils', () => { - let columns = null; - let db: Database; - let app: MockServer; - - beforeEach(async () => { - app = mockServer(); - db = app.db; - }); - - afterEach(async () => { - await app.destroy(); - }); - - it('first columns2Appends', async () => { - columns = [ - { dataIndex: ['f_kp6gk63udss'], defaultTitle: '商品名称' }, - { - dataIndex: ['f_brjkofr2mbt'], - enum: [ - { value: 'lzjqrrw2vdl', label: '节' }, - { value: 'i0qarqlm87m', label: '胡' }, - { value: '1fpb8x0swq1', label: '一一' }, - ], - defaultTitle: '工在在', - }, - { dataIndex: ['f_qhvvfuignh2', 'createdBy', 'id'], defaultTitle: 'ID' }, - { dataIndex: ['f_wu28mus1c65', 'roles', 'title'], defaultTitle: '角色名称' }, - ]; - }); - - it('second columns2Appends', async () => { - columns = [ - { dataIndex: ['f_kp6gk63udss'], defaultTitle: '商品名称' }, - { - dataIndex: ['f_brjkofr2mbt'], - enum: [ - { value: 'lzjqrrw2vdl', label: '节' }, - { value: 'i0qarqlm87m', label: '胡' }, - { value: '1fpb8x0swq1', label: '一一' }, - ], - defaultTitle: '工在在', - }, - { dataIndex: ['f_qhvvfuignh2', 'createdBy', 'id'], defaultTitle: 'ID' }, - { dataIndex: ['f_qhvvfuignh2', 'createdBy', 'nickname'], defaultTitle: '角色名称' }, - ]; - }); -}); diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts index 6e3c255da9..ebcef7a732 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/actions/export-xlsx.ts @@ -9,46 +9,61 @@ import { Context, Next } from '@nocobase/actions'; import { Repository } from '@nocobase/database'; -import xlsx from 'node-xlsx'; -import render from '../renders'; -import { columns2Appends } from '../utils'; -export async function exportXlsx(ctx: Context, next: Next) { +import XlsxExporter from '../xlsx-exporter'; +import XLSX from 'xlsx'; +import { Mutex } from 'async-mutex'; +import { DataSource } from '@nocobase/data-source-manager'; + +const mutex = new Mutex(); + +async function exportXlsxAction(ctx: Context, next: Next) { const { title, filter, sort, fields, except } = ctx.action.params; let columns = ctx.action.params.values?.columns || ctx.action.params?.columns; + if (typeof columns === 'string') { columns = JSON.parse(columns); } + const repository = ctx.getCurrentRepository() as Repository; + const dataSource = ctx.dataSource as DataSource; + const collection = repository.collection; - columns = columns?.filter((col) => collection.hasField(col.dataIndex[0]) && col?.dataIndex?.length > 0); - const appends = columns2Appends(columns, ctx); - const data = await repository.find({ - filter, - fields, - appends, - except, - sort, - limit: 200, - context: ctx, - }); - const collectionFields = columns.map((col) => collection.fields.get(col.dataIndex[0])); - const { rows, ranges } = await render({ columns, fields: collectionFields, data }, ctx); - ctx.body = xlsx.build([ - { - name: 'Sheet 1', - data: rows, - options: { - '!merges': ranges, - }, + + const xlsxExporter = new XlsxExporter({ + collectionManager: dataSource.collectionManager, + collection, + columns, + findOptions: { + filter, + fields, + except, + sort, }, - ]); + }); + + const wb = await xlsxExporter.run(); + + ctx.body = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }); ctx.set({ 'Content-Type': 'application/octet-stream', - // to avoid "invalid character" error in header (RFC) 'Content-Disposition': `attachment; filename=${encodeURI(title)}.xlsx`, }); +} + +export async function exportXlsx(ctx: Context, next: Next) { + if (mutex.isLocked()) { + throw new Error(`another export action is running, please try again later.`); + } + + const release = await mutex.acquire(); + + try { + await exportXlsxAction(ctx, next); + } finally { + release(); + } await next(); } diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/collections/.gitkeep b/packages/plugins/@nocobase/plugin-action-export/src/server/collections/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts index d34186a3eb..3e6cb8c10e 100644 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/index.ts @@ -7,7 +7,7 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -import { InstallOptions, Plugin } from '@nocobase/server'; +import { Plugin } from '@nocobase/server'; import { exportXlsx } from './actions'; export class PluginActionExportServer extends Plugin { @@ -15,11 +15,6 @@ export class PluginActionExportServer extends Plugin { async load() { this.app.dataSourceManager.afterAddDataSource((dataSource) => { - // @ts-ignore - if (!dataSource.collectionManager?.db) { - return; - } - dataSource.resourceManager.registerActionHandler('export', exportXlsx); dataSource.acl.setAvailableAction('export', { displayName: '{{t("Export")}}', @@ -27,8 +22,6 @@ export class PluginActionExportServer extends Plugin { }); }); } - - async install(options: InstallOptions) {} } export default PluginActionExportServer; diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/models/.gitkeep b/packages/plugins/@nocobase/plugin-action-export/src/server/models/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/renders/index.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/renders/index.ts deleted file mode 100644 index e6b5c2ae58..0000000000 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/renders/index.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import * as renders from './renders'; - -function getInterfaceRender(name: string): Function { - return renders[name] || renders._; -} - -function renderHeader(params, ctx) { - const { columns, fields, headers = [], rowIndex = 0 } = params; - - const { colIndex = 0 } = params; - - if (!headers[rowIndex]) { - headers.push([]); - } - const row = headers[rowIndex]; - fields.forEach((field, i) => { - const nextColIndex = colIndex + i; - row.push({ - column: columns[i], - field, - rowIndex, - colIndex: nextColIndex, - }); - // if (field.interface === 'subTable') { - // const subTable = ctx.db.getTable(field.target); - // const subFields = subTable.getOptions().fields.filter((field) => Boolean(field.__index)); - // renderHeader( - // { - // fields: subFields, - // headers, - // rowIndex: rowIndex + 1, - // colIndex: nextColIndex, - // }, - // ctx, - // ); - // colIndex += subFields.length; - // } - }); - - Object.assign(params, { headers }); -} - -async function renderRows({ columns, fields, data }, ctx) { - return await data.reduce(async (preResult, row) => { - const result = await preResult; - const thisRow = []; - const rowIndex = 0; - let colOffset = 0; - for (let i = 0, iLen = fields.length; i < iLen; i++) { - const field = fields[i]; - - if (!thisRow[rowIndex]) { - thisRow.push([]); - } - const cells = thisRow[rowIndex]; - if (field.options.interface !== 'subTable') { - const render = getInterfaceRender(field.options.interface); - const value = await render(field, row, ctx, columns[i]); - cells.push({ - value, - rowIndex: result.length + rowIndex, - colIndex: i + colOffset, - }); - } else { - const subTable = ctx.db.getTable(field.target); - const subFields = subTable.getOptions().fields.filter((item) => Boolean(item.__index)); - //TODO: must provide sub-table columns - const subTableColumns = []; - const subRows = await renderRows( - { columns: subTableColumns, fields: subFields, data: row.get(field.name) || [] }, - ctx, - ); - - // const { rows: subRowGroups } = subTableRows; - subRows.forEach((cells, j) => { - const subRowIndex = rowIndex + j; - if (!thisRow[subRowIndex]) { - thisRow.push([]); - } - const subCells = thisRow[subRowIndex]; - subCells.push( - ...cells.map((cell) => ({ - ...cell, - rowIndex: result.length + subRowIndex, - colIndex: cell.colIndex + i, - })), - ); - }); - colOffset += subFields.length; - } - } - thisRow.forEach((cells) => { - cells.forEach((cell) => { - const relRowIndex = cell.rowIndex - result.length; - Object.assign(cell, { - rowSpan: - relRowIndex >= thisRow.length - 1 || - thisRow[relRowIndex + 1].find((item) => item.colIndex === cell.colIndex) - ? 1 - : thisRow.length - relRowIndex, - }); - }); - }); - - return result.concat(thisRow); - }, Promise.resolve([])); -} - -export default async function ({ columns, fields, data }, ctx) { - const headers = []; - renderHeader({ columns, fields, headers }, ctx); - const ranges = []; - // 计算全表最大的列索引（由于无论如何最大列都是单个单元格，所以等价于长度） - const maxColIndex = Math.max(...headers.map((row) => row[row.length - 1].colIndex)); - // 遍历所有单元格，计算需要合并的坐标范围 - headers.forEach((row, rowIndex) => { - row.forEach((cell, cellIndex) => { - // 跨行合并的行数为 - cell.rowSpan = - cell.rowIndex >= headers.length - 1 || - headers[cell.rowIndex + 1].find((item) => item.colIndex === cell.colIndex) - ? 1 - : headers.length - cell.rowIndex; - - const nextCell = headers - .slice(0, rowIndex + 1) - .map((r) => r.find((item) => item.colIndex > cell.colIndex)) - .filter((c) => Boolean(c)) - .reduce((min, c) => (min && Math.min(min.colIndex, c.colIndex) === min.colIndex ? min : c), null); - cell.colSpan = nextCell ? nextCell.colIndex - cell.colIndex : maxColIndex - cell.colIndex + 1; - - if (cell.rowSpan > 1 || cell.colSpan > 1) { - ranges.push({ - s: { c: cell.colIndex, r: cell.rowIndex }, - e: { c: cell.colIndex + cell.colSpan - 1, r: cell.rowIndex + cell.rowSpan - 1 }, - }); - } - }); - }); - - const rows = (await renderRows({ columns, fields, data }, ctx)).map((row) => { - const cells = Array(maxColIndex).fill(null); - row.forEach((cell) => { - cells.splice(cell.colIndex, 1, cell.value); - if (cell.rowSpan > 1) { - ranges.push({ - s: { c: cell.colIndex, r: cell.rowIndex + headers.length }, - e: { c: cell.colIndex, r: cell.rowIndex + cell.rowSpan - 1 + headers.length }, - }); - } - }); - return cells; - }); - - return { - rows: [ - ...headers.map((row) => { - // 补齐无数据单元格，以供合并 - const cells = Array(maxColIndex).fill(null); - row.forEach((cell) => - cells.splice( - cell.colIndex, - 1, - cell.column.title ?? cell.column.defaultTitle ?? cell.column.dataIndex[cell.column.dataIndex.length - 1], - ), - ); - return cells; - }), - ...rows, - ], - ranges, - }; -} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/renders/renders.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/renders/renders.ts deleted file mode 100644 index 1b9ee8125d..0000000000 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/renders/renders.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import { getDefaultFormat, str2moment, toFixedByStep } from '@nocobase/utils'; - -export async function _(field, row, ctx, column?: any) { - if (column?.dataIndex.length > 1) { - const result = column.dataIndex.reduce((result, col) => { - if (Array.isArray(result)) { - const subResults = []; - for (const r of result) { - subResults.push(r?.[col]); - } - return subResults; - } else { - if (Array.isArray(result?.[col])) { - const subResults = []; - if (!result?.[col]) { - return subResults; - } - for (const r of result[col]) { - subResults.push(r); - } - return subResults; - } else { - return result?.[col]; - } - } - }, row); - if (Array.isArray(result)) { - return result.join(','); - } else { - return result; - } - } else { - return row.get(field.name); - } -} - -export async function datetime(field, row, ctx) { - const value = row.get(field.name); - if (!value) { - return ''; - } - const utcOffset = ctx.get('X-Timezone'); - const props = field.options?.uiSchema?.['x-component-props'] ?? {}; - const format = getDefaultFormat(props); - const m = str2moment(value, { ...props, utcOffset }); - return m ? m.format(format) : ''; -} - -export async function percent(field, row, ctx) { - const value = row.get(field.name); - const step = field.options?.uiSchema?.['x-component-props']?.['step'] ?? 0; - return value && `${toFixedByStep(value * 100, step)}%`; -} - -export async function boolean(field, row, ctx, column?: any) { - const value = row.get(field.name); - const { enum: enumData } = column ?? {}; - if (enumData?.length > 0) { - const option = enumData.find((item) => item.value === value); - return option?.label; - } else { - // FIXME: i18n - return value ? '是' : value === null || value === undefined ? '' : '否'; - } -} - -export const checkbox = boolean; - -export async function select(field, row, ctx, column?: any) { - const value = row.get(field.name); - let { enum: enumData } = column ?? {}; - if (!enumData) { - const repository = ctx.db.getCollection('uiSchemas').repository; - const model = await repository.findById(field.options.uiSchemaUid); - enumData = model.get('enum'); - } - const option = enumData.find((item) => item.value === value); - return option?.label; -} - -export async function multipleSelect(field, row, ctx, column?: any) { - const values = row.get(field.name); - let { enum: enumData } = column ?? {}; - if (!enumData) { - const repository = ctx.db.getCollection('uiSchemas').repository; - const model = await repository.findById(field.options.uiSchemaUid); - enumData = model.get('enum'); - } - return values - ?.map((value) => { - const option = enumData.find((item) => item.value === value); - return option?.label; - }) - ?.join(); -} - -export const radio = select; - -export const radioGroup = select; - -export const checkboxes = multipleSelect; - -export const checkboxGroup = multipleSelect; - -export async function subTable(field, row, ctx) { - // TODO: need title field to be defined - return (row.get(field.name) || []).map((item) => item[field.sourceKey]); -} - -export async function linkTo(field, row, ctx, column?: any) { - return (row.get(field.name) || []).map((item) => { - return column.dataIndex.reduce((buf, cur) => { - buf = item[cur]; - return buf; - }); - }); - // return (row.get(field.name) || []).map((item) => item[field.labelField]); -} - -export async function attachment(field, row, ctx) { - return (row.get(field.name) || []).map((item) => item[field.url]).join(' '); -} - -export async function chinaRegion(field, row, ctx, column?: any) { - const value = row.get(field.name); - const values = (Array.isArray(value) ? value : [value]).sort((a, b) => - a.level !== b.level ? a.level - b.level : a.sort - b.sort, - ); - return values.map((item) => item.name).join('/'); -} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/repositories/.gitkeep b/packages/plugins/@nocobase/plugin-action-export/src/server/repositories/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/utils/columns2Appends.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/utils/columns2Appends.ts deleted file mode 100644 index 6b048db95d..0000000000 --- a/packages/plugins/@nocobase/plugin-action-export/src/server/utils/columns2Appends.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -export function columns2Appends(columns, ctx) { - const { resourceName } = ctx.action; - const appends = new Set([]); - for (const column of columns) { - let collection = ctx.dataSource.collectionManager.getCollection(resourceName); - const appendColumns = []; - for (let i = 0, iLen = column.dataIndex.length; i < iLen; i++) { - const field = collection.getField(column.dataIndex[i]); - if (field?.target) { - appendColumns.push(column.dataIndex[i]); - collection = ctx.dataSource.collectionManager.getCollection(field.target); - } - } - if (appendColumns.length > 0) { - appends.add(appendColumns.join('.')); - } - } - return Array.from(appends); -} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/utils/deep-get.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/utils/deep-get.ts new file mode 100644 index 0000000000..e07db7c2e3 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/utils/deep-get.ts @@ -0,0 +1,25 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import _ from 'lodash'; + +export function deepGet(object, path) { + const pathParts = Array.isArray(path) ? path : path.split('.'); + + let current = object; + for (const part of pathParts) { + if (Array.isArray(current)) { + current = current.map((item) => _.get(item, part)); + } else { + current = _.get(current, part); + } + } + + return current; +} diff --git a/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts b/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts new file mode 100644 index 0000000000..c40aacc2f4 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-export/src/server/xlsx-exporter.ts @@ -0,0 +1,200 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { + FindOptions, + ICollection, + ICollectionManager, + IField, + IModel, + IRelationField, +} from '@nocobase/data-source-manager'; + +import XLSX from 'xlsx'; +import { deepGet } from './utils/deep-get'; + +type ExportColumn = { + dataIndex: Array; + defaultTitle: string; +}; + +type ExportOptions = { + collectionManager: ICollectionManager; + collection: ICollection; + columns: Array; + findOptions?: FindOptions; + chunkSize?: number; +}; + +class XlsxExporter { + /** + * You can adjust the maximum number of exported rows based on business needs and system + * available resources. However, please note that you need to fully understand the risks + * after the modification. Increasing the maximum number of rows that can be exported may + * increase system resource usage, leading to increased processing delays for other + * requests, or even server processes being recycled by the operating system. + * + * 您可以根据业务需求和系统可用资源等参数，调整最大导出数量的限制。但请注意，您需要充分了解修改之后的风险， + * 增加最大可导出的行数可能会导致系统资源占用率升高，导致其他请求处理延迟增加、无法处理、甚至 + * 服务端进程被操作系统回收等问题。 + */ + limit = process.env['EXPORT_LIMIT'] ? parseInt(process.env['EXPORT_LIMIT']) : 2000; + + constructor(private options: ExportOptions) {} + + async run(ctx?): Promise { + const { collection, columns, chunkSize } = this.options; + + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.sheet_new(); + + // write headers + XLSX.utils.sheet_add_aoa(worksheet, [this.renderHeaders()], { + origin: 'A1', + }); + + let startRowNumber = 2; + + await collection.repository.chunk({ + ...this.getFindOptions(), + chunkSize: chunkSize || 200, + callback: async (rows, options) => { + const chunkData = rows.map((r) => { + return columns.map((col) => { + return this.renderCellValue(r, col, ctx); + }); + }); + + XLSX.utils.sheet_add_aoa(worksheet, chunkData, { + origin: `A${startRowNumber}`, + }); + + startRowNumber += rows.length; + + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + }, + }); + + XLSX.utils.book_append_sheet(workbook, worksheet, 'Data'); + return workbook; + } + + private getAppendOptionsFromColumns() { + return this.options.columns + .map((col) => { + if (col.dataIndex.length > 1) { + return col.dataIndex.join('.'); + } + + const field = this.options.collection.getField(col.dataIndex[0]); + + if (field.isRelationField()) { + return col.dataIndex[0]; + } + + return null; + }) + .filter(Boolean); + } + + private getFindOptions() { + const { findOptions = {} } = this.options; + + findOptions.limit = this.limit; + + const appendOptions = this.getAppendOptionsFromColumns(); + + if (appendOptions.length) { + return { + ...findOptions, + appends: appendOptions, + }; + } + + return findOptions; + } + + private findFieldByDataIndex(dataIndex: Array): IField { + const { collection } = this.options; + const currentField = collection.getField(dataIndex[0]); + + if (dataIndex.length > 1) { + let targetCollection: ICollection; + + for (let i = 0; i < dataIndex.length - 1; i++) { + const isLast = i === dataIndex.length - 1; + + if (isLast) { + return targetCollection.getField(dataIndex[i]); + } + + targetCollection = (currentField as IRelationField).targetCollection(); + } + } + + return currentField; + } + + private renderHeaders() { + return this.options.columns.map((col) => { + const field = this.findFieldByDataIndex(col.dataIndex); + return field?.options.title || col.defaultTitle; + }); + } + + private renderRawValue(value) { + if (typeof value === 'object' && value !== null) { + return JSON.stringify(value); + } + + return value; + } + + private renderCellValue(rowData: IModel, column: ExportColumn, ctx?) { + const { dataIndex } = column; + rowData = rowData.toJSON(); + const value = rowData[dataIndex[0]]; + + if (dataIndex.length > 1) { + const deepValue = deepGet(rowData, dataIndex); + + if (Array.isArray(deepValue)) { + return deepValue.join(','); + } + + return deepValue; + } + + const field = this.findFieldByDataIndex(dataIndex); + + if (!field) { + return this.renderRawValue(value); + } + + const fieldOptions = field.options; + const interfaceName = fieldOptions['interface']; + + if (!interfaceName) { + return this.renderRawValue(value); + } + + const InterfaceClass = this.options.collectionManager.getFieldInterface(interfaceName); + + if (!InterfaceClass) { + return this.renderRawValue(value); + } + + const interfaceInstance = new InterfaceClass(fieldOptions); + return interfaceInstance.toString(value, ctx); + } +} + +export default XlsxExporter; diff --git a/packages/plugins/@nocobase/plugin-action-import/package.json b/packages/plugins/@nocobase/plugin-action-import/package.json index f055f3a149..3accced240 100644 --- a/packages/plugins/@nocobase/plugin-action-import/package.json +++ b/packages/plugins/@nocobase/plugin-action-import/package.json @@ -24,7 +24,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-i18next": "^11.15.1", - "xlsx": "^0.17.0" + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" }, "peerDependencies": { "@nocobase/actions": "1.x", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx index faf6c8ea95..e1cf95b14c 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportActionInitializer.tsx @@ -11,8 +11,8 @@ import type { ISchema } from '@formily/react'; import { Schema } from '@formily/react'; import { merge } from '@formily/shared'; import { - SchemaInitializerItem, css, + SchemaInitializerItem, useCollection_deprecated, useSchemaInitializer, useSchemaInitializerItem, @@ -43,7 +43,12 @@ const initImportSettings = (fields) => { export const ImportWarning = () => { const { t } = useImportTranslation(); - return ; + return ; +}; + +export const DownloadTips = () => { + const { t } = useImportTranslation(); + return ; }; export const ImportActionInitializer = () => { @@ -96,18 +101,7 @@ export const ImportActionInitializer = () => { properties: { tip: { type: 'void', - 'x-component': 'Markdown.Void', - 'x-editable': false, - 'x-component-props': { - style: { - padding: `var(--paddingContentVerticalSM)`, - backgroundColor: `var(--colorInfoBg)`, - border: `1px solid var(--colorInfoBorder)`, - color: `var(--colorText)`, - marginBottom: `var(--marginSM)`, - }, - content: `{{ t("Download tip", {ns: "${NAMESPACE}" }) }}`, - }, + 'x-component': 'DownloadTips', }, downloadAction: { type: 'void', diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportModal.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportModal.tsx index 4baeaadcfc..3fcc4a22f2 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportModal.tsx +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportModal.tsx @@ -58,7 +58,7 @@ export const ImportModal = (props: any) => {

- {t('Import done, total success have {{successCount}} , total failure have {{failureCount}}', { + {t('{{successCount}} records have been successfully imported', { ...(meta ?? {}), })}

diff --git a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx index b86a251e91..d7e2b76e8d 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx +++ b/packages/plugins/@nocobase/plugin-action-import/src/client/ImportPluginProvider.tsx @@ -10,7 +10,7 @@ import { SchemaComponentOptions } from '@nocobase/client'; import React, { useState } from 'react'; import { createPortal } from 'react-dom'; -import { ImportActionInitializer, ImportDesigner, ImportWarning } from '.'; +import { ImportActionInitializer, ImportDesigner, ImportWarning, DownloadTips } from '.'; import { ImportContext } from './context'; import { ImportModal, ImportStatus } from './ImportModal'; import { useDownloadXlsxTemplateAction, useImportStartAction } from './useImportAction'; @@ -20,7 +20,7 @@ export const ImportPluginProvider = (props: any) => { const { uploadValidator, beforeUploadHandler, validateUpload } = useShared(); return ( { let schema = s; @@ -103,6 +104,10 @@ export const useImportStartAction = () => { const form = useForm(); const { setVisible, fieldSchema } = useActionContext(); const { setImportModalVisible, setImportStatus, setImportResult } = useImportContext(); + const { upload } = form.values; + useEffect(() => { + form.reset(); + }, []); return { async run() { const { importColumns, explain } = lodash.cloneDeep( @@ -149,5 +154,6 @@ export const useImportStartAction = () => { setVisible(true); } }, + disabled: upload?.length === 0 || form.errors?.length > 0, }; }; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json index e41e6c71ba..1cb15d0e73 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/en-US.json @@ -8,11 +8,11 @@ "Download template": "Download template", "Step 1: Download template": "Step 1: Download template", "Step 2: Upload Excel": "Step 2: Upload Excel", - "Download tip": "- Download the template and fill in the data according to the format \r\n - Import only the first worksheet \r\n - Support single import of up to 1,000 rows of data \r\n - Do not change the header of the template to prevent import failure", - "Import warning": "You can import up to 200 rows of data at a time, any excess will be ignored.", + "Download tips": "- Download the template and fill in the data according to the format \r\n - Import only the first worksheet \r\n - Do not change the header of the template to prevent import failure", + "Import warnings": "You can import up to {{limit}} rows of data at a time, any excess will be ignored.", "Upload placeholder": "Drag and drop the file here or click to upload, file size should not exceed 30M", "Excel data importing": "Excel data importing", - "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "Import is complete, with a total of {{successCount}} successful and {{failureCount}} failed", + "{{successCount}} records have been successfully imported": "{{successCount}} records have been successfully imported", "To download the failure data": "To download the failure data", "Add importable field": "Add importable field", "Done": "Done", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/es-ES.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/es-ES.json index 23eb52d83a..04e112de29 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/es-ES.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/es-ES.json @@ -8,10 +8,10 @@ "Download template": "Descargar plantilla", "Step 1: Download template": "Paso 1: Descargar plantilla", "Step 2: Upload Excel": "Paso 2: Cargar Excel", - "Download tip": "- Descargar la plantilla y rellenar los datos según el formato \r\n - Importar sólo la primera hoja de cálculo \r\n - Soportar una única importación de hasta 10.000 filas de datos \r\n - No cambiar la cabecera de la plantilla para evitar fallos en la importación", + "Download tips": "- Descargar la plantilla y rellenar los datos según el formato \r\n - Importar sólo la primera hoja de cálculo \r\n - No cambiar la cabecera de la plantilla para evitar fallos en la importación", "Upload placeholder": "Arrastra y suelta el archivo aquí o haga clic para cargarlo, el tamaño del archivo no debe superar los 10M", "Excel data importing": "Importación de datos Excel", - "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "La importación se ha completado, con un total de {{successCount}} correctos y {{failureCount}} fallidos", + "{{successCount}} records have been successfully imported": "{{successCount}} registros han sido importados exitosamente", "To download the failure data": "Para descargar los datos de fallo", "Add importable field": "Añadir campo importable", "Done": "Hecho", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/ko_KR.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/ko_KR.json index f6a5566c75..f06d9ce6e7 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/ko_KR.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/ko_KR.json @@ -8,10 +8,10 @@ "Download template": "템플릿 다운로드", "Step 1: Download template": "단계 1: 템플릿 다운로드", "Step 2: Upload Excel": "단계 2: Excel 업로드", - "Download tip": "- 템플릿을 다운로드하고 형식에 맞게 데이터를 작성합니다.\r\n - 첫 번째 시트만 가져옵니다.\r\n - 단일 가져오기로 10000행 이하의 데이터를 지원합니다.\r\n - 템플릿 헤더를 수정하지 마세요. 가져오기 실패를 방지합니다.", + "Download tips": "- 템플릿을 다운로드하고 형식에 맞게 데이터를 작성합니다.\r\n - 첫 번째 시트만 가져옵니다.\r\n - 템플릿 헤더를 수정하지 마세요. 가져오기 실패를 방지합니다.", "Upload placeholder": "파일을 여기에 드래그하거나 클릭하여 업로드하십시오. 파일 크기는 10M을 초과할 수 없습니다.", "Excel data importing": "Excel 데이터 가져오기 중입니다. 창을 닫지 마십시오.", - "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "가져오기 완료, 총 성공 {{successCount}} 건, 총 실패 {{failureCount}} 건", + "{{successCount}} records have been successfully imported": "{{successCount}} 개의 데이터를 성공적으로 가져왔습니다.", "To download the failure data": "실패한 데이터를 다운로드하려면", "Add importable field": "가져올 수 있는 필드 추가", "Done": "완료", @@ -24,5 +24,5 @@ "Incorrect date format": "잘못된 날짜 형식", "Incorrect email format": "잘못된 이메일 형식", "Illegal percentage format": "잘못된 백분율 형식", - "Imported template does not match, please download again.": "가져온 템플릿이 일치하지 않습니다. 다시 다운로드하세요." + "Imported template does not match, please download again.": "가져온 템플릿이 일치하지 않습니다. 다시 다운로드하세요." } diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/pt-BR.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/pt-BR.json index aa0efe96bc..225e9eb92e 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/pt-BR.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/pt-BR.json @@ -8,10 +8,10 @@ "Download template": "Baixar modelo", "Step 1: Download template": "Passo 1: Baixar modelo", "Step 2: Upload Excel": "Passo 2: Enviar Excel", - "Download tip": "- Baixe o modelo e preencha os dados de acordo com o formato \r\n - Importe apenas a primeira planilha \r\n - Suporte de importação única de até 10.000 linhas de dados \r\n - Não altere o cabeçalho do modelo para evitar falhas de importação", + "Download tips": "- Baixe o modelo e preencha os dados de acordo com o formato \r\n - Importe apenas a primeira planilha \r\n - Não altere o cabeçalho do modelo para evitar falhas de importação", "Upload placeholder": "Arraste e solte o arquivo aqui ou clique para enviar, o tamanho do arquivo não deve exceder 10 MB", "Excel data importing": "Importando dados do Excel", - "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "Importação concluída, com um total de {{successCount}} sucesso e {{failureCount}} falhas", + "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "{{successCount}} dados foram importados com sucesso", "To download the failure data": "Para baixar os dados que falharam", "Add importable field": "Adicionar campo importável", "Done": "Concluído", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json b/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json index a36b07cdef..f567667956 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json +++ b/packages/plugins/@nocobase/plugin-action-import/src/locale/zh-CN.json @@ -8,11 +8,11 @@ "Download template": "下载模板", "Step 1: Download template": "1.下载模板", "Step 2: Upload Excel": "2.上传完善后的表格", - "Download tip": "- 下载模板后，按格式填写数据\r\n - 只导入第一张工作表\r\n - 支持单次导入不超过1000行数据\r\n - 请勿改模板表头，防止导入失败", - "Import warning": "每次最多导入 200 行数据，超出的将被忽略。", + "Download tips": "- 下载模板后，按格式填写数据\r\n - 只导入第一张工作表\r\n - 请勿改模板表头，防止导入失败", + "Import warnings": "每次最多导入 {{limit}} 行数据，超出的将被忽略。", "Upload placeholder": "将文件拖曳到此处或点击上传，文件大小不超过10M", "Excel data importing": "数据导入中，请勿关闭窗口", - "Import done, total success have {{successCount}} , total failure have {{failureCount}}": "导入完成，共导入成功{{successCount}}条数据，共导入失败{{failureCount}}条数据", + "{{successCount}} records have been successfully imported": "已成功导入 {{successCount}} 条数据", "To download the failure data": "下载导入失败的数据", "Add importable field": "添加可导入字段", "Done": "完成", diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts new file mode 100644 index 0000000000..4ecbb0f5ff --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/download-template.test.ts @@ -0,0 +1,108 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { createMockServer, MockServer } from '@nocobase/test'; +import { TemplateCreator } from '../services/template-creator'; +import XLSX from 'xlsx'; + +describe('download template', () => { + let app: MockServer; + beforeEach(async () => { + app = await createMockServer({}); + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should render template with explain', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + title: 'Name', + }, + { + type: 'string', + name: 'email', + title: 'Email', + }, + ], + }); + + const explain = 'Please fill in the following information:'; + + const templateCreator = new TemplateCreator({ + collection: User, + explain, + title: 'UsersImportTemplate', + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + { + dataIndex: ['email'], + defaultTitle: 'Email', + }, + ], + }); + + const workbook = await templateCreator.run(); + const sheet0 = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false }); + + const explainData = sheetData[0]; + expect(explainData[0]).toEqual(explain); + + const headerData = sheetData[1]; + expect(headerData).toEqual(['Name', 'Email']); + }); + + it('should render template', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + title: 'Name', + }, + { + type: 'string', + name: 'email', + title: 'Email', + }, + ], + }); + + const templateCreator = new TemplateCreator({ + collection: User, + title: 'UsersImportTemplate', + columns: [ + { + dataIndex: ['name'], + defaultTitle: 'Name', + }, + { + dataIndex: ['email'], + defaultTitle: 'Email', + }, + ], + }); + + const workbook = await templateCreator.run(); + const sheet0 = workbook.Sheets[workbook.SheetNames[0]]; + const sheetData = XLSX.utils.sheet_to_json(sheet0, { header: 1, defval: null, raw: false }); + const headerData = sheetData[0]; + expect(headerData).toEqual(['Name', 'Email']); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts new file mode 100644 index 0000000000..00e75c9734 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/__tests__/xlsx-importer.test.ts @@ -0,0 +1,706 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { createMockServer, MockServer } from '@nocobase/test'; +import { TemplateCreator } from '../services/template-creator'; +import { XlsxImporter } from '../services/xlsx-importer'; +import XLSX from 'xlsx'; +import * as process from 'node:process'; + +describe('xlsx importer', () => { + let app: MockServer; + beforeEach(async () => { + app = await createMockServer({ + plugins: ['field-china-region'], + }); + }); + + afterEach(async () => { + await app.destroy(); + }); + + it('should import china region field', async () => { + const Post = app.db.collection({ + name: 'posts', + fields: [ + { type: 'string', name: 'title' }, + { + type: 'belongsToMany', + target: 'chinaRegions', + through: 'userRegions', + targetKey: 'code', + interface: 'chinaRegion', + name: 'region', + }, + ], + }); + + await app.db.sync(); + + const columns = [ + { dataIndex: ['title'], defaultTitle: 'Title' }, + { + dataIndex: ['region'], + defaultTitle: 'region', + }, + ]; + + const templateCreator = new TemplateCreator({ + collection: Post, + columns, + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa(worksheet, [['post0', '山西省/长治市/潞城区']], { + origin: 'A2', + }); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + columns, + workbook: template, + }); + + await importer.run(); + + expect(await Post.repository.count()).toBe(1); + + const post = await Post.repository.findOne({ + appends: ['region'], + }); + + expect(post.get('region').map((item: any) => item.code)).toEqual(['14', '1404', '140406']); + }); + + it.skipIf(process.env['DB_DIALECT'] === 'sqlite')('should import with number field', async () => { + const User = app.db.collection({ + name: 'users', + autoGenId: false, + fields: [ + { + type: 'bigInt', + name: 'id', + primaryKey: true, + autoIncrement: true, + }, + { + type: 'bigInt', + interface: 'integer', + name: 'bigInt', + }, + { + type: 'float', + interface: 'percent', + name: 'percent', + }, + { + type: 'float', + interface: 'float', + name: 'float', + }, + { + type: 'boolean', + interface: 'boolean', + name: 'boolean', + }, + ], + }); + + await app.db.sync(); + + const columns = [ + { + dataIndex: ['id'], + defaultTitle: 'ID', + }, + { + dataIndex: ['bigInt'], + defaultTitle: 'bigInt', + }, + { + dataIndex: ['percent'], + defaultTitle: '百分比', + }, + { + dataIndex: ['float'], + defaultTitle: '浮点数', + }, + { + dataIndex: ['boolean'], + defaultTitle: '布尔值', + }, + ]; + + const templateCreator = new TemplateCreator({ + collection: User, + columns, + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + [1, '1238217389217389217', '10%', 0.1, '是'], + [2, 123123, '20%', 0.2, '0'], + ], + { + origin: 'A2', + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns, + workbook: template, + }); + + await importer.run(); + + expect(await User.repository.count()).toBe(2); + + const user1 = await User.repository.findOne({ + filter: { + id: 1, + }, + }); + + const user1Json = user1.toJSON(); + + expect(user1Json['bigInt']).toBe('1238217389217389217'); + expect(user1Json['percent']).toBe(0.1); + expect(user1Json['float']).toBe(0.1); + expect(user1Json['boolean']).toBe(true); + + const user2 = await User.repository.findOne({ + filter: { + id: 2, + }, + }); + + const user2Json = user2.toJSON(); + expect(user2Json['bigInt']).toBe(123123); + expect(user2Json['percent']).toBe(0.2); + expect(user2Json['float']).toBe(0.2); + expect(user2Json['boolean']).toBe(false); + }); + + it('should reset id seq after import', async () => { + const User = app.db.collection({ + name: 'users', + autoGenId: false, + fields: [ + { + type: 'bigInt', + name: 'id', + primaryKey: true, + autoIncrement: true, + }, + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['id'], + defaultTitle: 'ID', + }, + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + [1, 'User1', 'test@test.com'], + [2, 'User2', 'test2@test.com'], + ], + { + origin: 'A2', + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['id'], + defaultTitle: 'ID', + }, + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook: template, + }); + + await importer.run(); + + expect(await User.repository.count()).toBe(2); + + const user3 = await User.repository.create({ + values: { + name: 'User3', + email: 'test3@test.com', + }, + }); + + expect(user3.get('id')).toBe(3); + }); + + it('should validate workbook with error', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.sheet_new(); + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + ['column1', 'column2'], + ['row21', 'row22'], + ], + { + origin: 'A1', + }, + ); + + XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet 1'); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook, + }); + + let error; + try { + importer.getData(); + } catch (e) { + error = e; + } + + expect(error).toBeTruthy(); + }); + + it('should validate workbook true', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + }); + + const template = await templateCreator.run(); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook: template, + }); + + let error; + try { + importer.getData(); + } catch (e) { + error = e; + } + + expect(error).toBeFalsy(); + }); + + it('should import with associations', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + const Tag = app.db.collection({ + name: 'tags', + fields: [ + { + type: 'string', + name: 'name', + }, + ], + }); + + const Comments = app.db.collection({ + name: 'comments', + fields: [ + { + type: 'string', + name: 'content', + }, + ], + }); + + const Post = app.db.collection({ + name: 'posts', + fields: [ + { + type: 'string', + name: 'title', + }, + { + type: 'string', + name: 'content', + }, + { + type: 'belongsTo', + name: 'user', + interface: 'm2o', + }, + { + type: 'belongsToMany', + name: 'tags', + through: 'postsTags', + interface: 'm2m', + }, + { + type: 'hasMany', + name: 'comments', + interface: 'o2m', + }, + ], + }); + + await app.db.sync(); + + await User.repository.create({ + values: { + name: 'User1', + email: 'u1@test.com', + }, + }); + + await Tag.repository.create({ + values: [ + { + name: 'Tag1', + }, + { + name: 'Tag2', + }, + { + name: 'Tag3', + }, + ], + }); + + await Comments.repository.create({ + values: [ + { + content: 'Comment1', + }, + { + content: 'Comment2', + }, + { + content: 'Comment3', + }, + ], + }); + + const importColumns = [ + { + dataIndex: ['title'], + defaultTitle: '标题', + }, + { + dataIndex: ['content'], + defaultTitle: '内容', + }, + { + dataIndex: ['user', 'name'], + defaultTitle: '作者', + }, + { + dataIndex: ['tags', 'name'], + defaultTitle: '标签', + }, + { + dataIndex: ['comments', 'content'], + defaultTitle: '评论', + }, + ]; + + const templateCreator = new TemplateCreator({ + collection: Post, + columns: importColumns, + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + ['Post1', 'Content1', 'User1', 'Tag1,Tag2', 'Comment1,Comment2'], + ['Post2', 'Content2', 'User1', 'Tag2,Tag3', 'Comment3'], + ['Post3', 'Content3', 'UserNotExist', 'Tag3,TagNotExist', ''], + ], + { + origin: 'A2', + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: Post, + columns: importColumns, + workbook: template, + }); + + await importer.run(); + + expect(await Post.repository.count()).toBe(3); + }); + + it('should import data with xlsx', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + explain: 'test', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + ['User1', 'test@test.com'], + ['User2', 'test2@test.com'], + ], + { + origin: 'A3', + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + explain: 'test', + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook: template, + }); + + await importer.run(); + + expect(await User.repository.count()).toBe(2); + }); + + it('should throw error when import failed', async () => { + const User = app.db.collection({ + name: 'users', + fields: [ + { + type: 'string', + name: 'name', + unique: true, + }, + { + type: 'string', + name: 'email', + }, + ], + }); + + await app.db.sync(); + + const templateCreator = new TemplateCreator({ + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + }); + + const template = await templateCreator.run(); + + const worksheet = template.Sheets[template.SheetNames[0]]; + + XLSX.utils.sheet_add_aoa( + worksheet, + [ + ['User1', 'test@test.com'], + ['User1', 'test2@test.com'], + ], + { + origin: 'A2', + }, + ); + + const importer = new XlsxImporter({ + collectionManager: app.mainDataSource.collectionManager, + collection: User, + columns: [ + { + dataIndex: ['name'], + defaultTitle: '姓名', + }, + { + dataIndex: ['email'], + defaultTitle: '邮箱', + }, + ], + workbook: template, + }); + + let error; + try { + await importer.run(); + } catch (e) { + error = e; + } + + expect(await User.repository.count()).toBe(0); + expect(error).toBeTruthy(); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/downloadXlsxTemplate.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts similarity index 69% rename from packages/plugins/@nocobase/plugin-action-import/src/server/actions/downloadXlsxTemplate.ts rename to packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts index c9948de88f..29750be444 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/downloadXlsxTemplate.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/download-xlsx-template.ts @@ -8,7 +8,8 @@ */ import { Context, Next } from '@nocobase/actions'; -import xlsx from 'node-xlsx'; +import { TemplateCreator } from '../services/template-creator'; +import XLSX from 'xlsx'; export async function downloadXlsxTemplate(ctx: Context, next: Next) { let { columns } = ctx.request.body as any; @@ -16,23 +17,21 @@ export async function downloadXlsxTemplate(ctx: Context, next: Next) { if (typeof columns === 'string') { columns = JSON.parse(columns); } - const header = columns?.map((column) => column.defaultTitle); - const data = [header]; - if (explain?.trim() !== '') { - data.unshift([explain]); - } - ctx.body = xlsx.build([ - { - name: 'Sheet 1', - data, - options: {}, - }, - ]); + const templateCreator = new TemplateCreator({ + explain, + title, + columns, + }); + + const workbook = await templateCreator.run(); + + const buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }); + + ctx.body = buffer; ctx.set({ 'Content-Type': 'application/octet-stream', - // to avoid "invalid character" error in header (RFC) 'Content-Disposition': `attachment; filename=${encodeURIComponent(title)}.xlsx`, }); diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/import-xlsx.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/import-xlsx.ts new file mode 100644 index 0000000000..53d7d6a99c --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/import-xlsx.ts @@ -0,0 +1,73 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Context, Next } from '@nocobase/actions'; +import { Repository } from '@nocobase/database'; +import XLSX from 'xlsx'; +import { XlsxImporter } from '../services/xlsx-importer'; +import { Mutex } from 'async-mutex'; +import { DataSource } from '@nocobase/data-source-manager'; + +const mutex = new Mutex(); + +const IMPORT_LIMIT_COUNT = 2000; + +async function importXlsxAction(ctx: Context, next: Next) { + let columns = (ctx.request.body as any).columns as any[]; + if (typeof columns === 'string') { + columns = JSON.parse(columns); + } + + let readLimit = IMPORT_LIMIT_COUNT; + + // add header raw + readLimit += 1; + + if ((ctx.request.body as any).explain) { + readLimit += 1; + } + + const workbook = XLSX.read(ctx.file.buffer, { + type: 'buffer', + sheetRows: readLimit, + }); + + const repository = ctx.getCurrentRepository() as Repository; + const dataSource = ctx.dataSource as DataSource; + + const collection = repository.collection; + + const importer = new XlsxImporter({ + collectionManager: dataSource.collectionManager, + collection, + columns, + workbook, + }); + + const importedCount = await importer.run(); + + ctx.bodyMeta = { successCount: importedCount }; + ctx.body = ctx.bodyMeta; +} + +export async function importXlsx(ctx: Context, next: Next) { + if (mutex.isLocked()) { + throw new Error(`another import action is running, please try again later.`); + } + + const release = await mutex.acquire(); + + try { + await importXlsxAction(ctx, next); + } finally { + release(); + } + + await next(); +} diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/importXlsx.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/importXlsx.ts deleted file mode 100644 index 4355448fbb..0000000000 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/importXlsx.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import { Context, Next } from '@nocobase/actions'; -import { Collection, Repository } from '@nocobase/database'; -import { uid } from '@nocobase/utils'; -import xlsx from 'node-xlsx'; -import XLSX from 'xlsx'; -import { namespace } from '../../'; - -const IMPORT_LIMIT_COUNT = 200; - -class Importer { - repository: Repository; - collection: Collection; - columns: any[]; - items: any[][] = []; - headerRow; - context: Context; - - constructor(ctx: Context) { - const { resourceName, resourceOf } = ctx.action; - this.context = ctx; - this.repository = ctx.db.getRepository(resourceName, resourceOf); - this.collection = this.repository.collection; - this.parseXlsx(); - } - - getRows() { - const workbook = XLSX.read(this.context.file.buffer, { - type: 'buffer', - sheetRows: IMPORT_LIMIT_COUNT, - // cellDates: true, - // raw: false, - }); - const r = workbook.Sheets[workbook.SheetNames[0]]; - const rows = XLSX.utils.sheet_to_json(r, { header: 1, defval: null, raw: false }); - return rows; - } - - parseXlsx() { - const rows = this.getRows(); - let columns = (this.context.request.body as any).columns as any[]; - if (typeof columns === 'string') { - columns = JSON.parse(columns); - } - this.columns = columns - .map((column) => { - return { - ...column, - field: this.collection.fields.get(column.dataIndex[0]), - }; - }) - .filter((col) => col.field); - const str = this.columns.map((column) => column.defaultTitle).join('||'); - for (const row of rows) { - if (this.hasHeaderRow()) { - if (row && row.join('').trim()) { - this.items.push(row); - } - } - if (str === row.filter((r) => r).join('||')) { - this.headerRow = row; - } - } - } - - getFieldByIndex(index) { - return this.columns[index].field; - } - - async getItems() { - const items: any[] = []; - for (const row of this.items) { - const values = {}; - const errors = []; - for (let index = 0; index < row.length; index++) { - if (!this.columns[index]) { - continue; - } - const column = this.columns[index]; - const { field, defaultTitle } = column; - let value = row[index]; - if (value === undefined || value === null) { - continue; - } - const parser = this.context.db.buildFieldValueParser(field, { ...this.context, column }); - await parser.setValue(typeof value === 'string' ? value.trim() : value); - value = parser.getValue(); - if (parser.errors.length > 0) { - errors.push(`${defaultTitle}: ${parser.errors.join(';')}`); - } - if (value === undefined) { - continue; - } - values[field.name] = value; - } - items.push({ - row, - values, - errors, - }); - } - return items; - } - - hasSortField() { - return !!this.collection.options.sortable; - } - - async run() { - return await this.context.db.sequelize.transaction(async (transaction) => { - let sort = 0; - if (this.hasSortField()) { - sort = await this.repository.model.max('sort', { transaction }); - } - const result: any = [[], []]; - for (const { row, values, errors } of await this.getItems()) { - if (errors.length > 0) { - row.push(errors.join(';')); - result[1].push(row); - continue; - } - if (this.hasSortField()) { - values['sort'] = ++sort; - } - try { - const instance = await this.repository.create({ - values, - transaction, - logging: false, - context: this.context, - }); - result[0].push(instance); - } catch (error) { - this.context.log.error(error, row); - row.push(error.message); - result[1].push(row); - } - } - return result; - }); - } - - hasHeaderRow() { - return !!this.headerRow; - } -} - -export async function importXlsx(ctx: Context, next: Next) { - const importer = new Importer(ctx); - - if (!importer.hasHeaderRow()) { - ctx.throw(400, ctx.t('Imported template does not match, please download again.', { ns: namespace })); - } - - const [success, failure] = await importer.run(); - - ctx.body = { - rows: xlsx.build([ - { - name: `${uid()}.xlsx`, - data: [importer.headerRow].concat(failure), - }, - ]), - successCount: success.length, - failureCount: failure.length, - }; - - await next(); -} diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/index.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/index.ts index 060f50e207..0122e620c1 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/actions/index.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/actions/index.ts @@ -7,5 +7,5 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -export * from './downloadXlsxTemplate'; -export * from './importXlsx'; +export * from './download-xlsx-template'; +export * from './import-xlsx'; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/collections/.gitkeep b/packages/plugins/@nocobase/plugin-action-import/src/server/collections/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts index c0a25a6535..017a04eea5 100644 --- a/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/index.ts @@ -7,7 +7,7 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -import { InstallOptions, Plugin } from '@nocobase/server'; +import { Plugin } from '@nocobase/server'; import { namespace } from '..'; import { downloadXlsxTemplate, importXlsx } from './actions'; import { enUS, zhCN } from './locale'; @@ -21,11 +21,6 @@ export class PluginActionImportServer extends Plugin { async load() { this.app.dataSourceManager.afterAddDataSource((dataSource) => { - // @ts-ignore - if (!dataSource.collectionManager?.db) { - return; - } - dataSource.resourceManager.use(importMiddleware); dataSource.resourceManager.registerActionHandler('downloadXlsxTemplate', downloadXlsxTemplate); dataSource.resourceManager.registerActionHandler('importXlsx', importXlsx); @@ -40,10 +35,6 @@ export class PluginActionImportServer extends Plugin { dataSource.acl.allow('*', 'downloadXlsxTemplate', 'loggedIn'); }); } - - async install(options: InstallOptions) { - // TODO - } } export default PluginActionImportServer; diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/models/.gitkeep b/packages/plugins/@nocobase/plugin-action-import/src/server/models/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/repositories/.gitkeep b/packages/plugins/@nocobase/plugin-action-import/src/server/repositories/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts new file mode 100644 index 0000000000..7d68279006 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/services/template-creator.ts @@ -0,0 +1,48 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { ICollection } from '@nocobase/data-source-manager'; +import { ImportColumn } from './xlsx-importer'; +import XLSX, { WorkBook } from 'xlsx'; + +type TemplateCreatorOptions = { + collection?: ICollection; + title?: string; + explain?: string; + columns: Array; +}; + +export class TemplateCreator { + constructor(private options: TemplateCreatorOptions) {} + + async run(): Promise { + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.sheet_new(); + + const data = [this.renderHeaders()]; + + if (this.options.explain && this.options.explain?.trim() !== '') { + data.unshift([this.options.explain]); + } + + // write headers + XLSX.utils.sheet_add_aoa(worksheet, data, { + origin: 'A1', + }); + + XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet 1'); + return workbook; + } + + renderHeaders() { + return this.options.columns.map((col) => { + return col.defaultTitle; + }); + } +} diff --git a/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts new file mode 100644 index 0000000000..0168df95bc --- /dev/null +++ b/packages/plugins/@nocobase/plugin-action-import/src/server/services/xlsx-importer.ts @@ -0,0 +1,222 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import XLSX, { WorkBook } from 'xlsx'; +import lodash from 'lodash'; +import { ICollection, ICollectionManager, IRelationField } from '@nocobase/data-source-manager'; +import { Collection as DBCollection, Database } from '@nocobase/database'; +import { Transaction } from 'sequelize'; + +export type ImportColumn = { + dataIndex: Array; + defaultTitle: string; +}; + +type ImporterOptions = { + collectionManager: ICollectionManager; + collection: ICollection; + columns: Array; + workbook: WorkBook; + chunkSize?: number; + explain?: string; +}; + +type RunOptions = { + transaction?: Transaction; +}; + +export class XlsxImporter { + constructor(private options: ImporterOptions) { + if (options.columns.length == 0) { + throw new Error(`columns is empty`); + } + } + + async run(options: RunOptions = {}) { + let transaction = options.transaction; + + // @ts-ignore + if (!transaction && this.options.collectionManager.db) { + // @ts-ignore + transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction(); + } + + try { + const imported = await this.performImport(options); + + // @ts-ignore + if (this.options.collectionManager.db) { + await this.resetSeq(options); + } + + transaction && (await transaction.commit()); + + return imported; + } catch (error) { + transaction && (await transaction.rollback()); + + throw error; + } + } + + async resetSeq(options?: RunOptions) { + const { transaction } = options; + // @ts-ignore + const db: Database = this.options.collectionManager.db; + const collection: DBCollection = this.options.collection as DBCollection; + + // @ts-ignore + const autoIncrementAttribute = collection.model.autoIncrementAttribute; + if (!autoIncrementAttribute) { + return; + } + + let tableInfo = collection.getTableNameWithSchema(); + if (typeof tableInfo === 'string') { + tableInfo = { + tableName: tableInfo, + }; + } + + const autoIncrInfo = await db.queryInterface.getAutoIncrementInfo({ + tableInfo, + fieldName: autoIncrementAttribute, + transaction, + }); + + const maxVal = (await collection.model.max(autoIncrementAttribute, { transaction })) as number; + + const queryInterface = db.queryInterface; + await queryInterface.setAutoIncrementVal({ + tableInfo, + columnName: collection.model.rawAttributes[autoIncrementAttribute].field, + currentVal: maxVal, + seqName: autoIncrInfo.seqName, + transaction, + }); + } + + async performImport(options?: RunOptions) { + const transaction = options?.transaction; + const rows = this.getData(); + const chunks = lodash.chunk(rows, this.options.chunkSize || 200); + + let handingRowIndex = 1; + + if (this.options.explain) { + handingRowIndex += 1; + } + + let imported = 0; + + for (const chunkRows of chunks) { + for (const row of chunkRows) { + const rowValues = {}; + handingRowIndex += 1; + try { + for (let index = 0; index < this.options.columns.length; index++) { + const column = this.options.columns[index]; + + const field = this.options.collection.getField(column.dataIndex[0]); + + if (!field) { + throw new Error(`Field not found: ${column.dataIndex[0]}`); + } + + const str = row[index]; + + const dataKey = column.dataIndex[0]; + + const fieldOptions = field.options; + const interfaceName = fieldOptions.interface; + + const InterfaceClass = this.options.collectionManager.getFieldInterface(interfaceName); + + if (!InterfaceClass) { + rowValues[dataKey] = str; + continue; + } + + const interfaceInstance = new InterfaceClass(field.options); + + const ctx: any = { + transaction, + field, + }; + + if (column.dataIndex.length > 1) { + ctx.targetCollection = (field as IRelationField).targetCollection(); + ctx.filterKey = column.dataIndex[1]; + } + + rowValues[dataKey] = await interfaceInstance.toValue(this.trimString(str), ctx); + } + + await this.options.collection.repository.create({ + values: rowValues, + transaction, + }); + + imported += 1; + + await new Promise((resolve) => setTimeout(resolve, 5)); + } catch (error) { + console.log(error); + throw new Error( + `failed to import row ${handingRowIndex}, rowData: ${JSON.stringify(rowValues)} message: ${error.message}`, + { cause: error }, + ); + } + } + + // await to prevent high cpu usage + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + return imported; + } + + trimString(str: string) { + if (typeof str === 'string') { + return str.trim(); + } + + return str; + } + + getData() { + const firstSheet = this.firstSheet(); + const rows = XLSX.utils.sheet_to_json(firstSheet, { header: 1, defval: null, raw: false }); + + if (this.options.explain) { + rows.shift(); + } + + const headers = rows[0]; + + const columns = this.options.columns; + + // validate headers + for (let i = 0; i < columns.length; i++) { + const column = columns[i]; + if (column.defaultTitle !== headers[i]) { + throw new Error(`Invalid header: ${column.defaultTitle} !== ${headers[i]}`); + } + } + + // remove header + rows.shift(); + + return rows; + } + + firstSheet() { + return this.options.workbook.Sheets[this.options.workbook.SheetNames[0]]; + } +} diff --git a/packages/plugins/@nocobase/plugin-client/src/server/server.ts b/packages/plugins/@nocobase/plugin-client/src/server/server.ts index 34e1846f4d..e0772d7bb7 100644 --- a/packages/plugins/@nocobase/plugin-client/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-client/src/server/server.ts @@ -12,6 +12,7 @@ import { resolve } from 'path'; import { getAntdLocale } from './antd'; import { getCronLocale } from './cron'; import { getCronstrueLocale } from './cronstrue'; +import * as process from 'node:process'; async function getLang(ctx) { const SystemSetting = ctx.db.getRepository('systemSettings'); @@ -81,7 +82,8 @@ export class PluginClientServer extends Plugin { if (enabledLanguages.includes(currentUser?.appLang)) { lang = currentUser?.appLang; } - ctx.body = { + + const info: any = { database: { dialect, }, @@ -90,6 +92,11 @@ export class PluginClientServer extends Plugin { name: ctx.app.name, theme: currentUser?.systemSettings?.theme || systemSetting?.options?.theme || 'default', }; + + if (process.env['EXPORT_LIMIT']) { + info.exportLimit = parseInt(process.env['EXPORT_LIMIT']); + } + ctx.body = info; await next(); }, async getLang(ctx, next) { diff --git a/packages/plugins/@nocobase/plugin-field-china-region/src/server/index.ts b/packages/plugins/@nocobase/plugin-field-china-region/src/server/index.ts index a789982644..652a51db02 100644 --- a/packages/plugins/@nocobase/plugin-field-china-region/src/server/index.ts +++ b/packages/plugins/@nocobase/plugin-field-china-region/src/server/index.ts @@ -9,6 +9,7 @@ import { Plugin } from '@nocobase/server'; import { resolve } from 'path'; +import { ChinaRegionInterface } from './interfaces/china-region-interface'; function getChinaDivisionData(key: string) { try { @@ -38,6 +39,8 @@ export class PluginFieldChinaRegionServer extends Plugin { await next(); } }); + + this.app.db.interfaceManager.registerInterfaceType('chinaRegion', ChinaRegionInterface); } async importData() { diff --git a/packages/plugins/@nocobase/plugin-field-china-region/src/server/interfaces/china-region-interface.ts b/packages/plugins/@nocobase/plugin-field-china-region/src/server/interfaces/china-region-interface.ts new file mode 100644 index 0000000000..f1f68e208d --- /dev/null +++ b/packages/plugins/@nocobase/plugin-field-china-region/src/server/interfaces/china-region-interface.ts @@ -0,0 +1,42 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface, Repository } from '@nocobase/database'; + +export class ChinaRegionInterface extends BaseInterface { + async toValue(str: string, ctx?: any): Promise { + const { field } = ctx; + const items = str.split('/'); + const repository = field.database.getRepository(field.target) as Repository; + + const instances = await repository.find({ + filter: { + name: items, + }, + }); + + for (let i = 0; i < items.length; i++) { + const instance = instances.find((item) => item.name === items[i]); + if (!instance) { + throw new Error(`china region "${items[i]}" does not exist`); + } + items[i] = instance.get('code'); + } + + return items; + } + + toString(value: any, ctx?: any) { + const values = (Array.isArray(value) ? value : [value]).sort((a, b) => + a.level !== b.level ? a.level - b.level : a.sort - b.sort, + ); + + return values.map((item) => item.name).join('/'); + } +} diff --git a/packages/plugins/@nocobase/plugin-field-formula/src/server/__tests__/formula-field.test.ts b/packages/plugins/@nocobase/plugin-field-formula/src/server/__tests__/formula-field.test.ts index 4a28f2f3d5..187f767874 100644 --- a/packages/plugins/@nocobase/plugin-field-formula/src/server/__tests__/formula-field.test.ts +++ b/packages/plugins/@nocobase/plugin-field-formula/src/server/__tests__/formula-field.test.ts @@ -1124,7 +1124,7 @@ describe('formula field', () => { await db.sync(); const test = await Test.model.create({ - a: BigInt(now.valueOf()), + a: now.valueOf(), }); expect(test.get('result')).toEqual(new Date(now)); }); diff --git a/packages/plugins/@nocobase/plugin-file-manager/src/server/interfaces/attachment-interface.ts b/packages/plugins/@nocobase/plugin-file-manager/src/server/interfaces/attachment-interface.ts new file mode 100644 index 0000000000..e1c2b2ed31 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-file-manager/src/server/interfaces/attachment-interface.ts @@ -0,0 +1,32 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from '@nocobase/database'; +import lodash from 'lodash'; +import { basename, extname } from 'path'; + +export class AttachmentInterface extends BaseInterface { + async toValue(value: any, ctx?: any) { + return this.castArray(value).map((url: string) => { + return { + title: basename(url), + extname: extname(url), + filename: basename(url), + url, + }; + }); + } + + toString(value: any, ctx?: any) { + return lodash + .castArray(value) + .map((item) => item.url) + .join(','); + } +} diff --git a/packages/plugins/@nocobase/plugin-file-manager/src/server/server.ts b/packages/plugins/@nocobase/plugin-file-manager/src/server/server.ts index 671e991218..36492fde63 100644 --- a/packages/plugins/@nocobase/plugin-file-manager/src/server/server.ts +++ b/packages/plugins/@nocobase/plugin-file-manager/src/server/server.ts @@ -20,6 +20,7 @@ import StorageTypeLocal from './storages/local'; import StorageTypeAliOss from './storages/ali-oss'; import StorageTypeS3 from './storages/s3'; import StorageTypeTxCos from './storages/tx-cos'; +import { AttachmentInterface } from './interfaces/attachment-interface'; export type * from './storages'; @@ -137,5 +138,7 @@ export default class PluginFileManagerServer extends Plugin { this.app.acl.addFixedParams('attachments', 'update', ownMerger); this.app.acl.addFixedParams('attachments', 'create', ownMerger); this.app.acl.addFixedParams('attachments', 'destroy', ownMerger); + + this.app.db.interfaceManager.registerInterfaceType('attachment', AttachmentInterface); } } diff --git a/packages/plugins/@nocobase/plugin-map/src/server/__tests__/interfaces.test.ts b/packages/plugins/@nocobase/plugin-map/src/server/__tests__/interfaces.test.ts new file mode 100644 index 0000000000..77c2a2a9f2 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-map/src/server/__tests__/interfaces.test.ts @@ -0,0 +1,78 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { CircleInterface, LineStringInterface, PointInterface, PolygonInterface } from '../interfaces'; + +describe('interfaces', () => { + describe('point', () => { + it('should toString', async () => { + const interfaceInstance = new PointInterface(); + expect(await interfaceInstance.toString([1, 2])).toBe('1,2'); + }); + + it('should toValue', async () => { + const interfaceInstance = new PointInterface(); + expect(await interfaceInstance.toValue('1,2')).toMatchObject([1, 2]); + }); + }); + + describe('lineString', () => { + it('should toString', async () => { + const interfaceInstance = new LineStringInterface(); + expect( + interfaceInstance.toString([ + [1, 2], + [3, 4], + ]), + ).toBe('(1,2),(3,4)'); + }); + + it('should toValue', async () => { + const interfaceInstance = new LineStringInterface(); + expect(await interfaceInstance.toValue('(1,2),(3,4)')).toMatchObject([ + [1, 2], + [3, 4], + ]); + }); + }); + + describe('polygon', () => { + it('should toString', async () => { + const interfaceInstance = new PolygonInterface(); + expect( + interfaceInstance.toString([ + [1, 2], + [3, 4], + [5, 6], + ]), + ).toBe('(1,2),(3,4),(5,6)'); + }); + + it('should toValue', async () => { + const interfaceInstance = new PolygonInterface(); + expect(await interfaceInstance.toValue('(1,2),(3,4),(5,6)')).toMatchObject([ + [1, 2], + [3, 4], + [5, 6], + ]); + }); + }); + + describe('circle', () => { + it('should toString', async () => { + const interfaceInstance = new CircleInterface(); + expect(interfaceInstance.toString([1, 2, 0.5])).toBe('1,2,0.5'); + }); + + it('should toValue', async () => { + const interfaceInstance = new CircleInterface(); + expect(await interfaceInstance.toValue('1,2,0.5')).toMatchObject([1, 2, 0.5]); + }); + }); +}); diff --git a/packages/plugins/@nocobase/plugin-map/src/server/interfaces/index.ts b/packages/plugins/@nocobase/plugin-map/src/server/interfaces/index.ts new file mode 100644 index 0000000000..6c03d13810 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-map/src/server/interfaces/index.ts @@ -0,0 +1,53 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { BaseInterface } from '@nocobase/database'; + +export class PointInterface extends BaseInterface { + async toValue(str: any, ctx?: any): Promise { + if (!str) return null; + return str.split(',').map((v: string) => parseFloat(v)); + } + + toString(value: any, ctx?: any) { + if (!value) return null; + + return value.join(','); + } +} + +export class PolygonInterface extends BaseInterface { + async toValue(str: any, ctx?: any): Promise { + if (!str) return null; + + return str + .substring(1, str.length - 1) + .split('),(') + .map((v: string) => v.split(',').map((v: string) => parseFloat(v))); + } + + toString(value: any, ctx?: any) { + if (!value) return null; + return `(${value.map((v: any) => v.join(',')).join('),(')})`; + } +} + +export class LineStringInterface extends PolygonInterface {} + +export class CircleInterface extends BaseInterface { + async toValue(str: any, ctx?: any): Promise { + if (!str) return null; + return str.split(',').map((v: string) => parseFloat(v)); + } + + toString(value: any, ctx?: any) { + if (!value) return null; + return value.join(','); + } +} diff --git a/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts index 1831226a6e..08a9ceb90c 100644 --- a/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-map/src/server/plugin.ts @@ -12,6 +12,7 @@ import path from 'path'; import { getConfiguration, setConfiguration } from './actions'; import { CircleField, LineStringField, PointField, PolygonField } from './fields'; import { CircleValueParser, LineStringValueParser, PointValueParser, PolygonValueParser } from './value-parsers'; +import { CircleInterface, LineStringInterface, PointInterface, PolygonInterface } from './interfaces'; export class PluginMapServer extends Plugin { afterAdd() {} @@ -30,6 +31,11 @@ export class PluginMapServer extends Plugin { lineString: LineStringValueParser, circle: CircleValueParser, }); + + this.db.interfaceManager.registerInterfaceType('point', PointInterface); + this.db.interfaceManager.registerInterfaceType('polygon', PolygonInterface); + this.db.interfaceManager.registerInterfaceType('lineString', LineStringInterface); + this.db.interfaceManager.registerInterfaceType('circle', CircleInterface); } async load() { diff --git a/yarn.lock b/yarn.lock index ec74e6c960..66bdd55d03 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26586,6 +26586,10 @@ xlsx@^0.17.0: wmf "~1.0.1" word "~0.3.0" +"xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz": + version "0.20.2" + resolved "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz#0f64eeed3f1a46e64724620c3553f2dbd3cd2d7d" + xml-lexer@^0.2.2: version "0.2.2" resolved "https://registry.npmmirror.com/xml-lexer/-/xml-lexer-0.2.2.tgz#518193a4aa334d58fc7d248b549079b89907e046"