From 5d5f455b3c31124e274692d058c8808dd4753d75 Mon Sep 17 00:00:00 2001 From: Katherine Date: Wed, 8 Jan 2025 09:32:49 +0800 Subject: [PATCH] feat: supports configuring dynamic environment variables and secrets (#5966) * feat: environments plugin * feat: improve code * fix: improve code * feat: improve code * refactor: package description * feat: bulk import * fix: remove * refactor: file manager support environment variables * refactor: file manager support environment variables * refactor: map manager support environment variables * refactor: support environment variables * refactor: support environment variables * refactor: support delete environment variables * fix: bug * refactor: workflow support environment variables * refactor: email environment variables * refactor: support bulk import * refactor: support bulk import * refactor: support bulk import * refactor: support bulk import * refactor: code improve * feat: env * chore: update * feat: environment * fix: bug * fix: acl snippet * fix: acl snippets * chore: map manager * refactor: support line break * refactor: support password * chore: environment variables * fix: bug * fix: bug * chore: enviroment variables * chore: system settings * fix: improve code * feat: verification * feat: map * feat: file-manager * feat: notification * fix: bug * feat: workflow * fix: improve code * fix: bug * feat: data-source * feat: auth * fix: error * fix: bug * refactor: description * refactor: locale * refactor: locale * refactor: locale * refactor: code improve * refactor: locale * refactor: locale * style: style improve * fix: error * fix: bug * fix: bug * refactor: environment * fix: ellipsis * refactor: password * fix: bug * fix: bug * fix: bug * fix: bug * fix: bug * chore: test * fix: cache * fix: mysql dialect options * refactor: email config form * fix: bug * fix: bug * fix: authenticator.dataValues parse * fix: include undefined * fix: json * fix: json parse * chore: enviromentProvider * fix: acl * fix: rowKey * fix: update ProviderOptions.tsx * feat: get app instance * fix: bug * fix: text * fix: build error * fix: error * chore: migration rules options * chore: migration rules * refactor: code improve * feat: env v2 * chore: environment varibales * chore: environment serve * fix: getVariables * feat: improve code * fix: bug * chore: collection options for migration * chore: tree collection options * chore: migration rules * chore: migration rules * chore: env api * chore: env api * fix: optionsKeysNotAllowedInEnv * fix: required true * fix: improve code * fix: app refresh * fix: remove db.import * fix: type error * fix: map * refactor: locale improve * refactor: tx-cos * fix: undefined * refactor: code improve * chore: use bookworm * fix: npm add user * fix: npm login * fix: npm adduser * fix: npm adduser * fix: expect * fix: expect * fix: environmentVariables * refactor: support bulk delete & filter * refactor: locale improve * feat: filter * refactor: useGlobalVariable * fix: scope * fix: bug * fix: optionsKeysNotAllowedInEnv * fix: test error * fix: test * fix: test * feat: improve code --------- Co-authored-by: chenos Co-authored-by: Chareice --- .gitignore | 14 - Dockerfile | 17 +- docker/nocobase/Dockerfile | 4 +- lerna.json | 4 +- packages/core/auth/src/auth.ts | 4 + .../client/src/application/Application.tsx | 9 + .../client/src/application/hooks/index.ts | 1 + .../application/hooks/useGlobalVariable.ts | 31 ++ .../Input/inputComponentSettings.tsx | 2 +- .../src/schema-component/antd/input/Json.tsx | 15 +- .../antd/variable/TextArea.tsx | 18 +- .../antd/variable/TextAreaWithGlobalScope.tsx | 61 +++ .../schema-component/antd/variable/index.ts | 1 + .../SystemSettingsProvider.tsx | 7 +- .../SystemSettingsShortcut.tsx | 9 +- .../src/variables/VariablesProvider.tsx | 1 - .../src/collection-manager.ts | 8 +- .../data-source-manager/src/collection.ts | 4 +- .../src/data-source-factory.ts | 14 +- .../src/data-source-manager.ts | 5 +- .../data-source-manager/src/data-source.ts | 13 +- .../src/sequelize-collection-manager.ts | 6 + .../core/data-source-manager/src/types.ts | 6 + .../src/__tests__/underscored-options.test.ts | 2 +- packages/core/database/src/collection.ts | 5 +- packages/core/database/src/database.ts | 1 + .../database/src/dialects/mysql-dialect.ts | 6 + .../src/fields/belongs-to-many-field.ts | 3 + packages/core/server/src/application.ts | 8 + packages/core/server/src/environment.ts | 47 ++ .../server/src/helpers/application-version.ts | 1 + .../src/plugin-manager/options/collection.ts | 1 + packages/core/test/src/server/index.ts | 1 + .../src/server/collections/roles-users.ts | 2 + .../src/server/collections/roles.ts | 2 + .../src/server/collections/rolesResources.ts | 1 + .../collections/rolesResourcesActions.ts | 1 + .../collections/rolesResourcesScopes.ts | 1 + .../src/server/collections/users.ts | 1 + .../CustomRequestActionDesigner.tsx | 17 +- .../hooks/useCustomRequestVariableOptions.ts | 5 +- .../src/server/actions/send.ts | 3 +- .../src/server/collections/customRequest.ts | 1 + .../server/collections/customRequestsRoles.ts | 1 + .../src/server/plugin.ts | 5 +- .../src/collections/apiKeys.ts | 1 + .../src/server/collections/auditLogs.ts | 1 + .../src/server/collections/authenticators.ts | 1 + .../src/server/collections/token-blacklist.ts | 1 + .../collections/users-authenticators.ts | 1 + .../plugin-auth/src/server/plugin.ts | 4 +- .../plugin-auth/src/server/storer.ts | 39 +- .../src/server/collections/iframe-html.ts | 1 + .../plugin-client/src/server/server.ts | 12 +- .../src/server/plugin.ts | 4 + .../collections/collectionCategories.ts | 1 + .../src/server/collections/collections.ts | 1 + .../src/server/collections/fields.ts | 1 + .../src/server/server.ts | 9 +- .../collections/data-sources-collections.ts | 1 + .../server/collections/data-sources-fields.ts | 1 + .../data-sources-roles-resources-actions.ts | 1 + .../data-sources-roles-resources-scopes.ts | 1 + .../data-sources-roles-resources.ts | 1 + .../server/collections/data-sources-roles.ts | 1 + .../src/server/collections/data-sources.ts | 1 + .../src/server/models/data-source.ts | 10 +- .../src/server/plugin.ts | 10 +- .../plugin-environment-variables/.npmignore | 2 + .../plugin-environment-variables/README.md | 1 + .../plugin-environment-variables/client.d.ts | 2 + .../plugin-environment-variables/client.js | 1 + .../plugin-environment-variables/package.json | 18 + .../plugin-environment-variables/server.d.ts | 2 + .../plugin-environment-variables/server.js | 1 + ...EnvironmentVariablesAndSecretsProvider.tsx | 36 ++ .../src/client/client.d.ts | 249 +++++++++ .../src/client/components/EnvironmentPage.tsx | 15 + .../src/client/components/EnvironmentTabs.tsx | 510 ++++++++++++++++++ .../src/client/index.tsx | 27 + .../src/client/locale.ts | 21 + .../src/client/utils.ts | 33 ++ .../plugin-environment-variables/src/index.ts | 11 + .../src/locale/en-US.json | 1 + .../src/locale/zh-CN.json | 16 + .../plugin-environment-variables/src/re.ts | 10 + .../src/server/AesEncryptor.tsx | 76 +++ .../src/server/collections/.gitkeep | 0 .../collections/environmentVariables.ts | 35 ++ .../src/server/index.ts | 10 + .../src/server/plugin.ts | 175 ++++++ .../src/server/collections/chinaRegions.ts | 1 + .../src/server/collections/sequences.ts | 1 + .../src/client/StorageOptions.tsx | 28 +- .../src/client/schemas/storage.ts | 4 +- .../client/schemas/storageTypes/ali-oss.tsx | 11 +- .../src/client/schemas/storageTypes/s3.ts | 11 +- .../client/schemas/storageTypes/tx-cos.tsx | 13 +- .../src/client/templates/file.ts | 2 +- .../src/server/actions/attachments.ts | 9 +- .../src/server/collections/attachments.ts | 1 + .../src/server/collections/storages.ts | 1 + .../plugin-file-manager/src/server/server.ts | 24 +- .../src/server/collections/graphPositions.ts | 1 + .../server/collections/localization-texts.ts | 1 + .../collections/localization-translations.ts | 1 + .../src/client/components/Configuration.tsx | 32 +- .../src/client/components/GoogleMaps/Map.tsx | 2 +- .../src/client/hooks/useMapConfiguration.ts | 10 +- .../@nocobase/plugin-map/src/client/index.tsx | 2 +- .../plugin-map/src/locale/en-US.json | 2 +- .../plugin-map/src/locale/ja-JP.json | 2 +- .../plugin-map/src/locale/ko-KR.json | 2 +- .../plugin-map/src/locale/pt-BR.json | 2 +- .../plugin-map/src/locale/zh-CN.json | 2 +- .../plugin-map/src/server/actions/index.ts | 6 +- .../server/collections/mapConfiguration.ts | 1 + .../@nocobase/plugin-map/src/server/plugin.ts | 7 +- .../src/server/collections/mobileRoutes.ts | 1 + .../src/server/collections/applications.ts | 1 + .../src/client/ConfigForm.tsx | 52 +- .../src/types/messages.ts | 1 + .../src/collections/channel.ts | 1 + .../src/collections/messageLog.ts | 1 + .../src/server/collections/channels.ts | 4 +- .../src/server/collections/messageLogs.ts | 4 +- .../src/server/manager.ts | 10 +- .../plugin-notifications/src/server/server.ts | 7 +- .../src/client/collections/publicForms.ts | 3 +- .../client/components/AdminPublicFormPage.tsx | 41 +- .../src/server/collections/publicForms.ts | 1 + .../plugin-public-forms/src/server/plugin.ts | 2 +- .../src/server/collections/systemSettings.ts | 1 + .../src/server/server.ts | 43 +- .../src/server/collections/theme-config.ts | 1 + .../server/collections/uiSchemaServerHooks.ts | 1 + .../server/collections/uiSchemaTemplates.ts | 1 + .../server/collections/uiSchemaTreePath.ts | 1 + .../src/server/collections/uiSchemas.ts | 1 + .../user-data-sync-records-resources.ts | 1 + .../collections/user-data-sync-records.ts | 1 + .../collections/user-data-sync-sources.ts | 1 + .../collections/user-data-sync-tasks.ts | 1 + .../src/server/collections/users.ts | 1 + .../src/client/providerTypes/sms-aliyun.ts | 11 +- .../src/client/providerTypes/sms-tencent.ts | 15 +- .../src/server/__tests__/Plugin.test.ts | 2 +- .../src/server/__tests__/index.ts | 15 +- .../src/server/collections/verifications.ts | 1 + .../collections/verifications_providers.ts | 1 + .../src/server/providers/Provider.ts | 7 +- .../src/server/collections/1-users_jobs.ts | 1 + .../plugin-workflow/src/client/variable.tsx | 13 +- .../plugin-workflow/src/server/Processor.ts | 1 + .../src/server/collections/executions.ts | 1 + .../src/server/collections/flow_nodes.ts | 1 + .../src/server/collections/jobs.ts | 1 + .../src/server/collections/workflows.ts | 1 + packages/presets/nocobase/package.json | 2 + storage/.gitignore | 14 +- storage/verdaccio/config.yaml | 213 -------- storage/verdaccio/htpasswd | 1 - 162 files changed, 1902 insertions(+), 486 deletions(-) create mode 100644 packages/core/client/src/application/hooks/useGlobalVariable.ts create mode 100644 packages/core/client/src/schema-component/antd/variable/TextAreaWithGlobalScope.tsx create mode 100644 packages/core/server/src/environment.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/.npmignore create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/README.md create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/client.d.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/client.js create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/package.json create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/server.d.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/server.js create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/EnvironmentVariablesAndSecretsProvider.tsx create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/client.d.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/components/EnvironmentPage.tsx create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/components/EnvironmentTabs.tsx create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/index.tsx create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/locale.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/client/utils.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/index.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/locale/en-US.json create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/locale/zh-CN.json create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/re.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/server/AesEncryptor.tsx create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/server/collections/.gitkeep create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/server/collections/environmentVariables.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/server/index.ts create mode 100644 packages/plugins/@nocobase/plugin-environment-variables/src/server/plugin.ts delete mode 100644 storage/verdaccio/config.yaml delete mode 100644 storage/verdaccio/htpasswd diff --git a/.gitignore b/.gitignore index 63dd4bf4ff..4ff2ca8a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,27 +20,13 @@ docs-dist/ dist/ docker/**/storage cache/diskstore-* -*.nbdump -storage/duplicator/* -storage/backups/* **/.dumi/tmp **/.dumi/tmp-test **/.dumi/tmp-production packages/core/client/docs/contributing.md packages/core/app/client/src/.plugins -storage/plugins -storage/tar -storage/tmp -storage/print-templates -storage/cache -storage/app.watch.ts -storage/.upgrading -storage/logs-e2e -storage/uploads-e2e -storage/.pm2-* tsconfig.paths.json /playwright -/storage/playwright .swc ncc-cache/ yarn--** diff --git a/Dockerfile b/Dockerfile index a94d3e8f45..662dfdbdb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.13-bullseye as builder +FROM node:20-bookworm as builder ARG VERDACCIO_URL=http://host.docker.internal:10104/ ARG COMMIT_HASH ARG APPEND_PRESET_LOCAL_PLUGINS @@ -7,10 +7,17 @@ ARG PLUGINS_DIRS ENV PLUGINS_DIRS=${PLUGINS_DIRS} +RUN apt-get update && apt-get install -y jq expect -RUN npx npm-cli-adduser --username test --password test -e test@nocobase.com -r $VERDACCIO_URL +RUN expect < /etc/apt/sources.list.d/pgdg.list' +RUN sh -c 'echo "deb http://mirrors.ustc.edu.cn/postgresql/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' RUN wget --quiet -O - http://mirrors.ustc.edu.cn/postgresql/repos/apt/ACCC4CF8.asc | apt-key add - RUN apt-get update && apt-get -y --no-install-recommends install nginx libaio1 postgresql-client-16 postgresql-client-17 \ diff --git a/docker/nocobase/Dockerfile b/docker/nocobase/Dockerfile index 41841cb7d5..f81c515592 100644 --- a/docker/nocobase/Dockerfile +++ b/docker/nocobase/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-bullseye-slim as builder +FROM node:20-bookworm-slim as builder ARG CNA_VERSION @@ -14,7 +14,7 @@ RUN cd /app \ && rm -rf nocobase.tar.gz \ && tar -zcf ./nocobase.tar.gz -C /app/my-nocobase-app . -FROM node:18-bullseye-slim +FROM node:20-bookworm-slim # COPY ./sources.list /etc/apt/sources.list RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ diff --git a/lerna.json b/lerna.json index d2dd1e12e9..129fb50861 100644 --- a/lerna.json +++ b/lerna.json @@ -2,9 +2,7 @@ "version": "1.6.0-alpha.9", "npmClient": "yarn", "useWorkspaces": true, - "npmClientArgs": [ - "--ignore-engines" - ], + "npmClientArgs": ["--ignore-engines"], "command": { "version": { "forcePublish": true, diff --git a/packages/core/auth/src/auth.ts b/packages/core/auth/src/auth.ts index 7c1de54a4e..b0622576ee 100644 --- a/packages/core/auth/src/auth.ts +++ b/packages/core/auth/src/auth.ts @@ -31,6 +31,10 @@ interface IAuth { } export abstract class Auth implements IAuth { + /** + * options keys that are not allowed to use environment variables + */ + public static optionsKeysNotAllowedInEnv: string[]; abstract user: Model; protected authenticator: Authenticator; protected options: { diff --git a/packages/core/client/src/application/Application.tsx b/packages/core/client/src/application/Application.tsx index f66810ace7..f4062bda93 100644 --- a/packages/core/client/src/application/Application.tsx +++ b/packages/core/client/src/application/Application.tsx @@ -99,6 +99,7 @@ export class Application { public schemaSettingsManager: SchemaSettingsManager; public dataSourceManager: DataSourceManager; public name: string; + public globalVars: Record = {}; loading = true; maintained = false; @@ -465,4 +466,12 @@ export class Application { componentOption, ); } + + addGlobalVar(key: string, value: any) { + set(this.globalVars, key, value); + } + + getGlobalVar(key) { + return get(this.globalVars, key); + } } diff --git a/packages/core/client/src/application/hooks/index.ts b/packages/core/client/src/application/hooks/index.ts index e7d4f06ee0..57fb89faff 100644 --- a/packages/core/client/src/application/hooks/index.ts +++ b/packages/core/client/src/application/hooks/index.ts @@ -11,3 +11,4 @@ export * from './useApp'; export * from './useAppSpin'; export * from './usePlugin'; export * from './useRouter'; +export * from './useGlobalVariable'; diff --git a/packages/core/client/src/application/hooks/useGlobalVariable.ts b/packages/core/client/src/application/hooks/useGlobalVariable.ts new file mode 100644 index 0000000000..1ef7ef76d1 --- /dev/null +++ b/packages/core/client/src/application/hooks/useGlobalVariable.ts @@ -0,0 +1,31 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { isFunction } from 'lodash'; +import { useMemo } from 'react'; +import { useApp } from './'; + +export const useGlobalVariable = (key: string) => { + const app = useApp(); + + const variable = useMemo(() => { + return app.getGlobalVar(key); + }, [app, key]); + + if (isFunction(variable)) { + try { + return variable(); + } catch (error) { + console.error(`Error calling global variable function for key: ${key}`, error); + return undefined; + } + } + + return variable; +}; diff --git a/packages/core/client/src/modules/fields/component/Input/inputComponentSettings.tsx b/packages/core/client/src/modules/fields/component/Input/inputComponentSettings.tsx index deae4f1a29..8f53455a71 100644 --- a/packages/core/client/src/modules/fields/component/Input/inputComponentSettings.tsx +++ b/packages/core/client/src/modules/fields/component/Input/inputComponentSettings.tsx @@ -72,7 +72,7 @@ export const enableLinkSettingsItem: SchemaSettingsItemType = { const { fieldSchema: columnSchema } = useColumnSchema(); const schema = useFieldSchema(); const fieldSchema = columnSchema || schema; - const { name } = useBlockContext(); + const { name } = useBlockContext() || {}; return name !== 'kanban' && (fieldSchema?.['x-read-pretty'] || field.readPretty); }, useComponentProps() { diff --git a/packages/core/client/src/schema-component/antd/input/Json.tsx b/packages/core/client/src/schema-component/antd/input/Json.tsx index 5d5a5510c2..eb1a66c9b0 100644 --- a/packages/core/client/src/schema-component/antd/input/Json.tsx +++ b/packages/core/client/src/schema-component/antd/input/Json.tsx @@ -7,13 +7,13 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ +import { css, cx } from '@emotion/css'; import { Field } from '@formily/core'; import { useField } from '@formily/react'; import { Input } from 'antd'; import { TextAreaProps } from 'antd/es/input'; -import React, { useState, useEffect, Ref } from 'react'; -import { cx, css } from '@emotion/css'; import JSON5 from 'json5'; +import React, { Ref, useEffect, useState } from 'react'; export type JSONTextAreaProps = TextAreaProps & { value?: string; space?: number; json5?: boolean }; @@ -30,7 +30,16 @@ export const Json = React.forwardRef( useEffect(() => { try { if (value != null) { - setText(_JSON.stringify(value, null, space)); + if (typeof value === 'string') { + try { + _JSON.parse(value); + setText(value); + } catch (error) { + setText(_JSON.stringify(value, null, space)); + } + } else { + setText(_JSON.stringify(value, null, space)); + } } else { setText(undefined); } diff --git a/packages/core/client/src/schema-component/antd/variable/TextArea.tsx b/packages/core/client/src/schema-component/antd/variable/TextArea.tsx index 519e814866..0c0b127468 100644 --- a/packages/core/client/src/schema-component/antd/variable/TextArea.tsx +++ b/packages/core/client/src/schema-component/antd/variable/TextArea.tsx @@ -223,7 +223,7 @@ function useVariablesFromValue(value: string, delimiters: [string, string] = ['{ export function TextArea(props) { const { wrapSSR, hashId, componentCls } = useStyles(); - const { scope, onChange, changeOnSelect, style, fieldNames, delimiters = ['{{', '}}'] } = props; + const { scope, onChange, changeOnSelect, style, fieldNames, delimiters = ['{{', '}}'], addonBefore } = props; const value = typeof props.value === 'string' ? props.value : props.value == null ? '' : props.value.toString(); const variables = useVariablesFromValue(value, delimiters); const inputRef = useRef(null); @@ -397,7 +397,6 @@ export function TextArea(props) { }, [onChange, delimitersString], ); - const disabled = props.disabled || form.disabled; return wrapSSR( + {addonBefore && ( +
+ {addonBefore} +
+ )}
{ + const environmentVariables = useGlobalVariable('$env'); + return useMemo(() => { + if (environmentVariables) { + return [environmentVariables].filter(Boolean); + } + return scope; + }, [environmentVariables, scope]); +}; + +const isVariable = (value) => { + const regex = /{{.*?}}/; + return regex.test(value); +}; +interface TextAreaWithGlobalScopeProps { + supportsLineBreak?: boolean; + password?: boolean; + number?: boolean; + boolean?: boolean; + value?: any; + scope?: string | object; + [key: string]: any; +} + +export const TextAreaWithGlobalScope = connect((props: TextAreaWithGlobalScopeProps) => { + const { supportsLineBreak, password, number, boolean, ...others } = props; + const scope = useEnvironmentVariableOptions(props.scope); + const fieldNames = { value: 'name', label: 'title' }; + + if (supportsLineBreak) { + return ; + } + if (number) { + return ; + } + if (password && props.value && !isVariable(props.value)) { + return ; + } + if (boolean) { + return ; + } + return