From b36586f324c8ab4cde232f8264cf9df202b584d7 Mon Sep 17 00:00:00 2001 From: YANG QIA <2013xile@gmail.com> Date: Wed, 23 Oct 2024 10:18:58 +0800 Subject: [PATCH] fix(data-vi): fix issues when using external oracle data source for data visualization (#5436) * fix(data-vi): fix issues when using external oracle data source for data visualization * fix: bug * fix: bug * fix: oracle group by * fix: oracle formatter --- .../src/server/__tests__/api.test.ts | 9 +- .../src/server/__tests__/formatter.test.ts | 18 ++- .../src/server/__tests__/query.test.ts | 27 ++-- .../src/server/actions/query.ts | 126 +----------------- .../src/server/formatter/index.ts | 26 ---- .../src/server/formatter/oracle-formatter.ts | 55 ++++++++ .../src/server/plugin.ts | 3 +- .../src/server/query-parser/index.ts | 32 +++++ .../server/query-parser/mysql-query-parser.ts | 21 +++ .../query-parser/oracle-query-parser.ts | 41 ++++++ .../query-parser/postgres-query-parser.ts | 21 +++ .../src/server/query-parser/query-parser.ts | 120 +++++++++++++++++ .../query-parser/sqlite-query-parser.ts | 21 +++ .../src/server/types.ts | 51 +++++++ 14 files changed, 400 insertions(+), 171 deletions(-) delete mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/index.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/oracle-formatter.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/index.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/mysql-query-parser.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/oracle-query-parser.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/postgres-query-parser.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/query-parser.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/sqlite-query-parser.ts create mode 100644 packages/plugins/@nocobase/plugin-data-visualization/src/server/types.ts diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/api.test.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/api.test.ts index 0335e08992..9a72fa4ba8 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/api.test.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/api.test.ts @@ -10,7 +10,8 @@ import { Database, Repository } from '@nocobase/database'; import { MockServer, createMockServer } from '@nocobase/test'; import compose from 'koa-compose'; -import { parseBuilder, parseFieldAndAssociations, queryData } from '../actions/query'; +import { parseFieldAndAssociations, queryData } from '../actions/query'; +import { createQueryParser } from '../query-parser'; describe('api', () => { let app: MockServer; @@ -91,7 +92,8 @@ describe('api', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); }); @@ -125,7 +127,8 @@ describe('api', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ createdAt: '2023-01' }, { createdAt: '2023-02' }]); }); diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/formatter.test.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/formatter.test.ts index f87283e133..0230c0f74c 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/formatter.test.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/formatter.test.ts @@ -10,7 +10,8 @@ import { Database } from '@nocobase/database'; import { MockServer, createMockServer } from '@nocobase/test'; import compose from 'koa-compose'; -import { parseBuilder, parseFieldAndAssociations, queryData } from '../actions/query'; +import { parseFieldAndAssociations, queryData } from '../actions/query'; +import { createQueryParser } from '../query-parser'; describe('formatter', () => { let app: MockServer; @@ -85,7 +86,8 @@ describe('formatter', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ date: '2024-05-15 01:02:30' }]); }); @@ -125,7 +127,8 @@ describe('formatter', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ dateOnly: '2024-05-14' }]); }); @@ -165,7 +168,8 @@ describe('formatter', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ datetimeNoTz: '2024-05-14 19:32:30' }]); }); @@ -213,7 +217,8 @@ describe('formatter', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ unixTs: '2023-01-01 10:04:56' }]); }); @@ -260,7 +265,8 @@ describe('formatter', () => { }, }, } as any; - await compose([parseFieldAndAssociations, parseBuilder, queryData])(ctx, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse(), queryData])(ctx, async () => {}); expect(ctx.action.params.values.data).toBeDefined(); expect(ctx.action.params.values.data).toMatchObject([{ unixTsMs: '2023-01-01 10:04:56' }]); }); diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts index 15fc2554d2..4c39e0c306 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/__tests__/query.test.ts @@ -13,13 +13,14 @@ import { vi } from 'vitest'; import { cacheMiddleware, checkPermission, - parseBuilder, parseFieldAndAssociations, parseVariables, postProcess, } from '../actions/query'; import { Database } from '@nocobase/database'; import * as formatter from '../formatter'; +import { createQueryParser } from '../query-parser'; +import { QueryParser } from '../query-parser/query-parser'; describe('query', () => { describe('parseBuilder', () => { @@ -157,7 +158,8 @@ describe('query', () => { }, }, }; - await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse()])(context, async () => {}); expect(context.action.params.values.queryParams.attributes).toEqual([ [db.sequelize.col('orders.price'), 'price'], ]); @@ -179,7 +181,7 @@ describe('query', () => { }, }, }; - await compose([parseFieldAndAssociations, parseBuilder])(context2, async () => {}); + await compose([parseFieldAndAssociations, queryParser.parse()])(context2, async () => {}); expect(context2.action.params.values.queryParams.attributes).toEqual([ [db.sequelize.fn('sum', db.sequelize.col('orders.price')), 'price-alias'], ]); @@ -203,20 +205,17 @@ describe('query', () => { }, }, }; + const queryParser = createQueryParser(db); try { - await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {}); + await compose([parseFieldAndAssociations, queryParser.parse()])(context, async () => {}); } catch (error) { expect(error.message).toBe('Invalid aggregation function: if(1=2,sleep(1),sleep(3)) and sum'); } }); it('should parse dimensions', async () => { - vi.spyOn(formatter, 'createFormatter').mockImplementation( - () => - ({ - format: () => 'formatted-field' as any, - }) as any, - ); + const queryParser = createQueryParser(db); + vi.spyOn(queryParser.formatter, 'format').mockImplementation(() => 'formatted-field' as any); const dimensions = [ { field: ['createdAt'], @@ -235,7 +234,7 @@ describe('query', () => { }, }, }; - await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {}); + await compose([parseFieldAndAssociations, queryParser.parse()])(context, async () => {}); expect(context.action.params.values.queryParams.attributes).toEqual([['formatted-field', 'Created at']]); expect(context.action.params.values.queryParams.group).toEqual([]); const measures = [ @@ -256,7 +255,7 @@ describe('query', () => { }, }, }; - await compose([parseFieldAndAssociations, parseBuilder])(context2, async () => {}); + await compose([parseFieldAndAssociations, queryParser.parse()])(context2, async () => {}); expect(context2.action.params.values.queryParams.group).toEqual(['formatted-field']); }); @@ -277,7 +276,8 @@ describe('query', () => { }, }, }; - await compose([parseFieldAndAssociations, parseBuilder])(context, async () => {}); + const queryParser = createQueryParser(db); + await compose([parseFieldAndAssociations, queryParser.parse()])(context, async () => {}); expect(context.action.params.values.queryParams.where.createdAt).toBeDefined(); }); @@ -332,7 +332,6 @@ describe('query', () => { await parseVariables(context, async () => {}); const { filter } = context.action.params.values; const dateOn = filter.$and[0].createdAt.$dateOn; - console.log(dateOn); expect(new Date(dateOn).getTime()).toBeLessThanOrEqual(new Date().getTime()); const userId = filter.$and[1].userId.$eq; expect(userId).toBe(1); diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts index 2770dd48d7..631937a59a 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/actions/query.ts @@ -12,52 +12,8 @@ import { BelongsToArrayAssociation, Field, FilterParser } from '@nocobase/databa import compose from 'koa-compose'; import { Cache } from '@nocobase/cache'; import { middlewares } from '@nocobase/server'; -import { createFormatter } from '../formatter'; - -type MeasureProps = { - field: string | string[]; - type?: string; - aggregation?: string; - alias?: string; - distinct?: boolean; -}; - -type DimensionProps = { - field: string | string[]; - type?: string; - alias?: string; - format?: string; - options?: any; -}; - -type OrderProps = { - field: string | string[]; - alias?: string; - order?: 'asc' | 'desc'; -}; - -type QueryParams = Partial<{ - uid: string; - dataSource: string; - collection: string; - measures: MeasureProps[]; - dimensions: DimensionProps[]; - orders: OrderProps[]; - filter: any; - limit: number; - sql: { - fields?: string; - clauses?: string; - }; - cache: { - enabled: boolean; - ttl: number; - }; - // Get the latest data from the database - refresh: boolean; -}>; - -const AllowedAggFuncs = ['sum', 'count', 'avg', 'min', 'max']; +import { QueryParams } from '../types'; +import { createQueryParser } from '../query-parser'; const getDB = (ctx: Context, dataSource: string) => { const ds = ctx.app.dataSourceManager.dataSources.get(dataSource); @@ -109,79 +65,6 @@ export const queryData = async (ctx: Context, next: Next) => { // return data; }; -export const parseBuilder = async (ctx: Context, next: Next) => { - const { dataSource, measures, dimensions, orders, include, where, limit } = ctx.action.params.values; - const db = getDB(ctx, dataSource) || ctx.db; - const { sequelize } = db; - const attributes = []; - const group = []; - const order = []; - const fieldMap = {}; - let hasAgg = false; - - measures.forEach((measure: MeasureProps & { field: string }) => { - const { field, aggregation, alias, distinct } = measure; - const attribute = []; - const col = sequelize.col(field); - if (aggregation) { - if (!AllowedAggFuncs.includes(aggregation)) { - throw new Error(`Invalid aggregation function: ${aggregation}`); - } - hasAgg = true; - attribute.push(sequelize.fn(aggregation, distinct ? sequelize.fn('DISTINCT', col) : col)); - } else { - attribute.push(col); - } - if (alias) { - attribute.push(alias); - } - attributes.push(attribute.length > 1 ? attribute : attribute[0]); - fieldMap[alias || field] = measure; - }); - - dimensions.forEach((dimension: DimensionProps & { field: string }) => { - const { field, format, alias, type, options } = dimension; - const attribute = []; - const col = sequelize.col(field); - if (format) { - const formatter = createFormatter(sequelize); - attribute.push(formatter.format({ type, field, format, timezone: ctx.timezone, options })); - } else { - attribute.push(col); - } - if (alias) { - attribute.push(alias); - } - attributes.push(attribute.length > 1 ? attribute : attribute[0]); - if (hasAgg) { - group.push(attribute[0]); - } - fieldMap[alias || field] = dimension; - }); - - orders.forEach((item: OrderProps) => { - const alias = sequelize.getQueryInterface().quoteIdentifier(item.alias); - const name = hasAgg ? sequelize.literal(alias) : sequelize.col(item.field as string); - order.push([name, item.order || 'ASC']); - }); - - ctx.action.params.values = { - ...ctx.action.params.values, - queryParams: { - where, - attributes, - include, - group, - order, - limit: limit || 2000, - subQuery: false, - raw: true, - }, - fieldMap, - }; - await next(); -}; - export const parseFieldAndAssociations = async (ctx: Context, next: Next) => { const { dataSource, @@ -323,13 +206,16 @@ export const checkPermission = (ctx: Context, next: Next) => { }; export const query = async (ctx: Context, next: Next) => { + const { dataSource } = ctx.action.params.values as QueryParams; + const db = getDB(ctx, dataSource) || ctx.db; + const queryParser = createQueryParser(db); try { await compose([ checkPermission, cacheMiddleware, parseVariables, parseFieldAndAssociations, - parseBuilder, + queryParser.parse(), queryData, postProcess, ])(ctx, next); diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/index.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/index.ts deleted file mode 100644 index 3644ba183e..0000000000 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * This file is part of the NocoBase (R) project. - * Copyright (c) 2020-2024 NocoBase Co., Ltd. - * Authors: NocoBase Team. - * - * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. - * For more information, please refer to: https://www.nocobase.com/agreement. - */ - -import { Sequelize } from 'sequelize'; -import { SQLiteFormatter } from './sqlite-formatter'; -import { PostgresFormatter } from './postgres-formatter'; -import { MySQLFormatter } from './mysql-formatter'; - -export const createFormatter = (sequelize: Sequelize) => { - const dialect = sequelize.getDialect(); - switch (dialect) { - case 'sqlite': - return new SQLiteFormatter(sequelize); - case 'postgres': - return new PostgresFormatter(sequelize); - case 'mysql': - case 'mariadb': - return new MySQLFormatter(sequelize); - } -}; diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/oracle-formatter.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/oracle-formatter.ts new file mode 100644 index 0000000000..a60e163b04 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/formatter/oracle-formatter.ts @@ -0,0 +1,55 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Col, Formatter } from './formatter'; + +export class OracleFormatter extends Formatter { + convertFormat(format: string) { + return format.replace(/hh/g, 'HH24').replace(/mm/g, 'MI').replace(/ss/g, 'SS'); + } + formatDate(field: Col, format: string, timezoneOffset?: string) { + format = this.convertFormat(format); + const timezone = this.getTimezoneByOffset(timezoneOffset); + if (timezone) { + const col = this.sequelize.getQueryInterface().quoteIdentifiers((field as Col).col); + const fieldWithTZ = this.sequelize.literal(`(${col} AT TIME ZONE '${timezone}')`); + return this.sequelize.fn('to_char', fieldWithTZ, format); + } + return this.sequelize.fn('to_char', field, format); + } + + formatUnixTimeStamp( + field: string, + format: string, + accuracy: 'second' | 'millisecond' = 'second', + timezoneOffset?: string, + ) { + format = this.convertFormat(format); + const col = this.sequelize.getQueryInterface().quoteIdentifiers(field); + const timezone = this.getTimezoneByOffset(timezoneOffset); + if (timezone) { + if (accuracy === 'millisecond') { + return this.sequelize.fn( + 'to_char', + this.sequelize.literal(`to_timestamp(ROUND(${col} / 1000)) AT TIME ZONE '${timezone}'`), + format, + ); + } + return this.sequelize.fn( + 'to_char', + this.sequelize.literal(`to_timestamp(${col}) AT TIME ZONE '${timezone}'`), + format, + ); + } + if (accuracy === 'millisecond') { + return this.sequelize.fn('to_char', this.sequelize.literal(`to_timestamp(ROUND(${col} / 1000)`), format); + } + return this.sequelize.fn('to_char', this.sequelize.literal(`to_timestamp(${col})`), format); + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts index 6732980ecd..285ec787dc 100644 --- a/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/plugin.ts @@ -10,7 +10,6 @@ import { Cache } from '@nocobase/cache'; import { InstallOptions, Plugin } from '@nocobase/server'; import { query } from './actions/query'; -import { resolve } from 'path'; export class PluginDataVisualizationServer extends Plugin { cache: Cache; @@ -18,7 +17,7 @@ export class PluginDataVisualizationServer extends Plugin { afterAdd() {} beforeLoad() { - this.app.resource({ + this.app.resourceManager.define({ name: 'charts', actions: { query, diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/index.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/index.ts new file mode 100644 index 0000000000..a5c2a8e0c1 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/index.ts @@ -0,0 +1,32 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Database } from '@nocobase/database'; +import { SQLiteQueryParser } from './sqlite-query-parser'; +import { PostgresQueryParser } from './postgres-query-parser'; +import { MySQLQueryParser } from './mysql-query-parser'; +import { QueryParser } from './query-parser'; +import { OracleQueryParser } from './oracle-query-parser'; + +export const createQueryParser = (db: Database) => { + const dialect = db.sequelize.getDialect(); + switch (dialect) { + case 'sqlite': + return new SQLiteQueryParser(db); + case 'postgres': + return new PostgresQueryParser(db); + case 'mysql': + case 'mariadb': + return new MySQLQueryParser(db); + case 'oracle': + return new OracleQueryParser(db); + default: + return new QueryParser(db); + } +}; diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/mysql-query-parser.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/mysql-query-parser.ts new file mode 100644 index 0000000000..3845fe078f --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/mysql-query-parser.ts @@ -0,0 +1,21 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Database } from '@nocobase/database'; +import { MySQLFormatter } from '../formatter/mysql-formatter'; +import { QueryParser } from './query-parser'; + +export class MySQLQueryParser extends QueryParser { + declare formatter: MySQLFormatter; + + constructor(db: Database) { + super(db); + this.formatter = new MySQLFormatter(db.sequelize); + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/oracle-query-parser.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/oracle-query-parser.ts new file mode 100644 index 0000000000..f03922af91 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/oracle-query-parser.ts @@ -0,0 +1,41 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { QueryParser } from './query-parser'; +import { OrderProps, QueryParams } from '../types'; +import { Context } from '@nocobase/actions'; +import { OracleFormatter } from '../formatter/oracle-formatter'; +import { Database } from '@nocobase/database'; + +export class OracleQueryParser extends QueryParser { + declare formatter: OracleFormatter; + + constructor(db: Database) { + super(db); + this.formatter = new OracleFormatter(db.sequelize); + } + + parseOrders(ctx: Context, orders: OrderProps[], hasAgg: boolean) { + const { collection: collectionName, dimensions } = ctx.action.params.values as QueryParams; + const collection = this.db.getCollection(collectionName); + if (!orders.length) { + if (dimensions.length) { + orders.push(dimensions[0]); + } else { + let filterTks = collection.filterTargetKey; + if (!Array.isArray(filterTks)) { + filterTks = [filterTks]; + } + orders.push(...filterTks.map((field) => ({ field, alias: field }))); + } + } + const order = super.parseOrders(ctx, orders, hasAgg); + return order; + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/postgres-query-parser.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/postgres-query-parser.ts new file mode 100644 index 0000000000..3a77daefcb --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/postgres-query-parser.ts @@ -0,0 +1,21 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Database } from '@nocobase/database'; +import { PostgresFormatter } from '../formatter/postgres-formatter'; +import { QueryParser } from './query-parser'; + +export class PostgresQueryParser extends QueryParser { + declare formatter: PostgresFormatter; + + constructor(db: Database) { + super(db); + this.formatter = new PostgresFormatter(db.sequelize); + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/query-parser.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/query-parser.ts new file mode 100644 index 0000000000..a593591b76 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/query-parser.ts @@ -0,0 +1,120 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Context, Next } from '@nocobase/actions'; +import { DimensionProps, MeasureProps, OrderProps } from '../types'; +import { Formatter } from '../formatter/formatter'; +import { Database } from '@nocobase/database'; + +const AllowedAggFuncs = ['sum', 'count', 'avg', 'min', 'max']; + +export class QueryParser { + db: Database; + formatter: Formatter; + + constructor(db: Database) { + this.db = db; + this.formatter = { + format: ({ field }) => db.sequelize.col(field), + } as Formatter; + } + + parseMeasures(ctx: Context, measures: MeasureProps[]) { + let hasAgg = false; + const sequelize = this.db.sequelize; + const attributes = []; + const fieldMap = {}; + measures.forEach((measure: MeasureProps & { field: string }) => { + const { field, aggregation, alias, distinct } = measure; + const attribute = []; + const col = sequelize.col(field); + if (aggregation) { + if (!AllowedAggFuncs.includes(aggregation)) { + throw new Error(`Invalid aggregation function: ${aggregation}`); + } + hasAgg = true; + attribute.push(sequelize.fn(aggregation, distinct ? sequelize.fn('DISTINCT', col) : col)); + } else { + attribute.push(col); + } + if (alias) { + attribute.push(alias); + } + attributes.push(attribute.length > 1 ? attribute : attribute[0]); + fieldMap[alias || field] = measure; + }); + return { attributes, fieldMap, hasAgg }; + } + + parseDimensions(ctx: Context, dimensions: (DimensionProps & { field: string })[], hasAgg: boolean, timezone: string) { + const sequelize = this.db.sequelize; + const attributes = []; + const group = []; + const fieldMap = {}; + dimensions.forEach((dimension: DimensionProps & { field: string }) => { + const { field, format, alias, type, options } = dimension; + const attribute = []; + const col = sequelize.col(field); + if (format) { + attribute.push(this.formatter.format({ type, field, format, timezone, options })); + } else { + attribute.push(col); + } + if (alias) { + attribute.push(alias); + } + attributes.push(attribute.length > 1 ? attribute : attribute[0]); + if (hasAgg) { + group.push(attribute[0]); + } + fieldMap[alias || field] = dimension; + }); + return { attributes, group, fieldMap }; + } + + parseOrders(ctx: Context, orders: OrderProps[], hasAgg: boolean) { + const sequelize = this.db.sequelize; + const order = []; + orders.forEach((item: OrderProps) => { + const alias = sequelize.getQueryInterface().quoteIdentifier(item.alias); + const name = hasAgg ? sequelize.literal(alias) : sequelize.col(item.field as string); + order.push([name, item.order || 'ASC']); + }); + return order; + } + + parse() { + return async (ctx: Context, next: Next) => { + const { measures, dimensions, orders, include, where, limit } = ctx.action.params.values; + const { attributes: measureAttributes, fieldMap: measureFieldMap, hasAgg } = this.parseMeasures(ctx, measures); + const { + attributes: dimensionAttributes, + group, + fieldMap: dimensionFieldMap, + } = this.parseDimensions(ctx, dimensions, hasAgg, ctx.timezone); + const order = this.parseOrders(ctx, orders, hasAgg); + + ctx.action.params.values = { + ...ctx.action.params.values, + queryParams: { + where, + attributes: [...measureAttributes, ...dimensionAttributes], + include, + group, + order, + limit: limit || 2000, + subQuery: false, + raw: true, + }, + fieldMap: { ...measureFieldMap, ...dimensionFieldMap }, + }; + await next(); + }; + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/sqlite-query-parser.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/sqlite-query-parser.ts new file mode 100644 index 0000000000..872a7153b8 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/query-parser/sqlite-query-parser.ts @@ -0,0 +1,21 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +import { Database } from '@nocobase/database'; +import { SQLiteFormatter } from '../formatter/sqlite-formatter'; +import { QueryParser } from './query-parser'; + +export class SQLiteQueryParser extends QueryParser { + declare formatter: SQLiteFormatter; + + constructor(db: Database) { + super(db); + this.formatter = new SQLiteFormatter(db.sequelize); + } +} diff --git a/packages/plugins/@nocobase/plugin-data-visualization/src/server/types.ts b/packages/plugins/@nocobase/plugin-data-visualization/src/server/types.ts new file mode 100644 index 0000000000..7aff7351d6 --- /dev/null +++ b/packages/plugins/@nocobase/plugin-data-visualization/src/server/types.ts @@ -0,0 +1,51 @@ +/** + * This file is part of the NocoBase (R) project. + * Copyright (c) 2020-2024 NocoBase Co., Ltd. + * Authors: NocoBase Team. + * + * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License. + * For more information, please refer to: https://www.nocobase.com/agreement. + */ + +export type MeasureProps = { + field: string | string[]; + type?: string; + aggregation?: string; + alias?: string; + distinct?: boolean; +}; + +export type DimensionProps = { + field: string | string[]; + type?: string; + alias?: string; + format?: string; + options?: any; +}; + +export type OrderProps = { + field: string | string[]; + alias?: string; + order?: 'asc' | 'desc'; +}; + +export type QueryParams = Partial<{ + uid: string; + dataSource: string; + collection: string; + measures: MeasureProps[]; + dimensions: DimensionProps[]; + orders: OrderProps[]; + filter: any; + limit: number; + sql: { + fields?: string; + clauses?: string; + }; + cache: { + enabled: boolean; + ttl: number; + }; + // Get the latest data from the database + refresh: boolean; +}>;