chore: split sql collection (#3650)

* chore: split sql collection

* chore: package json

* chore: test

* chore: build

* chore: move sql resourcer into plugin-collection-sql

* fix: server

* fix: ast parser, fix T-4236

* fix: fix T-4236

* fix: fields

* fix: test

* fix: test

* fix: test

* fix: test

* chore: add keyword

* chore: node sql parser version

* chore: yarn.lock

* fix: types

* fix: remove column named `*`

* fix: package.json

* fix: version

* chore: update homepage

---------

Co-authored-by: xilesun <2013xile@gmail.com>
This commit is contained in:
ChengLei Shao 2024-05-17 15:39:01 +08:00 committed by GitHub
parent caffcc4b9b
commit 3d000d395e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
37 changed files with 284 additions and 39809 deletions

View File

@ -2,9 +2,7 @@
"version": "1.0.0-alpha.14",
"npmClient": "yarn",
"useWorkspaces": true,
"npmClientArgs": [
"--ignore-engines"
],
"npmClientArgs": ["--ignore-engines"],
"command": {
"version": {
"forcePublish": true,

View File

@ -136,6 +136,7 @@ export const FieldsConfigure = observer(
Object.entries(sourceFields || {}).forEach(([col, val]: [string, any]) =>
fieldsMp.set(col, {
name: col,
type: 'string', // default
...val,
uiSchema: {
title: col,
@ -217,7 +218,7 @@ export const FieldsConfigure = observer(
placeholder={t('Select field source')}
onChange={(value: string[]) => {
let sourceField = sourceFields[value?.[1]];
if (!sourceField) {
if (!sourceField?.interface) {
sourceField = getCollectionField(value?.join('.') || '');
}
handleFieldChange(

View File

@ -6,6 +6,7 @@
"types": "./lib/index.d.ts",
"license": "AGPL-3.0",
"dependencies": {
"node-sql-parser": "^4.18.0",
"@nocobase/logger": "1.0.0-alpha.14",
"@nocobase/utils": "1.0.0-alpha.14",
"async-mutex": "^0.3.2",

View File

@ -54,7 +54,6 @@ import QueryInterface from './query-interface/query-interface';
import buildQueryInterface from './query-interface/query-interface-builder';
import { RelationRepository } from './relation-repository/relation-repository';
import { Repository } from './repository';
import { SqlCollection } from './sql-collection/sql-collection';
import {
AfterDefineCollectionListener,
BeforeDefineCollectionListener,
@ -1011,20 +1010,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
return;
},
});
this.collectionFactory.registerCollectionType(SqlCollection, {
condition: (options) => {
return options.sql;
},
async onSync() {
return;
},
async onDump(dumper, collection: Collection) {
return;
},
});
}
}

View File

@ -50,5 +50,5 @@ export { snakeCase } from './utils';
export * from './value-parsers';
export * from './view-collection';
export * from './view/view-inference';
export * from './sql-collection';
export * from './helpers';
export { default as sqlParser, SQLParserTypes } from './sql-parser';

View File

@ -9,7 +9,7 @@
import lodash from 'lodash';
import { Collection } from '../collection';
import sqlParser from '../sql-parser/postgres';
import sqlParser from '../sql-parser';
import QueryInterface, { TableInfo } from './query-interface';
import { Transaction } from 'sequelize';
@ -124,7 +124,9 @@ export default class PostgresQueryInterface extends QueryInterface {
}
parseSQL(sql: string): any {
return sqlParser.parse(sql);
return sqlParser.parse(sql, {
database: 'Postgresql',
});
}
async viewColumnUsage(options): Promise<{

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,14 @@
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { Parser } from 'node-sql-parser';
import type * as SQLParserTypes from 'node-sql-parser';
export default new Parser();
export type { SQLParserTypes };

File diff suppressed because it is too large Load Diff

View File

@ -1,2 +0,0 @@
use peggy to transform pegjs to sql parser
https://github.com/peggyjs/peggy

File diff suppressed because it is too large Load Diff

View File

@ -11,7 +11,7 @@ import { createMockServer } from '@nocobase/test';
export default async function createApp() {
const app = await createMockServer({
plugins: ['nocobase'],
plugins: ['nocobase', 'collection-sql'],
});
return app;
}

View File

@ -0,0 +1,2 @@
/node_modules
/src

View File

@ -0,0 +1 @@
# @nocobase/plugin-collection-sql

View File

@ -0,0 +1,2 @@
export * from './dist/client';
export { default } from './dist/client';

View File

@ -0,0 +1 @@
module.exports = require('./dist/client/index.js');

View File

@ -0,0 +1,21 @@
{
"name": "@nocobase/plugin-collection-sql",
"displayName": "Collection: SQL",
"displayName.zh-CN": "数据表: SQL",
"description": "Provides SQL collection template",
"description.zh-CN": "提供 SQL 数据表模板",
"version": "1.0.0-alpha.14",
"homepage": "https://docs-cn.nocobase.com/handbook/collection-sql",
"homepage.zh-CN": "https://docs-cn.nocobase.com/handbook/collection-sql",
"main": "dist/server/index.js",
"license": "AGPL-3.0",
"dependencies": {},
"peerDependencies": {
"@nocobase/client": "0.x",
"@nocobase/server": "0.x",
"@nocobase/test": "0.x"
},
"keywords": [
"Collections"
]
}

View File

@ -0,0 +1,2 @@
export * from './dist/server';
export { default } from './dist/server';

View File

@ -0,0 +1 @@
module.exports = require('./dist/server/index.js');

View File

@ -0,0 +1,21 @@
import { Plugin } from '@nocobase/client';
export class PluginCollectionSqlClient extends Plugin {
async afterAdd() {
// await this.app.pm.add()
}
async beforeLoad() {}
// You can get and modify the app instance here
async load() {
console.log(this.app);
// this.app.addComponents({})
// this.app.addScopes({})
// this.app.addProvider()
// this.app.addProviders()
// this.app.router.add()
}
}
export default PluginCollectionSqlClient;

View File

@ -0,0 +1,2 @@
export * from './server';
export { default } from './server';

View File

@ -7,7 +7,8 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { Database, SQLModel } from '@nocobase/database';
import { Database } from '@nocobase/database';
import { SQLModel } from '../sql-collection';
import { MockServer, createMockServer } from '@nocobase/test';
describe('sql collection', () => {
@ -17,7 +18,7 @@ describe('sql collection', () => {
beforeEach(async () => {
app = await createMockServer({
plugins: ['data-source-main', 'error-handler'],
plugins: ['data-source-main', 'error-handler', 'collection-sql'],
});
db = app.db;
db.options.underscored = false;

View File

@ -7,9 +7,8 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import Database from '../../database';
import { mockDatabase } from '../../mock-database';
import { SQLModel } from '../../sql-collection/sql-model';
import { Database, mockDatabase } from '@nocobase/database';
import { SQLModel } from '../sql-collection';
describe('infer fields', () => {
let db: Database;
@ -103,4 +102,18 @@ left join roles r on ru.role_name=r.name`;
name: { type: 'string', source: 'roles.name' },
});
});
it('should infer fields for without collection', async () => {
const model = class extends SQLModel {};
model.init(null, {
modelName: 'test',
tableName: 'test',
sequelize: db.sequelize,
});
model.database = db;
model.sql = `select a from a3`;
expect(model.inferFields()).toMatchObject({
a: {},
});
});
});

View File

@ -7,7 +7,7 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { SQLModel } from '../../sql-collection/sql-model';
import { SQLModel } from '../sql-collection';
import { Sequelize } from 'sequelize';
describe('select query', () => {

View File

@ -7,15 +7,28 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { mockDatabase } from '../../mock-database';
import { SqlCollection } from '../../sql-collection';
import { Collection, mockDatabase } from '@nocobase/database';
import { SQLCollection } from '../sql-collection';
test('sql-collection', async () => {
const db = mockDatabase({ tablePrefix: '' });
await db.clean({ drop: true });
const collection = db.collectionFactory.createCollection<SqlCollection>({
db.collectionFactory.registerCollectionType(SQLCollection, {
condition: (options) => {
return options.sql;
},
async onSync() {
return;
},
async onDump(dumper, collection: Collection) {
return;
},
});
const collection = db.collectionFactory.createCollection<SQLCollection>({
name: 'test',
sql: true,
sql: 'SELECT * FROM test;',
});
expect(collection.isSql()).toBe(true);
expect(collection.collectionSchema()).toBeUndefined();

View File

@ -0,0 +1 @@
export { default } from './plugin';

View File

@ -0,0 +1,40 @@
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { Plugin } from '@nocobase/server';
import { Collection } from '@nocobase/database';
import { SQLCollection } from './sql-collection';
import sqlResourcer from './resources/sql';
export class PluginCollectionSQLServer extends Plugin {
async beforeLoad() {
this.app.db.collectionFactory.registerCollectionType(SQLCollection, {
condition: (options) => {
return options.sql;
},
async onSync() {
return;
},
async onDump(dumper, collection: Collection) {
return;
},
});
this.app.resourceManager.define(sqlResourcer);
this.app.acl.registerSnippet({
name: `pm.data-source-manager.collection-sql `,
actions: ['sqlCollection:*'],
});
}
}
export default PluginCollectionSQLServer;

View File

@ -8,13 +8,12 @@
*/
import { Context, Next } from '@nocobase/actions';
import { SQLModel, SqlCollection } from '@nocobase/database';
import { CollectionModel } from '../models';
import { SQLCollection, SQLModel } from '../sql-collection';
const updateCollection = async (ctx: Context, transaction: any) => {
const { filterByTk, values } = ctx.action.params;
const repo = ctx.db.getRepository('collections');
const collection: CollectionModel = await repo.findOne({
const collection = await repo.findOne({
filter: {
name: filterByTk,
},
@ -47,7 +46,7 @@ export default {
if (!/^select/i.test(sql) && !/^with([\s\S]+)select([\s\S]+)/i.test(sql)) {
ctx.throw(400, ctx.t('Only supports SELECT statements or WITH clauses'));
}
const tmpCollection = new SqlCollection({ name: 'tmp', sql }, { database: ctx.db });
const tmpCollection = new SQLCollection({ name: 'tmp', sql }, { database: ctx.db });
const model = tmpCollection.model as typeof SQLModel;
// The result is for preview only, add limit clause to avoid too many results
const data = await model.findAll({ attributes: ['*'], limit: 5, raw: true });
@ -65,7 +64,13 @@ export default {
ctx.logger.warn(`resource: sql-collection, action: execute, error: ${err}`);
fields = {};
}
const sources = Array.from(new Set(Object.values(fields).map((field) => field.collection)));
const sources = Array.from(
new Set(
Object.values(fields)
.map((field) => field.collection)
.filter((c) => c),
),
);
ctx.body = { data, fields, sources };
await next();
},
@ -90,7 +95,7 @@ export default {
try {
const { upRes } = await updateCollection(ctx, transaction);
const [collection] = upRes;
await (collection as CollectionModel).load({ transaction, resetFields: true });
await collection.load({ transaction, resetFields: true });
await transaction.commit();
ctx.body = upRes;
} catch (e) {

View File

@ -10,7 +10,7 @@
import { GroupOption, Order, ProjectionAlias, WhereOptions } from 'sequelize';
import { SQLModel } from './sql-model';
import { lodash } from '@nocobase/utils';
import { Collection } from '../collection';
import { Collection } from '@nocobase/database';
export function selectQuery(
tableName: string,

View File

@ -7,10 +7,11 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { Collection, CollectionContext, CollectionOptions } from '../collection';
import { Collection, CollectionContext, CollectionOptions } from '@nocobase/database';
import { SQLModel } from './sql-model';
import { QueryInterfaceDropTableOptions } from 'sequelize';
export class SqlCollection extends Collection {
export class SQLCollection extends Collection {
constructor(options: CollectionOptions, context: CollectionContext) {
options.autoGenId = false;
options.timestamps = false;
@ -51,7 +52,7 @@ export class SqlCollection extends Collection {
model.removeAttribute('id');
}
model.sql = sql;
model.sql = sql?.endsWith(';') ? sql.slice(0, -1) : sql;
model.database = this.context.database;
model.collection = this;
@ -64,4 +65,10 @@ export class SqlCollection extends Collection {
},
});
}
async removeFromDb(options?: QueryInterfaceDropTableOptions & { dropCollection?: boolean }) {
if (options?.dropCollection !== false) {
return this.remove();
}
}
}

View File

@ -7,8 +7,7 @@
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
import { Model } from '../model';
import sqlParser from '../sql-parser';
import { Model, sqlParser, SQLParserTypes } from '@nocobase/database';
import { selectQuery } from './query-generator';
export class SQLModel extends Model {
@ -37,6 +36,78 @@ export class SQLModel extends Model {
static async sync(): Promise<any> {}
private static getTableNameWithSchema(table: string) {
if (this.database.inDialect('postgres') && !table.includes('.')) {
const schema = process.env.DB_SCHEMA || 'public';
return `${schema}.${table}`;
}
return table;
}
private static parseSelectAST(ast: SQLParserTypes.Select) {
const tablesMap: { [table: string]: { name: string; as?: string }[] } = {}; // table => columns
const tableAliases = {};
ast.from.forEach((fromItem: SQLParserTypes.From) => {
tablesMap[fromItem.table] = [];
if (fromItem.as) {
tableAliases[fromItem.as] = fromItem.table;
}
});
ast.columns.forEach((column: SQLParserTypes.Column) => {
const expr = column.expr as SQLParserTypes.ColumnRef;
if (expr.type !== 'column_ref') {
return;
}
const table = expr.table;
const name = tableAliases[table] || table;
const columnAttr = { name: expr.column as string, as: column.as };
if (!name) {
Object.keys(tablesMap).forEach((n) => {
tablesMap[n].push(columnAttr);
});
} else if (tablesMap[name]) {
tablesMap[name].push(columnAttr);
}
});
return tablesMap;
}
private static parseTablesAndColumns(): {
table: string;
columns: { name: string; as?: string }[];
}[] {
let { ast: _ast } = sqlParser.parse(this.sql);
if (Array.isArray(_ast)) {
_ast = _ast[0];
}
const ast = _ast as SQLParserTypes.Select;
ast.from = ast.from || [];
ast.columns = ast.columns || [];
if (ast.with) {
// The type definition of the AST is not accurate in node-sql-parser 4.18.0
// So we need to use any here temporarily
const withAST = ast.with as any;
withAST.forEach((withItem: any) => {
const as = withItem.name.value;
const withAst = withItem.stmt.ast;
ast.from.push(...withAst.from.map((f: any) => ({ ...f, as })));
ast.columns.push(
...withAst.columns.map((c: any) => ({
...c,
expr: {
...c.expr,
table: as,
},
})),
);
});
}
const tablesMap = this.parseSelectAST(ast);
return Object.entries(tablesMap)
.filter(([_, columns]) => columns)
.map(([table, columns]) => ({ table, columns }));
}
static inferFields(): {
[field: string]: {
type: string;
@ -50,11 +121,19 @@ export class SQLModel extends Model {
const tableName = this.getTableNameWithSchema(table);
const collection = this.database.tableNameCollectionMap.get(tableName);
if (!collection) {
return fields;
const originFields = {};
columns.forEach((column) => {
if (column.name === '*') {
return;
}
originFields[column.as || column.name] = {};
});
return { ...fields, ...originFields };
}
const all = columns.some((column) => column.name === '*');
const attributes = collection.model.getAttributes();
const sourceFields = {};
if (columns === '*') {
if (all) {
Object.values(attributes).forEach((attribute) => {
const field = collection.getField((attribute as any).fieldName);
if (!field?.options.interface) {
@ -69,105 +148,25 @@ export class SQLModel extends Model {
};
});
} else {
(columns as { name: string; as: string }[]).forEach((column) => {
columns.forEach((column) => {
let options = {};
const modelField = Object.values(attributes).find((attribute) => attribute.field === column.name);
if (!modelField) {
return;
if (modelField) {
const field = collection.getField((modelField as any).fieldName);
if (field?.options.interface) {
options = {
collection: field.collection.name,
type: field.type,
source: `${field.collection.name}.${field.name}`,
interface: field.options.interface,
uiSchema: field.options.uiSchema,
};
}
}
const field = collection.getField((modelField as any).fieldName);
if (!field?.options.interface) {
return;
}
sourceFields[column.as || column.name] = {
collection: field.collection.name,
type: field.type,
source: `${field.collection.name}.${field.name}`,
interface: field.options.interface,
uiSchema: field.options.uiSchema,
};
sourceFields[column.as || column.name] = options;
});
}
return { ...fields, ...sourceFields };
}, {});
}
private static parseTablesAndColumns(): {
table: string;
columns: string | { name: string; as: string }[];
}[] {
let { ast } = sqlParser.parse(this.sql);
if (Array.isArray(ast)) {
ast = ast[0];
}
ast.from = ast.from || [];
ast.columns = ast.columns || [];
if (ast.with) {
ast.with.forEach((withItem: any) => {
const as = withItem.name;
const withAst = withItem.stmt.ast;
ast.from.push(...withAst.from.map((f: any) => ({ ...f, as })));
ast.columns.push(
...withAst.columns.map((c: any) => ({
...c,
expr: {
...c.expr,
table: as,
},
})),
);
});
}
if (ast.columns === '*') {
const tables = new Set<string>();
ast.from.forEach((fromItem: { table: string; as: string }) => {
tables.add(fromItem.table);
});
return Array.from(tables).map((table) => ({ table, columns: '*' }));
}
const tableAliases = {};
ast.from.forEach((fromItem: { table: string; as: string }) => {
if (!fromItem.as) {
return;
}
tableAliases[fromItem.as] = fromItem.table;
});
const columns: string[] = ast.columns.reduce(
(
tableMp: { [table: string]: { name: string; as: string }[] },
column: {
as: string;
expr: {
table: string;
column: string;
type: string;
};
},
) => {
if (column.expr.type !== 'column_ref') {
return tableMp;
}
const table = column.expr.table;
const name = tableAliases[table] || table;
const columnAttr = { name: column.expr.column, as: column.as };
if (!tableMp[name]) {
tableMp[name] = [columnAttr];
} else {
tableMp[name].push(columnAttr);
}
return tableMp;
},
{},
);
return Object.entries(columns)
.filter(([_, columns]) => columns)
.map(([table, columns]) => ({ table, columns }));
}
private static getTableNameWithSchema(table: string) {
if (this.database.inDialect('postgres') && !table.includes('.')) {
const schema = process.env.DB_SCHEMA || 'public';
return `${schema}.${table}`;
}
return table;
}
}

View File

@ -26,7 +26,6 @@ import { beforeCreateForValidateField, beforeUpdateForValidateField } from './ho
import { beforeCreateForViewCollection } from './hooks/beforeCreateForViewCollection';
import { CollectionModel, FieldModel } from './models';
import collectionActions from './resourcers/collections';
import sqlResourcer from './resourcers/sql';
import viewResourcer from './resourcers/views';
export class PluginDataSourceMainServer extends Plugin {
@ -293,11 +292,6 @@ export class PluginDataSourceMainServer extends Plugin {
name: `pm.data-source-manager.collection-view `,
actions: ['dbViews:*'],
});
this.app.acl.registerSnippet({
name: `pm.data-source-manager.collection-sql `,
actions: ['sqlCollection:*'],
});
}
async load() {
@ -324,7 +318,6 @@ export class PluginDataSourceMainServer extends Plugin {
});
this.app.resource(viewResourcer);
this.app.resource(sqlResourcer);
this.app.actions(collectionActions);
const handleFieldSource = (fields) => {

View File

@ -23,6 +23,7 @@
"@nocobase/plugin-calendar": "1.0.0-alpha.14",
"@nocobase/plugin-charts": "1.0.0-alpha.14",
"@nocobase/plugin-client": "1.0.0-alpha.14",
"@nocobase/plugin-collection-sql": "1.0.0-alpha.14",
"@nocobase/plugin-data-source-main": "1.0.0-alpha.14",
"@nocobase/plugin-data-source-manager": "1.0.0-alpha.14",
"@nocobase/plugin-data-visualization": "1.0.0-alpha.14",

View File

@ -51,6 +51,7 @@ export class PresetNocoBase extends Plugin {
'kanban',
'action-duplicate',
'action-print',
'collection-sql',
];
localPlugins = [

View File

@ -8947,7 +8947,7 @@ bessel@^1.0.2:
resolved "https://registry.npmmirror.com/bessel/-/bessel-1.0.2.tgz#828812291e0b62e94959cdea43fac186e8a7202d"
integrity sha512-Al3nHGQGqDYqqinXhQzmwmcRToe/3WyBv4N8aZc5Pef8xw2neZlR9VPi84Sa23JtgWcucu18HxVZrnI0fn2etw==
big-integer@^1.6.44:
big-integer@^1.6.44, big-integer@^1.6.48:
version "1.6.52"
resolved "https://registry.npmmirror.com/big-integer/-/big-integer-1.6.52.tgz#60a887f3047614a8e1bffe5d7173490a97dc8c85"
integrity sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==
@ -19039,6 +19039,13 @@ node-releases@^2.0.14:
resolved "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b"
integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==
node-sql-parser@^4.18.0:
version "4.18.0"
resolved "https://registry.npmmirror.com/node-sql-parser/-/node-sql-parser-4.18.0.tgz#516b6e633c55c5abbba1ca588ab372db81ae9318"
integrity sha512-2YEOR5qlI1zUFbGMLKNfsrR5JUvFg9LxIRVE+xJe962pfVLH0rnItqLzv96XVs1Y1UIR8FxsXAuvX/lYAWZ2BQ==
dependencies:
big-integer "^1.6.48"
node-xlsx@^0.16.1:
version "0.16.2"
resolved "https://registry.npmmirror.com/node-xlsx/-/node-xlsx-0.16.2.tgz#40f580187eae0e032cac96e958e97cb6ceca09f6"