feat: improve pm.add

jest.config.js

@@ -25,7 +25,7 @@ module.exports = {
       },
     ],
   },
-  modulePathIgnorePatterns: ['/esm/', '/es/', '/dist/', '/lib/', '/client/', '/sdk/', '\\.test\\.tsx$'],
+  modulePathIgnorePatterns: ['/storage/', '/esm/', '/es/', '/dist/', '/lib/', '/client/', '/sdk/', '\\.test\\.tsx$'],
   coveragePathIgnorePatterns: [
     '/node_modules/',
     '/__tests__/',

packages/core/cli/src/commands/postinstall.js

@@ -1,6 +1,6 @@
 const { Command } = require('commander');
-const { run, isDev, isPackageValid } = require('../util');
+const { run, isDev, isPackageValid, createStoragePluginsSymlink } = require('../util');
-const { resolve } = require('path');
+const { resolve, dirname } = require('path');
 const { existsSync } = require('fs');
 const { readFile, writeFile } = require('fs').promises;
 
@@ -13,10 +13,11 @@ module.exports = (cli) => {
     .command('postinstall')
     .allowUnknownOption()
     .action(async () => {
+      const cwd = process.cwd();
+      await createStoragePluginsSymlink();
       if (!isDev()) {
         return;
       }
-      const cwd = process.cwd();
       if (!existsSync(resolve(cwd, '.env')) && existsSync(resolve(cwd, '.env.example'))) {
         const content = await readFile(resolve(cwd, '.env.example'), 'utf-8');
         await writeFile(resolve(cwd, '.env'), content, 'utf-8');

packages/core/cli/src/util.js

@@ -2,8 +2,8 @@ const net = require('net');
 const chalk = require('chalk');
 const execa = require('execa');
 const { dirname, resolve } = require('path');
-const { readFile, writeFile } = require('fs').promises;
 const { existsSync, mkdirSync, cpSync } = require('fs');
+const { readFile, writeFile, readdir, symlink, unlink, mkdir, stat } = require('fs').promises;
 
 exports.isPackageValid = (package) => {
   try {
@@ -174,3 +174,56 @@ exports.generateAppDir = function generateAppDir() {
     process.env.APP_PACKAGE_ROOT = appPkgPath;
   }
 };
+
+async function getStoragePluginNames(target) {
+  const plugins = [];
+  const items = await readdir(target);
+  for (const item of items) {
+    if (item.startsWith('@')) {
+      const children = await getStoragePluginNames(resolve(target, item));
+      plugins.push(
+        ...children.map((child) => {
+          return `${item}/${child}`;
+        }),
+      );
+    } else if (await fsExists(resolve(target, item, 'package.json'))) {
+      plugins.push(item);
+    }
+  }
+  return plugins;
+}
+
+async function fsExists(path) {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+async function createStoragePluginSymLink(pluginName) {
+  const storagePluginsPath = resolve(process.cwd(), 'storage/plugins');
+  // const nodeModulesPath = resolve(dirname(require.resolve('@nocobase/server/package.json')), 'node_modules');
+  const nodeModulesPath = resolve(process.cwd(), 'node_modules');
+  if (pluginName.startsWith('@')) {
+    const [orgName] = pluginName.split('/');
+    if (!(await fsExists(resolve(nodeModulesPath, orgName)))) {
+      await mkdir(resolve(nodeModulesPath, orgName), { recursive: true });
+    }
+  }
+  const link = resolve(nodeModulesPath, pluginName);
+  if (await fsExists(link)) {
+    await unlink(link);
+  }
+  await symlink(resolve(storagePluginsPath, pluginName), link);
+}
+
+async function createStoragePluginsSymlink() {
+  const pluginNames = await getStoragePluginNames(resolve(process.cwd(), 'storage/plugins'));
+  for (const pluginName of pluginNames) {
+    await createStoragePluginSymLink(pluginName);
+  }
+}
+
+exports.createStoragePluginsSymlink = createStoragePluginsSymlink;

packages/core/database/src/repository.ts

@@ -3,16 +3,16 @@ import lodash from 'lodash';
 import {
   Association,
   BulkCreateOptions,
-  CountOptions as SequelizeCountOptions,
-  CreateOptions as SequelizeCreateOptions,
-  DestroyOptions as SequelizeDestroyOptions,
-  FindAndCountOptions as SequelizeAndCountOptions,
-  FindOptions as SequelizeFindOptions,
   ModelStatic,
   Op,
   Sequelize,
-  Transactionable,
+  FindAndCountOptions as SequelizeAndCountOptions,
+  CountOptions as SequelizeCountOptions,
+  CreateOptions as SequelizeCreateOptions,
+  DestroyOptions as SequelizeDestroyOptions,
+  FindOptions as SequelizeFindOptions,
   UpdateOptions as SequelizeUpdateOptions,
+  Transactionable,
   WhereOperators,
 } from 'sequelize';
 import { Collection } from './collection';
@@ -37,6 +37,7 @@ import { UpdateGuard } from './update-guard';
 
 const debug = require('debug')('noco-database');
 
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface IRepository {}
 
 interface CreateManyOptions extends BulkCreateOptions {
@@ -216,7 +217,12 @@ export interface AggregateOptions {
   distinct?: boolean;
 }
 
-interface FirstOrCreateOptions extends Transactionable {
+export interface FirstOrCreateOptions extends Transactionable {
+  filterKeys: string[];
+  values?: Values;
+}
+
+export interface UpdateOrCreateOptions extends Transactionable {
   filterKeys: string[];
   values?: Values;
 }
@@ -242,7 +248,7 @@ export class Repository<TModelAttributes extends {} = any, TCreationAttributes e
       const chunks = key.split('.');
       return chunks
         .filter((chunk) => {
-          return !Boolean(chunk.match(/\d+/));
+          return !chunk.match(/\d+/);
         })
         .join('.');
     };
@@ -503,7 +509,7 @@ export class Repository<TModelAttributes extends {} = any, TCreationAttributes e
     return this.create({ values, transaction });
   }
 
-  async updateOrCreate(options: FirstOrCreateOptions) {
+  async updateOrCreate(options: UpdateOrCreateOptions) {
     const { filterKeys, values, transaction } = options;
     const filter = Repository.valuesToFilter(values, filterKeys);
 

packages/core/server/package.json

@@ -27,6 +27,7 @@
     "commander": "^9.2.0",
     "cronstrue": "^2.11.0",
     "dayjs": "^1.11.8",
+    "decompress": "^4.2.1",
     "find-package-json": "^1.2.0",
     "i18next": "^22.4.9",
     "koa": "^2.13.4",

packages/core/server/src/plugin-manager/plugin-manager-repository.ts

@@ -1,9 +1,10 @@
-import { Repository } from '@nocobase/database';
+import { Repository, UpdateOrCreateOptions } from '@nocobase/database';
-import lodash from 'lodash';
+import lodash, { isBoolean } from 'lodash';
 import { PluginManager } from './plugin-manager';
 
 export class PluginManagerRepository extends Repository {
   pm: PluginManager;
+  _authenticated: boolean;
 
   setPluginManager(pm: PluginManager) {
     this.pm = pm;
@@ -68,6 +69,9 @@ export class PluginManagerRepository extends Repository {
   }
 
   async getItems() {
+    if (!(await this.authenticate())) {
+      return [];
+    }
     try {
       // sort plugins by id
       return await this.find({
@@ -86,9 +90,22 @@ export class PluginManagerRepository extends Repository {
     }
   }
 
+  async authenticate() {
+    if (!isBoolean(this._authenticated)) {
+      this._authenticated = await this.collection.existsInDb();
+    }
+    return this._authenticated;
+  }
+
+  async updateOrCreate(options: UpdateOrCreateOptions) {
+    if (!(await this.authenticate())) {
+      return;
+    }
+    return super.updateOrCreate(options);
+  }
+
   async init() {
-    const exists = await this.collection.existsInDb();
+    if (!(await this.authenticate())) {
-    if (!exists) {
       return;
     }
 

packages/core/server/src/plugin-manager/plugin-manager.ts

@@ -6,10 +6,11 @@ import net from 'net';
 import { resolve } from 'path';
 import Application from '../application';
 import { createAppProxy } from '../helper';
-import { Plugin } from '../plugin';
+import { Plugin, PluginOptions } from '../plugin';
 import collectionOptions from './options/collection';
 import resourceOptions from './options/resource';
 import { PluginManagerRepository } from './plugin-manager-repository';
+import { PluginResolver } from './plugin-resolver';
 
 export interface PluginManagerOptions {
   app: Application;
@@ -22,17 +23,39 @@ export interface InstallOptions {
   sync?: SyncOptions;
 }
 
+export type PluginResolverResult = {
+  name?: string;
+  packageName?: string;
+  version?: string;
+  Plugin: any;
+};
+
+export type PluginResolverHandleOptions = {
+  isPackageName?: boolean;
+  pluginName?: string;
+  packageName?: string;
+  authToken?: string;
+  registry?: string;
+  upgrading?: boolean;
+};
+
+export type AddingPluginType = string | typeof Plugin;
+
+export type PMAddOptions = PluginOptions & {};
+
 export class AddPresetError extends Error {}
 
 export class PluginManager {
   app: Application;
-  collection: Collection;
-  _repository: PluginManagerRepository;
-  pluginInstances = new Map<typeof Plugin, Plugin>();
-  pluginAliases = new Map<string, Plugin>();
   server: net.Server;
+  collection: Collection;
+  protected pluginInstances = new Map<typeof Plugin, Plugin>();
+  protected pluginAliases = new Map<string, Plugin>();
+  protected PluginResolver: typeof PluginResolver;
+  protected _repository: PluginManagerRepository;
 
   constructor(public options: PluginManagerOptions) {
+    this.PluginResolver = PluginResolver;
     this.app = options.app;
     this.app.db.registerRepositories({
       PluginManagerRepository,
@@ -75,6 +98,8 @@ export class PluginManager {
   }
 
   static getPackageJson(packageName: string) {
+    const file = require.resolve(`${packageName}/package.json`);
+    delete require.cache[file];
     return require(`${packageName}/package.json`);
   }
 
@@ -121,6 +146,16 @@ export class PluginManager {
     throw new Error(`No available packages found, ${name} plugin does not exist`);
   }
 
+  setPluginResolver(Resolver: typeof PluginResolver) {
+    this.PluginResolver = Resolver;
+  }
+
+  async resolvePlugin(pluginName: AddingPluginType, options) {
+    const Resolver = this.PluginResolver;
+    const resolver = new Resolver(this.app);
+    return resolver.handle(pluginName, options);
+  }
+
   static resolvePlugin(pluginName: string | typeof Plugin) {
     if (typeof pluginName === 'string') {
       const packageName = this.getPackageName(pluginName);
@@ -189,7 +224,7 @@ export class PluginManager {
     await run('yarn', ['install']);
   }
 
-  async add(plugin?: any, options: any = {}) {
+  async add(plugin?: AddingPluginType, options: PMAddOptions = {}) {
     if (this.has(plugin)) {
       const name = typeof plugin === 'string' ? plugin : plugin.name;
       this.app.log.warn(`plugin [${name}] added`);
@@ -199,19 +234,24 @@ export class PluginManager {
       options.name = plugin;
     }
     this.app.log.debug(`adding plugin [${options.name}]...`);
-    let P: any;
     try {
-      P = PluginManager.resolvePlugin(plugin);
+      const { Plugin: P, ...opts } = await this.resolvePlugin(plugin, options);
-    } catch (error) {
+      const instance: Plugin = new P(createAppProxy(this.app), opts);
-      this.app.log.warn('plugin not found', error);
-      return;
-    }
-    const instance: Plugin = new P(createAppProxy(this.app), options);
       this.pluginInstances.set(P, instance);
-    if (options.name) {
+      if (!options.isPreset && opts.name) {
+        await this.repository.updateOrCreate({
+          values: {
+            ...options,
+            ...opts,
+          },
+          filterKeys: ['name'],
+        });
         this.pluginAliases.set(options.name, instance);
       }
       await instance.afterAdd();
+    } catch (error) {
+      this.app.log.warn('plugin not found', error);
+    }
   }
 
   async initPlugins() {

packages/core/server/src/plugin-manager/plugin-resolver.ts

@@ -0,0 +1,135 @@
+import { CleanOptions, SyncOptions } from '@nocobase/database';
+import { requireModule } from '@nocobase/utils';
+import Application from '../application';
+import { Plugin } from '../plugin';
+import { PluginManager } from './plugin-manager';
+
+export interface PluginManagerOptions {
+  app: Application;
+  plugins?: any[];
+}
+
+export interface InstallOptions {
+  cliArgs?: any[];
+  clean?: CleanOptions | boolean;
+  sync?: SyncOptions;
+}
+
+export type PluginResolverResult = {
+  name?: string;
+  packageName?: string;
+  version?: string;
+  Plugin: any;
+};
+
+export type PluginResolverHandleOptions = {
+  isPackageName?: boolean;
+  pluginName?: string;
+  packageName?: string;
+  authToken?: string;
+  registry?: string;
+  upgrading?: boolean;
+};
+
+export type PluginType = string | typeof Plugin;
+
+export class PluginResolver {
+  constructor(protected app: Application) {}
+
+  async handlePluginClass(pluginName: PluginType, options: PluginResolverHandleOptions): Promise<PluginResolverResult> {
+    return {
+      Plugin: pluginName,
+    };
+  }
+
+  async handleLocalPackage(pluginName: string, options: PluginResolverHandleOptions): Promise<PluginResolverResult> {
+    const { packageName } = options;
+    const packageJson = PluginManager.getPackageJson(packageName);
+    return {
+      name: pluginName,
+      packageName,
+      version: packageJson.version,
+      Plugin: requireModule(packageName),
+    };
+  }
+
+  async handleCompressedPackage(pkgPath: string, options: PluginResolverHandleOptions): Promise<PluginResolverResult> {
+    return {
+      Plugin: '',
+    };
+  }
+
+  async handle(pluginName: PluginType, options: PluginResolverHandleOptions): Promise<PluginResolverResult> {
+    if (typeof pluginName !== 'string') {
+      return this.handlePluginClass(pluginName, options);
+    }
+
+    if (!options.isPackageName) {
+      options.packageName = PluginManager.getPackageName(pluginName);
+      return this.handleLocalPackage(pluginName, options);
+    }
+
+    let pkgPath = pluginName;
+
+    if (options.isPackageName) {
+      if (options.upgrading && !this.isStoragePackage(pluginName)) {
+        options.packageName = pluginName;
+        return this.handleLocalPackage(this.getPluginNameViaPackageName(pluginName), options);
+      }
+      if (this.packageExists(pluginName)) {
+        options.packageName = pluginName;
+        return this.handleLocalPackage(this.getPluginNameViaPackageName(pluginName), options);
+      }
+      pkgPath = await this.downloadNpmPackage(pluginName, options);
+    } else if (this.isURL(pluginName)) {
+      pkgPath = await this.download(pluginName, options);
+    }
+
+    if (!this.isCompressedPackage(pkgPath)) {
+      throw new Error('invalid');
+    }
+
+    return this.handleCompressedPackage(pkgPath, options);
+  }
+
+  getPluginNameViaPackageName(packageName: string) {
+    return packageName;
+  }
+
+  protected async download(url: string, options) {
+    return '';
+  }
+
+  protected async downloadNpmPackage(packageName: string, options) {
+    return '';
+  }
+
+  isStoragePackage(packageName: string) {
+    return true;
+  }
+
+  packageExists(packageName: string) {
+    try {
+      require.resolve(packageName);
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  protected isCompressedPackage(file) {
+    return true;
+  }
+
+  protected isURL(string) {
+    let url;
+
+    try {
+      url = new URL(string);
+    } catch (e) {
+      return false;
+    }
+
+    return url.protocol === 'http:' || url.protocol === 'https:';
+  }
+}

packages/core/server/src/plugin.ts

@@ -11,25 +11,22 @@ export interface PluginInterface {
 }
 
 export interface PluginOptions {
-  activate?: boolean;
+  name?: string;
-  displayName?: string;
+  packageName?: string;
-  description?: string;
   version?: string;
   enabled?: boolean;
-  install?: (this: Plugin) => void;
+  installed?: boolean;
-  load?: (this: Plugin) => void;
+  isPreset?: boolean;
-  plugin?: typeof Plugin;
-
   [key: string]: any;
 }
 
-export abstract class Plugin<O = any> implements PluginInterface {
+export abstract class Plugin implements PluginInterface {
   options: any;
   app: Application;
   model: Model;
   state: any = {};
 
-  constructor(app: Application, options?: any) {
+  constructor(app: Application, options?: PluginOptions) {
     this.app = app;
     this.setOptions(options);
   }

storage/.gitignore

@@ -1 +1,2 @@
 .pm2
+plugins