ChengLei Shao fa97d0a642
feat: application backup and restore (#3268)
* fix: perform load action on boot main app

* feat: add dataType option in collection duplicator

* chore: reset optional dumpable config

* chore: dump command

* chore: dump & restore command

* chore: delay restore

* fix: dump test

* chore: restore command

* chore: dump command action

* chore: dumpable collection api

* chore: client collection option

* feat: backup& restore client

* chore: content disposition header in dump response

* chore: download backup field

* feat: collection origin option

* fix: test

* chore: collection manager collection origin

* chore: upload  backup field

* chore: upload restore file

* chore: upload restore file

* fix: test

* chore: backup and restore support learn more

* refactor: upload restore file

* refactor: upload restore file

* fix: test

* fix: test

* chore: dumpable collection with title

* chore: pg only test

* chore: test

* fix: test

* chore: test sleep

* style: locale improve

* refactor: download backup file

* refactor: start restore

* fix: restore key name

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* refactor: start restore

* chore: unify duplicator option

* fix: dump empty collection

* chore: test

* chore: test

* style: style improve

* refactor: locale improve

* chore: dumpalbe collection orders

* style: style improve

* style: style improve

* style: icon adjust

* chore: nginx body size

* chore: get file status

* feat: run dump task

* feat: download api

* chore: backup files resourcer

* feat: restore destroy api

* chore: backup files resoucer

* feat: list backup files action

* chore: get collection meta from dumped file

* fix: dump file name

* fix: test

* chore: backup and restore ui

* chore: swagger api for backup & restore

* chore: api doc

* chore: api doc

* chore: api doc

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* chore: backup and restore ui

* fix: restore values

* style: style improve

* fix: download field respontype

* fix: restore form local file

* refactor: local improve

* refactor: delete backup file

* fix: in progress status

* refactor: locale improve

* refactor: locale improve

* refactor: style improve

* refactor: style improve

* refactor: style improve

* test: dump collection table attribute

* chore: dump collection with table attributes

* chore: test

* chore: create new table in restore

* fix: import error

* chore: restore table from backup file

* chore: sync collection after restore collections

* fix: restore json data

* style: style improve

* chore: restore with fields

* chore: test

* fix: test

* fix: test with underscored

* style: style improve

* fix: lock file state

* chore: add test file

* refactor: backup & restore plugin

* fix: mysql test

* chore: skip import view collection

* chore: restore collection with inherits topo order

* fix: import

* style: style improve

* fix: restore sequence fields

* fix: themeConfig collection duplicator option

* fix: restore with dialectOnly meta

* fix: throw error

* fix: restore

* fix: import backup file created in postgres into mysql

* fix: repeated items in inherits

* chore: upgrade after restore

* feat: check database env before restore

* feat: handle autoincr val in postgres

* chore: sqlite & mysql queryInterface

* chore: test

* fix: test

* chore: test

* fix: build

* fix: pg test

* fix: restore with date field

* chore: theme-config collection

* chore: chage import collections method to support collection origin

* chore: fallback get autoincr value in mysql

* fix: dataType normalize

* chore: delay restore

* chore: test

* fix: build

* feat: collectin onDump

* feat: collection onDump interface

* chore: dump with view collection

* chore: sync in restore

* refactor: locale improve

* refactor: code improve

* fix: test

* fix: data sync

* chore: rename backup & restore plugin

* chore: skip test

* style: style improve

* style: style improve

* style: style improve

* style: style improve

* chore: import version check

* chore: backup file dir

* chore: build

* fix: bugs

* fix: error

* fix: pageSize

* fix: import origin

* fix: improve code

* fix: remove namespace

* chore: dump rules config

* fix: dump custom collection

* chore: version

* fix: test

* fix: test

* fix: test

* fix: test

* chore: test

* fix: load custom collection

* fix: client

* fix: translation

* chore: code

* fix: bug

* fix:  support shared option

* fix: roles collection dumpRules

* chore: test

* fix: define collections

* chore: collection group

* fix: translation

* fix: translation

* fix: restore options

* chore: restore command

* chore: dump error

* fix: too many open files

---------

Co-authored-by: katherinehhh <katherine_15995@163.com>
Co-authored-by: chenos <chenlinxh@gmail.com>
2024-01-08 18:59:56 +08:00

213 lines
5.2 KiB
TypeScript

import { Dumper } from '../dumper';
import { DumpRulesGroupType } from '@nocobase/database';
import fs from 'fs';
import { koaMulter as multer } from '@nocobase/utils';
import os from 'os';
import path from 'path';
import fsPromises from 'fs/promises';
import { Restorer } from '../restorer';
import { DEFAULT_PAGE, DEFAULT_PER_PAGE } from '@nocobase/actions';
export default {
name: 'backupFiles',
middleware: async (ctx, next) => {
if (ctx.action.actionName !== 'upload') {
return next();
}
const storage = multer.diskStorage({
destination: os.tmpdir(),
filename: function (req, file, cb) {
const randomName = Date.now().toString() + Math.random().toString().slice(2); // 随机生成文件名
cb(null, randomName);
},
});
const upload = multer({ storage }).single('file');
return upload(ctx, next);
},
actions: {
async list(ctx, next) {
const { page = DEFAULT_PAGE, pageSize = DEFAULT_PER_PAGE } = ctx.action.params;
const dumper = new Dumper(ctx.app);
const backupFiles = await dumper.allBackUpFilePaths({
includeInProgress: true,
});
// handle pagination
const count = backupFiles.length;
const rows = await Promise.all(
backupFiles.slice((page - 1) * pageSize, page * pageSize).map(async (file) => {
// if file is lock file, remove lock extension
return await Dumper.getFileStatus(file.endsWith('.lock') ? file.replace('.lock', '') : file);
}),
);
ctx.body = {
count,
rows,
page: Number(page),
pageSize: Number(pageSize),
totalPage: Math.ceil(count / pageSize),
};
await next();
},
async get(ctx, next) {
const { filterByTk } = ctx.action.params;
const dumper = new Dumper(ctx.app);
const filePath = dumper.backUpFilePath(filterByTk);
async function sendError(message, status = 404) {
ctx.body = { status: 'error', message };
ctx.status = status;
}
try {
const fileState = await Dumper.getFileStatus(filePath);
if (fileState.status !== 'ok') {
await sendError(`Backup file ${filterByTk} not found`);
} else {
const restorer = new Restorer(ctx.app, {
backUpFilePath: filePath,
});
const restoreMeta = await restorer.parseBackupFile();
ctx.body = {
...fileState,
meta: restoreMeta,
};
}
} catch (e) {
if (e.code === 'ENOENT') {
await sendError(`Backup file ${filterByTk} not found`);
}
}
await next();
},
/**
* create dump task
* @param ctx
* @param next
*/
async create(ctx, next) {
const data = <
{
dataTypes: string[];
}
>ctx.request.body;
const dumper = new Dumper(ctx.app);
const taskId = await dumper.runDumpTask({
groups: new Set(data.dataTypes) as Set<DumpRulesGroupType>,
});
ctx.body = {
key: taskId,
};
await next();
},
/**
* download backup file
* @param ctx
* @param next
*/
async download(ctx, next) {
const { filterByTk } = ctx.action.params;
const dumper = new Dumper(ctx.app);
const filePath = dumper.backUpFilePath(filterByTk);
const fileState = await Dumper.getFileStatus(filePath);
if (fileState.status !== 'ok') {
throw new Error(`Backup file ${filterByTk} not found`);
}
ctx.attachment(filePath);
ctx.body = fs.createReadStream(filePath);
await next();
},
async restore(ctx, next) {
const { dataTypes, filterByTk, key } = ctx.action.params.values;
const filePath = (() => {
if (key) {
const tmpDir = os.tmpdir();
return path.resolve(tmpDir, key);
}
if (filterByTk) {
const dumper = new Dumper(ctx.app);
return dumper.backUpFilePath(filterByTk);
}
})();
if (!filePath) {
throw new Error(`Backup file ${filterByTk} not found`);
}
const args = ['restore', '-f', filePath];
for (const dataType of dataTypes) {
args.push('-g', dataType);
}
await ctx.app.runCommand(...args);
await next();
},
async destroy(ctx, next) {
const { filterByTk } = ctx.action.params;
const dumper = new Dumper(ctx.app);
const filePath = dumper.backUpFilePath(filterByTk);
await fsPromises.unlink(filePath);
// remove file
ctx.body = {
status: 'ok',
};
await next();
},
async upload(ctx, next) {
const file = ctx.file;
const fileName = file.filename;
const restorer = new Restorer(ctx.app, {
backUpFilePath: file.path,
});
const restoreMeta = await restorer.parseBackupFile();
ctx.body = {
key: fileName,
meta: restoreMeta,
};
await next();
},
async dumpableCollections(ctx, next) {
ctx.withoutDataWrapping = true;
const dumper = new Dumper(ctx.app);
ctx.body = await dumper.dumpableCollectionsGroupByGroup();
await next();
},
},
};