diff --git a/packages/app-server/src/config.ts b/packages/app-server/src/config.ts index 455a64c331..93553075b5 100644 --- a/packages/app-server/src/config.ts +++ b/packages/app-server/src/config.ts @@ -1,13 +1,17 @@ import * as path from 'path' import * as dotenv from 'dotenv' + +/** + * parse environment vars from the `.env` file if existing + */ dotenv.config() /** - * 应用运行配置管理 + * configuration management */ export default class Config { /** - * 获取数据库连接配置 + * mongodb connection configuration */ static get db() { if (!process.env['DB']) { @@ -21,12 +25,12 @@ export default class Config { return { database: process.env['DB'], uri: process.env['DB_URI'], - poolSize: (process.env['DB_POOL_LIMIT'] ?? 100) as number, + maxPoolSize: (process.env['DB_POOL_LIMIT'] ?? 10) as number, } } /** - * 指定服务端密钥,用于生成 token + * the server secret salt, mainly used for generating tokens */ static get SERVER_SECRET_SALT(): string { const secret_salt = process.env['SERVER_SECRET_SALT'] ?? process.env['SERVER_SALT'] @@ -36,40 +40,54 @@ export default class Config { return secret_salt } - // 本地上传文件存储目录 + /** + * the root path of local file system driver, only used while `FILE_SYSTEM_DRIVER` equals to 'local' + */ static get LOCAL_STORAGE_ROOT_PATH(): string { return process.env['LOCAL_STORAGE_ROOT_PATH'] ?? path.join(process.cwd(), "data") } - // 临时文件目录 + /** + * the file system driver: 'local', 'gridfs' + */ + static get FILE_SYSTEM_DRIVER(): string { + return process.env['FILE_SYSTEM_DRIVER'] ?? 'gridfs' + } + + /** + * the `temp path` + */ static get TMP_PATH(): string { const tmp_path = process.env['TMP_PATH'] ?? path.join(process.cwd(), "tmp") return tmp_path } /** - * 指定日志级别 + * the logger level : 'fatal', 'error', 'warning', 'debug', 'info', 'trace' */ static get LOG_LEVEL(): string { return process.env['LOG_LEVEL'] ?? (this.isProd ? 'info' : 'debug') } /** - * 指定服务监听端口号,缺省为 8000 + * the serving port, default is 8000 */ static get PORT(): number { return (process.env.PORT ?? 8000) as number } /** - * 指定开启云函数日志: "always" | "debug" | "none" + * enable cloud function logging, default is `always` + * - `always` means that all cloud functions' execution will be logged + * - `debug` means that only logging for debug invokes + * - `never` no logging any case */ static get ENABLE_CLOUD_FUNCTION_LOG(): string { - return (process.env.ENABLE_CLOUD_FUNCTION_LOG ?? 'debug') + return (process.env.ENABLE_CLOUD_FUNCTION_LOG ?? 'always') } /** - * 是否生产环境 + * in production deploy or not */ static get isProd(): boolean { return process.env.NODE_ENV === 'production' diff --git a/packages/app-server/src/lib/globals/globals.ts b/packages/app-server/src/lib/globals/globals.ts index f4700b6bdb..2aa0f03ec0 100644 --- a/packages/app-server/src/lib/globals/globals.ts +++ b/packages/app-server/src/lib/globals/globals.ts @@ -90,7 +90,7 @@ export class Globals { * @returns */ private static _createAccessor() { - const accessor = new MongoAccessor(Config.db.database, Config.db.uri, { directConnection: true, maxPoolSize: Config.db.poolSize }) + const accessor = new MongoAccessor(Config.db.database, Config.db.uri, { directConnection: true, maxPoolSize: Config.db.maxPoolSize }) accessor.setLogger(createLogger('db', 'warning')) accessor.init() diff --git a/packages/app-server/src/lib/storage/local_file_storage.ts b/packages/app-server/src/lib/storage/local_file_storage.ts index 98b1779466..4c250f54d3 100644 --- a/packages/app-server/src/lib/storage/local_file_storage.ts +++ b/packages/app-server/src/lib/storage/local_file_storage.ts @@ -8,9 +8,9 @@ export class LocalFileStorage implements FileStorageInterface { rootPath: string namespace: string - constructor(rootPath: string, namespace: string = 'default') { + constructor(rootPath: string, bucket: string = 'public') { this.rootPath = rootPath - this.namespace = namespace + this.namespace = bucket } /** @@ -104,7 +104,7 @@ export class LocalFileStorage implements FileStorageInterface { * @param name * @returns */ - checkSafeDirectoryName(name: string): boolean { + checkSafeDirectoryName(name: string): boolean { assert(typeof name === 'string', 'name must be a string') const reg = /^([0-9]|[A-z]|_|-){3,64}$/ @@ -118,7 +118,7 @@ export class LocalFileStorage implements FileStorageInterface { assert(typeof name === 'string', 'name must be a string') const reg = /^([0-9]|[A-z]|_|-|\.){3,64}$/ return reg.test(name) - } + } /** * 生成文件名 diff --git a/packages/app-server/src/router/file/gridfs.ts b/packages/app-server/src/router/file/gridfs.ts new file mode 100644 index 0000000000..cc9a0f7c97 --- /dev/null +++ b/packages/app-server/src/router/file/gridfs.ts @@ -0,0 +1,105 @@ + +import * as fs from 'fs' +import * as express from 'express' +import { GridFSBucket } from 'mongodb' +import { Globals } from '../../lib/globals' +import { checkFileOperationToken, FS_OPERATION } from './utils' + + +export const GridFSHandlers = { + handleUploadFile, + handleDownloadFile +} + +/** + * upload file to the gridfs + * @param {string} bucket the bucket of file to store into, for example if `bucket = public`, the file path would be `/public/xxx.png` + */ +async function handleUploadFile(req: express.Request, res: express.Response) { + const auth = req['auth'] + const bucket_name = req.params.bucket + + // check file operation token if bucket is not 'public' + if (bucket_name !== 'public') { + const [code, message] = checkFileOperationToken(bucket_name, req.query?.token as string, FS_OPERATION.CREATE) + if (code) { + return res.status(code).send(message) + } + } + + // check given file + const file = req.file + if (!file) { + return res.status(422).send('file cannot be empty') + } + + // create a local file system driver + const bucket = new GridFSBucket(Globals.accessor.db, { bucketName: bucket_name }) + + const stream = bucket.openUploadStream(file.filename, { + metadata: { original_name: file.originalname, created_by: auth?.uid, bucket: bucket_name, created_at: Date.now() }, + contentType: file.mimetype + }) + + + // save to gridfs + fs.createReadStream(file.path) + .pipe(stream as any) + .on('finish', () => { + res.send({ + code: 0, + data: { + id: stream.id, + filename: stream.filename, + bucket: bucket_name, + contentType: file.mimetype + } + }) + }) + .on('error', (err: Error) => { + Globals.logger.error('upload file to gridfs stream occurred error', err) + res.status(500).send('internal server error') + }) +} + + +/** + * download file from gridfs by pointed bucket name and filename + * @param {string} bucket the bucket of file to store into, for example if `bucket = public`, the file path would be `/public/xxx.png` + */ +async function handleDownloadFile(req: express.Request, res: express.Response) { + + const { bucket: bucket_name, filename } = req.params + + // check file operation token if bucket is not 'public' + if (bucket_name !== 'public') { + const [code, message] = checkFileOperationToken(bucket_name, req.query?.token as string, FS_OPERATION.READ) + if (code) { + return res.status(code).send(message) + } + } + + + try { + + const bucket = new GridFSBucket(Globals.accessor.db, { bucketName: bucket_name }) + const stream = bucket.openDownloadStreamByName(filename) + + const files = await bucket.find({ filename: filename }).toArray() + const file = files.shift() + if (!file) { + return res.status(404).send('Not Found') + } + + if (file?.contentType) { + res.contentType(file.contentType) + } + + res.set('x-bucket', bucket_name) + res.set('x-uri', `/${bucket_name}/${filename}`) + return stream.pipe(res) + } catch (error) { + Globals.logger.error('get file info failed', error) + return res.status(404).send('Not Found') + } +} \ No newline at end of file diff --git a/packages/app-server/src/router/file/index.ts b/packages/app-server/src/router/file/index.ts index 5ddbcf6fb0..c427f03c0c 100644 --- a/packages/app-server/src/router/file/index.ts +++ b/packages/app-server/src/router/file/index.ts @@ -2,14 +2,12 @@ import * as express from 'express' import * as path from 'path' import * as multer from 'multer' import Config from '../../config' -import { LocalFileStorage } from '../../lib/storage/local_file_storage' import { v4 as uuidv4 } from 'uuid' -import { parseToken } from '../../lib/utils/token' +import { LocalFileSystemHandlers } from './localfs' +import { GridFSHandlers } from './gridfs' -export const FileRouter = express.Router() - -// multer 上传配置 +// config multer const uploader = multer({ storage: multer.diskStorage({ filename: (_req, file, cb) => { @@ -19,107 +17,61 @@ const uploader = multer({ }) }) -FileRouter.use('/public', express.static(path.join(Config.LOCAL_STORAGE_ROOT_PATH, 'public'))) +export const FileRouter = express.Router() + /** - * 上传文件 - * @namespace {string} 上传的名字空间,做为二级目录使用,只支持一级,名字可以为数字或字母; 如 namespace=public,则文件路径为 /public/xxx.png + * if use GridFS driver */ -FileRouter.post('/upload/:namespace', uploader.single('file'), async (req, res) => { - - const namespace = req.params.namespace - if (!checkNamespace(namespace)) { - return res.status(422).send('invalid namespace') - } - - // 验证访问 token - if (namespace !== 'public') { - // 验证上传 token - const uploadToken = req.query?.token - if (!uploadToken) { - return res.status(401).send('Unauthorized') - } - - const parsedToken = parseToken(uploadToken as string) - if (!parsedToken) { - return res.status(403).send('Invalid upload token') - } - - if (!['create', 'all'].includes(parsedToken?.op)) { - return res.status(403).send('Permission denied') - } - - if (parsedToken?.ns != namespace) { - return res.status(403).send('Permission denied') - } - } - - // 文件不可为空 - const file = req['file'] - if (!file) { - return res.status(422).send('file cannot be empty') - } - - // 存储上传文件 - const localStorage = new LocalFileStorage(Config.LOCAL_STORAGE_ROOT_PATH, namespace) - - const filepath = path.join(file.destination, `${file.filename}`) - const info = await localStorage.saveFile(filepath) +if (Config.FILE_SYSTEM_DRIVER === "gridfs") { + /** + * upload file + * @see GridFSHandlers.handleUploadFile() + */ + FileRouter.post('/upload/:bucket', uploader.single('file'), GridFSHandlers.handleUploadFile) + + + /** + * download file + * @see GridFSHandlers.handleDownloadFile() + */ + FileRouter.get('/download/:bucket/:filename', GridFSHandlers.handleDownloadFile) + + /** + * Alias URL for downloading file + * @see GridFSHandlers.handleDownloadFile() + */ + FileRouter.get('/:bucket/:filename', GridFSHandlers.handleDownloadFile) +} - // 不得暴露全路径给客户端 - delete info.fullpath - return res.send({ - code: 0, - data: info +/** + * if use local file system driver + */ +if (Config.FILE_SYSTEM_DRIVER === "local") { + // config multer + const uploader = multer({ + storage: multer.diskStorage({ + filename: (_req, file, cb) => { + const { ext } = path.parse(file.originalname) + cb(null, uuidv4() + ext) + } + }) }) -}) - -FileRouter.get('/download/:namespace/:filename', async (req, res) => { - const { namespace, filename } = req.params - if (!checkNamespace(namespace)) { - return res.status(422).send('invalid namespace') - } - - if (!checkFilename(filename)) { - return res.status(422).send('invalid filename') - } - - // 验证访问 token - if (namespace !== 'public') { - const token = req.query?.token - if (!token) { - return res.status(401).send('Unauthorized') - } - const prasedToken = parseToken(token as string) - if (!prasedToken) { - return res.status(403).send('Invalid token') - } - - if (prasedToken?.ns != namespace) { - return res.status(403).send('Permission denied') - } - - if (['read', 'all'].includes(prasedToken?.op)) { - return res.status(403).send('Permission denied') - } + FileRouter.use('/public', express.static(path.join(Config.LOCAL_STORAGE_ROOT_PATH, 'public'))) - if (prasedToken?.fn && prasedToken?.fn != filename) { - return res.status(403).send('Permission denied') - } - } + /** + * upload file + * @see LocalFileSystemHandlers.handleUploadFile() + */ + FileRouter.post('/upload/:bucket', uploader.single('file'), LocalFileSystemHandlers.handleUploadFile) - const localStorage = new LocalFileStorage(Config.LOCAL_STORAGE_ROOT_PATH, namespace) - const info = await localStorage.getFileInfo(filename) - return res.download(info.fullpath) -}) + /** + * download file + * @see LocalFileSystemHandlers.handleDownloadFile() + */ + FileRouter.get('/download/:bucket/:filename', LocalFileSystemHandlers.handleDownloadFile) -function checkNamespace(namespace: string) { - return (new LocalFileStorage('')).checkSafeDirectoryName(namespace) } - -function checkFilename(name: string) { - return (new LocalFileStorage('')).checkSafeFilename(name) -} \ No newline at end of file diff --git a/packages/app-server/src/router/file/localfs.ts b/packages/app-server/src/router/file/localfs.ts new file mode 100644 index 0000000000..c4ad2e55ea --- /dev/null +++ b/packages/app-server/src/router/file/localfs.ts @@ -0,0 +1,94 @@ +import * as express from 'express' +import * as path from 'path' +import Config from '../../config' +import { LocalFileStorage } from '../../lib/storage/local_file_storage' +import { Globals } from '../../lib/globals' +import { checkFileOperationToken, FS_OPERATION } from './utils' + + +export const LocalFileSystemHandlers = { + handleUploadFile, + handleDownloadFile +} + +/** + * Upload file into the local file system + * @param {string} bucket the bucket of file to store into, for example if `bucket = public`, the file path would be `/public/xxx.png` + */ +async function handleUploadFile(req: express.Request, res: express.Response) { + + const bucket = req.params.bucket + if (!checkBucketName(bucket)) { + return res.status(422).send('invalid bucket name') + } + + // check file operation token if bucket is not 'public' + if (bucket !== 'public') { + const [code, message] = checkFileOperationToken(bucket, req.query?.token as string, FS_OPERATION.CREATE) + if (code) { + return res.status(code).send(message) + } + } + + // check given file + const file = req.file + if (!file) { + return res.status(422).send('file cannot be empty') + } + + // create a local file system driver + const localStorage = new LocalFileStorage(Config.LOCAL_STORAGE_ROOT_PATH, bucket) + const filepath = path.join(file.destination, `${file.filename}`) + const info = await localStorage.saveFile(filepath) + + // do not expose server path to client + delete info.fullpath + + return res.send({ + code: 0, + data: info + }) +} + + +/** + * Download file from the local file system by pointed bucket name and filename + * @param {string} bucket the bucket of file to store into, for example if `bucket = public`, the file path would be `/public/xxx.png` + */ +async function handleDownloadFile(req: express.Request, res: express.Response) { + + const { bucket, filename } = req.params + if (!checkBucketName(bucket)) { + return res.status(422).send('invalid bucket') + } + + if (!checkFilename(filename)) { + return res.status(422).send('invalid filename') + } + + // check file operation token if bucket is not 'public' + if (bucket !== 'public') { + const [code, message] = checkFileOperationToken(bucket, req.query?.token as string, FS_OPERATION.READ) + if (code) { + return res.status(code).send(message) + } + } + + const localStorage = new LocalFileStorage(Config.LOCAL_STORAGE_ROOT_PATH, bucket) + + try { + const info = await localStorage.getFileInfo(filename) + return res.download(info.fullpath) + } catch (error) { + Globals.logger.error('get file info failed', error) + return res.status(404).send('Not Found') + } +} + +function checkBucketName(namespace: string) { + return (new LocalFileStorage('')).checkSafeDirectoryName(namespace) +} + +function checkFilename(name: string) { + return (new LocalFileStorage('')).checkSafeFilename(name) +} \ No newline at end of file diff --git a/packages/app-server/src/router/file/utils.ts b/packages/app-server/src/router/file/utils.ts new file mode 100644 index 0000000000..dcda62c2ae --- /dev/null +++ b/packages/app-server/src/router/file/utils.ts @@ -0,0 +1,68 @@ +import assert = require("assert") +import { parseToken } from "../../lib/utils/token" + + +/** + * file operations + */ +export enum FS_OPERATION { + /** + * upload a file + */ + CREATE = 'create', + + /** + * read a file + */ + READ = 'read', + + /** + * delete a file + */ + DELETE = 'delete', + + /** + * read file list + */ + LIST = 'list' +} + +/** + * check file operation token, the payload of the file token would have these fields: + * ``` + * { + * bucket: string, // indicated that this token only valid for this `bucket` + * ops: string[], // operation permissions granted, values can be one or more of: 'create' | 'read' | 'delete' | 'list' + * + * } + * ``` + * @param bucket the bucket name + * @param token the file operation token + * @param operation the operation: 'create' | 'read' | 'delete' | 'list' + * @returns + */ +export function checkFileOperationToken(bucket: string, token: string, operation: FS_OPERATION): [number, string] { + assert(bucket, 'empty `bucket` got') + assert(token, 'empty `token` got') + assert(operation, 'empty `operation` got') + + if (!token) { + return [401, 'Unauthorized'] + } + + const payload = parseToken(token) + if (!payload) { + return [403, 'invalid upload token'] + } + + const operations = payload.ops ?? [] + if (!operations.includes(operation)) { + return [403, 'permission denied'] + } + + if (payload?.bucket != bucket) { + return [403, 'permission denied'] + } + + return [0, null] +} \ No newline at end of file