From 89bc827a7abed80dc99abaacbbccfb836c9ba88e Mon Sep 17 00:00:00 2001 From: monoid Date: Fri, 22 Jan 2021 15:13:05 +0900 Subject: [PATCH] event driven watcher --- .gitignore | 3 +- package.json | 12 +-- src/diff/MangaConfig.ts | 6 -- src/diff/content_handler.ts | 29 ++++--- src/diff/diff.ts | 10 +-- src/diff/watcher.ts | 75 +++---------------- .../{ => watcher}/MangaConfig.schema.json | 0 src/diff/watcher/MangaConfig.ts | 8 ++ src/diff/watcher/common_watcher.ts | 45 +++++++++++ src/diff/watcher/compositer.ts | 24 ++++++ src/diff/watcher/manga_watcher.ts | 17 +++++ src/diff/watcher/recursive_watcher.ts | 59 +++++++++++++++ src/diff/watcher/util.ts | 35 +++++++++ src/diff/watcher/watcher_filter.ts | 45 +++++++++++ src/server.ts | 5 +- src/util/configRW.ts | 51 +++++++++++++ src/util/type_check.js | 22 ------ 17 files changed, 330 insertions(+), 116 deletions(-) delete mode 100644 src/diff/MangaConfig.ts rename src/diff/{ => watcher}/MangaConfig.schema.json (100%) create mode 100644 src/diff/watcher/MangaConfig.ts create mode 100644 src/diff/watcher/common_watcher.ts create mode 100644 src/diff/watcher/compositer.ts create mode 100644 src/diff/watcher/manga_watcher.ts create mode 100644 src/diff/watcher/recursive_watcher.ts create mode 100644 src/diff/watcher/util.ts create mode 100644 src/diff/watcher/watcher_filter.ts create mode 100644 src/util/configRW.ts delete mode 100644 src/util/type_check.js diff --git a/.gitignore b/.gitignore index bca9d65..70854a8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ package-lock.json devdb.sqlite3 build/** app/** -settings.json \ No newline at end of file +settings.json +*config.json \ No newline at end of file diff --git a/package.json b/package.json index c6e8727..a0d55e4 100644 --- a/package.json +++ b/package.json @@ -31,14 +31,16 @@ "author": "", "license": "ISC", "dependencies": { + "chokidar": "^3.5.1", + "jsonschema": "^1.4.0", "jsonwebtoken": "^8.5.1", - "knex": "^0.21.14", - "koa": "^2.13.0", + "knex": "^0.21.16", + "koa": "^2.13.1", "koa-bodyparser": "^4.3.0", "koa-router": "^10.0.0", "natural-orderby": "^2.0.3", "node-stream-zip": "^1.12.0", - "sqlite3": "^5.0.0" + "sqlite3": "^5.0.1" }, "devDependencies": { "@types/jsonwebtoken": "^8.5.0", @@ -46,8 +48,8 @@ "@types/koa": "^2.11.6", "@types/koa-bodyparser": "^4.3.0", "@types/koa-router": "^7.4.1", - "@types/node": "^14.14.16", - "electron": "^11.1.1", + "@types/node": "^14.14.22", + "electron": "^11.2.0", "electron-builder": "^22.9.1", "eslint-plugin-node": "^11.1.0", "ts-json-schema-generator": "^0.82.0", diff --git a/src/diff/MangaConfig.ts b/src/diff/MangaConfig.ts deleted file mode 100644 index d3964d0..0000000 --- a/src/diff/MangaConfig.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Schema from './MangaConfig.schema.json'; - -export interface MangaConfig{ - watch:string[] -} - diff --git a/src/diff/content_handler.ts b/src/diff/content_handler.ts index b240086..afd6109 100644 --- a/src/diff/content_handler.ts +++ b/src/diff/content_handler.ts @@ -1,4 +1,4 @@ -import {join as pathjoin} from 'path'; +import {basename, dirname, join as pathjoin} from 'path'; import {Document, DocumentAccessor} from '../model/mod'; import { ContentFile, createContentFile } from '../content/mod'; import {IDiffWatcher} from './watcher'; @@ -23,12 +23,13 @@ export class ContentDiffHandler{ } } register(diff:IDiffWatcher){ - diff.on('create',(filename)=>this.OnCreated(diff.path,filename)) - .on('delete',(filename)=>this.OnDeleted(diff.path,filename)) - .on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename)); + diff.on('create',(path)=>this.OnCreated(path)) + .on('delete',(path)=>this.OnDeleted(path)) + .on('change',(prev,cur)=>this.OnChanged(prev,cur)); } - private async OnDeleted(basepath:string,filename:string){ - const cpath = pathjoin(basepath,filename); + private async OnDeleted(cpath: string){ + const basepath = dirname(cpath); + const filename = basename(cpath); if(this.waiting_list.hasPath(cpath)){ this.waiting_list.deletePath(cpath); return; @@ -52,7 +53,9 @@ export class ContentDiffHandler{ }); this.tombstone.set(dbc[0].content_hash, dbc[0]); } - private async OnCreated(basepath:string,filename:string){ + private async OnCreated(cpath:string){ + const basepath = dirname(cpath); + const filename = basename(cpath); const content = createContentFile(this.content_type,pathjoin(basepath,filename)); const hash = await content.getHash(); const c = this.tombstone.get(hash); @@ -67,8 +70,14 @@ export class ContentDiffHandler{ } this.waiting_list.set(content); } - private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){ - const doc = await this.doc_cntr.findByPath(basepath,prev_filename); - await this.doc_cntr.update({...doc[0],filename:cur_filename}); + private async OnChanged(prev_path:string,cur_path:string){ + const prev_basepath = dirname(prev_path); + const prev_filename = basename(prev_path); + const cur_basepath = dirname(cur_path); + const cur_filename = basename(cur_path); + const doc = await this.doc_cntr.findByPath(prev_basepath,prev_filename); + await this.doc_cntr.update({...doc[0], + basepath:cur_basepath, + filename:cur_filename}); } } \ No newline at end of file diff --git a/src/diff/diff.ts b/src/diff/diff.ts index 354e5a4..db39a4f 100644 --- a/src/diff/diff.ts +++ b/src/diff/diff.ts @@ -1,6 +1,7 @@ import { DocumentAccessor } from '../model/doc'; import {ContentDiffHandler} from './content_handler'; -import { CommonDiffWatcher } from './watcher'; +import { IDiffWatcher } from './watcher'; + //import {join as pathjoin} from 'path'; export class DiffManager{ watching: {[content_type:string]:ContentDiffHandler}; @@ -9,15 +10,12 @@ export class DiffManager{ this.watching = {}; this.doc_cntr = contorller; } - async register(content_type:string,path:string){ + async register(content_type:string,watcher:IDiffWatcher){ if(this.watching[content_type] === undefined){ this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type); } - const watcher = new CommonDiffWatcher(path); this.watching[content_type].register(watcher); - const initial_doc = await this.doc_cntr.findByPath(path); - await watcher.setup(initial_doc.map(x=>x.filename)); - watcher.watch(); + await watcher.setup(this.doc_cntr); } async commit(type:string,path:string){ const list = this.watching[type].waiting_list; diff --git a/src/diff/watcher.ts b/src/diff/watcher.ts index 0c69220..fbac687 100644 --- a/src/diff/watcher.ts +++ b/src/diff/watcher.ts @@ -1,78 +1,25 @@ import { FSWatcher, watch } from 'fs'; import { promises } from 'fs'; import event from 'events'; - +import { join } from 'path'; +import { DocumentAccessor } from '../model/doc'; const readdir = promises.readdir; -interface DiffWatcherEvent{ - 'create':(filename:string)=>void, - 'delete':(filename:string)=>void, - 'change':(prev_filename:string,cur_filename:string)=>void, +export interface DiffWatcherEvent{ + 'create':(path:string)=>void, + 'delete':(path:string)=>void, + 'change':(prev_path:string,cur_path:string)=>void, } export interface IDiffWatcher extends event.EventEmitter { on(event:U,listener:DiffWatcherEvent[U]): this; emit(event:U,...arg:Parameters): boolean; - readonly path: string; + setup(cntr:DocumentAccessor):Promise; } -export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{ - on(event:U,listener:DiffWatcherEvent[U]): this{ - return super.on(event,listener); - } - emit(event:U,...arg:Parameters): boolean{ - return super.emit(event,...arg); - } - private _path:string; - private _watcher: FSWatcher|null; - - constructor(path:string){ - super(); - this._path = path; - this._watcher = null; - } - public get path(){ - return this._path; - } - /** - * setup - * @argument initial_filenames filename in path - */ - async setup(initial_filenames:string[]){ - const cur = (await readdir(this._path,{ - encoding:"utf8", - withFileTypes: true, - })).filter(x=>x.isFile).map(x=>x.name); - //Todo : reduce O(nm) to O(n+m) using hash map. - let added = cur.filter(x => !initial_filenames.includes(x)); - let deleted = initial_filenames.filter(x=>!cur.includes(x)); - for (const iterator of added) { - this.emit('create',iterator); - } - for (const iterator of deleted){ - this.emit('delete',iterator); - } - } - watch():FSWatcher{ - this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{ - if(eventType === "rename"){ - const cur = (await readdir(this._path,{ - encoding:"utf8", - withFileTypes: true, - })).filter(x=>x.isFile).map(x=>x.name); - //add - if(cur.includes(filename)){ - this.emit('create',filename); - } - else{ - this.emit('delete',filename) - } - } - }); - return this._watcher; - } - watchClose(){ - this._watcher?.close() - } +export function linkWatcher(fromWatcher :IDiffWatcher, toWatcher: IDiffWatcher){ + fromWatcher.on("create",p=>toWatcher.emit("create",p)); + fromWatcher.on("delete",p=>toWatcher.emit("delete",p)); + fromWatcher.on("change",(p,c)=>toWatcher.emit("change",p,c)); } \ No newline at end of file diff --git a/src/diff/MangaConfig.schema.json b/src/diff/watcher/MangaConfig.schema.json similarity index 100% rename from src/diff/MangaConfig.schema.json rename to src/diff/watcher/MangaConfig.schema.json diff --git a/src/diff/watcher/MangaConfig.ts b/src/diff/watcher/MangaConfig.ts new file mode 100644 index 0000000..dad7876 --- /dev/null +++ b/src/diff/watcher/MangaConfig.ts @@ -0,0 +1,8 @@ +import {ConfigManager} from '../../util/configRW'; +import MangaSchema from "./MangaConfig.schema.json" +export interface MangaConfig{ + watch:string[] +} + +export const MangaConfig = new ConfigManager("manga_config.json",{watch:[]},MangaSchema); + diff --git a/src/diff/watcher/common_watcher.ts b/src/diff/watcher/common_watcher.ts new file mode 100644 index 0000000..7395d63 --- /dev/null +++ b/src/diff/watcher/common_watcher.ts @@ -0,0 +1,45 @@ +import event from 'events'; +import {FSWatcher,watch,promises} from 'fs'; +import {IDiffWatcher, DiffWatcherEvent} from '../watcher'; +import {join} from 'path'; +import { DocumentAccessor } from '../../model/doc'; +import { setupHelp } from './util'; + +const {readdir} = promises; + +export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{ + on(event:U,listener:DiffWatcherEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + return super.emit(event,...arg); + } + private _path:string; + private _watcher: FSWatcher; + + constructor(path:string){ + super(); + this._path = path; + this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{ + if(eventType === "rename"){ + const cur = await readdir(this._path); + //add + if(cur.includes(filename)){ + this.emit('create',join(this.path,filename)); + } + else{ + this.emit('delete',join(this.path,filename)) + } + } + }); + } + async setup(cntr: DocumentAccessor): Promise { + await setupHelp(this,this.path,cntr); + } + public get path(){ + return this._path; + } + watchClose(){ + this._watcher.close() + } +} \ No newline at end of file diff --git a/src/diff/watcher/compositer.ts b/src/diff/watcher/compositer.ts new file mode 100644 index 0000000..020fb70 --- /dev/null +++ b/src/diff/watcher/compositer.ts @@ -0,0 +1,24 @@ +import { EventEmitter } from "events"; +import { DocumentAccessor } from "../../model/doc"; +import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher"; + + +export class WatcherCompositer extends EventEmitter implements IDiffWatcher{ + refWatchers : IDiffWatcher[]; + on(event:U,listener:DiffWatcherEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + return super.emit(event,...arg); + } + constructor(refWatchers:IDiffWatcher[]){ + super(); + this.refWatchers = refWatchers; + for(const refWatcher of this.refWatchers){ + linkWatcher(refWatcher,this); + } + } + async setup(cntr: DocumentAccessor): Promise { + await Promise.all(this.refWatchers.map(x=>x.setup(cntr))); + } +} \ No newline at end of file diff --git a/src/diff/watcher/manga_watcher.ts b/src/diff/watcher/manga_watcher.ts new file mode 100644 index 0000000..0e48ebc --- /dev/null +++ b/src/diff/watcher/manga_watcher.ts @@ -0,0 +1,17 @@ +import {IDiffWatcher, DiffWatcherEvent} from '../watcher'; +import {EventEmitter} from 'events'; +import { DocumentAccessor } from '../../model/doc'; +import { WatcherFilter } from './watcher_filter'; +import { RecursiveWatcher } from './recursive_watcher'; +import { MangaConfig } from './MangaConfig'; +import {WatcherCompositer} from './compositer' + + +const createMangaWatcherBase = (path:string)=> { + return new WatcherFilter(new RecursiveWatcher(path),(x)=>x.endsWith(".zip")); +} +export const createMangaWatcher = ()=>{ + const file = MangaConfig.get_config_file(); + console.log(`register manga ${file.watch.join(",")}`) + return new WatcherCompositer(file.watch.map(path=>createMangaWatcherBase(path))); +} \ No newline at end of file diff --git a/src/diff/watcher/recursive_watcher.ts b/src/diff/watcher/recursive_watcher.ts new file mode 100644 index 0000000..d77e3cb --- /dev/null +++ b/src/diff/watcher/recursive_watcher.ts @@ -0,0 +1,59 @@ +import {watch, FSWatcher} from 'chokidar'; +import { EventEmitter } from 'events'; +import { join } from 'path'; +import { DocumentAccessor } from '../../model/doc'; +import { DiffWatcherEvent, IDiffWatcher } from '../watcher'; +import { setupHelp, setupRecursive } from './util'; + +type RecursiveWatcherOption={ + /** @default true */ + watchFile?:boolean, + /** @default false */ + watchDir?:boolean, +} + +export class RecursiveWatcher extends EventEmitter implements IDiffWatcher { + on(event:U,listener:DiffWatcherEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + return super.emit(event,...arg); + } + readonly path: string; + private watcher: FSWatcher + + constructor(path:string, option?:RecursiveWatcherOption){ + super(); + this.path = path; + option = option || { + watchDir:false, + watchFile:true, + } + this.watcher = watch(path,{ + persistent:true, + ignoreInitial:true, + depth:100, + }); + if(option.watchFile === undefined || option.watchFile){ + this.watcher.on("add",path=>{ + const cpath = join(this.path,path); + this.emit("create",cpath); + }).on("unlink",path=>{ + const cpath = join(this.path,path); + this.emit("delete",cpath); + }); + } + if(option.watchDir){ + this.watcher.on("addDir",path=>{ + const cpath = join(this.path,path); + this.emit("create",cpath); + }).on("unlinkDir",path=>{ + const cpath = join(this.path,path); + this.emit("delete",cpath); + }) + } + } + async setup(cntr: DocumentAccessor): Promise { + await setupRecursive(this,this.path,cntr); + } +} diff --git a/src/diff/watcher/util.ts b/src/diff/watcher/util.ts new file mode 100644 index 0000000..16e13d9 --- /dev/null +++ b/src/diff/watcher/util.ts @@ -0,0 +1,35 @@ +import { EventEmitter } from "events"; +import { promises } from "fs"; +import { join } from "path"; +const {readdir} = promises; +import { DocumentAccessor } from "../../model/doc"; +import { IDiffWatcher } from "../watcher"; + + +function setupCommon(watcher:IDiffWatcher,basepath:string,initial_filenames:string[],cur:string[]){ + //Todo : reduce O(nm) to O(n+m) using hash map. + let added = cur.filter(x => !initial_filenames.includes(x)); + let deleted = initial_filenames.filter(x=>!cur.includes(x)); + for (const it of added) { + const cpath = join(basepath,it); + watcher.emit('create',cpath); + } + for (const it of deleted){ + const cpath = join(basepath,it); + watcher.emit('delete',cpath); + } +} +export async function setupHelp(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){ + const initial_document = await cntr.findByPath(basepath); + const initial_filenames = initial_document.map(x=>x.filename); + const cur = await readdir(basepath); + setupCommon(watcher,basepath,initial_filenames,cur); +} +export async function setupRecursive(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){ + const initial_document = await cntr.findByPath(basepath); + const initial_filenames = initial_document.map(x=>x.filename); + const cur = await readdir(basepath,{withFileTypes:true}); + setupCommon(watcher,basepath,initial_filenames,cur.map(x=>x.name)); + await Promise.all([cur.filter(x=>x.isDirectory()) + .map(x=>setupHelp(watcher,join(basepath,x.name),cntr))]); +} diff --git a/src/diff/watcher/watcher_filter.ts b/src/diff/watcher/watcher_filter.ts new file mode 100644 index 0000000..bed3049 --- /dev/null +++ b/src/diff/watcher/watcher_filter.ts @@ -0,0 +1,45 @@ +import { EventEmitter } from "events"; +import { DocumentAccessor } from "../../model/doc"; +import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher"; + + +export class WatcherFilter extends EventEmitter implements IDiffWatcher{ + refWatcher : IDiffWatcher; + filter : (filename:string)=>boolean;; + on(event:U,listener:DiffWatcherEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + if(event === "change"){ + const prev = arg[0]; + const cur = arg[1] as string; + if(this.filter(prev)){ + if(this.filter(cur)){ + return super.emit("change",prev,cur); + } + else{ + return super.emit("delete",cur); + } + } + else{ + if(this.filter(cur)){ + return super.emit("create",cur); + } + } + return false; + } + else if(!this.filter(arg[0])){ + return false; + } + else return super.emit(event,...arg); + } + constructor(refWatcher:IDiffWatcher, filter:(filename:string)=>boolean){ + super(); + this.refWatcher = refWatcher; + this.filter = filter; + linkWatcher(refWatcher,this); + } + setup(cntr:DocumentAccessor): Promise { + return this.refWatcher.setup(cntr); + } +} \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index 45b4af5..86fb577 100644 --- a/src/server.ts +++ b/src/server.ts @@ -14,6 +14,7 @@ import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, Logo import {createInterface as createReadlineInterface} from 'readline'; import { DocumentAccessor, UserAccessor } from './model/mod'; +import { createMangaWatcher } from './diff/watcher/manga_watcher'; class ServerApplication{ readonly userController: UserAccessor; @@ -21,7 +22,7 @@ class ServerApplication{ readonly diffManger; readonly app: Koa; private index_html:Buffer; - constructor(userController: UserAccessor,documentController:DocumentAccessor){ + private constructor(userController: UserAccessor,documentController:DocumentAccessor){ this.userController = userController; this.documentController = documentController; this.diffManger = new DiffManager(documentController); @@ -49,7 +50,7 @@ class ServerApplication{ app.use(createUserMiddleWare(this.userController)); let diff_router = createDiffRouter(this.diffManger); - this.diffManger.register("manga","testdata"); + this.diffManger.register("manga",createMangaWatcher()); let router = new Router(); router.use('/api/diff',diff_router.routes()); diff --git a/src/util/configRW.ts b/src/util/configRW.ts new file mode 100644 index 0000000..c2db258 --- /dev/null +++ b/src/util/configRW.ts @@ -0,0 +1,51 @@ +import {readFileSync, existsSync, writeFileSync, promises as fs} from 'fs'; +import {validate} from 'jsonschema'; + +export class ConfigManager{ + path:string; + default_config: T; + config: T| null; + schema:object; + constructor(path:string,default_config:T,schema:object){ + this.path = path; + this.default_config = default_config; + this.config = null; + this.schema = schema; + } + get_config_file(): T{ + if(this.config !== null) return this.config; + this.config = {...this.read_config_file()}; + return this.config; + } + private emptyToDefault(target:T){ + let occur = false; + for(const key in this.default_config){ + if(key === undefined || key in target){ + continue; + } + target[key] = this.default_config[key]; + occur = true; + } + return occur; + } + read_config_file():T{ + if(!existsSync(this.path)){ + writeFileSync(this.path,JSON.stringify(this.default_config)); + return this.default_config; + } + const ret = JSON.parse(readFileSync(this.path,{encoding:"utf8"})); + if(this.emptyToDefault(ret)){ + writeFileSync(this.path,JSON.stringify(ret)); + } + const result = validate(ret,this.schema); + if(!result.valid){ + throw new Error(result.toString()); + } + return ret; + } + async write_config_file(new_config:T){ + this.config = new_config; + await fs.writeFile(`${this.path}.temp`,JSON.stringify(new_config)); + await fs.rename(`${this.path}.temp`,this.path); + } +} \ No newline at end of file diff --git a/src/util/type_check.js b/src/util/type_check.js deleted file mode 100644 index 0035642..0000000 --- a/src/util/type_check.js +++ /dev/null @@ -1,22 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.check_type = void 0; -function check_type(obj, check_proto) { - for (const it in check_proto) { - let defined = check_proto[it]; - if (defined === undefined) - return false; - defined = defined.trim(); - if (defined.endsWith("[]")) { - if (!(obj[it] instanceof Array)) { - return false; - } - } - else if (defined !== typeof obj[it]) { - return false; - } - } - return true; -} -exports.check_type = check_type; -;