From 9f24f812c910d2dbc4f76f43601836a98d88aa00 Mon Sep 17 00:00:00 2001 From: monoid Date: Sat, 16 Nov 2024 04:14:41 +0900 Subject: [PATCH] feat: modified file check --- packages/server/src/content/file.ts | 20 +++++++++----- packages/server/src/diff/content_handler.ts | 26 ++++++++++++++++++- packages/server/src/diff/watcher.ts | 2 ++ .../server/src/diff/watcher/common_watcher.ts | 3 +++ .../src/diff/watcher/recursive_watcher.ts | 5 ++++ .../server/src/diff/watcher/watcher_filter.ts | 6 +++-- packages/server/src/util/oshash.ts | 10 ++++++- 7 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/server/src/content/file.ts b/packages/server/src/content/file.ts index 9497db4..c26d5f9 100644 --- a/packages/server/src/content/file.ts +++ b/packages/server/src/content/file.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { promises, type Stats } from "node:fs"; import path, { extname } from "node:path"; import type { DocumentBody } from "dbtype"; -import { oshash } from "src/util/oshash.ts"; +import { oshash, OshashError } from "src/util/oshash.ts"; /** * content file or directory referrer */ @@ -30,7 +30,7 @@ export const createDefaultClass = (type: string): ContentFileConstructor => { protected hash: string | undefined; protected stat: Stats | undefined; - protected getStat(){ + protected getStat() { return this.stat; } @@ -42,7 +42,7 @@ export const createDefaultClass = (type: string): ContentFileConstructor => { async createDocumentBody(): Promise { console.log(`createDocumentBody: ${this.path}`); const { base, dir, name } = path.parse(this.path); - + const ret = { title: name, basepath: dir, @@ -61,9 +61,17 @@ export const createDefaultClass = (type: string): ContentFileConstructor => { } async getHash(): Promise { if (this.hash !== undefined) return this.hash; - const hash = await oshash(this.path); - this.hash = hash.toString(); - return this.hash; + try { + const hash = await oshash(this.path); + this.hash = hash.toString(); + return this.hash; + } catch (error) { + if (error instanceof OshashError) { + // too short to hash + return "0"; + } + throw error; + } } async getMtime(): Promise { const oldStat = this.getStat(); diff --git a/packages/server/src/diff/content_handler.ts b/packages/server/src/diff/content_handler.ts index bdc301c..b4b7865 100644 --- a/packages/server/src/diff/content_handler.ts +++ b/packages/server/src/diff/content_handler.ts @@ -34,7 +34,8 @@ export class ContentDiffHandler { diff .on("create", (path) => this.OnCreated(path)) .on("delete", (path) => this.OnDeleted(path)) - .on("change", (prev, cur) => this.OnChanged(prev, cur)); + .on("change", (prev, cur) => this.OnChanged(prev, cur)) + .on("modify", (path) => this.onModified(path)); } private async OnDeleted(cpath: string) { const basepath = dirname(cpath); @@ -126,4 +127,27 @@ export class ContentDiffHandler { filename: cur_filename, }); } + async onModified(path: string) { + const content = createContentFile(this.content_type, path); + // if it is in waiting list, update it. + if (this.waiting_list.hasByPath(path)) { + this.waiting_list.set(content); + return; + } + // if it is not in waiting list, check it in db. + const basepath = dirname(path); + const filename = basename(path); + const doc = await this.doc_cntr.findByPath(basepath, filename); + if (doc.length === 0) { + // if it is not in db, add it to waiting list. because it can happen that the file creation event occurs after the file modification event. + this.waiting_list.set(content); + return; + } + // if it is in db, update it. + await this.doc_cntr.update({ + ...doc[0], + content_hash: await content.getHash(), + modified_at: Date.now(), + }); + } } diff --git a/packages/server/src/diff/watcher.ts b/packages/server/src/diff/watcher.ts index 62e306a..7e28346 100644 --- a/packages/server/src/diff/watcher.ts +++ b/packages/server/src/diff/watcher.ts @@ -10,6 +10,7 @@ export interface DiffWatcherEvent { create: (path: string) => void; delete: (path: string) => void; change: (prev_path: string, cur_path: string) => void; + modify: (path: string) => void; } export interface IDiffWatcher extends event.EventEmitter { @@ -22,4 +23,5 @@ export function linkWatcher(fromWatcher: IDiffWatcher, toWatcher: IDiffWatcher) fromWatcher.on("create", (p) => toWatcher.emit("create", p)); fromWatcher.on("delete", (p) => toWatcher.emit("delete", p)); fromWatcher.on("change", (p, c) => toWatcher.emit("change", p, c)); + fromWatcher.on("modify", (p) => toWatcher.emit("modify", p)); } diff --git a/packages/server/src/diff/watcher/common_watcher.ts b/packages/server/src/diff/watcher/common_watcher.ts index cdeebbf..aac0821 100644 --- a/packages/server/src/diff/watcher/common_watcher.ts +++ b/packages/server/src/diff/watcher/common_watcher.ts @@ -33,6 +33,9 @@ export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatche this.emit("delete", join(this.path, filename)); } } + if (eventType === "change") { + this.emit("modify", join(this.path, filename)); + } }); } async setup(cntr: DocumentAccessor): Promise { diff --git a/packages/server/src/diff/watcher/recursive_watcher.ts b/packages/server/src/diff/watcher/recursive_watcher.ts index 66c71bc..08b04b1 100644 --- a/packages/server/src/diff/watcher/recursive_watcher.ts +++ b/packages/server/src/diff/watcher/recursive_watcher.ts @@ -47,6 +47,11 @@ export class RecursiveWatcher extends EventEmitter implements IDiffWatcher { const cpath = path; // console.log("unlink ", cpath); this.emit("delete", cpath); + }) + .on("change", (path) => { + const cpath = path; + // console.log("change ", cpath); + this.emit("modify", cpath); }); } if (option.watchDir) { diff --git a/packages/server/src/diff/watcher/watcher_filter.ts b/packages/server/src/diff/watcher/watcher_filter.ts index ea8caba..ecf0c7b 100644 --- a/packages/server/src/diff/watcher/watcher_filter.ts +++ b/packages/server/src/diff/watcher/watcher_filter.ts @@ -28,9 +28,11 @@ export class WatcherFilter extends EventEmitter implements IDiffWatcher { return super.emit("create", cur); } return false; - }if (!this.filter(arg[0])) { + } + if (!this.filter(arg[0])) { return false; - }return super.emit(event, ...arg); + } + return super.emit(event, ...arg); } constructor(refWatcher: IDiffWatcher, filter: (filename: string) => boolean) { super(); diff --git a/packages/server/src/util/oshash.ts b/packages/server/src/util/oshash.ts index c2a9105..08369c4 100644 --- a/packages/server/src/util/oshash.ts +++ b/packages/server/src/util/oshash.ts @@ -1,4 +1,12 @@ import { open } from "fs/promises"; + +export class OshashError extends Error { + constructor(message: string) { + super(message); + this.name = "OshashError"; + } +} + /** * get hash of file or directory. * @@ -22,7 +30,7 @@ export async function oshash( if (st.size < minFileSize){ if (st.size < 8) { fd.close(); - throw new Error("Too short to hash"); + throw new OshashError("Too short to hash"); } // Although the original oshash algorithm should throw an exception, // just applying the hash function.