diff --git a/packages/dbtype/src/api.ts b/packages/dbtype/src/api.ts index 120f01f..1634c7e 100644 --- a/packages/dbtype/src/api.ts +++ b/packages/dbtype/src/api.ts @@ -11,6 +11,7 @@ export const DocumentBodySchema = z.object({ additional: z.record(z.unknown()), tags: z.array(z.string()), pagenum: z.number().int(), + gid: z.number().nullable(), }); export type DocumentBody = z.infer; diff --git a/packages/dbtype/src/types.ts b/packages/dbtype/src/types.ts index 63152e7..cbdb075 100644 --- a/packages/dbtype/src/types.ts +++ b/packages/dbtype/src/types.ts @@ -21,6 +21,7 @@ export interface Document { modified_at: number; title: string; pagenum: number; + gid: number | null; } export interface Permissions { diff --git a/packages/server/migrations/initial.ts b/packages/server/migrations/initial.ts index c54a41e..36f097e 100644 --- a/packages/server/migrations/initial.ts +++ b/packages/server/migrations/initial.ts @@ -23,7 +23,7 @@ export async function up(db: Kysely) { .addColumn('filename', 'varchar(512)', col => col.notNull()) .addColumn('content_hash', 'varchar') .addColumn('additional', 'json') - .addColumn("pagenum", "integer", col => col.notNull()) + .addColumn("pagenum", "integer", col => col.notNull()) .addColumn('created_at', 'integer', col => col.notNull()) .addColumn('modified_at', 'integer', col => col.notNull()) .addColumn('deleted_at', 'integer') @@ -62,13 +62,27 @@ export async function up(db: Kysely) { }) .execute(); - await db - .insertInto('schema_migration') - .values({ - version: '0.0.1', - dirty: false, - }) - .execute(); + await db + .insertInto('schema_migration') + .values({ + version: '0.0.1', + dirty: false, + }) + .execute(); + + // create indexes + await db.schema.createIndex("index_document_basepath_filename") + .on("document") + .columns(["basepath", "filename"]) + .execute(); + await db.schema.createIndex("index_document_content_hash") + .on("document") + .columns(["content_hash"]) + .execute(); + await db.schema.createIndex("index_document_created_at") + .on("document") + .columns(["created_at"]) + .execute(); } export async function down(db: Kysely) { diff --git a/packages/server/src/content/file.ts b/packages/server/src/content/file.ts index c26d5f9..caf9f15 100644 --- a/packages/server/src/content/file.ts +++ b/packages/server/src/content/file.ts @@ -3,6 +3,7 @@ import { promises, type Stats } from "node:fs"; import path, { extname } from "node:path"; import type { DocumentBody } from "dbtype"; import { oshash, OshashError } from "src/util/oshash.ts"; +import { extractGidFromFilename } from "src/util/gid.ts"; /** * content file or directory referrer */ @@ -53,6 +54,7 @@ export const createDefaultClass = (type: string): ContentFileConstructor => { content_hash: await this.getHash(), modified_at: await this.getMtime(), pagenum: 0, + gid: extractGidFromFilename(base), } as DocumentBody; return ret; } diff --git a/packages/server/src/db/doc.ts b/packages/server/src/db/doc.ts index cd4238e..301a6d1 100644 --- a/packages/server/src/db/doc.ts +++ b/packages/server/src/db/doc.ts @@ -88,7 +88,7 @@ class SqliteDocumentAccessor implements DocumentAccessor { .executeTakeFirst() as { id: number }; const id = id_lst.id; - if (tags.length > 0){ + if (tags.length > 0) { // add tags await trx.insertInto("tags") .values(tags.map((x) => ({ name: x }))) @@ -202,6 +202,19 @@ class SqliteDocumentAccessor implements DocumentAccessor { tags: JSON.parse(x.tags ?? "[]"), //?.map((x: { tag_name: string }) => x.tag_name) ?? [], })); } + async findByGidList(gid_list: number[]) { + const result = await this.kysely + .selectFrom("document") + .selectAll() + .where("gid", "in", gid_list) + .execute(); + return result.map((x) => ({ + ...x, + content_hash: x.content_hash ?? "", + tags: [], + additional: {}, + })); + } async findByPath(path: string, filename?: string): Promise { const results = await this.kysely .selectFrom("document") @@ -216,6 +229,7 @@ class SqliteDocumentAccessor implements DocumentAccessor { additional: {}, })); } + async update(c: Partial & { id: number }) { const { id, tags, additional, ...rest } = c; const r = await this.kysely.updateTable("document") @@ -322,7 +336,7 @@ class SqliteDocumentAccessor implements DocumentAccessor { .values(tags.map((x) => ({ tag_name: x, doc_id: c.id })) ) .execute(); - + }); } } diff --git a/packages/server/src/db/kysely.ts b/packages/server/src/db/kysely.ts index 75ca907..0c0c4c0 100644 --- a/packages/server/src/db/kysely.ts +++ b/packages/server/src/db/kysely.ts @@ -25,11 +25,11 @@ export function getKysely() { log: (event) => { if (event.level === "error") { console.error("Query failed : ", { - durationMs: event.queryDurationMillis, - error: event.error, - sql: event.query.sql, - params: event.query.parameters, - }); + durationMs: event.queryDurationMillis, + error: event.error, + sql: event.query.sql, + params: event.query.parameters, + }); } } }); diff --git a/packages/server/src/diff/content_handler.ts b/packages/server/src/diff/content_handler.ts index b4b7865..61dd451 100644 --- a/packages/server/src/diff/content_handler.ts +++ b/packages/server/src/diff/content_handler.ts @@ -52,7 +52,7 @@ export class ContentDiffHandler { console.log("its not in waiting_list and db!!!: ", cpath); return; } - console.log("deleted ",cpath, dbc[0].content_hash) + console.log("deleted ",cpath, dbc[0].content_hash) const content_hash = dbc[0].content_hash; // When a path is changed, it takes into account when the // creation event occurs first and the deletion occurs, not @@ -62,14 +62,12 @@ export class ContentDiffHandler { if (cf) { // if a path is changed, update the changed path. console.log("update path from", cpath, "to", cf.path); - const newFilename = basename(cf.path); - const newBasepath = dirname(cf.path); + const new_doc = createContentFile(this.content_type, cf.path); this.waiting_list.deleteByHash(content_hash); + await this.doc_cntr.update({ - id: dbc[0].id, - deleted_at: null, - filename: newFilename, - basepath: newBasepath, + ...dbc[0], + ...new_doc.createDocumentBody(), }); return; } @@ -91,21 +89,17 @@ export class ContentDiffHandler { if (c !== undefined) { await this.doc_cntr.update({ id: c.id, - deleted_at: null, - filename: filename, - basepath: basepath, + ...content.createDocumentBody(), }); } if (this.waiting_list.hasByHash(hash)) { - console.log("Hash Conflict!!!"); + console.log("Hash Conflict!!!", cpath); } this.waiting_list.set(content); } private async OnChanged(prev_path: string, cur_path: string) { const prev_basepath = dirname(prev_path); const prev_filename = basename(prev_path); - const cur_basepath = dirname(cur_path); - const cur_filename = basename(cur_path); console.log("modify", cur_path, "from", prev_path); const c = this.waiting_list.getByPath(prev_path); if (c !== undefined) { @@ -123,8 +117,7 @@ export class ContentDiffHandler { await this.doc_cntr.update({ ...doc[0], - basepath: cur_basepath, - filename: cur_filename, + ...createContentFile(this.content_type, cur_path).createDocumentBody(), }); } async onModified(path: string) { @@ -146,7 +139,7 @@ export class ContentDiffHandler { // if it is in db, update it. await this.doc_cntr.update({ ...doc[0], - content_hash: await content.getHash(), + ...content.createDocumentBody(), modified_at: Date.now(), }); } diff --git a/packages/server/src/diff/watcher/ComicConfig.ts b/packages/server/src/diff/watcher/ComicConfig.ts index 23dd6ab..da42a85 100644 --- a/packages/server/src/diff/watcher/ComicConfig.ts +++ b/packages/server/src/diff/watcher/ComicConfig.ts @@ -1,5 +1,5 @@ import { ConfigManager } from "../../util/configRW.ts"; -import ComicSchema from "./ComicConfig.schema.json"; +import ComicSchema from "./ComicConfig.schema.json" assert { type: "json" }; export interface ComicConfig { watch: string[]; } diff --git a/packages/server/src/model/doc.ts b/packages/server/src/model/doc.ts index 0758041..0e193bf 100644 --- a/packages/server/src/model/doc.ts +++ b/packages/server/src/model/doc.ts @@ -55,6 +55,12 @@ export interface DocumentAccessor { * if you call this function with filename, its return array length is 0 or 1. */ findByPath: (basepath: string, filename?: string) => Promise; + /** + * find by gid list + * @param gid_list + * @returns Document list + */ + findByGidList: (gid_list: number[]) => Promise; /** * find deleted content */ diff --git a/packages/server/src/route/contents.ts b/packages/server/src/route/contents.ts index fdcb12b..2a8c4e0 100644 --- a/packages/server/src/route/contents.ts +++ b/packages/server/src/route/contents.ts @@ -213,6 +213,15 @@ function getRescanDocumentHandler(controller: DocumentAccessor) { export const getContentRouter = (controller: DocumentAccessor) => { const ret = new Router(); ret.get("/search", PerCheck(Per.QueryContent), ContentQueryHandler(controller)); + ret.get("/_gid", PerCheck(Per.QueryContent), async (ctx, next) => { + const gid_list = ParseQueryArray(ctx.query.gid).map((x) => Number.parseInt(x)) + if (gid_list.some((x) => Number.isNaN(x))) { + return sendError(400, "gid is not a number"); + } + const r = await controller.findByGidList(gid_list); + ctx.body = r; + ctx.type = "json"; + }); ret.get("/:num(\\d+)", PerCheck(Per.QueryContent), ContentIDHandler(controller)); ret.all("/:num(\\d+)/(.*)", PerCheck(Per.QueryContent), ContentHandler(controller)); ret.post("/:num(\\d+)", AdminOnly, UpdateContentHandler(controller)); diff --git a/packages/server/src/util/gid.ts b/packages/server/src/util/gid.ts new file mode 100644 index 0000000..d58f3c7 --- /dev/null +++ b/packages/server/src/util/gid.ts @@ -0,0 +1,5 @@ +export function extractGidFromFilename(filename: string) { + const match = filename.match(/^([0-9]+)(:?\.|\s)/); + if (match === null) return null; + return Number.parseInt(match[1]); +} \ No newline at end of file