simple-fs-server/src/collect.ts

import { basename, extname, join, relative } from "path/mod.ts";
import { readMarkdownDoc } from "./readDoc.ts";
import { Index } from "./client_search.ts";

export interface Doc {
  path: string;
  content: string;
  attributes: {
    title?: string;
    japanese_title?: string;
    tags?: string[];
    rjcode?: string;
    author?: string;
  };
}

export async function loadDocuments(path: string): Promise<Doc[]> {
  const doc_json = await Deno.readTextFile(path);
  return JSON.parse(doc_json) as Doc[];
}

export interface DocCollectorOptions {
  summaryOnly?: boolean;
  dropContent?: boolean;
}

export class DocCollector {
  private doc_map: Map<string, Doc>;
  private options: DocCollectorOptions;

  constructor(options: DocCollectorOptions = {}) {
    this.doc_map = new Map();
    this.options = options;
  }

  public getDocs(): Doc[] {
    return [...this.doc_map.values()];
  }

  public setDoc(doc: Doc) {
    if (this.options.dropContent) {
      doc.content = "";
    }
    this.doc_map.set(doc.path, doc);
  }
  public setDocs(docs: Doc[]) {
    for (const doc of docs) {
      this.setDoc(doc);
    }
  }
  public removeDoc(path: string) {
    this.doc_map.delete(path);
  }

  public async walkDir(path: string) {
    const dir = Deno.readDir(path);
    const fileList = [];
    for await (const entry of dir) {
      fileList.push(entry);
    }

    if (fileList.some((entry) => entry.name === "SUMMARY.md")) {
      const { content, metadata } = await readMarkdownDoc(
        join(path, "SUMMARY.md"),
      );
      this.setDoc({
        path: join(path, "SUMMARY.md"),
        content: content,
        attributes: metadata,
      });
    } else {
      for (const entry of fileList) {
        if (entry.isDirectory) {
          await this.walkDir(join(path, entry.name));
        } else if (entry.isFile && !this.options.summaryOnly) {
          const doc = await this.readDoc(join(path, entry.name));
          this.setDoc(doc);
        }
      }
    }
  }

  public async readDoc(path: string): Promise<Doc> {
    const ext = extname(path);
    if (ext === ".md") {
      return await this.readMarkdown(path);
    } else if (ext === ".html" || ext === ".htm" || ext === ".xhtml") {
      return await this.readHTML(path);
    } else if (ext === ".txt") {
      return await this.readText(path);
    } else {
      return {
        path: path,
        content: "",
        attributes: {},
      };
    }
  }

  public async readHTML(path: string): Promise<Doc> {
    const content = await Deno.readTextFile(path);
    return {
      path: path,
      content: content,
      attributes: {},
    };
  }
  public async readText(path: string): Promise<Doc> {
    const content = await Deno.readTextFile(path);
    return {
      path: path,
      content: content,
      attributes: {},
    };
  }
  public async readMarkdown(path: string): Promise<Doc> {
    const { content, metadata } = await readMarkdownDoc(path);

    return {
      path: path,
      content: content,
      attributes: metadata,
    };
  }
  async watchDir(path: string, {
    onRemove = (_path: string) => {},
    onAdd = (_doc: Doc) => {},
    onChange = (_doc: Doc) => {},
    abort = undefined,
  }: {
    onRemove?: (path: string) => void | Promise<void>;
    onAdd?: (doc: Doc) => void | Promise<void>;
    onChange?: (doc: Doc) => void | Promise<void>;
    abort?: AbortSignal;
  }) {
    const watcher = Deno.watchFs(path);
    if (abort) {
      abort.addEventListener("abort", () => {
        watcher.close();
      });
    }
    for await (const event of watcher) {
      if (
        event.kind === "access" || event.kind === "other" ||
        event.kind === "any"
      ) {
        continue;
      }
      if (event.paths.length === 0) {
        continue;
      }
      for (const path of event.paths) {
        const relpath = relative(Deno.cwd(), path);
        const filename = basename(relpath);
        if (filename === "SUMMARY.md") {
          if (event.kind === "remove") {
            this.doc_map.delete(relpath);
            await onRemove(relpath);
          } else if (event.kind === "create" || event.kind === "modify") {
            const { content, metadata } = await readMarkdownDoc(relpath);
            const doc = {
              path: relpath,
              content: content,
              attributes: metadata,
            };
            this.setDoc(doc);
            if (event.kind === "create") {
              await onAdd(doc);
            } else if (event.kind === "modify") {
              await onChange(doc);
            }
          }
        }
      }
    }
  }

  makeIndex(options?: {
    onUpdate?: (() => void) | (() => Promise<void>);
    abort?: AbortSignal;
    watch?: boolean;
  }) {
    const opt = options ?? {};
    const index = Index.createIndex(this.getDocs());
    if (!opt.watch) {
      return index;
    }
    const update = async () => {
      index.setDocs(this.getDocs());
      if (opt.onUpdate) {
        await opt.onUpdate();
      }
    };
    this.watchDir(".", {
      async onAdd(_doc) {
        await update();
      },
      async onRemove(_path) {
        await update();
      },
      async onChange(_doc) {
        await update();
      },
      abort: opt.abort,
    });
    return index;
  }
}