From fa58d6eec471563a246f1bf06838549bead04cb1 Mon Sep 17 00:00:00 2001 From: monoid Date: Wed, 9 Oct 2024 00:14:22 +0900 Subject: [PATCH] db --- deno.lock | 175 +++++++++++++++ packages/client/src/App.tsx | 2 + .../src/components/gallery/WorkQueue.tsx | 206 ++++++++++++++++++ packages/client/src/components/layout/nav.tsx | 9 +- packages/client/src/components/ui/chart.tsx | 5 - packages/client/src/page/taskQueuePage.tsx | 9 + packages/dbtype/src/api.ts | 9 - packages/server/pt.ts | 60 +++++ packages/server/src/content/file.ts | 11 +- packages/server/src/content/video.ts | 6 - 10 files changed, 460 insertions(+), 32 deletions(-) create mode 100644 deno.lock create mode 100644 packages/client/src/components/gallery/WorkQueue.tsx create mode 100644 packages/client/src/page/taskQueuePage.tsx create mode 100644 packages/server/pt.ts delete mode 100644 packages/server/src/content/video.ts diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..77efb12 --- /dev/null +++ b/deno.lock @@ -0,0 +1,175 @@ +{ + "version": "3", + "packages": { + "specifiers": { + "jsr:@db/sqlite": "jsr:@db/sqlite@0.11.1", + "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.5", + "jsr:@std/assert@^0.214.0": "jsr:@std/assert@0.214.0", + "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", + "jsr:@std/encoding@0.214": "jsr:@std/encoding@0.214.0", + "jsr:@std/fmt@0.214": "jsr:@std/fmt@0.214.0", + "jsr:@std/fs@0.214": "jsr:@std/fs@0.214.0", + "jsr:@std/path": "jsr:@std/path@0.217.0", + "jsr:@std/path@0.214": "jsr:@std/path@0.214.0", + "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", + "jsr:@std/path@^0.214.0": "jsr:@std/path@0.214.0" + }, + "jsr": { + "@db/sqlite@0.11.1": { + "integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5", + "dependencies": [ + "jsr:@denosaurs/plug@1", + "jsr:@std/path@0.217" + ] + }, + "@denosaurs/plug@1.0.5": { + "integrity": "04cd988da558adc226202d88c3a434d5fcc08146eaf4baf0cea0c2284b16d2bf", + "dependencies": [ + "jsr:@std/encoding@0.214", + "jsr:@std/fmt@0.214", + "jsr:@std/fs@0.214", + "jsr:@std/path@0.214" + ] + }, + "@std/assert@0.214.0": { + "integrity": "55d398de76a9828fd3b1aa653f4dba3eee4c6985d90c514865d2be9bd082b140" + }, + "@std/assert@0.217.0": { + "integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642" + }, + "@std/encoding@0.214.0": { + "integrity": "30a8713e1db22986c7e780555ffd2fefd1d4f9374d734bb41f5970f6c3352af5" + }, + "@std/fmt@0.214.0": { + "integrity": "40382cff88a0783b347b4d69b94cf931ab8e549a733916718cb866c08efac4d4" + }, + "@std/fs@0.214.0": { + "integrity": "bc880fea0be120cb1550b1ed7faf92fe071003d83f2456a1e129b39193d85bea", + "dependencies": [ + "jsr:@std/assert@^0.214.0", + "jsr:@std/path@^0.214.0" + ] + }, + "@std/path@0.214.0": { + "integrity": "d5577c0b8d66f7e8e3586d864ebdf178bb326145a3611da5a51c961740300285", + "dependencies": [ + "jsr:@std/assert@^0.214.0" + ] + }, + "@std/path@0.217.0": { + "integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11", + "dependencies": [ + "jsr:@std/assert@^0.217.0" + ] + } + } + }, + "remote": { + "https://deno.land/x/sqlite@v3.9.1/build/sqlite.js": "2afc7875c7b9c85d89730c4a311ab3a304e5d1bf761fbadd8c07bbdf130f5f9b", + "https://deno.land/x/sqlite@v3.9.1/build/vfs.js": "7f7778a9fe499cd10738d6e43867340b50b67d3e39142b0065acd51a84cd2e03", + "https://deno.land/x/sqlite@v3.9.1/mod.ts": "e09fc79d8065fe222578114b109b1fd60077bff1bb75448532077f784f4d6a83", + "https://deno.land/x/sqlite@v3.9.1/src/constants.ts": "90f3be047ec0a89bcb5d6fc30db121685fc82cb00b1c476124ff47a4b0472aa9", + "https://deno.land/x/sqlite@v3.9.1/src/db.ts": "03d0c860957496eadedd86e51a6e650670764630e64f56df0092e86c90752401", + "https://deno.land/x/sqlite@v3.9.1/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a", + "https://deno.land/x/sqlite@v3.9.1/src/function.ts": "bc778cab7a6d771f690afa27264c524d22fcb96f1bb61959ade7922c15a4ab8d", + "https://deno.land/x/sqlite@v3.9.1/src/query.ts": "d58abda928f6582d77bad685ecf551b1be8a15e8e38403e293ec38522e030cad", + "https://deno.land/x/sqlite@v3.9.1/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487" + }, + "workspace": { + "packageJson": { + "dependencies": [ + "npm:@biomejs/biome@1.6.3" + ] + }, + "members": { + "packages/client": { + "packageJson": { + "dependencies": [ + "npm:@radix-ui/react-icons@^1.3.0", + "npm:@radix-ui/react-label@^2.1.0", + "npm:@radix-ui/react-popover@^1.1.2", + "npm:@radix-ui/react-progress@^1.1.0", + "npm:@radix-ui/react-radio-group@^1.2.1", + "npm:@radix-ui/react-scroll-area@^1.2.0", + "npm:@radix-ui/react-select@^2.1.2", + "npm:@radix-ui/react-separator@^1.1.0", + "npm:@radix-ui/react-slot@^1.1.0", + "npm:@radix-ui/react-tabs@^1.1.1", + "npm:@radix-ui/react-tooltip@^1.1.3", + "npm:@tanstack/react-virtual@^3.10.8", + "npm:@types/node@^22.7.4", + "npm:@types/react-dom@^18.3.0", + "npm:@types/react@^18.3.11", + "npm:@typescript-eslint/eslint-plugin@^7.18.0", + "npm:@typescript-eslint/parser@^7.18.0", + "npm:@vitejs/plugin-react-swc@^3.7.1", + "npm:autoprefixer@^10.4.20", + "npm:class-variance-authority@^0.7.0", + "npm:clsx@^2.1.1", + "npm:eslint-plugin-react-hooks@^4.6.2", + "npm:eslint-plugin-react-refresh@^0.4.12", + "npm:eslint@^8.57.1", + "npm:jotai@^2.10.0", + "npm:lucide-react@^0.451.0", + "npm:postcss@^8.4.47", + "npm:react-dom@^18.3.1", + "npm:react-resizable-panels@^2.1.4", + "npm:react@^18.3.1", + "npm:recharts@^2.12.7", + "npm:shadcn-ui@^0.8.0", + "npm:swr@^2.2.5", + "npm:tailwind-merge@^2.5.3", + "npm:tailwindcss-animate@^1.0.7", + "npm:tailwindcss@^3.4.13", + "npm:typescript@^5.6.2", + "npm:usehooks-ts@^3.1.0", + "npm:vite@^5.4.8", + "npm:vitest@^2.1.2", + "npm:wouter@^3.3.5" + ] + } + }, + "packages/dbtype": { + "packageJson": { + "dependencies": [ + "npm:@types/better-sqlite3@^7.6.9", + "npm:better-sqlite3@^9.4.3", + "npm:kysely-codegen@^0.14.1", + "npm:kysely@^0.27.3", + "npm:typescript@^5.4.3", + "npm:zod@^3.23.8" + ] + } + }, + "packages/server": { + "packageJson": { + "dependencies": [ + "npm:@types/better-sqlite3@^7.6.11", + "npm:@types/jsonwebtoken@^8.5.9", + "npm:@types/koa-bodyparser@^4.3.12", + "npm:@types/koa-compose@^3.2.8", + "npm:@types/koa-router@^7.4.8", + "npm:@types/koa@^2.15.0", + "npm:@types/node@^22.7.4", + "npm:@types/tiny-async-pool@^1.0.5", + "npm:@zip.js/zip.js@^2.7.52", + "npm:better-sqlite3@^9.6.0", + "npm:chokidar@^3.6.0", + "npm:dotenv@^16.4.5", + "npm:jose@^5.9.3", + "npm:koa-bodyparser@^4.4.1", + "npm:koa-compose@^4.1.0", + "npm:koa-router@^12.0.1", + "npm:koa@^2.15.3", + "npm:kysely@^0.27.4", + "npm:natural-orderby@^2.0.3", + "npm:nodemon@^3.1.7", + "npm:tiny-async-pool@^1.3.0", + "npm:tsx@^4.19.1", + "npm:typescript@^5.6.2" + ] + } + } + } + } +} diff --git a/packages/client/src/App.tsx b/packages/client/src/App.tsx index 78e992f..9c500c1 100644 --- a/packages/client/src/App.tsx +++ b/packages/client/src/App.tsx @@ -16,6 +16,7 @@ import SettingPage from "@/page/settingPage.tsx"; import ComicPage from "@/page/reader/comicPage.tsx"; import DifferencePage from "./page/differencePage.tsx"; import TagsPage from "./page/tagsPage.tsx"; +import TaskQueuePage from "./page/taskQueuePage.tsx"; const App = () => { const { isDarkMode } = useTernaryDarkMode(); @@ -42,6 +43,7 @@ const App = () => { + diff --git a/packages/client/src/components/gallery/WorkQueue.tsx b/packages/client/src/components/gallery/WorkQueue.tsx new file mode 100644 index 0000000..bb89fb8 --- /dev/null +++ b/packages/client/src/components/gallery/WorkQueue.tsx @@ -0,0 +1,206 @@ +import { useState, useEffect } from 'react' +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" +import { Button } from "@/components/ui/button" +import { Progress } from "@/components/ui/progress" +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer } from 'recharts' + +interface Task { + id: string; + status: "Processed" | "Processing" | "Queued" | "Exception"; + createdAt: Date; + title: string; + description: string; + expectedProgress: number; + currentJobDescription: string; +} + +const generateMockTasks = (): Task[] => { + const statuses: Task['status'][] = ["Processed", "Processing", "Queued", "Exception"]; + return Array.from({ length: 50 }, (_, i) => ({ + id: `task-${i + 1}`, + status: statuses[Math.floor(Math.random() * statuses.length)], + createdAt: new Date(Date.now() - Math.random() * 10000000000), + title: `Task ${i + 1}`, + description: `This is a description for Task ${i + 1}`, + expectedProgress: Math.random(), + currentJobDescription: `Current job for Task ${i + 1}` + })); +}; + +export default function DynamicWorkQueue() { + const [tasks, setTasks] = useState([]) + + useEffect(() => { + setTasks(generateMockTasks()); + + const intervalId = setInterval(() => { + setTasks(prevTasks => { + let newTasks: Task[] = prevTasks; + // if processed tasks are more than 10, remove the oldest one + if (newTasks.filter(task => task.status === "Processed").length > 10) { + newTasks = newTasks.filter(task => task.status !== "Processed"); + } + // update the progress of each task + newTasks = newTasks.map(task => ({ + ...task, + expectedProgress: Math.min(1, task.expectedProgress + Math.random() * 0.2), + status: task.expectedProgress >= 1 ? "Processed" : task.status + })); + // if there are no queued tasks, add a new one + if (newTasks.filter(task => task.status === "Queued").length === 0) { + newTasks.push({ + id: `task-${newTasks.length + 1}`, + status: "Queued", + createdAt: new Date(), + title: `Task ${newTasks.length + 1}`, + description: `This is a description for Task ${newTasks.length + 1}`, + expectedProgress: 0, + currentJobDescription: `Current job for Task ${newTasks.length + 1}` + }); + } + return newTasks; + }); + }, 5000); + + return () => clearInterval(intervalId); + }, []) + + const updateTaskStatus = (taskId: string, newStatus: Task['status']) => { + setTasks(prevTasks => + prevTasks.map(task => + task.id === taskId ? { ...task, status: newStatus } : task + ) + ); + } + + const renderTaskList = (status: Task['status']) => { + const filteredTasks = tasks.filter(task => task.status === status) + + return ( + <> + {filteredTasks.length === 0 ? ( +

No tasks

+ ) : ( +
    + {filteredTasks.map(task => ( +
  • +

    {task.title}

    +

    {task.description}

    +

    Created: {task.createdAt.toLocaleString()}

    +
    + +

    {Math.round(task.expectedProgress * 100)}%

    +
    +

    {task.currentJobDescription}

    + {status === "Queued" && ( + + )} + {status === "Processing" && ( + + )} +
    • + ))} +
+ )} + + ) + } + + const renderProcessedTaskSummary = () => { + const processedTasks = tasks.filter(task => task.status === "Processed"); + const totalProcessed = processedTasks.length; + const averageProgress = processedTasks.reduce((sum, task) => sum + task.expectedProgress, 0) / totalProcessed || 0; + + const chartData = [ + { name: 'Processed', value: totalProcessed }, + { name: 'Remaining', value: tasks.length - totalProcessed }, + ]; + + return ( +
+
+ + + Total Processed + + +

{totalProcessed}

+ + + + + Average Progress + + +

{(averageProgress * 100).toFixed(2)}%

+ + +
+ + + Processed vs Remaining Tasks + + + + + + + + + + + + +
+ ) + } + + return ( +
+ + + Dynamic Work Queue + + + + + Queued ({tasks.filter(t => t.status === "Queued").length}) + Processing ({tasks.filter(t => t.status === "Processing").length}) + Processed ({tasks.filter(t => t.status === "Processed").length}) + Exception ({tasks.filter(t => t.status === "Exception").length}) + Summary + + + {renderTaskList("Queued")} + + + {renderTaskList("Processing")} + + + {renderTaskList("Processed")} + + + {renderTaskList("Exception")} + + + {renderProcessedTaskSummary()} + + + + +
+ ) +} diff --git a/packages/client/src/components/layout/nav.tsx b/packages/client/src/components/layout/nav.tsx index 89da992..5ea962f 100644 --- a/packages/client/src/components/layout/nav.tsx +++ b/packages/client/src/components/layout/nav.tsx @@ -1,5 +1,5 @@ import { Link } from "wouter" -import { Search, Settings, Tags, ArchiveIcon, UserIcon } from "lucide-react" +import { SearchIcon, SettingsIcon, TagsIcon, ArchiveIcon, UserIcon, LayoutListIcon } from "lucide-react" import { Button, buttonVariants } from "@/components/ui/button.tsx" import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip.tsx" import { useLogin } from "@/state/user.ts"; @@ -66,13 +66,14 @@ export function NavList() { return } \ No newline at end of file diff --git a/packages/client/src/components/ui/chart.tsx b/packages/client/src/components/ui/chart.tsx index b58b494..a21d77e 100644 --- a/packages/client/src/components/ui/chart.tsx +++ b/packages/client/src/components/ui/chart.tsx @@ -1,10 +1,5 @@ import * as React from "react" import * as RechartsPrimitive from "recharts" -import { - NameType, - Payload, - ValueType, -} from "recharts/types/component/DefaultTooltipContent" import { cn } from "@/lib/utils" diff --git a/packages/client/src/page/taskQueuePage.tsx b/packages/client/src/page/taskQueuePage.tsx new file mode 100644 index 0000000..9e8de0b --- /dev/null +++ b/packages/client/src/page/taskQueuePage.tsx @@ -0,0 +1,9 @@ +import DynamicWorkQueue from "@/components/gallery/WorkQueue"; + +export default function TaskQueuePage(){ + return ( +
+ +
+ ) +} \ No newline at end of file diff --git a/packages/dbtype/src/api.ts b/packages/dbtype/src/api.ts index 99ef984..120f01f 100644 --- a/packages/dbtype/src/api.ts +++ b/packages/dbtype/src/api.ts @@ -59,16 +59,7 @@ export const SchemaMigrationSchema = z.object({ export type SchemaMigration = z.infer; -const WorkStatusEnum = z.enum(["pending", "done", "error"]); -export const WorkSchema = z.object({ - uuid: z.string(), - type: z.literal("rehash"), - status: WorkStatusEnum, - detail: z.string(), -}); - -export type Work = z.infer; export const QueryListOptionSchema = z.object({ /** diff --git a/packages/server/pt.ts b/packages/server/pt.ts new file mode 100644 index 0000000..c02b131 --- /dev/null +++ b/packages/server/pt.ts @@ -0,0 +1,60 @@ + import { Database } from "jsr:@db/sqlite"; +import {join} from "jsr:@std/path"; +let db = new Database("./db.sqlite3") +const stmt = db.prepare("SELECT id, basepath, filename from document"); +let ds = [...stmt.all()]; + +async function oshash( + path: string +){ + const chunkSize = 4096; + const minFileSize = chunkSize * 2; + + const fd = await Deno.open(path); + const st = await fd.stat(); + let hash = BigInt(st.size); + + if (st.size < minFileSize){ + throw new Error("File is too small to hash"); + } + + // read first and last chunk + const firstChunk = new Uint8Array(chunkSize); + await fd.read(firstChunk, 0, chunkSize, 0); + const lastChunk = new Uint8Array(chunkSize); + await fd.read(lastChunk, 0, chunkSize, st.size - chunkSize); + // iterate over first and last chunk. + // for each uint64_t, add it to the hash. + const firstChunkView = new DataView(firstChunk.buffer); + for (let i = 0; i < chunkSize; i += 8){ + hash += firstChunkView.getBigUint64(i, true); + // prevent overflow + hash = (hash & 0xFFFFFFFFFFFFFFFFn); + } + const lastChunkView = new DataView(lastChunk.buffer); + for (let i = 0; i < chunkSize; i += 8){ + hash += lastChunkView.getBigUint64(i, true); + // prevent overflow + hash = (hash & 0xFFFFFFFFFFFFFFFFn); + } + return hash; +} + +async function updateHash(ds: {id: number, basepath: string, filename: string}[]) { + const content_hashs = await Promise.all(ds.map(async (d) => { + const p = join(d.basepath, d.filename); + return await oshash(p); + })); + db.transaction(() => { + for (let i = 0; i < ds.length; i++) { + db.run(`UPDATE document SET content_hash = ? where id = ?`, content_hashs[i].toString(), ds[i].id) + } + })(); +} + +for (let i = 0; i < ds.length; i += 32) { + const d = ds.slice(i, i + 32); + console.log(d.map(x => x.id)); + await updateHash(d); +} +db.close(); \ No newline at end of file diff --git a/packages/server/src/content/file.ts b/packages/server/src/content/file.ts index cd9ce15..d33bbc4 100644 --- a/packages/server/src/content/file.ts +++ b/packages/server/src/content/file.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { promises, type Stats } from "node:fs"; import path, { extname } from "node:path"; import type { DocumentBody } from "dbtype"; +import { oshash } from "src/util/oshash.ts"; /** * content file or directory referrer */ @@ -60,14 +61,8 @@ export const createDefaultClass = (type: string): ContentFileConstructor => { } async getHash(): Promise { if (this.hash !== undefined) return this.hash; - this.stat = await promises.stat(this.path); - const hash = createHash("sha512"); - hash.update(extname(this.path)); - hash.update(this.stat.mode.toString()); - // if(this.desc !== undefined) - // hash.update(JSON.stringify(this.desc)); - hash.update(this.stat.size.toString()); - this.hash = hash.digest("base64"); + const hash = await oshash(this.path); + this.hash = hash.toString(); return this.hash; } async getMtime(): Promise { diff --git a/packages/server/src/content/video.ts b/packages/server/src/content/video.ts deleted file mode 100644 index d0897c8..0000000 --- a/packages/server/src/content/video.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { registerContentReferrer } from "./file.ts"; -import { createDefaultClass } from "./file.ts"; - -export class VideoReferrer extends createDefaultClass("video") { -} -registerContentReferrer(VideoReferrer);