60 lines
1.9 KiB
TypeScript
60 lines
1.9 KiB
TypeScript
|
import { Database } from "jsr:@db/sqlite";
|
||
|
|
import {join} from "jsr:@std/path";
|
||
|
|
let db = new Database("./db.sqlite3")
|
||
|
|
const stmt = db.prepare("SELECT id, basepath, filename from document");
|
||
|
|
let ds = [...stmt.all()];
|
||
|
|
|
||
|
|
async function oshash(
|
||
|
|
path: string
|
||
|
|
){
|
||
|
|
const chunkSize = 4096;
|
||
|
|
const minFileSize = chunkSize * 2;
|
||
|
|
|
||
|
|
const fd = await Deno.open(path);
|
||
|
|
const st = await fd.stat();
|
||
|
|
let hash = BigInt(st.size);
|
||
|
|
|
||
|
|
if (st.size < minFileSize){
|
||
|
|
throw new Error("File is too small to hash");
|
||
|
|
}
|
||
|
|
|
||
|
|
// read first and last chunk
|
||
|
|
const firstChunk = new Uint8Array(chunkSize);
|
||
|
|
await fd.read(firstChunk, 0, chunkSize, 0);
|
||
|
|
const lastChunk = new Uint8Array(chunkSize);
|
||
|
|
await fd.read(lastChunk, 0, chunkSize, st.size - chunkSize);
|
||
|
|
// iterate over first and last chunk.
|
||
|
|
// for each uint64_t, add it to the hash.
|
||
|
|
const firstChunkView = new DataView(firstChunk.buffer);
|
||
|
|
for (let i = 0; i < chunkSize; i += 8){
|
||
|
|
hash += firstChunkView.getBigUint64(i, true);
|
||
|
|
// prevent overflow
|
||
|
|
hash = (hash & 0xFFFFFFFFFFFFFFFFn);
|
||
|
|
}
|
||
|
|
const lastChunkView = new DataView(lastChunk.buffer);
|
||
|
|
for (let i = 0; i < chunkSize; i += 8){
|
||
|
|
hash += lastChunkView.getBigUint64(i, true);
|
||
|
|
// prevent overflow
|
||
|
|
hash = (hash & 0xFFFFFFFFFFFFFFFFn);
|
||
|
|
}
|
||
|
|
return hash;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function updateHash(ds: {id: number, basepath: string, filename: string}[]) {
|
||
|
|
const content_hashs = await Promise.all(ds.map(async (d) => {
|
||
|
|
const p = join(d.basepath, d.filename);
|
||
|
|
return await oshash(p);
|
||
|
|
}));
|
||
|
|
db.transaction(() => {
|
||
|
|
for (let i = 0; i < ds.length; i++) {
|
||
|
|
db.run(`UPDATE document SET content_hash = ? where id = ?`, content_hashs[i].toString(), ds[i].id)
|
||
|
|
}
|
||
|
|
})();
|
||
|
|
}
|
||
|
|
|
||
|
|
for (let i = 0; i < ds.length; i += 32) {
|
||
|
|
const d = ds.slice(i, i + 32);
|
||
|
|
console.log(d.map(x => x.id));
|
||
|
|
await updateHash(d);
|
||
|
|
}
|
||
|
|
db.close();
|