feat: optimize addList method to handle large content_list
This commit is contained in:
parent
e5d410d809
commit
2a5cb909b5
@ -21,6 +21,20 @@ class SqliteDocumentAccessor implements DocumentAccessor {
|
||||
throw new Error("Method not implemented.");
|
||||
}
|
||||
async addList(content_list: DocumentBody[]): Promise<number[]> {
|
||||
if (content_list.length === 0) return [];
|
||||
if (content_list.length > 256) {
|
||||
// if content_list is too long, split it.
|
||||
const chunkLength = 256;
|
||||
const chunked = [];
|
||||
for (let i = 0; i < content_list.length; i += chunkLength) {
|
||||
chunked.push(content_list.slice(i, i + chunkLength));
|
||||
}
|
||||
const ids = [];
|
||||
for (const chunk of chunked) {
|
||||
ids.push(await this.addList(chunk));
|
||||
}
|
||||
return ids.flat();
|
||||
}
|
||||
return await this.kysely.transaction().execute(async (trx) => {
|
||||
// add tags
|
||||
const tagCollected = new Set<string>();
|
||||
|
Loading…
Reference in New Issue
Block a user