monoid/ionian
1
0
Fork 0
Code Issues 9 Pull Requests Projects Releases Wiki Activity

add diff

Browse Source
This commit is contained in:
monoid 2021-01-15 18:43:36 +09:00
parent ec5465f2d2
commit cade73da87
32 changed files with 639 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
app.ts
View File

@ -1,5 +1,5 @@
import { app, BrowserWindow, session, dialog } from "electron"; import { app, BrowserWindow, session, dialog } from "electron";
import { get_setting } from "./src/setting"; import { get_setting } from "./src/SettingConfig";
import { create_server, start_server } from "./src/server"; import { create_server, start_server } from "./src/server";
import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login"; import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login";
@ -14,8 +14,8 @@ if (!setting.cli) {
center: true, center: true,
useContentSize: true, useContentSize: true,
}); });
//await window.loadURL(`data:text/html;base64,`+Buffer.from(get_loading_html()).toString('base64')); await wnd.loadURL(`data:text/html;base64,`+Buffer.from(loading_html).toString('base64'));
await wnd.loadFile('../loading.html'); //await wnd.loadURL('../loading.html');
await session.defaultSession.cookies.set({ await session.defaultSession.cookies.set({
url:`http://localhost:${setting.port}`, url:`http://localhost:${setting.port}`,
name:accessTokenName, name:accessTokenName,
@ -88,3 +88,35 @@ if (!setting.cli) {
start_server(server); start_server(server);
})(); })();
} }
const loading_html = `<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>`;

48
gen_conf_schema.ts Normal file
View File

@ -0,0 +1,48 @@
import { promises } from 'fs';
const { readdir, writeFile } = promises;
import {createGenerator} from 'ts-json-schema-generator';
import {dirname,join} from 'path';
async function genSchema(path:string,typename:string){
const gen = createGenerator({
path:path,
type:typename,
tsconfig:"tsconfig.json"
});
const schema = gen.createSchema(typename);
if(schema.definitions != undefined){
const definitions = schema.definitions;
const definition = definitions[typename];
if(typeof definition == "object" ){
let property = definition.properties;
if(property){
property['$schema'] = {
type:"string"
};
}
}
}
const text = JSON.stringify(schema);
await writeFile(join(dirname(path),`${typename}.schema.json`),text);
}
function capitalize(s:string){
return s.charAt(0).toUpperCase() + s.slice(1);
}
async function setToALL(path:string) {
console.log(`scan ${path}`)
const direntry = await readdir(path,{withFileTypes:true});
const works = direntry.filter(x=>x.isFile()&&x.name.endsWith("Config.ts")).map(x=>{
const name = x.name;
const m = /(.+)\.ts/.exec(name);
if(m !== null){
const typename = m[1];
return genSchema(join(path,typename),capitalize(typename));
}
})
await Promise.all(works);
const subdir = direntry.filter(x=>x.isDirectory()).map(x=>x.name);
for(const x of subdir){
await setToALL(join(path,x));
}
}
setToALL("src")

32
loading.html
View File

@ -1,32 +0,0 @@
<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>

3
migrations/initial.ts
View File

@ -14,7 +14,8 @@ export async function up(knex:Knex) {
b.string("filename",256).notNullable().comment("filename"); b.string("filename",256).notNullable().comment("filename");
b.string("content_hash").nullable(); b.string("content_hash").nullable();
b.json("additional").nullable(); b.json("additional").nullable();
b.timestamps(); b.integer("created_at").notNullable();
b.integer("deleted_at");
b.index("content_type","content_type_index"); b.index("content_type","content_type_index");
}); });
await knex.schema.createTable("tags", (b)=>{ await knex.schema.createTable("tags", (b)=>{

4
package.json
View File

@ -18,7 +18,8 @@
"files": [ "files": [
"build/**/*", "build/**/*",
"node_modules/**/*", "node_modules/**/*",
"package.json" "package.json",
"!node_modules/@material-ui/**/*"
], ],
"appId": "com.prelude.ionian.app", "appId": "com.prelude.ionian.app",
"productName": "Ionian", "productName": "Ionian",
@ -82,6 +83,7 @@
"eslint-plugin-node": "^11.1.0", "eslint-plugin-node": "^11.1.0",
"mini-css-extract-plugin": "^1.3.3", "mini-css-extract-plugin": "^1.3.3",
"style-loader": "^2.0.0", "style-loader": "^2.0.0",
"ts-json-schema-generator": "^0.82.0",
"ts-node": "^9.1.1", "ts-node": "^9.1.1",
"typescript": "^4.1.3", "typescript": "^4.1.3",
"webpack": "^5.11.0", "webpack": "^5.11.0",

1
preload.ts Normal file
View File

@ -0,0 +1 @@
import {} from 'electron';

66
src/SettingConfig.schema.json Normal file
View File

@ -0,0 +1,66 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/SettingConfig",
"definitions": {
"SettingConfig": {
"type": "object",
"properties": {
"localmode": {
"type": "boolean",
"description": "if true, server will bind on '127.0.0.1' rather than '0.0.0.0'"
},
"guest": {
"type": "array",
"items": {
"$ref": "#/definitions/Permission"
},
"description": "guest permission"
},
"jwt_secretkey": {
"type": "string",
"description": "JWT secret key. if you change its value, all access tokens are invalidated."
},
"port": {
"type": "number",
"description": "the port which running server is binding on."
},
"mode": {
"type": "string",
"enum": [
"development",
"production"
]
},
"cli": {
"type": "boolean",
"description": "if true, do not show 'electron' window and show terminal only."
},
"forbid_remote_admin_login": {
"type": "boolean",
"description": "forbid to login admin from remote client. but, it do not invalidate access token. \r if you want to invalidate access token, change 'jwt_secretkey'."
},
"$schema": {
"type": "string"
}
},
"required": [
"localmode",
"guest",
"jwt_secretkey",
"port",
"mode",
"cli",
"forbid_remote_admin_login"
],
"additionalProperties": false
},
"Permission": {
"type": "string",
"enum": [
"ModifyTag",
"QueryContent",
"ModifyTagDesc"
]
}
}
}

15
src/setting.ts → src/SettingConfig.ts
View File

@ -1,9 +1,8 @@
import { Settings } from '@material-ui/icons';
import { randomBytes } from 'crypto'; import { randomBytes } from 'crypto';
import { existsSync, readFileSync, writeFileSync } from 'fs'; import { existsSync, readFileSync, writeFileSync } from 'fs';
import { Permission } from './permission/permission'; import { Permission } from './permission/permission';
export type Setting = { export interface SettingConfig {
/** /**
* if true, server will bind on '127.0.0.1' rather than '0.0.0.0' * if true, server will bind on '127.0.0.1' rather than '0.0.0.0'
*/ */
@ -31,7 +30,7 @@ export type Setting = {
* if you want to invalidate access token, change 'jwt_secretkey'.*/ * if you want to invalidate access token, change 'jwt_secretkey'.*/
forbid_remote_admin_login:boolean, forbid_remote_admin_login:boolean,
} }
const default_setting:Setting = { const default_setting:SettingConfig = {
localmode: true, localmode: true,
guest:[], guest:[],
@ -41,15 +40,15 @@ const default_setting:Setting = {
cli:false, cli:false,
forbid_remote_admin_login:true, forbid_remote_admin_login:true,
} }
let setting: null|Setting = null; let setting: null|SettingConfig = null;
const setEmptyToDefault = (target:any,default_table:Setting)=>{ const setEmptyToDefault = (target:any,default_table:SettingConfig)=>{
let diff_occur = false; let diff_occur = false;
for(const key in default_table){ for(const key in default_table){
if(key === undefined || key in target){ if(key === undefined || key in target){
continue; continue;
} }
target[key] = default_table[key as keyof Setting]; target[key] = default_table[key as keyof SettingConfig];
diff_occur = true; diff_occur = true;
} }
return diff_occur; return diff_occur;
@ -61,9 +60,9 @@ export const read_setting_from_file = ()=>{
if(partial_occur){ if(partial_occur){
writeFileSync("settings.json",JSON.stringify(ret)); writeFileSync("settings.json",JSON.stringify(ret));
} }
return ret as Setting; return ret as SettingConfig;
} }
export function get_setting():Setting{ export function get_setting():SettingConfig{
if(setting === null){ if(setting === null){
setting = read_setting_from_file(); setting = read_setting_from_file();
const env = process.env.NODE_ENV || 'development'; const env = process.env.NODE_ENV || 'development';

1
src/client/accessor/document.ts
View File

@ -44,6 +44,7 @@ export class ClientDocumentAccessor implements DocumentAccessor{
return ret; return ret;
} }
async add(c: DocumentBody): Promise<number>{ async add(c: DocumentBody): Promise<number>{
throw new Error("not allow");
const res = await fetch(`${baseurl}`,{ const res = await fetch(`${baseurl}`,{
method: "POST", method: "POST",
body: JSON.stringify(c) body: JSON.stringify(c)

46
src/client/page/difference.tsx
View File

@ -1,11 +1,51 @@
import React from 'react'; import React, { useContext, useEffect, useState } from 'react';
import { CommonMenuList, Headline } from "../component/mod"; import { CommonMenuList, Headline } from "../component/mod";
import { UserContext } from "../state"; import { UserContext } from "../state";
import { Grid, Typography } from "@material-ui/core"; import { Grid, Paper, Typography } from "@material-ui/core";
export function DifferencePage(){ export function DifferencePage(){
const ctx = useContext(UserContext);
const [diffList,setDiffList] = useState<
{type:string,value:{path:string,type:string}[]}[]
>([]);
const doLoad = async ()=>{
const list = await fetch('/api/diff/list');
if(list.ok){
const inner = await list.json();
setDiffList(inner);
}
else{
//setDiffList([]);
}
};
useEffect(
()=>{
doLoad();
const i = setInterval(doLoad,5000);
return ()=>{
clearInterval(i);
}
},[]
)
const Commit = async(x:{type:string,path:string})=>{
const res = await fetch('/api/diff/commit',{
method:'POST',
body: JSON.stringify([{...x}]),
headers:{
'content-type':'application/json'
}
});
const bb = await res.json();
if(bb.ok){
doLoad();
}
}
const menu = CommonMenuList(); const menu = CommonMenuList();
return (<Headline menu={menu}> return (<Headline menu={menu}>
<div>Not implemented</div> {diffList.map(x=><Paper key={x.type}>
<Typography variant='h3'>{x.type}</Typography>
{x.value.map(y=><Typography variant='h5' onClick={()=>Commit(y)}>{y.path}</Typography>)}
</Paper>)}
</Headline>) </Headline>)
} }

46
src/content/file.ts
View File

@ -3,37 +3,65 @@ import Router from 'koa-router';
import {createHash} from 'crypto'; import {createHash} from 'crypto';
import {promises} from 'fs' import {promises} from 'fs'
import {extname} from 'path'; import {extname} from 'path';
import { DocumentBody } from '../model/mod';
import path from 'path';
/** /**
* content file or directory referrer * content file or directory referrer
*/ */
export interface ContentFile{ export interface ContentFile{
getHash():Promise<string>; getHash():Promise<string>;
getDesc():Promise<object|null>; createDocumentBody():Promise<DocumentBody>;
readonly path: string; readonly path: string;
readonly type: string; readonly type: string;
} }
type ContentFileConstructor = (new (path:string,desc?:object) => ContentFile)&{content_type:string}; export type ContentConstructOption = {
hash: string,
tags: string[],
title: string,
additional: JSONMap
}
type ContentFileConstructor = (new (path:string,option?:ContentConstructOption) => ContentFile)&{content_type:string};
export const createDefaultClass = (type:string):ContentFileConstructor=>{ export const createDefaultClass = (type:string):ContentFileConstructor=>{
let cons = class implements ContentFile{ let cons = class implements ContentFile{
readonly path: string; readonly path: string;
type = type; //type = type;
static content_type = type; static content_type = type;
protected hash: string| undefined;
constructor(path:string,option?:object){ constructor(path:string,option?:ContentConstructOption){
this.path = path; this.path = path;
this.hash = option?.hash;
}
async createDocumentBody(): Promise<DocumentBody> {
const {base,dir, name} = path.parse(this.path);
const ret = {
title : name,
basepath : dir,
additional: {},
content_type: cons.content_type,
filename: base,
tags: [],
content_hash: await this.getHash(),
} as DocumentBody;
return ret;
}
get type():string{
return cons.content_type;
} }
async getDesc(): Promise<object|null> { async getDesc(): Promise<object|null> {
return null; return null;
} }
async getHash():Promise<string>{ async getHash():Promise<string>{
if(this.hash !== undefined) return this.hash;
const stat = await promises.stat(this.path); const stat = await promises.stat(this.path);
const hash = createHash("sha512"); const hash = createHash("sha512");
hash.update(extname(this.type)); hash.update(extname(this.path));
hash.update(stat.mode.toString()); hash.update(stat.mode.toString());
//if(this.desc !== undefined) //if(this.desc !== undefined)
// hash.update(JSON.stringify(this.desc)); // hash.update(JSON.stringify(this.desc));
hash.update(stat.size.toString()); hash.update(stat.size.toString());
return hash.digest("base64"); this.hash = hash.digest("base64");
return this.hash;
} }
}; };
return cons; return cons;
@ -43,11 +71,11 @@ export function registerContentReferrer(s: ContentFileConstructor){
console.log(`registered content type: ${s.content_type}`) console.log(`registered content type: ${s.content_type}`)
ContstructorTable[s.content_type] = s; ContstructorTable[s.content_type] = s;
} }
export function createContentFile(type:string,path:string,option?:object){ export function createContentFile(type:string,path:string,option?:ContentConstructOption){
const constructorMethod = ContstructorTable[type]; const constructorMethod = ContstructorTable[type];
if(constructorMethod === undefined){ if(constructorMethod === undefined){
console.log(type); console.log(`${type} are not in ${JSON.stringify(ContstructorTable)}`);
throw new Error("undefined"); throw new Error("construction method of the content type is undefined");
} }
return new constructorMethod(path,option); return new constructorMethod(path,option);
} }

7
src/content/manga.ts
View File

@ -1,10 +1,11 @@
import {ContentFile} from './file'; import {ContentFile, createDefaultClass,registerContentReferrer, ContentConstructOption} from './file';
import {createDefaultClass,registerContentReferrer} from './file';
import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap'; import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap';
export class MangaReferrer extends createDefaultClass("manga"){ export class MangaReferrer extends createDefaultClass("manga"){
desc: object|null|undefined; desc: object|null|undefined;
constructor(path:string,option?:object|undefined){ additional: object| undefined;
constructor(path:string,option?:ContentConstructOption){
super(path); super(path);
this.additional = option;
} }
async getDesc(){ async getDesc(){
if(this.desc !== undefined){ if(this.desc !== undefined){

4
src/content/video.ts
View File

@ -1,8 +1,8 @@
import {ContentFile, registerContentReferrer} from './file'; import {ContentFile, registerContentReferrer, ContentConstructOption} from './file';
import {createDefaultClass} from './file'; import {createDefaultClass} from './file';
export class VideoReferrer extends createDefaultClass("video"){ export class VideoReferrer extends createDefaultClass("video"){
constructor(path:string,desc?:object){ constructor(path:string,desc?:ContentConstructOption){
super(path,desc); super(path,desc);
} }
} }

2
src/database.ts
View File

@ -1,7 +1,7 @@
import { existsSync } from 'fs'; import { existsSync } from 'fs';
import Knex from 'knex'; import Knex from 'knex';
import {Knex as KnexConfig} from './config'; import {Knex as KnexConfig} from './config';
import { get_setting } from './setting'; import { get_setting } from './SettingConfig';
export async function connectDB(){ export async function connectDB(){
const config = KnexConfig.config; const config = KnexConfig.config;

30
src/db/doc.ts
View File

@ -8,7 +8,7 @@ type DBTagContentRelation = {
tag_name:string tag_name:string
} }
class KnexContentsAccessor implements DocumentAccessor{ class KnexDocumentAccessor implements DocumentAccessor{
knex : Knex; knex : Knex;
tagController: TagAccessor; tagController: TagAccessor;
constructor(knex : Knex){ constructor(knex : Knex){
@ -19,6 +19,7 @@ class KnexContentsAccessor implements DocumentAccessor{
const {tags,additional, ...rest} = c; const {tags,additional, ...rest} = c;
const id_lst = await this.knex.insert({ const id_lst = await this.knex.insert({
additional:JSON.stringify(additional), additional:JSON.stringify(additional),
created_at:Date.now(),
...rest ...rest
}).into('document'); }).into('document');
const id = id_lst[0]; const id = id_lst[0];
@ -53,9 +54,20 @@ class KnexContentsAccessor implements DocumentAccessor{
return { return {
...first, ...first,
tags:ret_tags, tags:ret_tags,
additional: JSON.parse(first.additional || "{}"), additional: first.additional !== null ? JSON.parse(first.additional) : {},
}; };
}; };
async findDeleted(content_type:string){
const s = await this.knex.select("*")
.where({content_type:content_type})
.whereNotNull("update_at")
.from("document");
return s.map(x=>({
...x,
tags:[],
additional:{}
}));
}
async findList(option?:QueryListOption){ async findList(option?:QueryListOption){
option = option || {}; option = option || {};
const allow_tag = option.allow_tag || []; const allow_tag = option.allow_tag || [];
@ -94,6 +106,7 @@ class KnexContentsAccessor implements DocumentAccessor{
} }
} }
query = query.limit(limit); query = query.limit(limit);
query = query.orderBy('id',"desc");
return query; return query;
} }
let query = buildquery(); let query = buildquery();
@ -119,13 +132,14 @@ class KnexContentsAccessor implements DocumentAccessor{
} }
return result; return result;
}; };
async findListByBasePath(path:string):Promise<Document[]>{ async findByPath(path:string,filename?:string):Promise<Document[]>{
let results = await this.knex.select("*").from("document").where({basepath:path}); const e = filename == undefined ? {} : {filename:filename}
const results = await this.knex.select("*").from("document").where({basepath:path,...e});
return results.map(x=>({ return results.map(x=>({
...x, ...x,
tags:[], tags:[],
additional:JSON.parse(x.additional || "{}"), additional:{}
})); }))
} }
async update(c:Partial<Document> & { id:number }){ async update(c:Partial<Document> & { id:number }){
const {id,tags,...rest} = c; const {id,tags,...rest} = c;
@ -150,6 +164,6 @@ class KnexContentsAccessor implements DocumentAccessor{
return true; return true;
} }
} }
export const createKnexContentsAccessor = (knex:Knex): DocumentAccessor=>{ export const createKnexDocumentAccessor = (knex:Knex): DocumentAccessor=>{
return new KnexContentsAccessor(knex); return new KnexDocumentAccessor(knex);
} }

1
src/diff/MangaConfig.schema.json Normal file
View File

@ -0,0 +1 @@
{"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/MangaConfig","definitions":{"MangaConfig":{"type":"object","properties":{"watch":{"type":"array","items":{"type":"string"}},"$schema":{"type":"string"}},"required":["watch"],"additionalProperties":false}}}

6
src/diff/MangaConfig.ts Normal file
View File

@ -0,0 +1,6 @@
import Schema from './MangaConfig.schema.json';
export interface MangaConfig{
watch:string[]
}

74
src/diff/content_handler.ts Normal file
View File

@ -0,0 +1,74 @@
import {join as pathjoin} from 'path';
import {Document, DocumentAccessor} from '../model/mod';
import { ContentFile, createContentFile } from '../content/mod';
import {IDiffWatcher} from './watcher';
import {ContentList} from './content_list';
//refactoring needed.
export class ContentDiffHandler{
waiting_list:ContentList;
tombstone: Map<string,Document>;//hash, contentfile
doc_cntr: DocumentAccessor;
content_type: string;
constructor(cntr: DocumentAccessor,content_type:string){
this.waiting_list = new ContentList();
this.tombstone = new Map<string,Document>();
this.doc_cntr = cntr;
this.content_type = content_type;
}
async setup(){
const deleted = await this.doc_cntr.findDeleted(this.content_type);
for (const it of deleted) {
this.tombstone.set(it.content_hash,it);
}
}
register(diff:IDiffWatcher){
diff.on('create',(filename)=>this.OnCreated(diff.path,filename))
.on('delete',(filename)=>this.OnDeleted(diff.path,filename))
.on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename));
}
private async OnDeleted(basepath:string,filename:string){
const cpath = pathjoin(basepath,filename);
if(this.waiting_list.hasPath(cpath)){
this.waiting_list.deletePath(cpath);
return;
}
const dbc = await this.doc_cntr.findByPath(basepath,filename);
if(dbc.length === 0) return; //ignore
if(this.waiting_list.hasHash(dbc[0].content_hash)){
//if path changed, update changed path.
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
//db invalidate
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: Date.now(),
});
this.tombstone.set(dbc[0].content_hash, dbc[0]);
}
private async OnCreated(basepath:string,filename:string){
const content = createContentFile(this.content_type,pathjoin(basepath,filename));
const hash = await content.getHash();
const c = this.tombstone.get(hash);
if(c !== undefined){
this.doc_cntr.update({
id: c.id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
this.waiting_list.set(content);
}
private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){
const doc = await this.doc_cntr.findByPath(basepath,prev_filename);
await this.doc_cntr.update({...doc[0],filename:cur_filename});
}
}

62
src/diff/content_list.ts Normal file
View File

@ -0,0 +1,62 @@
import { ContentFile } from '../content/mod';
import event from 'events';
interface ContentListEvent{
'set':(c:ContentFile)=>void,
'delete':(c:ContentFile)=>void,
}
export class ContentList extends event.EventEmitter{
cl:Map<string,ContentFile>;
hl:Map<string,ContentFile>;
on<U extends keyof ContentListEvent>(event:U,listener:ContentListEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof ContentListEvent>(event:U,...arg:Parameters<ContentListEvent[U]>): boolean{
return super.emit(event,...arg);
}
constructor(){
super();
this.cl = new Map;
this.hl = new Map;
}
hasHash(s:string){
return this.hl.has(s);
}
hasPath(p:string){
return this.cl.has(p);
}
getHash(s:string){
return this.hl.get(s)
}
getPath(p:string){
return this.cl.get(p);
}
async set(c:ContentFile){
const path = c.path;
const hash = await c.getHash();
this.cl.set(path,c);
this.hl.set(hash,c);
this.emit('set',c);
}
async delete(c:ContentFile){
let r = true;
r &&= this.cl.delete(c.path);
r &&= this.hl.delete(await c.getHash());
this.emit('delete',c);
return r;
}
async deletePath(p:string){
const o = this.getPath(p);
if(o === undefined) return false;
return this.delete(o);
}
async deleteHash(s:string){
const o = this.getHash(s);
if(o === undefined) return false;
return this.delete(o);
}
getAll(){
return [...this.cl.values()];
}
}

109
src/diff/diff.ts
View File

@ -1,82 +1,39 @@
import { watch } from 'fs'; import { DocumentAccessor } from '../model/doc';
import { promises } from 'fs'; import {ContentDiffHandler} from './content_handler';
import { ContentFile, createContentReferrer, getContentRefererConstructor } from '../content/referrer' import { CommonDiffWatcher } from './watcher';
import path from 'path'; //import {join as pathjoin} from 'path';
export class DiffManager{
const readdir = promises.readdir; watching: {[content_type:string]:ContentDiffHandler};
doc_cntr: DocumentAccessor;
constructor(contorller: DocumentAccessor){
export class Watcher{ this.watching = {};
private _type: string; this.doc_cntr = contorller;
private _path:string;
/**
* @todo : alter type Map<string,ContentReferrer>
*/
private _added: ContentFile[];
private _deleted: ContentFile[];
constructor(path:string,type:string){
this._path = path;
this._added =[];
this._deleted =[];
this._type = type;
} }
public get added() : ContentFile[] { async register(content_type:string,path:string){
return this._added; if(this.watching[content_type] === undefined){
this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type);
} }
/*public set added(diff : FileDiff[]) { const watcher = new CommonDiffWatcher(path);
this._added = diff; this.watching[content_type].register(watcher);
}*/ const initial_doc = await this.doc_cntr.findByPath(path);
public get deleted(): ContentFile[]{ await watcher.setup(initial_doc.map(x=>x.filename));
return this._deleted; watcher.watch();
} }
/*public set deleted(diff : FileDiff[]){ async commit(type:string,path:string){
this._deleted = diff; const list = this.watching[type].waiting_list;
}*/ const c = list.getPath(path);
public get path(){ if(c===undefined){
return this._path; throw new Error("path is not exist");
} }
public get type(){ await list.delete(c);
return this._type; const body = await c.createDocumentBody();
const id = await this.doc_cntr.add(body);
return id;
} }
private createCR(filename: string){ getAdded(){
return createContentReferrer(`${this.path}/${filename}`,this.type); return Object.keys(this.watching).map(x=>({
} type:x,
/** value:this.watching[x].waiting_list.getAll(),
* }));
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
this._added = added.map(x=>this.createCR(x));
this._deleted = deleted.map(x=>this.createCR(x));
watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this._added.push(this.createCR(filename));
}
else{
//added has one
if(this._added.map(x=>x.path).includes(path.join(this.path,filename))){
this._added = this._added.filter(x=> x.path !== path.join(this.path,filename));
}
else {
this._deleted.push(this.createCR(filename));
}
}
}
});
}
}
export class DiffWatcher{
Watchers: {[basepath:string]:Watcher} = {};
} }
};

2
src/diff/mod.ts Normal file
View File

@ -0,0 +1,2 @@
export * from './router';
export * from './diff';

61
src/diff/router.ts Normal file
View File

@ -0,0 +1,61 @@
import Koa from 'koa';
import Router from 'koa-router';
import { ContentFile } from '../content/mod';
import { sendError } from '../route/error_handler';
import {DiffManager} from './diff';
function content_file_to_return(x:ContentFile){
return {path:x.path,type:x.type};
}
export const getAdded = (diffmgr:DiffManager)=> (ctx:Koa.Context,next:Koa.Next)=>{
const ret = diffmgr.getAdded();
ctx.body = ret.map(x=>({
type:x.type,
value:x.value.map(x=>({path:x.path,type:x.type})),
}));
ctx.type = 'json';
}
type PostAddedBody = {
type:string,
path:string,
}[];
function checkPostAddedBody(body: any): body is PostAddedBody{
if(body instanceof Array){
return body.map(x=> 'type' in x && 'path' in x).every(x=>x);
}
return false;
}
export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next)=>{
const reqbody = ctx.request.body;
console.log(reqbody);
if(!checkPostAddedBody(reqbody)){
sendError(400,"format exception");
return;
}
const allWork = reqbody.map(op=>diffmgr.commit(op.type,op.path));
const results = await Promise.all(allWork);
ctx.body = {
ok:true,
docs:results,
}
ctx.type = 'json';
}
/*
export const getNotWatched = (diffmgr : DiffManager)=> (ctx:Router.IRouterContext,next:Koa.Next)=>{
ctx.body = {
added: diffmgr.added.map(content_file_to_return),
deleted: diffmgr.deleted.map(content_file_to_return),
};
ctx.type = 'json';
}*/
export function createDiffRouter(diffmgr: DiffManager){
const ret = new Router();
ret.get("/list",getAdded(diffmgr));
ret.post("/commit",postAdded(diffmgr));
return ret;
}

78
src/diff/watcher.ts Normal file
View File

@ -0,0 +1,78 @@
import { FSWatcher, watch } from 'fs';
import { promises } from 'fs';
import event from 'events';
const readdir = promises.readdir;
interface DiffWatcherEvent{
'create':(filename:string)=>void,
'delete':(filename:string)=>void,
'change':(prev_filename:string,cur_filename:string)=>void,
}
export interface IDiffWatcher extends event.EventEmitter {
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this;
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean;
readonly path: string;
}
export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
private _path:string;
private _watcher: FSWatcher|null;
constructor(path:string){
super();
this._path = path;
this._watcher = null;
}
public get path(){
return this._path;
}
/**
* setup
* @argument initial_filenames filename in path
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//Todo : reduce O(nm) to O(n+m) using hash map.
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
for (const iterator of added) {
this.emit('create',iterator);
}
for (const iterator of deleted){
this.emit('delete',iterator);
}
}
watch():FSWatcher{
this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this.emit('create',filename);
}
else{
this.emit('delete',filename)
}
}
});
return this._watcher;
}
watchClose(){
this._watcher?.close()
}
}

2
src/login.ts
View File

@ -5,7 +5,7 @@ import { sendError } from "./route/error_handler";
import Knex from "knex"; import Knex from "knex";
import { createKnexUserController } from "./db/mod"; import { createKnexUserController } from "./db/mod";
import { request } from "http"; import { request } from "http";
import { get_setting } from "./setting"; import { get_setting } from "./SettingConfig";
import { IUser, UserAccessor } from "./model/mod"; import { IUser, UserAccessor } from "./model/mod";
type PayloadInfo = { type PayloadInfo = {

18
src/model/doc.ts
View File

@ -1,17 +1,12 @@
import {TagAccessor} from './tag'; import {TagAccessor} from './tag';
import {check_type} from '../util/type_check' import {check_type} from '../util/type_check'
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;
export interface DocumentBody{ export interface DocumentBody{
title : string, title : string,
content_type : string, content_type : string,
basepath : string, basepath : string,
filename : string, filename : string,
content_hash? : string, content_hash : string,
additional : JSONMap, additional : JSONMap,
tags : string[],//eager loading tags : string[],//eager loading
} }
@ -32,6 +27,8 @@ export const isDocBody = (c : any):c is DocumentBody =>{
export interface Document extends DocumentBody{ export interface Document extends DocumentBody{
readonly id: number; readonly id: number;
readonly created_at:number;
readonly deleted_at:number|null;
}; };
export const isDoc = (c: any):c is Document =>{ export const isDoc = (c: any):c is Document =>{
@ -88,9 +85,14 @@ export interface DocumentAccessor{
*/ */
findById: (id:number,tagload?:boolean)=> Promise<Document| undefined>, findById: (id:number,tagload?:boolean)=> Promise<Document| undefined>,
/** /**
* * find by base path and filename.
* if you call this function with filename, its return array length is 0 or 1.
*/ */
findListByBasePath:(basepath: string)=>Promise<Document[]>; findByPath:(basepath: string,filename?:string)=>Promise<Document[]>;
/**
* find deleted content
*/
findDeleted:(content_type:string)=>Promise<Document[]>;
/** /**
* update document except tag. * update document except tag.
*/ */

6
src/route/contents.ts
View File

@ -71,7 +71,7 @@ const UpdateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
ctx.body = JSON.stringify(success); ctx.body = JSON.stringify(success);
ctx.type = 'json'; ctx.type = 'json';
} }
const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => { /*const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => {
const content_desc = ctx.request.body; const content_desc = ctx.request.body;
if(!isDocBody(content_desc)){ if(!isDocBody(content_desc)){
return sendError(400,"it is not a valid format"); return sendError(400,"it is not a valid format");
@ -79,7 +79,7 @@ const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
const id = await controller.add(content_desc); const id = await controller.add(content_desc);
ctx.body = JSON.stringify(id); ctx.body = JSON.stringify(id);
ctx.type = 'json'; ctx.type = 'json';
}; };*/
const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{ const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{
let tag_name = ctx.params['tag']; let tag_name = ctx.params['tag'];
const num = Number.parseInt(ctx.params['num']); const num = Number.parseInt(ctx.params['num']);
@ -137,7 +137,7 @@ export const getContentRouter = (controller: DocumentAccessor)=>{
ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller)); ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller));
ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller)); ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller));
//ret.use("/:num(\\d+)/:content_type"); //ret.use("/:num(\\d+)/:content_type");
ret.post("/",AdminOnly,CreateContentHandler(controller)); //ret.post("/",AdminOnly,CreateContentHandler(controller));
ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller)); ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller));
ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller)); ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller));
ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller)); ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller));

2
src/route/context.ts
View File

@ -1,5 +1,3 @@
import {ContentReferrer} from '../content/mod';
export type ContentLocation = { export type ContentLocation = {
path:string, path:string,
type:string, type:string,

20
src/server.ts
View File

@ -1,25 +1,30 @@
import Koa from 'koa'; import Koa from 'koa';
import Router from 'koa-router'; import Router from 'koa-router';
import {get_setting} from './setting'; import {get_setting} from './SettingConfig';
import {connectDB} from './database'; import {connectDB} from './database';
import {Watcher} from './diff/diff' import {DiffManager, createDiffRouter} from './diff/mod';
import { createReadStream, readFileSync } from 'fs'; import { createReadStream, readFileSync } from 'fs';
import getContentRouter from './route/contents'; import getContentRouter from './route/contents';
import { createKnexContentsAccessor } from './db/doc'; import { createKnexDocumentAccessor } from './db/mod';
import bodyparser from 'koa-bodyparser'; import bodyparser from 'koa-bodyparser';
import {error_handler} from './route/error_handler'; import {error_handler} from './route/error_handler';
import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login'; import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login';
import {createInterface as createReadlineInterface} from 'readline'; import {createInterface as createReadlineInterface} from 'readline';
//let Koa = require("koa"); //let Koa = require("koa");
export async function create_server(){ export async function create_server(){
let setting = get_setting(); const setting = get_setting();
let db = await connectDB(); let db = await connectDB();
let diffmgr = new DiffManager(createKnexDocumentAccessor(db));
let diff_router = createDiffRouter(diffmgr);
diffmgr.register("manga","testdata");
if(setting.cli){ if(setting.cli){
const userAdmin = await getAdmin(db); const userAdmin = await getAdmin(db);
if(await isAdminFirst(userAdmin)){ if(await isAdminFirst(userAdmin)){
@ -38,9 +43,12 @@ export async function create_server(){
app.use(createUserMiddleWare(db)); app.use(createUserMiddleWare(db));
//app.use(ctx=>{ctx.state['setting'] = settings}); //app.use(ctx=>{ctx.state['setting'] = settings});
const index_html = readFileSync("index.html"); const index_html = readFileSync("index.html");
let router = new Router(); let router = new Router();
router.use('/api/diff',diff_router.routes());
router.use('/api/diff',diff_router.allowedMethods());
//let watcher = new Watcher(setting.path[0]); //let watcher = new Watcher(setting.path[0]);
//await watcher.setup([]); //await watcher.setup([]);
@ -63,7 +71,7 @@ export async function create_server(){
if(setting.mode === "development") if(setting.mode === "development")
static_file_server('dist/js/bundle.js.map','text'); static_file_server('dist/js/bundle.js.map','text');
const content_router = getContentRouter(createKnexContentsAccessor(db)); const content_router = getContentRouter(createKnexDocumentAccessor(db));
router.use('/api/doc',content_router.routes()); router.use('/api/doc',content_router.routes());
router.use('/api/doc',content_router.allowedMethods()); router.use('/api/doc',content_router.allowedMethods());

6
src/types/db.d.ts vendored
View File

@ -17,8 +17,10 @@ declare module "knex" {
content_type: string; content_type: string;
basepath: string; basepath: string;
filename: string; filename: string;
content_hash?: string; created_at: number;
additional?: string; deleted_at: number|null;
content_hash: string;
additional: string|null;
}; };
doc_tag_relation: { doc_tag_relation: {
doc_id: number; doc_id: number;

5
src/types/json.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;

10
test.ts Normal file
View File

@ -0,0 +1,10 @@
import Knex from 'knex';
import {connectDB} from './src/database';
async function main() {
const db = await connectDB();
const query = db.update({deleted_at: null}).from('document');
console.log(query.toSQL());
}
main()

1
tsconfig.json
View File

@ -49,6 +49,7 @@
// "types": [], /* Type declaration files to be included in compilation. */ // "types": [], /* Type declaration files to be included in compilation. */
"allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
"resolveJsonModule": true,
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */