refactor(cli): add kvstore and platform interface (#36687)

This commit is contained in:
Yunlu Wen authored and GitHub committed 2026-05-27 05:30:12 +00:00
1 parent bee21c9f86
commit a8d380bcaf
95 files changed
+1035 -579

No files matched your search

+25 -18
View File
@@ -2,9 +2,17 @@ import type { AppMeta } from '../types/app-meta.js'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import yaml from 'js-yaml'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { CACHE_APP_INFO, cachePath } from '../store/manager.js'
import { YamlStore } from '../store/store.js'
import { platform } from '../sys/index.js'
import { FieldInfo, FieldParameters } from '../types/app-meta.js'
import { APP_INFO_TTL_MS, cachePath, loadAppInfoCache } from './app-info.js'
import { APP_INFO_TTL_MS, loadAppInfoCache } from './app-info.js'
function appInfoPath(dir: string): string {
return cachePath(dir, CACHE_APP_INFO)
}
function metaInfoOnly(): AppMeta {
return {
@@ -35,10 +43,10 @@ describe('app-info disk cache', () => {
})
it('round-trips an entry across reloads', async () => {
const c1 = await loadAppInfoCache({ configDir: dir })
const c1 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
await c1.set('http://localhost:9999', 'app-1', metaInfoOnly())
const c2 = await loadAppInfoCache({ configDir: dir })
const c2 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
const got = c2.get('http://localhost:9999', 'app-1')
expect(got).toBeDefined()
expect(got?.meta.info?.id).toBe('app-1')
@@ -47,7 +55,7 @@ describe('app-info disk cache', () => {
it('isFresh respects TTL', async () => {
const now = new Date('2026-05-09T00:00:00Z')
const c = await loadAppInfoCache({ configDir: dir, now: () => now })
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)), now: () => now })
await c.set('h', 'app-1', metaInfoOnly())
const r = c.get('h', 'app-1')
expect(r).toBeDefined()
@@ -58,45 +66,44 @@ describe('app-info disk cache', () => {
})
it('keys by (host, app_id) — different hosts isolate', async () => {
const c = await loadAppInfoCache({ configDir: dir })
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
await c.set('h1', 'app-1', metaInfoOnly())
expect(c.get('h2', 'app-1')).toBeUndefined()
expect(c.get('h1', 'app-1')).toBeDefined()
})
it('delete removes entry from disk', async () => {
const c1 = await loadAppInfoCache({ configDir: dir })
const c1 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
await c1.set('h', 'app-1', metaInfoOnly())
await c1.delete('h', 'app-1')
const c2 = await loadAppInfoCache({ configDir: dir })
const c2 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
expect(c2.get('h', 'app-1')).toBeUndefined()
})
it('writes file with 0600 permission', async () => {
const c = await loadAppInfoCache({ configDir: dir })
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
await c.set('h', 'app-1', metaInfoOnly())
const { stat } = await import('node:fs/promises')
const s = await stat(cachePath(dir))
if (process.platform !== 'win32')
const s = await stat(appInfoPath(dir))
if (platform() !== 'win32')
expect(s.mode & 0o777).toBe(0o600)
})
it('missing cache file is not an error', async () => {
const c = await loadAppInfoCache({ configDir: dir })
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
expect(c.get('h', 'app-1')).toBeUndefined()
})
it('corrupt cache file is treated as empty', async () => {
const { mkdir, writeFile } = await import('node:fs/promises')
await mkdir(join(dir, 'cache'), { recursive: true })
await writeFile(cachePath(dir), '{not json', 'utf8')
const c = await loadAppInfoCache({ configDir: dir })
const { writeFile } = await import('node:fs/promises')
await writeFile(appInfoPath(dir), ': : not valid yaml', 'utf8')
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
expect(c.get('h', 'app-1')).toBeUndefined()
})
it('updates same key in place (no growth)', async () => {
const c = await loadAppInfoCache({ configDir: dir })
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
await c.set('h', 'app-1', metaInfoOnly())
const slim: AppMeta = {
...metaInfoOnly(),
@@ -104,8 +111,8 @@ describe('app-info disk cache', () => {
parameters: { opening_statement: 'hi' },
}
await c.set('h', 'app-1', slim)
const raw = await readFile(cachePath(dir), 'utf8')
const parsed = JSON.parse(raw) as { entries: Record<string, unknown> }
const raw = await readFile(appInfoPath(dir), 'utf8')
const parsed = yaml.load(raw) as { entries: Record<string, unknown> }
expect(Object.keys(parsed.entries)).toHaveLength(1)
})
})
+24 -44
View File
@@ -1,15 +1,14 @@
import type { Store } from '../store/store.js'
import type { AppMeta, AppMetaCacheRecord, AppMetaFieldKey } from '../types/app-meta.js'
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
import { dirname, join } from 'node:path'
import { DIR_PERM, FILE_PERM } from '../config/dir.js'
import { CACHE_APP_INFO, getCache } from '../store/manager.js'
import { FieldInfo, FieldInputSchema, FieldParameters } from '../types/app-meta.js'
const CACHE_FILE = 'app-info.json'
export const APP_INFO_TTL_MS = 60 * 60 * 1000
type DiskShape = {
entries: Record<string, DiskEntry>
}
// All entries live under one top-level key; the inner record uses
// `host::appId` composites that contain `::` (never `.`), so they're
// safe as map keys without colliding with Store's dot-path semantics.
const ENTRIES_KEY = { key: 'entries', default: {} as Record<string, DiskEntry> } as const
type DiskEntry = {
meta: SerializedMeta
@@ -35,26 +34,25 @@ type State = {
}
export type AppInfoCacheOptions = {
readonly configDir: string
readonly store?: Store
readonly ttlMs?: number
readonly now?: () => Date
}
export async function loadAppInfoCache(opts: AppInfoCacheOptions): Promise<AppInfoCache> {
const path = cachePath(opts.configDir)
export async function loadAppInfoCache(opts: AppInfoCacheOptions = {}): Promise<AppInfoCache> {
const store = opts.store ?? getCache(CACHE_APP_INFO)
const ttlMs = opts.ttlMs ?? APP_INFO_TTL_MS
const state: State = { entries: new Map() }
await readDisk(path, state)
const state: State = { entries: readEntries(store) }
return {
get: (host, appId) => state.entries.get(key(host, appId)),
set: async (host, appId, meta) => {
const record: AppMetaCacheRecord = { meta, fetchedAt: (opts.now ?? (() => new Date()))().toISOString() }
state.entries.set(key(host, appId), record)
await persist(path, state)
writeEntries(store, state.entries)
},
delete: async (host, appId) => {
state.entries.delete(key(host, appId))
await persist(path, state)
writeEntries(store, state.entries)
},
isFresh: (record, now) => {
const t = (now ?? new Date()).getTime() - new Date(record.fetchedAt).getTime()
@@ -63,36 +61,22 @@ export async function loadAppInfoCache(opts: AppInfoCacheOptions): Promise<AppIn
}
}
export function cachePath(configDir: string): string {
return join(configDir, 'cache', CACHE_FILE)
}
function key(host: string, appId: string): string {
return `${host}::${appId}`
}
async function readDisk(path: string, state: State): Promise<void> {
let raw: string
function readEntries(store: Store): Map<string, AppMetaCacheRecord> {
const out = new Map<string, AppMetaCacheRecord>()
let raw: Record<string, DiskEntry>
try {
raw = await readFile(path, 'utf8')
}
catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT')
return
throw err
}
let parsed: DiskShape
try {
parsed = JSON.parse(raw) as DiskShape
raw = store.get(ENTRIES_KEY)
}
catch {
return
}
if (parsed.entries === undefined)
return
for (const [k, e] of Object.entries(parsed.entries)) {
state.entries.set(k, deserialize(e))
return out
}
for (const [k, e] of Object.entries(raw))
out.set(k, deserialize(e))
return out
}
function deserialize(e: DiskEntry): AppMetaCacheRecord {
@@ -127,12 +111,8 @@ function serialize(record: AppMetaCacheRecord): DiskEntry {
}
}
async function persist(path: string, state: State): Promise<void> {
const dir = dirname(path)
await mkdir(dir, { recursive: true, mode: DIR_PERM })
const disk: DiskShape = { entries: {} }
for (const [k, v] of state.entries) disk.entries[k] = serialize(v)
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`
await writeFile(tmp, JSON.stringify(disk), { mode: FILE_PERM })
await rename(tmp, path)
function writeEntries(store: Store, entries: Map<string, AppMetaCacheRecord>): void {
const out: Record<string, DiskEntry> = {}
for (const [k, v] of entries) out[k] = serialize(v)
store.set(ENTRIES_KEY, out)
}
+20 -21
View File
@@ -1,8 +1,15 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import yaml from 'js-yaml'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { loadNudgeStore, nudgeStorePath, WARN_INTERVAL_MS } from './nudge-store.js'
import { CACHE_NUDGE, cachePath } from '../store/manager.js'
import { YamlStore } from '../store/store.js'
import { loadNudgeStore, WARN_INTERVAL_MS } from './nudge-store.js'
function nudgeStorePath(dir: string): string {
return cachePath(dir, CACHE_NUDGE)
}
const HOST = 'https://cloud.dify.ai'
@@ -16,13 +23,13 @@ describe('NudgeStore', () => {
})
it('canWarn=true when no prior record exists', async () => {
const store = await loadNudgeStore({ configDir: dir })
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)) })
expect(store.canWarn(HOST)).toBe(true)
})
it('canWarn=false within the silence window, true past it', async () => {
const t0 = new Date('2026-05-19T12:00:00.000Z')
const store = await loadNudgeStore({ configDir: dir, now: () => t0 })
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
await store.markWarned(HOST)
expect(store.canWarn(HOST, new Date('2026-05-19T18:00:00.000Z'))).toBe(false)
expect(store.canWarn(HOST, new Date('2026-05-20T12:00:00.000Z'))).toBe(true)
@@ -30,7 +37,7 @@ describe('NudgeStore', () => {
it('canWarn clamps negative elapsed under clock skew (treats as still in window)', async () => {
const t0 = new Date('2026-05-19T12:00:00.000Z')
const store = await loadNudgeStore({ configDir: dir, now: () => t0 })
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
await store.markWarned(HOST)
const pastClock = new Date('2026-05-19T11:00:00.000Z') // clock moved backwards 1h
expect(store.canWarn(HOST, pastClock)).toBe(false)
@@ -38,33 +45,25 @@ describe('NudgeStore', () => {
it('markWarned persists across store reloads', async () => {
const t0 = new Date('2026-05-19T12:00:00.000Z')
const s1 = await loadNudgeStore({ configDir: dir, now: () => t0 })
const s1 = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
await s1.markWarned(HOST)
const s2 = await loadNudgeStore({ configDir: dir, now: () => t0 })
const s2 = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
expect(s2.canWarn(HOST)).toBe(false)
})
it('treats a corrupt cache file as empty', async () => {
const path = nudgeStorePath(dir)
await writeCacheFile(path, '{ not valid json')
const store = await loadNudgeStore({ configDir: dir })
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)) })
expect(store.canWarn(HOST)).toBe(true)
})
it('ignores file with mismatched schema', async () => {
const path = nudgeStorePath(dir)
await writeCacheFile(path, JSON.stringify({ schema: 99, warned: { [HOST]: '2026-05-19T12:00:00.000Z' } }))
const store = await loadNudgeStore({ configDir: dir })
expect(store.canWarn(HOST)).toBe(true)
})
it('writes ISO timestamps under schema:1/warned on disk', async () => {
it('writes ISO timestamps under warned/<host> on disk', async () => {
const t = new Date('2026-05-19T12:00:00.000Z')
const store = await loadNudgeStore({ configDir: dir, now: () => t })
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
await store.markWarned(HOST)
const raw = await readFile(nudgeStorePath(dir), 'utf8')
const parsed = JSON.parse(raw) as Record<string, unknown>
expect(parsed.schema).toBe(1)
const parsed = yaml.load(raw) as Record<string, unknown>
expect((parsed.warned as Record<string, string>)[HOST]).toBe(t.toISOString())
})
@@ -73,11 +72,11 @@ describe('NudgeStore', () => {
// warns about a different host. Without merge-on-write the second writer
// would clobber the first.
const t = new Date('2026-05-19T12:00:00.000Z')
const a = await loadNudgeStore({ configDir: dir, now: () => t })
const b = await loadNudgeStore({ configDir: dir, now: () => t })
const a = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
const b = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
await a.markWarned('https://a.example')
await b.markWarned('https://b.example')
const reread = await loadNudgeStore({ configDir: dir, now: () => t })
const reread = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
expect(reread.canWarn('https://a.example')).toBe(false)
expect(reread.canWarn('https://b.example')).toBe(false)
})
+21 -46
View File
@@ -1,37 +1,29 @@
import { randomUUID } from 'node:crypto'
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
import { dirname, join } from 'node:path'
import { DIR_PERM, FILE_PERM } from '../config/dir.js'
import type { Store } from '../store/store.js'
import { CACHE_NUDGE, getCache } from '../store/manager.js'
const CACHE_FILE = 'nudge.json'
const DISK_SCHEMA = 1
export const WARN_INTERVAL_MS = 24 * 60 * 60 * 1000
// Single top-level key holding host→ISO map. Hosts contain dots
// (cloud.dify.ai), so we cannot use them as Store paths directly —
// `doSet` would split on dots and create nested objects.
const WARNED_KEY = { key: 'warned', default: {} as Record<string, string> } as const
export type NudgeStore = {
readonly canWarn: (host: string, now?: Date) => boolean
readonly markWarned: (host: string, now?: Date) => Promise<void>
}
export type NudgeStoreOptions = {
readonly configDir: string
readonly store?: Store
readonly now?: () => Date
readonly intervalMs?: number
}
type DiskShape = {
schema?: number
warned?: Record<string, string>
}
export function nudgeStorePath(configDir: string): string {
return join(configDir, 'cache', CACHE_FILE)
}
export async function loadNudgeStore(opts: NudgeStoreOptions): Promise<NudgeStore> {
const path = nudgeStorePath(opts.configDir)
export async function loadNudgeStore(opts: NudgeStoreOptions = {}): Promise<NudgeStore> {
const store = opts.store ?? getCache(CACHE_NUDGE)
const intervalMs = opts.intervalMs ?? WARN_INTERVAL_MS
const clock = opts.now ?? (() => new Date())
const memory = await readDisk(path)
const memory = readWarned(store)
return {
canWarn: (host, now) => {
@@ -47,34 +39,23 @@ export async function loadNudgeStore(opts: NudgeStoreOptions): Promise<NudgeStor
// Re-read disk inside the write cycle so concurrent processes touching
// different hosts don't clobber each other's stamps. Same-host writers
// converge on a near-identical timestamp, so order doesn't matter.
const onDisk = await readDisk(path)
const onDisk = readWarned(store)
onDisk.set(host, stamp)
await persist(path, onDisk)
writeWarned(store, onDisk)
},
}
}
async function readDisk(path: string): Promise<Map<string, number>> {
function readWarned(store: Store): Map<string, number> {
const out = new Map<string, number>()
let raw: string
let raw: Record<string, string>
try {
raw = await readFile(path, 'utf8')
}
catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT')
return out
throw err
}
let parsed: DiskShape
try {
parsed = JSON.parse(raw) as DiskShape
raw = store.get(WARNED_KEY)
}
catch {
return out
}
if (parsed.schema !== DISK_SCHEMA || parsed.warned === undefined)
return out
for (const [host, iso] of Object.entries(parsed.warned)) {
for (const [host, iso] of Object.entries(raw)) {
const t = Date.parse(iso)
if (!Number.isNaN(t))
out.set(host, t)
@@ -82,15 +63,9 @@ async function readDisk(path: string): Promise<Map<string, number>> {
return out
}
async function persist(path: string, state: Map<string, number>): Promise<void> {
const dir = dirname(path)
await mkdir(dir, { recursive: true, mode: DIR_PERM })
const disk: DiskShape = { schema: DISK_SCHEMA, warned: {} }
function writeWarned(store: Store, state: Map<string, number>): void {
const warned: Record<string, string> = {}
for (const [host, t] of state)
disk.warned![host] = new Date(t).toISOString()
// randomUUID is collision-proof even when two writers stamp the same
// millisecond — pid+timestamp alone can still collide under tight loops.
const tmp = `${path}.${randomUUID()}.tmp`
await writeFile(tmp, JSON.stringify(disk), { mode: FILE_PERM })
await rename(tmp, path)
warned[host] = new Date(t).toISOString()
store.set(WARNED_KEY, warned)
}