mirror of
https://github.com/langgenius/dify.git
synced 2026-10-04 13:47:55 +08:00
refactor(cli): add kvstore and platform interface (#36687)
This commit is contained in:
1 parent
bee21c9f86
commit
a8d380bcaf
95 files changed
+1035
-579
No files matched your search
Vendored
+25
-18
@@ -2,9 +2,17 @@ import type { AppMeta } from '../types/app-meta.js'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import yaml from 'js-yaml'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { CACHE_APP_INFO, cachePath } from '../store/manager.js'
|
||||
import { YamlStore } from '../store/store.js'
|
||||
import { platform } from '../sys/index.js'
|
||||
import { FieldInfo, FieldParameters } from '../types/app-meta.js'
|
||||
import { APP_INFO_TTL_MS, cachePath, loadAppInfoCache } from './app-info.js'
|
||||
import { APP_INFO_TTL_MS, loadAppInfoCache } from './app-info.js'
|
||||
|
||||
function appInfoPath(dir: string): string {
|
||||
return cachePath(dir, CACHE_APP_INFO)
|
||||
}
|
||||
|
||||
function metaInfoOnly(): AppMeta {
|
||||
return {
|
||||
@@ -35,10 +43,10 @@ describe('app-info disk cache', () => {
|
||||
})
|
||||
|
||||
it('round-trips an entry across reloads', async () => {
|
||||
const c1 = await loadAppInfoCache({ configDir: dir })
|
||||
const c1 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
await c1.set('http://localhost:9999', 'app-1', metaInfoOnly())
|
||||
|
||||
const c2 = await loadAppInfoCache({ configDir: dir })
|
||||
const c2 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
const got = c2.get('http://localhost:9999', 'app-1')
|
||||
expect(got).toBeDefined()
|
||||
expect(got?.meta.info?.id).toBe('app-1')
|
||||
@@ -47,7 +55,7 @@ describe('app-info disk cache', () => {
|
||||
|
||||
it('isFresh respects TTL', async () => {
|
||||
const now = new Date('2026-05-09T00:00:00Z')
|
||||
const c = await loadAppInfoCache({ configDir: dir, now: () => now })
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)), now: () => now })
|
||||
await c.set('h', 'app-1', metaInfoOnly())
|
||||
const r = c.get('h', 'app-1')
|
||||
expect(r).toBeDefined()
|
||||
@@ -58,45 +66,44 @@ describe('app-info disk cache', () => {
|
||||
})
|
||||
|
||||
it('keys by (host, app_id) — different hosts isolate', async () => {
|
||||
const c = await loadAppInfoCache({ configDir: dir })
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
await c.set('h1', 'app-1', metaInfoOnly())
|
||||
expect(c.get('h2', 'app-1')).toBeUndefined()
|
||||
expect(c.get('h1', 'app-1')).toBeDefined()
|
||||
})
|
||||
|
||||
it('delete removes entry from disk', async () => {
|
||||
const c1 = await loadAppInfoCache({ configDir: dir })
|
||||
const c1 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
await c1.set('h', 'app-1', metaInfoOnly())
|
||||
await c1.delete('h', 'app-1')
|
||||
|
||||
const c2 = await loadAppInfoCache({ configDir: dir })
|
||||
const c2 = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
expect(c2.get('h', 'app-1')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('writes file with 0600 permission', async () => {
|
||||
const c = await loadAppInfoCache({ configDir: dir })
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
await c.set('h', 'app-1', metaInfoOnly())
|
||||
const { stat } = await import('node:fs/promises')
|
||||
const s = await stat(cachePath(dir))
|
||||
if (process.platform !== 'win32')
|
||||
const s = await stat(appInfoPath(dir))
|
||||
if (platform() !== 'win32')
|
||||
expect(s.mode & 0o777).toBe(0o600)
|
||||
})
|
||||
|
||||
it('missing cache file is not an error', async () => {
|
||||
const c = await loadAppInfoCache({ configDir: dir })
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
expect(c.get('h', 'app-1')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('corrupt cache file is treated as empty', async () => {
|
||||
const { mkdir, writeFile } = await import('node:fs/promises')
|
||||
await mkdir(join(dir, 'cache'), { recursive: true })
|
||||
await writeFile(cachePath(dir), '{not json', 'utf8')
|
||||
const c = await loadAppInfoCache({ configDir: dir })
|
||||
const { writeFile } = await import('node:fs/promises')
|
||||
await writeFile(appInfoPath(dir), ': : not valid yaml', 'utf8')
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
expect(c.get('h', 'app-1')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('updates same key in place (no growth)', async () => {
|
||||
const c = await loadAppInfoCache({ configDir: dir })
|
||||
const c = await loadAppInfoCache({ store: new YamlStore(cachePath(dir, CACHE_APP_INFO)) })
|
||||
await c.set('h', 'app-1', metaInfoOnly())
|
||||
const slim: AppMeta = {
|
||||
...metaInfoOnly(),
|
||||
@@ -104,8 +111,8 @@ describe('app-info disk cache', () => {
|
||||
parameters: { opening_statement: 'hi' },
|
||||
}
|
||||
await c.set('h', 'app-1', slim)
|
||||
const raw = await readFile(cachePath(dir), 'utf8')
|
||||
const parsed = JSON.parse(raw) as { entries: Record<string, unknown> }
|
||||
const raw = await readFile(appInfoPath(dir), 'utf8')
|
||||
const parsed = yaml.load(raw) as { entries: Record<string, unknown> }
|
||||
expect(Object.keys(parsed.entries)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
Vendored
+24
-44
@@ -1,15 +1,14 @@
|
||||
import type { Store } from '../store/store.js'
|
||||
import type { AppMeta, AppMetaCacheRecord, AppMetaFieldKey } from '../types/app-meta.js'
|
||||
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { DIR_PERM, FILE_PERM } from '../config/dir.js'
|
||||
import { CACHE_APP_INFO, getCache } from '../store/manager.js'
|
||||
import { FieldInfo, FieldInputSchema, FieldParameters } from '../types/app-meta.js'
|
||||
|
||||
const CACHE_FILE = 'app-info.json'
|
||||
export const APP_INFO_TTL_MS = 60 * 60 * 1000
|
||||
|
||||
type DiskShape = {
|
||||
entries: Record<string, DiskEntry>
|
||||
}
|
||||
// All entries live under one top-level key; the inner record uses
|
||||
// `host::appId` composites that contain `::` (never `.`), so they're
|
||||
// safe as map keys without colliding with Store's dot-path semantics.
|
||||
const ENTRIES_KEY = { key: 'entries', default: {} as Record<string, DiskEntry> } as const
|
||||
|
||||
type DiskEntry = {
|
||||
meta: SerializedMeta
|
||||
@@ -35,26 +34,25 @@ type State = {
|
||||
}
|
||||
|
||||
export type AppInfoCacheOptions = {
|
||||
readonly configDir: string
|
||||
readonly store?: Store
|
||||
readonly ttlMs?: number
|
||||
readonly now?: () => Date
|
||||
}
|
||||
|
||||
export async function loadAppInfoCache(opts: AppInfoCacheOptions): Promise<AppInfoCache> {
|
||||
const path = cachePath(opts.configDir)
|
||||
export async function loadAppInfoCache(opts: AppInfoCacheOptions = {}): Promise<AppInfoCache> {
|
||||
const store = opts.store ?? getCache(CACHE_APP_INFO)
|
||||
const ttlMs = opts.ttlMs ?? APP_INFO_TTL_MS
|
||||
const state: State = { entries: new Map() }
|
||||
await readDisk(path, state)
|
||||
const state: State = { entries: readEntries(store) }
|
||||
return {
|
||||
get: (host, appId) => state.entries.get(key(host, appId)),
|
||||
set: async (host, appId, meta) => {
|
||||
const record: AppMetaCacheRecord = { meta, fetchedAt: (opts.now ?? (() => new Date()))().toISOString() }
|
||||
state.entries.set(key(host, appId), record)
|
||||
await persist(path, state)
|
||||
writeEntries(store, state.entries)
|
||||
},
|
||||
delete: async (host, appId) => {
|
||||
state.entries.delete(key(host, appId))
|
||||
await persist(path, state)
|
||||
writeEntries(store, state.entries)
|
||||
},
|
||||
isFresh: (record, now) => {
|
||||
const t = (now ?? new Date()).getTime() - new Date(record.fetchedAt).getTime()
|
||||
@@ -63,36 +61,22 @@ export async function loadAppInfoCache(opts: AppInfoCacheOptions): Promise<AppIn
|
||||
}
|
||||
}
|
||||
|
||||
export function cachePath(configDir: string): string {
|
||||
return join(configDir, 'cache', CACHE_FILE)
|
||||
}
|
||||
|
||||
function key(host: string, appId: string): string {
|
||||
return `${host}::${appId}`
|
||||
}
|
||||
|
||||
async function readDisk(path: string, state: State): Promise<void> {
|
||||
let raw: string
|
||||
function readEntries(store: Store): Map<string, AppMetaCacheRecord> {
|
||||
const out = new Map<string, AppMetaCacheRecord>()
|
||||
let raw: Record<string, DiskEntry>
|
||||
try {
|
||||
raw = await readFile(path, 'utf8')
|
||||
}
|
||||
catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT')
|
||||
return
|
||||
throw err
|
||||
}
|
||||
let parsed: DiskShape
|
||||
try {
|
||||
parsed = JSON.parse(raw) as DiskShape
|
||||
raw = store.get(ENTRIES_KEY)
|
||||
}
|
||||
catch {
|
||||
return
|
||||
}
|
||||
if (parsed.entries === undefined)
|
||||
return
|
||||
for (const [k, e] of Object.entries(parsed.entries)) {
|
||||
state.entries.set(k, deserialize(e))
|
||||
return out
|
||||
}
|
||||
for (const [k, e] of Object.entries(raw))
|
||||
out.set(k, deserialize(e))
|
||||
return out
|
||||
}
|
||||
|
||||
function deserialize(e: DiskEntry): AppMetaCacheRecord {
|
||||
@@ -127,12 +111,8 @@ function serialize(record: AppMetaCacheRecord): DiskEntry {
|
||||
}
|
||||
}
|
||||
|
||||
async function persist(path: string, state: State): Promise<void> {
|
||||
const dir = dirname(path)
|
||||
await mkdir(dir, { recursive: true, mode: DIR_PERM })
|
||||
const disk: DiskShape = { entries: {} }
|
||||
for (const [k, v] of state.entries) disk.entries[k] = serialize(v)
|
||||
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`
|
||||
await writeFile(tmp, JSON.stringify(disk), { mode: FILE_PERM })
|
||||
await rename(tmp, path)
|
||||
function writeEntries(store: Store, entries: Map<string, AppMetaCacheRecord>): void {
|
||||
const out: Record<string, DiskEntry> = {}
|
||||
for (const [k, v] of entries) out[k] = serialize(v)
|
||||
store.set(ENTRIES_KEY, out)
|
||||
}
|
||||
Vendored
+20
-21
@@ -1,8 +1,15 @@
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import yaml from 'js-yaml'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { loadNudgeStore, nudgeStorePath, WARN_INTERVAL_MS } from './nudge-store.js'
|
||||
import { CACHE_NUDGE, cachePath } from '../store/manager.js'
|
||||
import { YamlStore } from '../store/store.js'
|
||||
import { loadNudgeStore, WARN_INTERVAL_MS } from './nudge-store.js'
|
||||
|
||||
function nudgeStorePath(dir: string): string {
|
||||
return cachePath(dir, CACHE_NUDGE)
|
||||
}
|
||||
|
||||
const HOST = 'https://cloud.dify.ai'
|
||||
|
||||
@@ -16,13 +23,13 @@ describe('NudgeStore', () => {
|
||||
})
|
||||
|
||||
it('canWarn=true when no prior record exists', async () => {
|
||||
const store = await loadNudgeStore({ configDir: dir })
|
||||
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)) })
|
||||
expect(store.canWarn(HOST)).toBe(true)
|
||||
})
|
||||
|
||||
it('canWarn=false within the silence window, true past it', async () => {
|
||||
const t0 = new Date('2026-05-19T12:00:00.000Z')
|
||||
const store = await loadNudgeStore({ configDir: dir, now: () => t0 })
|
||||
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
|
||||
await store.markWarned(HOST)
|
||||
expect(store.canWarn(HOST, new Date('2026-05-19T18:00:00.000Z'))).toBe(false)
|
||||
expect(store.canWarn(HOST, new Date('2026-05-20T12:00:00.000Z'))).toBe(true)
|
||||
@@ -30,7 +37,7 @@ describe('NudgeStore', () => {
|
||||
|
||||
it('canWarn clamps negative elapsed under clock skew (treats as still in window)', async () => {
|
||||
const t0 = new Date('2026-05-19T12:00:00.000Z')
|
||||
const store = await loadNudgeStore({ configDir: dir, now: () => t0 })
|
||||
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
|
||||
await store.markWarned(HOST)
|
||||
const pastClock = new Date('2026-05-19T11:00:00.000Z') // clock moved backwards 1h
|
||||
expect(store.canWarn(HOST, pastClock)).toBe(false)
|
||||
@@ -38,33 +45,25 @@ describe('NudgeStore', () => {
|
||||
|
||||
it('markWarned persists across store reloads', async () => {
|
||||
const t0 = new Date('2026-05-19T12:00:00.000Z')
|
||||
const s1 = await loadNudgeStore({ configDir: dir, now: () => t0 })
|
||||
const s1 = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
|
||||
await s1.markWarned(HOST)
|
||||
const s2 = await loadNudgeStore({ configDir: dir, now: () => t0 })
|
||||
const s2 = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t0 })
|
||||
expect(s2.canWarn(HOST)).toBe(false)
|
||||
})
|
||||
|
||||
it('treats a corrupt cache file as empty', async () => {
|
||||
const path = nudgeStorePath(dir)
|
||||
await writeCacheFile(path, '{ not valid json')
|
||||
const store = await loadNudgeStore({ configDir: dir })
|
||||
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)) })
|
||||
expect(store.canWarn(HOST)).toBe(true)
|
||||
})
|
||||
|
||||
it('ignores file with mismatched schema', async () => {
|
||||
const path = nudgeStorePath(dir)
|
||||
await writeCacheFile(path, JSON.stringify({ schema: 99, warned: { [HOST]: '2026-05-19T12:00:00.000Z' } }))
|
||||
const store = await loadNudgeStore({ configDir: dir })
|
||||
expect(store.canWarn(HOST)).toBe(true)
|
||||
})
|
||||
|
||||
it('writes ISO timestamps under schema:1/warned on disk', async () => {
|
||||
it('writes ISO timestamps under warned/<host> on disk', async () => {
|
||||
const t = new Date('2026-05-19T12:00:00.000Z')
|
||||
const store = await loadNudgeStore({ configDir: dir, now: () => t })
|
||||
const store = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
|
||||
await store.markWarned(HOST)
|
||||
const raw = await readFile(nudgeStorePath(dir), 'utf8')
|
||||
const parsed = JSON.parse(raw) as Record<string, unknown>
|
||||
expect(parsed.schema).toBe(1)
|
||||
const parsed = yaml.load(raw) as Record<string, unknown>
|
||||
expect((parsed.warned as Record<string, string>)[HOST]).toBe(t.toISOString())
|
||||
})
|
||||
|
||||
@@ -73,11 +72,11 @@ describe('NudgeStore', () => {
|
||||
// warns about a different host. Without merge-on-write the second writer
|
||||
// would clobber the first.
|
||||
const t = new Date('2026-05-19T12:00:00.000Z')
|
||||
const a = await loadNudgeStore({ configDir: dir, now: () => t })
|
||||
const b = await loadNudgeStore({ configDir: dir, now: () => t })
|
||||
const a = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
|
||||
const b = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
|
||||
await a.markWarned('https://a.example')
|
||||
await b.markWarned('https://b.example')
|
||||
const reread = await loadNudgeStore({ configDir: dir, now: () => t })
|
||||
const reread = await loadNudgeStore({ store: new YamlStore(cachePath(dir, CACHE_NUDGE)), now: () => t })
|
||||
expect(reread.canWarn('https://a.example')).toBe(false)
|
||||
expect(reread.canWarn('https://b.example')).toBe(false)
|
||||
})
|
||||
|
||||
Vendored
+21
-46
@@ -1,37 +1,29 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { DIR_PERM, FILE_PERM } from '../config/dir.js'
|
||||
import type { Store } from '../store/store.js'
|
||||
import { CACHE_NUDGE, getCache } from '../store/manager.js'
|
||||
|
||||
const CACHE_FILE = 'nudge.json'
|
||||
const DISK_SCHEMA = 1
|
||||
export const WARN_INTERVAL_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
// Single top-level key holding host→ISO map. Hosts contain dots
|
||||
// (cloud.dify.ai), so we cannot use them as Store paths directly —
|
||||
// `doSet` would split on dots and create nested objects.
|
||||
const WARNED_KEY = { key: 'warned', default: {} as Record<string, string> } as const
|
||||
|
||||
export type NudgeStore = {
|
||||
readonly canWarn: (host: string, now?: Date) => boolean
|
||||
readonly markWarned: (host: string, now?: Date) => Promise<void>
|
||||
}
|
||||
|
||||
export type NudgeStoreOptions = {
|
||||
readonly configDir: string
|
||||
readonly store?: Store
|
||||
readonly now?: () => Date
|
||||
readonly intervalMs?: number
|
||||
}
|
||||
|
||||
type DiskShape = {
|
||||
schema?: number
|
||||
warned?: Record<string, string>
|
||||
}
|
||||
|
||||
export function nudgeStorePath(configDir: string): string {
|
||||
return join(configDir, 'cache', CACHE_FILE)
|
||||
}
|
||||
|
||||
export async function loadNudgeStore(opts: NudgeStoreOptions): Promise<NudgeStore> {
|
||||
const path = nudgeStorePath(opts.configDir)
|
||||
export async function loadNudgeStore(opts: NudgeStoreOptions = {}): Promise<NudgeStore> {
|
||||
const store = opts.store ?? getCache(CACHE_NUDGE)
|
||||
const intervalMs = opts.intervalMs ?? WARN_INTERVAL_MS
|
||||
const clock = opts.now ?? (() => new Date())
|
||||
const memory = await readDisk(path)
|
||||
const memory = readWarned(store)
|
||||
|
||||
return {
|
||||
canWarn: (host, now) => {
|
||||
@@ -47,34 +39,23 @@ export async function loadNudgeStore(opts: NudgeStoreOptions): Promise<NudgeStor
|
||||
// Re-read disk inside the write cycle so concurrent processes touching
|
||||
// different hosts don't clobber each other's stamps. Same-host writers
|
||||
// converge on a near-identical timestamp, so order doesn't matter.
|
||||
const onDisk = await readDisk(path)
|
||||
const onDisk = readWarned(store)
|
||||
onDisk.set(host, stamp)
|
||||
await persist(path, onDisk)
|
||||
writeWarned(store, onDisk)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function readDisk(path: string): Promise<Map<string, number>> {
|
||||
function readWarned(store: Store): Map<string, number> {
|
||||
const out = new Map<string, number>()
|
||||
let raw: string
|
||||
let raw: Record<string, string>
|
||||
try {
|
||||
raw = await readFile(path, 'utf8')
|
||||
}
|
||||
catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT')
|
||||
return out
|
||||
throw err
|
||||
}
|
||||
let parsed: DiskShape
|
||||
try {
|
||||
parsed = JSON.parse(raw) as DiskShape
|
||||
raw = store.get(WARNED_KEY)
|
||||
}
|
||||
catch {
|
||||
return out
|
||||
}
|
||||
if (parsed.schema !== DISK_SCHEMA || parsed.warned === undefined)
|
||||
return out
|
||||
for (const [host, iso] of Object.entries(parsed.warned)) {
|
||||
for (const [host, iso] of Object.entries(raw)) {
|
||||
const t = Date.parse(iso)
|
||||
if (!Number.isNaN(t))
|
||||
out.set(host, t)
|
||||
@@ -82,15 +63,9 @@ async function readDisk(path: string): Promise<Map<string, number>> {
|
||||
return out
|
||||
}
|
||||
|
||||
async function persist(path: string, state: Map<string, number>): Promise<void> {
|
||||
const dir = dirname(path)
|
||||
await mkdir(dir, { recursive: true, mode: DIR_PERM })
|
||||
const disk: DiskShape = { schema: DISK_SCHEMA, warned: {} }
|
||||
function writeWarned(store: Store, state: Map<string, number>): void {
|
||||
const warned: Record<string, string> = {}
|
||||
for (const [host, t] of state)
|
||||
disk.warned![host] = new Date(t).toISOString()
|
||||
// randomUUID is collision-proof even when two writers stamp the same
|
||||
// millisecond — pid+timestamp alone can still collide under tight loops.
|
||||
const tmp = `${path}.${randomUUID()}.tmp`
|
||||
await writeFile(tmp, JSON.stringify(disk), { mode: FILE_PERM })
|
||||
await rename(tmp, path)
|
||||
warned[host] = new Date(t).toISOString()
|
||||
store.set(WARNED_KEY, warned)
|
||||
}
|
||||
Reference in new issue
Block a user